<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
	"runtime"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/GeertJohan/go.rice"
	"github.com/bmizerany/pat"

	"github.com/drone/drone/pkg/build/docker"
	"github.com/drone/drone/pkg/channel"
	"github.com/drone/drone/pkg/database"
	"github.com/drone/drone/pkg/handler"
	"github.com/drone/drone/pkg/queue"
)

var (
	// local path where the SQLite database
	// should be stored. By default this is
	// in the current working directory.
	path string

	// port the server will run on
	port string

	// database driver used to connect to the database
	driver string

	// driver specific connection information. In this
	// case, it should be the location of the SQLite file
	datasource string

	// optional flags for tls listener
	sslcert string
	sslkey  string

	// build will timeout after N milliseconds.
	// this will default to 500 minutes (6 hours)
	timeout time.Duration

	// commit sha for the current build.
	version string
)

func main() {
	// parse command line flags
	flag.StringVar(&path, "path", "", "")
	flag.StringVar(&port, "port", ":8080", "")
	flag.StringVar(&driver, "driver", "sqlite3", "")
	flag.StringVar(&datasource, "datasource", "drone.sqlite", "")
	flag.StringVar(&sslcert, "sslcert", "", "")
	flag.StringVar(&sslkey, "sslkey", "", "")
	flag.DurationVar(&timeout, "timeout", 300*time.Minute, "")
	flag.Parse()

	// validate the TLS arguments
	checkTLSFlags()

	// setup database and handlers
	if err := database.Init(driver, datasource); err != nil {
		log.Fatal("Can't initialize database: ", err)
	}
	setupStatic()
	setupHandlers()

	// debug
	log.Printf("starting drone version %s on port %s\n", version, port)

	// start webserver using HTTPS or HTTP
	if sslcert != "" && sslkey != "" {
		panic(http.ListenAndServeTLS(port, sslcert, sslkey, nil))
	} else {
		panic(http.ListenAndServe(port, nil))
	}
}

// checking if the TLS flags where supplied correctly.
func checkTLSFlags() {

	if sslcert != "" && sslkey == "" {
		log.Fatal("invalid configuration: -sslkey unspecified, but -sslcert was specified.")
	} else if sslcert == "" && sslkey != "" {
		log.Fatal("invalid configuration: -sslcert unspecified, but -sslkey was specified.")
	}

}

// setup routes for static assets. These assets may
// be directly embedded inside the application using
// the `rice embed` command, else they are served from disk.
func setupStatic() {
	box := rice.MustFindBox("assets")
	http.Handle("/css/", http.FileServer(box.HTTPBox()))
	http.Handle("/js/", http.FileServer(box.HTTPBox()))

	// we need to intercept all attempts to serve images
	// so that we can add a cache-control settings
	var images = http.FileServer(box.HTTPBox())
	http.HandleFunc("/img/", func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/img/build_") {
			w.Header().Add("Cache-Control", "no-cache")
		}

		// serce images
		images.ServeHTTP(w, r)
	})
}

// setup routes for serving dynamic content.
func setupHandlers() {
	queueRunner := queue.NewBuildRunner(docker.New(), timeout)
	queue := queue.Start(runtime.NumCPU(), queueRunner)

	hookHandler := handler.NewHookHandler(queue)

	m := pat.New()
	m.Get("/login", handler.ErrorHandler(handler.Login))
	m.Post("/login", handler.ErrorHandler(handler.Authorize))
	m.Get("/logout", handler.ErrorHandler(handler.Logout))
	m.Get("/forgot", handler.ErrorHandler(handler.Forgot))
	m.Post("/forgot", handler.ErrorHandler(handler.ForgotPost))
	m.Get("/reset", handler.ErrorHandler(handler.Reset))
	m.Post("/reset", handler.ErrorHandler(handler.ResetPost))
	m.Get("/signup", handler.ErrorHandler(handler.SignUp))
	m.Post("/signup", handler.ErrorHandler(handler.SignUpPost))
	m.Get("/register", handler.ErrorHandler(handler.Register))
	m.Post("/register", handler.ErrorHandler(handler.RegisterPost))
	m.Get("/accept", handler.UserHandler(handler.TeamMemberAccept))

	// handlers for setting up your GitHub repository
	m.Post("/new/github.com", handler.UserHandler(handler.RepoCreateGithub))
	m.Get("/new/github.com", handler.UserHandler(handler.RepoAdd))

	// handlers for linking your GitHub account
	m.Get("/auth/login/github", handler.UserHandler(handler.LinkGithub))

	// handlers for dashboard pages
	m.Get("/dashboard/team/:team", handler.UserHandler(handler.TeamShow))
	m.Get("/dashboard", handler.UserHandler(handler.UserShow))

	// handlers for user account management
	m.Get("/account/user/profile", handler.UserHandler(handler.UserEdit))
	m.Post("/account/user/profile", handler.UserHandler(handler.UserUpdate))
	m.Get("/account/user/delete", handler.UserHandler(handler.UserDeleteConfirm))
	m.Post("/account/user/delete", handler.UserHandler(handler.UserDelete))
	m.Get("/account/user/password", handler.UserHandler(handler.UserPass))
	m.Post("/account/user/password", handler.UserHandler(handler.UserPassUpdate))
	m.Get("/account/user/teams/add", handler.UserHandler(handler.TeamAdd))
	m.Post("/account/user/teams/add", handler.UserHandler(handler.TeamCreate))
	m.Get("/account/user/teams", handler.UserHandler(handler.UserTeams))

	// handlers for team managements
	m.Get("/account/team/:team/profile", handler.UserHandler(handler.TeamEdit))
	m.Post("/account/team/:team/profile", handler.UserHandler(handler.TeamUpdate))
	m.Get("/account/team/:team/delete", handler.UserHandler(handler.TeamDeleteConfirm))
	m.Post("/account/team/:team/delete", handler.UserHandler(handler.TeamDelete))
	m.Get("/account/team/:team/members/add", handler.UserHandler(handler.TeamMemberAdd))
	m.Post("/account/team/:team/members/add", handler.UserHandler(handler.TeamMemberInvite))
	m.Get("/account/team/:team/members/edit", handler.UserHandler(handler.TeamMemberEdit))
	m.Post("/account/team/:team/members/edit", handler.UserHandler(handler.TeamMemberUpdate))
	m.Post("/account/team/:team/members/delete", handler.UserHandler(handler.TeamMemberDelete))
	m.Get("/account/team/:team/members", handler.UserHandler(handler.TeamMembers))

	// handlers for system administration
	m.Get("/account/admin/settings", handler.AdminHandler(handler.AdminSettings))
	m.Post("/account/admin/settings", handler.AdminHandler(handler.AdminSettingsUpdate))
	m.Get("/account/admin/users/edit", handler.AdminHandler(handler.AdminUserEdit))
	m.Post("/account/admin/users/edit", handler.AdminHandler(handler.AdminUserUpdate))
	m.Post("/account/admin/users/delete", handler.AdminHandler(handler.AdminUserDelete))
	m.Get("/account/admin/users/add", handler.AdminHandler(handler.AdminUserAdd))
	m.Post("/account/admin/users", handler.AdminHandler(handler.AdminUserInvite))
	m.Get("/account/admin/users", handler.AdminHandler(handler.AdminUserList))

	// handlers for GitHub post-commit hooks
	m.Post("/hook/github.com", handler.ErrorHandler(hookHandler.Hook))

	// handlers for first-time installation
	m.Get("/install", handler.ErrorHandler(handler.Install))
	m.Post("/install", handler.ErrorHandler(handler.InstallPost))

	// handlers for repository, commits and build details
	m.Get("/:host/:owner/:name/commit/:commit/build/:label/out.txt", handler.RepoHandler(handler.BuildOut))
	m.Get("/:host/:owner/:name/commit/:commit/build/:label", handler.RepoHandler(handler.CommitShow))
	m.Get("/:host/:owner/:name/commit/:commit", handler.RepoHandler(handler.CommitShow))
	m.Get("/:host/:owner/:name/tree", handler.RepoHandler(handler.RepoDashboard))
	m.Get("/:host/:owner/:name/status.png", handler.ErrorHandler(handler.Badge))
	m.Get("/:host/:owner/:name/settings", handler.RepoAdminHandler(handler.RepoSettingsForm))
	m.Get("/:host/:owner/:name/params", handler.RepoAdminHandler(handler.RepoParamsForm))
	m.Get("/:host/:owner/:name/badges", handler.RepoAdminHandler(handler.RepoBadges))
	m.Get("/:host/:owner/:name/keys", handler.RepoAdminHandler(handler.RepoKeys))
	m.Get("/:host/:owner/:name/delete", handler.RepoAdminHandler(handler.RepoDeleteForm))
	m.Post("/:host/:owner/:name/delete", handler.RepoAdminHandler(handler.RepoDelete))
	m.Get("/:host/:owner/:name", handler.RepoHandler(handler.RepoDashboard))
	m.Post("/:host/:owner/:name", handler.RepoHandler(handler.RepoUpdate))
	http.Handle("/feed", websocket.Handler(channel.Read))

	// no routes are served at the root URL. Instead we will
	// redirect the user to his/her dashboard page.
	m.Get("/", http.RedirectHandler("/dashboard", http.StatusSeeOther))

	// the first time a page is requested we should record
	// the scheme and hostname.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// our multiplexer is a bit finnicky and therefore requires
		// us to strip any trailing slashes in order to correctly
		// find and match a route.
		if r.URL.Path != "/" && strings.HasSuffix(r.URL.Path, "/") {
			http.Redirect(w, r, r.URL.Path[:len(r.URL.Path)-1], http.StatusSeeOther)
			return
		}

		// standard header variables that should be set, for good measure.
		w.Header().Add("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate")
		w.Header().Add("X-Frame-Options", "DENY")
		w.Header().Add("X-Content-Type-Options", "nosniff")
		w.Header().Add("X-XSS-Protection", "1; mode=block")

		// ok, now we're ready to serve the request.
		m.ServeHTTP(w, r)
	})
}
<commit_msg>Remove unused -path<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	"runtime"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/GeertJohan/go.rice"
	"github.com/bmizerany/pat"

	"github.com/drone/drone/pkg/build/docker"
	"github.com/drone/drone/pkg/channel"
	"github.com/drone/drone/pkg/database"
	"github.com/drone/drone/pkg/handler"
	"github.com/drone/drone/pkg/queue"
)

var (
	// port the server will run on
	port string

	// database driver used to connect to the database
	driver string

	// driver specific connection information. In this
	// case, it should be the location of the SQLite file
	datasource string

	// optional flags for tls listener
	sslcert string
	sslkey  string

	// build will timeout after N milliseconds.
	// this will default to 500 minutes (6 hours)
	timeout time.Duration

	// commit sha for the current build.
	version string
)

func main() {
	// parse command line flags
	flag.StringVar(&port, "port", ":8080", "")
	flag.StringVar(&driver, "driver", "sqlite3", "")
	flag.StringVar(&datasource, "datasource", "drone.sqlite", "")
	flag.StringVar(&sslcert, "sslcert", "", "")
	flag.StringVar(&sslkey, "sslkey", "", "")
	flag.DurationVar(&timeout, "timeout", 300*time.Minute, "")
	flag.Parse()

	// validate the TLS arguments
	checkTLSFlags()

	// setup database and handlers
	if err := database.Init(driver, datasource); err != nil {
		log.Fatal("Can't initialize database: ", err)
	}
	setupStatic()
	setupHandlers()

	// debug
	log.Printf("starting drone version %s on port %s\n", version, port)

	// start webserver using HTTPS or HTTP
	if sslcert != "" && sslkey != "" {
		panic(http.ListenAndServeTLS(port, sslcert, sslkey, nil))
	} else {
		panic(http.ListenAndServe(port, nil))
	}
}

// checking if the TLS flags where supplied correctly.
func checkTLSFlags() {

	if sslcert != "" && sslkey == "" {
		log.Fatal("invalid configuration: -sslkey unspecified, but -sslcert was specified.")
	} else if sslcert == "" && sslkey != "" {
		log.Fatal("invalid configuration: -sslcert unspecified, but -sslkey was specified.")
	}

}

// setup routes for static assets. These assets may
// be directly embedded inside the application using
// the `rice embed` command, else they are served from disk.
func setupStatic() {
	box := rice.MustFindBox("assets")
	http.Handle("/css/", http.FileServer(box.HTTPBox()))
	http.Handle("/js/", http.FileServer(box.HTTPBox()))

	// we need to intercept all attempts to serve images
	// so that we can add a cache-control settings
	var images = http.FileServer(box.HTTPBox())
	http.HandleFunc("/img/", func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/img/build_") {
			w.Header().Add("Cache-Control", "no-cache")
		}

		// serce images
		images.ServeHTTP(w, r)
	})
}

// setup routes for serving dynamic content.
func setupHandlers() {
	queueRunner := queue.NewBuildRunner(docker.New(), timeout)
	queue := queue.Start(runtime.NumCPU(), queueRunner)

	hookHandler := handler.NewHookHandler(queue)

	m := pat.New()
	m.Get("/login", handler.ErrorHandler(handler.Login))
	m.Post("/login", handler.ErrorHandler(handler.Authorize))
	m.Get("/logout", handler.ErrorHandler(handler.Logout))
	m.Get("/forgot", handler.ErrorHandler(handler.Forgot))
	m.Post("/forgot", handler.ErrorHandler(handler.ForgotPost))
	m.Get("/reset", handler.ErrorHandler(handler.Reset))
	m.Post("/reset", handler.ErrorHandler(handler.ResetPost))
	m.Get("/signup", handler.ErrorHandler(handler.SignUp))
	m.Post("/signup", handler.ErrorHandler(handler.SignUpPost))
	m.Get("/register", handler.ErrorHandler(handler.Register))
	m.Post("/register", handler.ErrorHandler(handler.RegisterPost))
	m.Get("/accept", handler.UserHandler(handler.TeamMemberAccept))

	// handlers for setting up your GitHub repository
	m.Post("/new/github.com", handler.UserHandler(handler.RepoCreateGithub))
	m.Get("/new/github.com", handler.UserHandler(handler.RepoAdd))

	// handlers for linking your GitHub account
	m.Get("/auth/login/github", handler.UserHandler(handler.LinkGithub))

	// handlers for dashboard pages
	m.Get("/dashboard/team/:team", handler.UserHandler(handler.TeamShow))
	m.Get("/dashboard", handler.UserHandler(handler.UserShow))

	// handlers for user account management
	m.Get("/account/user/profile", handler.UserHandler(handler.UserEdit))
	m.Post("/account/user/profile", handler.UserHandler(handler.UserUpdate))
	m.Get("/account/user/delete", handler.UserHandler(handler.UserDeleteConfirm))
	m.Post("/account/user/delete", handler.UserHandler(handler.UserDelete))
	m.Get("/account/user/password", handler.UserHandler(handler.UserPass))
	m.Post("/account/user/password", handler.UserHandler(handler.UserPassUpdate))
	m.Get("/account/user/teams/add", handler.UserHandler(handler.TeamAdd))
	m.Post("/account/user/teams/add", handler.UserHandler(handler.TeamCreate))
	m.Get("/account/user/teams", handler.UserHandler(handler.UserTeams))

	// handlers for team managements
	m.Get("/account/team/:team/profile", handler.UserHandler(handler.TeamEdit))
	m.Post("/account/team/:team/profile", handler.UserHandler(handler.TeamUpdate))
	m.Get("/account/team/:team/delete", handler.UserHandler(handler.TeamDeleteConfirm))
	m.Post("/account/team/:team/delete", handler.UserHandler(handler.TeamDelete))
	m.Get("/account/team/:team/members/add", handler.UserHandler(handler.TeamMemberAdd))
	m.Post("/account/team/:team/members/add", handler.UserHandler(handler.TeamMemberInvite))
	m.Get("/account/team/:team/members/edit", handler.UserHandler(handler.TeamMemberEdit))
	m.Post("/account/team/:team/members/edit", handler.UserHandler(handler.TeamMemberUpdate))
	m.Post("/account/team/:team/members/delete", handler.UserHandler(handler.TeamMemberDelete))
	m.Get("/account/team/:team/members", handler.UserHandler(handler.TeamMembers))

	// handlers for system administration
	m.Get("/account/admin/settings", handler.AdminHandler(handler.AdminSettings))
	m.Post("/account/admin/settings", handler.AdminHandler(handler.AdminSettingsUpdate))
	m.Get("/account/admin/users/edit", handler.AdminHandler(handler.AdminUserEdit))
	m.Post("/account/admin/users/edit", handler.AdminHandler(handler.AdminUserUpdate))
	m.Post("/account/admin/users/delete", handler.AdminHandler(handler.AdminUserDelete))
	m.Get("/account/admin/users/add", handler.AdminHandler(handler.AdminUserAdd))
	m.Post("/account/admin/users", handler.AdminHandler(handler.AdminUserInvite))
	m.Get("/account/admin/users", handler.AdminHandler(handler.AdminUserList))

	// handlers for GitHub post-commit hooks
	m.Post("/hook/github.com", handler.ErrorHandler(hookHandler.Hook))

	// handlers for first-time installation
	m.Get("/install", handler.ErrorHandler(handler.Install))
	m.Post("/install", handler.ErrorHandler(handler.InstallPost))

	// handlers for repository, commits and build details
	m.Get("/:host/:owner/:name/commit/:commit/build/:label/out.txt", handler.RepoHandler(handler.BuildOut))
	m.Get("/:host/:owner/:name/commit/:commit/build/:label", handler.RepoHandler(handler.CommitShow))
	m.Get("/:host/:owner/:name/commit/:commit", handler.RepoHandler(handler.CommitShow))
	m.Get("/:host/:owner/:name/tree", handler.RepoHandler(handler.RepoDashboard))
	m.Get("/:host/:owner/:name/status.png", handler.ErrorHandler(handler.Badge))
	m.Get("/:host/:owner/:name/settings", handler.RepoAdminHandler(handler.RepoSettingsForm))
	m.Get("/:host/:owner/:name/params", handler.RepoAdminHandler(handler.RepoParamsForm))
	m.Get("/:host/:owner/:name/badges", handler.RepoAdminHandler(handler.RepoBadges))
	m.Get("/:host/:owner/:name/keys", handler.RepoAdminHandler(handler.RepoKeys))
	m.Get("/:host/:owner/:name/delete", handler.RepoAdminHandler(handler.RepoDeleteForm))
	m.Post("/:host/:owner/:name/delete", handler.RepoAdminHandler(handler.RepoDelete))
	m.Get("/:host/:owner/:name", handler.RepoHandler(handler.RepoDashboard))
	m.Post("/:host/:owner/:name", handler.RepoHandler(handler.RepoUpdate))
	http.Handle("/feed", websocket.Handler(channel.Read))

	// no routes are served at the root URL. Instead we will
	// redirect the user to his/her dashboard page.
	m.Get("/", http.RedirectHandler("/dashboard", http.StatusSeeOther))

	// the first time a page is requested we should record
	// the scheme and hostname.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// our multiplexer is a bit finnicky and therefore requires
		// us to strip any trailing slashes in order to correctly
		// find and match a route.
		if r.URL.Path != "/" && strings.HasSuffix(r.URL.Path, "/") {
			http.Redirect(w, r, r.URL.Path[:len(r.URL.Path)-1], http.StatusSeeOther)
			return
		}

		// standard header variables that should be set, for good measure.
		w.Header().Add("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate")
		w.Header().Add("X-Frame-Options", "DENY")
		w.Header().Add("X-Content-Type-Options", "nosniff")
		w.Header().Add("X-XSS-Protection", "1; mode=block")

		// ok, now we're ready to serve the request.
		m.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// Version is inserted at build using --ldflags -X
var Version = "(unknown version)"

const socketName = "/var/run/edgectl.socket"
const logfile = "/tmp/edgectl.log"
const apiVersion = 1

var failedToConnect = `Failed to connect to the daemon. Is it still running?
The daemon's log output in ` + logfile + ` may have more information.
Start the daemon using "sudo edgectl daemon" if it is not running.
`
var displayVersion = fmt.Sprintf("v%s (api v%d)", Version, apiVersion)
var edgectl string

func main() {
	// Figure out our executable and save it
	func() {
		var err error
		edgectl, err = os.Executable()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Internal error: %v", err)
			os.Exit(1)
		}
	}()

	// Notice early if we're impersonating Teleproxy
	if len(os.Args) == 3 && os.Args[1] == "teleproxy" {
		func() {
			var err error
			switch os.Args[2] {
			case "intercept":
				err = RunAsTeleproxyIntercept()
			case "bridge":
				err = RunAsTeleproxyBridge()
			default:
				return // Allow normal CLI error handling to proceed
			}
			if err != nil {
				fmt.Fprintf(os.Stderr, "error: %v", err)
				os.Exit(1)
			}
			os.Exit(0)
		}()
	}

	// Let the daemon take care of everything if possible
	failedToConnectErr := mainViaDaemon()

	// Couldn't reach the daemon. Try to handle things locally.
	// ...

	rootCmd := &cobra.Command{
		Use:          "edgectl",
		Short:        "Edge Control (daemon unavailable)",
		SilenceUsage: true, // https://github.com/spf13/cobra/issues/340
		Args:         cobra.ArbitraryArgs,
		RunE: func(_ *cobra.Command, _ []string) error {
			fmt.Println(failedToConnect)
			return failedToConnectErr
		},
	}

	rootCmd.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Show program's version number and exit",
		Args:  cobra.ExactArgs(0),
		RunE: func(_ *cobra.Command, _ []string) error {
			fmt.Println(failedToConnect)
			fmt.Println("Client", displayVersion)
			fmt.Println()
			return failedToConnectErr
		},
	})
	rootCmd.AddCommand(&cobra.Command{
		Use:    "daemon-foreground",
		Short:  "Launch Edge Control Daemon in the foreground (debug)",
		Args:   cobra.ExactArgs(0),
		Hidden: true,
		RunE: func(_ *cobra.Command, _ []string) error {
			return RunAsDaemon()
		},
	})
	rootCmd.AddCommand(&cobra.Command{
		Use:   "daemon",
		Short: "Launch Edge Control Daemon in the background (sudo)",
		Args:  cobra.ExactArgs(0),
		RunE:  launchDaemon,
	})
	rootCmd.SetHelpTemplate(rootCmd.HelpTemplate() + "\n" + failedToConnect)

	err := rootCmd.Execute()
	if err != nil {
		os.Exit(1)
	}
}

func launchDaemon(ccmd *cobra.Command, _ []string) error {
	if os.Geteuid() != 0 {
		fmt.Println("Edge Control Daemon must be launched as root.")
		fmt.Printf("\n  sudo %s\n\n", ccmd.CommandPath())
		return errors.New("root privileges required")
	}
	fmt.Println("Launching Edge Control Daemon", displayVersion)

	cmd := exec.Command(edgectl, "daemon-foreground")
	cmd.Env = os.Environ()
	cmd.Stdin = nil
	cmd.Stdout = nil
	cmd.Stderr = nil
	cmd.ExtraFiles = nil
	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
	err := cmd.Start()
	if err != nil {
		return errors.Wrap(err, "failed to launch the server")
	}

	success := false
	for count := 0; count < 40; count++ {
		if isServerRunning() {
			success = true
			break
		}
		if count == 4 {
			fmt.Println("Waiting for daemon to start...")
		}
		time.Sleep(250 * time.Millisecond)
	}
	if !success {
		fmt.Println("Server did not come up!")
		fmt.Printf("Take a look at %s for more information.\n", logfile)
		return errors.New("launch failed")
	}
	return nil
}
<commit_msg>Give Edge Control a version number when built directly<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// Version is inserted at build using --ldflags -X
// But we don't really build that way any longer.
// So include a fallback version number that's not useless.
var Version = "0.8.0"

const socketName = "/var/run/edgectl.socket"
const logfile = "/tmp/edgectl.log"
const apiVersion = 1

var failedToConnect = `Failed to connect to the daemon. Is it still running?
The daemon's log output in ` + logfile + ` may have more information.
Start the daemon using "sudo edgectl daemon" if it is not running.
`
var displayVersion = fmt.Sprintf("v%s (api v%d)", Version, apiVersion)

// edgectl is the full path to the Edge Control binary
var edgectl string

func main() {
	// Figure out our executable and save it
	func() {
		var err error
		edgectl, err = os.Executable()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Internal error: %v", err)
			os.Exit(1)
		}
	}()

	// Notice early if we're impersonating Teleproxy
	if len(os.Args) == 3 && os.Args[1] == "teleproxy" {
		func() {
			var err error
			switch os.Args[2] {
			case "intercept":
				err = RunAsTeleproxyIntercept()
			case "bridge":
				err = RunAsTeleproxyBridge()
			default:
				return // Allow normal CLI error handling to proceed
			}
			if err != nil {
				fmt.Fprintf(os.Stderr, "error: %v", err)
				os.Exit(1)
			}
			os.Exit(0)
		}()
	}

	// Let the daemon take care of everything if possible
	failedToConnectErr := mainViaDaemon()

	// Couldn't reach the daemon. Try to handle things locally.
	// ...

	rootCmd := &cobra.Command{
		Use:          "edgectl",
		Short:        "Edge Control (daemon unavailable)",
		SilenceUsage: true, // https://github.com/spf13/cobra/issues/340
		Args:         cobra.ArbitraryArgs,
		RunE: func(_ *cobra.Command, _ []string) error {
			fmt.Println(failedToConnect)
			return failedToConnectErr
		},
	}

	rootCmd.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Show program's version number and exit",
		Args:  cobra.ExactArgs(0),
		RunE: func(_ *cobra.Command, _ []string) error {
			fmt.Println(failedToConnect)
			fmt.Println("Client", displayVersion)
			fmt.Println()
			return failedToConnectErr
		},
	})
	rootCmd.AddCommand(&cobra.Command{
		Use:    "daemon-foreground",
		Short:  "Launch Edge Control Daemon in the foreground (debug)",
		Args:   cobra.ExactArgs(0),
		Hidden: true,
		RunE: func(_ *cobra.Command, _ []string) error {
			return RunAsDaemon()
		},
	})
	rootCmd.AddCommand(&cobra.Command{
		Use:   "daemon",
		Short: "Launch Edge Control Daemon in the background (sudo)",
		Args:  cobra.ExactArgs(0),
		RunE:  launchDaemon,
	})
	rootCmd.SetHelpTemplate(rootCmd.HelpTemplate() + "\n" + failedToConnect)

	err := rootCmd.Execute()
	if err != nil {
		os.Exit(1)
	}
}

func launchDaemon(ccmd *cobra.Command, _ []string) error {
	if os.Geteuid() != 0 {
		fmt.Println("Edge Control Daemon must be launched as root.")
		fmt.Printf("\n  sudo %s\n\n", ccmd.CommandPath())
		return errors.New("root privileges required")
	}
	fmt.Println("Launching Edge Control Daemon", displayVersion)

	cmd := exec.Command(edgectl, "daemon-foreground")
	cmd.Env = os.Environ()
	cmd.Stdin = nil
	cmd.Stdout = nil
	cmd.Stderr = nil
	cmd.ExtraFiles = nil
	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
	err := cmd.Start()
	if err != nil {
		return errors.Wrap(err, "failed to launch the server")
	}

	success := false
	for count := 0; count < 40; count++ {
		if isServerRunning() {
			success = true
			break
		}
		if count == 4 {
			fmt.Println("Waiting for daemon to start...")
		}
		time.Sleep(250 * time.Millisecond)
	}
	if !success {
		fmt.Println("Server did not come up!")
		fmt.Printf("Take a look at %s for more information.\n", logfile)
		return errors.New("launch failed")
	}
	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"context"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"

	"github.com/google/ent/utils"
	"github.com/ipfs/go-cid"
	format "github.com/ipfs/go-ipld-format"
	"github.com/ipfs/go-merkledag"
	"github.com/spf13/cobra"
)

var pullCmd = &cobra.Command{
	Use:  "pull [hash] [target directory]",
	Args: cobra.RangeArgs(1, 2),
	Run: func(cmd *cobra.Command, args []string) {
		hash := args[0]
		targetDir := "."
		if len(args) >= 2 {
			targetDir = args[1]
		}
		targetDir, err := filepath.Abs(targetDir)
		if err != nil {
			log.Fatalf("could not normalize target directory %q, %v", targetDir, err)
		}

		base, err := cid.Decode(hash)
		if err != nil {
			log.Fatalf("could not decode cid: %v", err)
		}

		pull(base, targetDir, false)

		log.Printf("pull %s %s", hash, targetDir)
	},
}

func pull(base cid.Cid, targetPath string, executable bool) {
	_, err := os.Stat(targetPath)
	if os.IsNotExist(err) {
		// Continue.
	} else if err != nil {
		log.Fatalf("could not stat target path: %v", err)
	}
	if !os.IsNotExist(err) {
		log.Printf("target path %s already exists; skipping", targetPath)
		// Skip?
		return
	}
	files, err := ioutil.ReadDir(targetPath)
	if os.IsNotExist(err) {
		// Continue.
	} else if err != nil {
		log.Fatalf("could not read target directory %q: %v", targetPath, err)
	}
	if len(files) > 0 {
		log.Fatalf("cannot pull to non-empty directory %q", targetPath)
	}
	traverseRemote(base, "", func(p string, node format.Node) error {
		fullPath := filepath.Join(targetPath, p)
		log.Printf("%s\n", fullPath)
		switch node := node.(type) {
		case *merkledag.ProtoNode:
			err := os.MkdirAll(fullPath, 0755)
			if err != nil {
				log.Fatalf("could not create directory %q: %v", fullPath, err)
			}
		case *merkledag.RawNode:
			err := os.MkdirAll(path.Dir(fullPath), 0755)
			if err != nil {
				log.Fatalf("could not create directory %q: %v", fullPath, err)
			}
			mode := 0644
			if executable {
				mode = 0755
			}
			err = ioutil.WriteFile(fullPath, node.RawData(), os.FileMode(mode))
			if err != nil {
				log.Fatalf("could not create file %q: %v", fullPath, err)
			}
		}
		return nil
	})
}

func traverseRemote(base cid.Cid, relativeFilename string, f func(string, format.Node) error) {
	obj, err := nodeService.GetObject(context.Background(), base.Hash())
	if err != nil {
		log.Fatal(err)
	}

	switch base.Prefix().Codec {
	case cid.DagProtobuf:
		node, err := utils.ParseProtoNode(obj)
		if err != nil {
			log.Fatal(err)
		}
		err = f(relativeFilename, node)
		if err != nil {
			log.Fatal(err)
		}

		for _, l := range node.Links() {
			if l.Name == "" {
				continue
			}
			newRelativeFilename := path.Join(relativeFilename, l.Name)
			traverseRemote(l.Cid, newRelativeFilename, f)
		}
	case cid.Raw:
		node, err := utils.ParseRawNode(obj)
		if err != nil {
			log.Fatal(err)
		}
		err = f(relativeFilename, node)
		if err != nil {
			log.Fatal(err)
		}
	default:
		log.Fatalf("invalid codec: %v", base.Prefix().Codec)
	}
}
<commit_msg>Fix formatting of pull logs<commit_after>package cmd

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"

	"github.com/fatih/color"
	"github.com/google/ent/utils"
	"github.com/ipfs/go-cid"
	format "github.com/ipfs/go-ipld-format"
	"github.com/ipfs/go-merkledag"
	"github.com/spf13/cobra"
)

var pullCmd = &cobra.Command{
	Use:  "pull [hash] [target directory]",
	Args: cobra.RangeArgs(1, 2),
	Run: func(cmd *cobra.Command, args []string) {
		hash := args[0]
		targetDir := "."
		if len(args) >= 2 {
			targetDir = args[1]
		}
		targetDir, err := filepath.Abs(targetDir)
		if err != nil {
			log.Fatalf("could not normalize target directory %q, %v", targetDir, err)
		}

		base, err := cid.Decode(hash)
		if err != nil {
			log.Fatalf("could not decode cid: %v", err)
		}

		pull(base, targetDir, false)

		log.Printf("pull %s %s", hash, targetDir)
	},
}

func pull(base cid.Cid, targetPath string, executable bool) {
	_, err := os.Stat(targetPath)
	if os.IsNotExist(err) {
		// Continue.
	} else if err != nil {
		log.Fatalf("could not stat target path: %v", err)
	}
	if !os.IsNotExist(err) {
		log.Printf("target path %s already exists; skipping", targetPath)
		// Skip?
		return
	}
	files, err := ioutil.ReadDir(targetPath)
	if os.IsNotExist(err) {
		// Continue.
	} else if err != nil {
		log.Fatalf("could not read target directory %q: %v", targetPath, err)
	}
	if len(files) > 0 {
		log.Fatalf("cannot pull to non-empty directory %q", targetPath)
	}
	traverseRemote(base, "", func(p string, node format.Node) error {
		fullPath := filepath.Join(targetPath, p)
		marker := color.BlueString("↓")
		fmt.Printf("%s %s %s\n", color.YellowString(node.Cid().String()), marker, fullPath)
		switch node := node.(type) {
		case *merkledag.ProtoNode:
			err := os.MkdirAll(fullPath, 0755)
			if err != nil {
				log.Fatalf("could not create directory %q: %v", fullPath, err)
			}
		case *merkledag.RawNode:
			err := os.MkdirAll(path.Dir(fullPath), 0755)
			if err != nil {
				log.Fatalf("could not create directory %q: %v", fullPath, err)
			}
			mode := 0644
			if executable {
				mode = 0755
			}
			err = ioutil.WriteFile(fullPath, node.RawData(), os.FileMode(mode))
			if err != nil {
				log.Fatalf("could not create file %q: %v", fullPath, err)
			}
		}
		return nil
	})
}

func traverseRemote(base cid.Cid, relativeFilename string, f func(string, format.Node) error) {
	obj, err := nodeService.GetObject(context.Background(), base.Hash())
	if err != nil {
		log.Fatal(err)
	}

	switch base.Prefix().Codec {
	case cid.DagProtobuf:
		node, err := utils.ParseProtoNode(obj)
		if err != nil {
			log.Fatal(err)
		}
		err = f(relativeFilename, node)
		if err != nil {
			log.Fatal(err)
		}

		for _, l := range node.Links() {
			if l.Name == "" {
				continue
			}
			newRelativeFilename := path.Join(relativeFilename, l.Name)
			traverseRemote(l.Cid, newRelativeFilename, f)
		}
	case cid.Raw:
		node, err := utils.ParseRawNode(obj)
		if err != nil {
			log.Fatal(err)
		}
		err = f(relativeFilename, node)
		if err != nil {
			log.Fatal(err)
		}
	default:
		log.Fatalf("invalid codec: %v", base.Prefix().Codec)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/spf13/cobra"

	"k8s.io/helm/pkg/helm"
)

const upgradeDesc = `
This command upgrades a release to a new version of a chart.

The upgrade arguments must be a release and a chart. The chart
argument can be a relative path to a packaged or unpackaged chart.
`

type upgradeCmd struct {
	release      string
	chart        string
	out          io.Writer
	client       helm.Interface
	dryRun       bool
	disableHooks bool
	valuesFile   string
}

func newUpgradeCmd(client helm.Interface, out io.Writer) *cobra.Command {

	upgrade := &upgradeCmd{
		out:    out,
		client: client,
	}

	cmd := &cobra.Command{
		Use:               "upgrade [RELEASE] [CHART]",
		Short:             "upgrade a release",
		Long:              upgradeDesc,
		PersistentPreRunE: setupConnection,
		RunE: func(cmd *cobra.Command, args []string) error {
			if err := checkArgsLength(2, len(args), "release name, chart path"); err != nil {
				return err
			}

			upgrade.release = args[0]
			upgrade.chart = args[1]
			upgrade.client = ensureHelmClient(upgrade.client)

			return upgrade.run()
		},
	}

	f := cmd.Flags()
	f.StringVarP(&upgrade.valuesFile, "values", "f", "", "path to a values YAML file")
	f.BoolVar(&upgrade.dryRun, "dry-run", false, "simulate an upgrade")
	f.BoolVar(&upgrade.disableHooks, "disable-hooks", false, "disable pre/post upgrade hooks")

	return cmd
}

func (u *upgradeCmd) run() error {
	chartPath, err := locateChartPath(u.chart)
	if err != nil {
		return err
	}

	rawVals := []byte{}
	if u.valuesFile != "" {
		rawVals, err = ioutil.ReadFile(u.valuesFile)
		if err != nil {
			return err
		}
	}

	_, err = u.client.UpdateRelease(u.release, chartPath, helm.UpdateValueOverrides(rawVals), helm.UpgradeDryRun(u.dryRun), helm.UpgradeDisableHooks(u.disableHooks))
	if err != nil {
		return prettyError(err)
	}

	success := u.release + " has been upgraded. Happy Helming!\n"
	fmt.Fprintf(u.out, success)

	return nil

}
<commit_msg>Add --set flag to `helm upgrade`<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"

	"github.com/spf13/cobra"

	"k8s.io/helm/pkg/helm"
)

const upgradeDesc = `
This command upgrades a release to a new version of a chart.

The upgrade arguments must be a release and a chart. The chart
argument can be a relative path to a packaged or unpackaged chart.

To override values in a chart, use either the '--values' flag and pass in a file
or use the '--set' flag and pass configuration from the command line.
`

type upgradeCmd struct {
	release      string
	chart        string
	out          io.Writer
	client       helm.Interface
	dryRun       bool
	disableHooks bool
	valuesFile   string
	values       *values
}

func newUpgradeCmd(client helm.Interface, out io.Writer) *cobra.Command {

	upgrade := &upgradeCmd{
		out:    out,
		client: client,
		values: new(values),
	}

	cmd := &cobra.Command{
		Use:               "upgrade [RELEASE] [CHART]",
		Short:             "upgrade a release",
		Long:              upgradeDesc,
		PersistentPreRunE: setupConnection,
		RunE: func(cmd *cobra.Command, args []string) error {
			if err := checkArgsLength(2, len(args), "release name, chart path"); err != nil {
				return err
			}

			upgrade.release = args[0]
			upgrade.chart = args[1]
			upgrade.client = ensureHelmClient(upgrade.client)

			return upgrade.run()
		},
	}

	f := cmd.Flags()
	f.StringVarP(&upgrade.valuesFile, "values", "f", "", "path to a values YAML file")
	f.BoolVar(&upgrade.dryRun, "dry-run", false, "simulate an upgrade")
	f.Var(upgrade.values, "set", "set values on the command line. Separate values with commas: key1=val1,key2=val2")
	f.BoolVar(&upgrade.disableHooks, "disable-hooks", false, "disable pre/post upgrade hooks")

	return cmd
}

func (u *upgradeCmd) vals() ([]byte, error) {
	var buffer bytes.Buffer

	// User specified a values file via -f/--values
	if u.valuesFile != "" {
		bytes, err := ioutil.ReadFile(u.valuesFile)
		if err != nil {
			return []byte{}, err
		}
		buffer.Write(bytes)
	}

	// User specified value pairs via --set
	// These override any values in the specified file
	if len(u.values.pairs) > 0 {
		bytes, err := u.values.yaml()
		if err != nil {
			return []byte{}, err
		}
		buffer.Write(bytes)
	}

	return buffer.Bytes(), nil
}

func (u *upgradeCmd) run() error {
	chartPath, err := locateChartPath(u.chart)
	if err != nil {
		return err
	}

	rawVals, err := u.vals()
	if err != nil {
		return err
	}

	_, err = u.client.UpdateRelease(u.release, chartPath, helm.UpdateValueOverrides(rawVals), helm.UpgradeDryRun(u.dryRun), helm.UpgradeDisableHooks(u.disableHooks))
	if err != nil {
		return prettyError(err)
	}

	success := u.release + " has been upgraded. Happy Helming!\n"
	fmt.Fprintf(u.out, success)

	return nil

}
<|endoftext|>


<commit_before>package main // import "vimagination.zapto.org/httpdir/cmd/httpdir"

import (
	"compress/flate"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/google/brotli/go/cbrotli"
	"vimagination.zapto.org/memio"
)

var (
	pkg      = flag.String("p", "main", "package name")
	in       = flag.String("i", "", "input filename")
	out      = flag.String("o", "", "output filename")
	odate    = flag.String("d", "", "modified date [seconds since epoch]")
	path     = flag.String("w", "", "http path")
	varname  = flag.String("v", "httpdir.Default", "http dir variable name")
	cvarname = flag.String("c", "httpdir.Default", "http dir compressed variable name")
	help     = flag.Bool("h", false, "show help")
	gzcomp   = flag.Bool("g", false, "compress using gzip")
	brcomp   = flag.Bool("b", false, "compress using brotli")
	flcomp   = flag.Bool("f", false, "compress using flate/deflate")
	single   = flag.Bool("s", false, "use single source var and decompress/compress for others")
)

type replacer struct {
	f *os.File
}

var hexArr = [16]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}

func (r replacer) Write(p []byte) (int, error) {
	n := 0
	toWrite := make([]byte, 0, 5)
	for len(p) > 0 {
		rn, s := utf8.DecodeRune(p)
		if rn == utf8.RuneError {
			s = 1
		}
		if s > 1 || (p[0] > 0 && p[0] < 0x7f && p[0] != '\n' && p[0] != '\\' && p[0] != '"') {
			toWrite = append(toWrite[:0], p[:s]...)
		} else {
			switch p[0] {
			case '\n':
				toWrite = append(toWrite[:0], '\\', 'n')
			case '\\':
				toWrite = append(toWrite[:0], '\\', '\\')
			case '"':
				toWrite = append(toWrite[:0], '\\', '"')
			default:
				toWrite = append(toWrite[:0], '\\', 'x', hexArr[p[0]>>4], hexArr[p[0]&15])
			}
		}
		_, err := r.f.Write(toWrite)
		n += s
		if err != nil {
			return n, err
		}
		p = p[s:]
	}
	return n, nil
}

type tickReplacer struct {
	f *os.File
}

func (r tickReplacer) Write(p []byte) (int, error) {
	var (
		m, n int
		err  error
	)
	hexes := make([]byte, 0, 32)
	toWrite := make([]byte, 0, 1024)
	for len(p) > 0 {
		dr, s := utf8.DecodeRune(p)
		if dr == utf8.RuneError || dr == '`' || dr == '\r' {
			hexes = append(hexes, p[0])
		} else {
			if len(hexes) > 0 {
				toWrite = toWrite[:0]
				toWrite = append(toWrite, '`', '+', '"')
				for _, b := range hexes {
					if b == '`' {
						toWrite = append(toWrite, '`')
					} else if b == '\r' {
						toWrite = append(toWrite, '\\', 'r')
					} else {
						toWrite = append(toWrite, '\\', 'x', hexArr[b>>4], hexArr[b&15])
					}
				}
				toWrite = append(toWrite, '"', '+', '`')
				_, err = r.f.Write(toWrite)
				n += len(hexes)
				if err != nil {
					break
				}
				hexes = hexes[:0]
			}
			m, err = r.f.Write(p[:s])
			n += m
			if err != nil {
				break
			}
		}
		p = p[s:]
	}
	return n, err
}

func e(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func ne(_ int, err error) {
	e(err)
}

type imports []string

func (im imports) Len() int {
	return len(im)
}

func (im imports) Less(i, j int) bool {
	si := strings.HasPrefix(im[i], "\"github.com")
	sj := strings.HasPrefix(im[j], "\"github.com")
	vi := strings.HasPrefix(im[i], "\"vimagination.zapto.org")
	vj := strings.HasPrefix(im[j], "\"vimagination.zapto.org")
	if si == sj && vi == vj {
		return im[i] < im[j]
	}
	return !si && !vi || si && vj
}

func (im imports) Swap(i, j int) {
	im[i], im[j] = im[j], im[i]
}

type encoding struct {
	Buffer                    []byte
	Compress, Decompress, Ext string
}

type encodings []encoding

func (e encodings) Len() int {
	return len(e)
}

func (e encodings) Less(i, j int) bool {
	return len(e[i].Buffer) < len(e[j].Buffer)
}

func (e encodings) Swap(i, j int) {
	e[i], e[j] = e[j], e[i]
}

func main() {
	flag.Parse()
	if *help {
		flag.Usage()
		return
	}
	var (
		f    *os.File
		err  error
		date int64
	)
	if *in == "-" || *in == "" {
		f = os.Stdin
		date = time.Now().Unix()
	} else {
		f, err = os.Open(*in)
		e(err)
		fi, err := f.Stat()
		e(err)
		date = fi.ModTime().Unix()
	}
	if *odate != "" {
		date, err = strconv.ParseInt(*odate, 10, 64)
		e(err)
	}

	data := make(memio.Buffer, 0, 1<<20)
	_, err = io.Copy(&data, f)
	e(err)
	e(f.Close())

	im := imports{"\"vimagination.zapto.org/httpdir\"", "\"time\""}

	encs := make(encodings, 1, 4)
	encs[0] = encoding{
		Buffer:     data,
		Compress:   identCompress,
		Decompress: identDecompress,
		Ext:        "",
	}

	if *brcomp {
		var b memio.Buffer
		br := cbrotli.NewWriter(&b, cbrotli.WriterOptions{Quality: 11})
		br.Write(data)
		br.Close()
		if *single {
			im = append(im, brotliImport)
		}
		encs = append(encs, encoding{
			Buffer:     b,
			Compress:   brotliCompress,
			Decompress: brotliDecompress,
			Ext:        ".br",
		})
	}
	if *flcomp {
		var b memio.Buffer
		fl, _ := flate.NewWriter(&b, flate.BestCompression)
		fl.Write(data)
		fl.Close()
		if *single {
			im = append(im, strings.Split(flateImport, "\n")...)
		}
		encs = append(encs, encoding{
			Buffer:     b,
			Compress:   flateCompress,
			Decompress: flateDecompress,
			Ext:        ".fl",
		})
	}
	if *gzcomp {
		var b memio.Buffer
		gz, _ := gzip.NewWriterLevel(&b, gzip.BestCompression)
		gz.Write(data)
		gz.Close()
		if *single {
			im = append(im, gzipImport)
		}
		encs = append(encs, encoding{
			Buffer:     b,
			Compress:   gzipCompress,
			Decompress: gzipDecompress,
			Ext:        ".gz",
		})
	}

	sort.Sort(encs)

	if *single && (*gzcomp || *brcomp || *flcomp) {
		im = append(im, "\"vimagination.zapto.org/memio\"")
		if encs[0].Ext != "" {
			im = append(im, "\"strings\"")
		}
	}
	if encs[0].Ext == ".fl" {
		im = append(im, "\"io\"")
	}

	sort.Sort(im)
	var (
		imports string
		ext     bool
	)
	for _, i := range im {
		if !ext && (strings.HasPrefix(i, "\"github.com") || strings.HasPrefix(i, "\"vimagination")) {
			imports += "\n"
			ext = true
		}
		imports += "	" + i + "\n"
	}
	if *out == "-" || *out == "" {
		f = os.Stdout
	} else {
		f, err = os.Create(*out)
		e(err)
	}
	fmt.Fprintf(f, packageStart, *pkg, imports, date)
	if *single {
		f.WriteString(stringStart)
		if encs[0].Ext == "" {
			ne(f.WriteString("`"))
			ne(tickReplacer{f}.Write(encs[0].Buffer))
			ne(f.WriteString("`"))
		} else {
			ne(f.WriteString("\""))
			ne(replacer{f}.Write(encs[0].Buffer))
			ne(f.WriteString("\""))
		}
		f.WriteString(stringEnd)
		for n, enc := range encs {
			var (
				templ string
				vars  = []interface{}{0, *cvarname, *path + enc.Ext}
			)
			if enc.Ext == "" {
				vars = vars[1:]
				vars[0] = *varname
				if n == 0 {
					templ = identDecompress
				} else {
					templ = identCompress
				}
			} else {
				if n == 0 {
					vars[0] = len(data)
					templ = enc.Decompress
				} else {
					vars[0] = len(enc.Buffer)
					templ = enc.Compress
				}
			}
			fmt.Fprintf(f, templ, vars...)
		}
	} else {
		for _, enc := range encs {
			filename := *path + enc.Ext
			ne(fmt.Fprintf(f, soloStart, *varname, filename))
			if enc.Ext == "" {
				ne(f.WriteString("`"))
				ne(tickReplacer{f}.Write(enc.Buffer))
				ne(f.WriteString("`"))
			} else {
				ne(f.WriteString("\""))
				ne(replacer{f}.Write(enc.Buffer))
				ne(f.WriteString("\""))
			}
			ne(f.WriteString(soloEnd))
		}
	}
	ne(fmt.Fprintf(f, packageEnd))
	e(f.Close())
}

const (
	packageStart = `package %s

import (
%s)

func init() {
	date := time.Unix(%d, 0)
`
	stringStart = `	s := `
	stringEnd   = `
`
	packageEnd = `}
`
	soloStart = `	%s.Create(%q, httpdir.FileString(`
	soloEnd   = `, date))
`
	identDecompress = `	b := []byte(s)
	%s.Create(%q, httpdir.FileString(s, date))
`
	identCompress = `	%s.Create(%q, httpdir.FileBytes(b, date))
`

	brotliImport     = "\"github.com/google/brotli/go/cbrotli\""
	brotliDecompress = `	b := make([]byte, %d)
	br := cbrotli.NewReader(strings.NewReader(s))
	br.Read(b)
	br.Close()
	%s.Create(%q, httpdir.FileString(s, date))
`
	brotliCompress = `	brb := make(memio.Buffer, 0, %d)
	br := cbrotli.NewWriter(&brb, cbrotli.WriterOptions{Quality: 11})
	br.Write(b)
	br.Close()
	%s.Create(%q, httpdir.FileBytes(brb, date))
`
	flateImport     = "\"compress/flate\""
	flateDecompress = `	b := make([]byte, %d)
	fl := flate.NewReader(strings.NewReader(s))
	io.ReadFull(fl, b)
	fl.Close()
	%s.Create(%q, httpdir.FileString(s, date))
`
	flateCompress = `	flb := make(memio.Buffer, 0, %d)
	fl, _ := flate.NewWriter(&flb, flate.BestCompression)
	fl.Write(b)
	fl.Close()
	%s.Create(%q, httpdir.FileBytes(flb, date))
`
	gzipImport     = "\"compress/gzip\""
	gzipDecompress = `	b := make([]byte, %d)
	gz := gzip.NewReader(strings.NewReader(s))
	gz.Read(b)
	gz.Close()
	%s.Create(%q, httpdir.FileString(s, date))
`
	gzipCompress = `	gzb := make(memio.Buffer, 0, %d)
	gz, _ := gzip.NewWriterLevel(&gzb, gzip.BestCompression)
	gz.Write(b)
	gz.Close()
	%s.Create(%q, httpdir.FileBytes(gzb, date))
`
)
<commit_msg>added zopfli compression option to replace gzip<commit_after>package main // import "vimagination.zapto.org/httpdir/cmd/httpdir"

import (
	"compress/flate"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/foobaz/go-zopfli/zopfli"
	"github.com/google/brotli/go/cbrotli"
	"vimagination.zapto.org/memio"
)

var (
	pkg      = flag.String("p", "main", "package name")
	in       = flag.String("i", "", "input filename")
	out      = flag.String("o", "", "output filename")
	odate    = flag.String("d", "", "modified date [seconds since epoch]")
	path     = flag.String("w", "", "http path")
	varname  = flag.String("v", "httpdir.Default", "http dir variable name")
	cvarname = flag.String("c", "httpdir.Default", "http dir compressed variable name")
	help     = flag.Bool("h", false, "show help")
	gzcomp   = flag.Bool("g", false, "compress using gzip")
	brcomp   = flag.Bool("b", false, "compress using brotli")
	flcomp   = flag.Bool("f", false, "compress using flate/deflate")
	single   = flag.Bool("s", false, "use single source var and decompress/compress for others")
	zpfcomp  = flag.Bool("z", false, "replace gzip with zopfli compression")
)

type replacer struct {
	f *os.File
}

var hexArr = [16]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}

func (r replacer) Write(p []byte) (int, error) {
	n := 0
	toWrite := make([]byte, 0, 5)
	for len(p) > 0 {
		rn, s := utf8.DecodeRune(p)
		if rn == utf8.RuneError {
			s = 1
		}
		if s > 1 || (p[0] > 0 && p[0] < 0x7f && p[0] != '\n' && p[0] != '\\' && p[0] != '"') {
			toWrite = append(toWrite[:0], p[:s]...)
		} else {
			switch p[0] {
			case '\n':
				toWrite = append(toWrite[:0], '\\', 'n')
			case '\\':
				toWrite = append(toWrite[:0], '\\', '\\')
			case '"':
				toWrite = append(toWrite[:0], '\\', '"')
			default:
				toWrite = append(toWrite[:0], '\\', 'x', hexArr[p[0]>>4], hexArr[p[0]&15])
			}
		}
		_, err := r.f.Write(toWrite)
		n += s
		if err != nil {
			return n, err
		}
		p = p[s:]
	}
	return n, nil
}

type tickReplacer struct {
	f *os.File
}

func (r tickReplacer) Write(p []byte) (int, error) {
	var (
		m, n int
		err  error
	)
	hexes := make([]byte, 0, 32)
	toWrite := make([]byte, 0, 1024)
	for len(p) > 0 {
		dr, s := utf8.DecodeRune(p)
		if dr == utf8.RuneError || dr == '`' || dr == '\r' {
			hexes = append(hexes, p[0])
		} else {
			if len(hexes) > 0 {
				toWrite = toWrite[:0]
				toWrite = append(toWrite, '`', '+', '"')
				for _, b := range hexes {
					if b == '`' {
						toWrite = append(toWrite, '`')
					} else if b == '\r' {
						toWrite = append(toWrite, '\\', 'r')
					} else {
						toWrite = append(toWrite, '\\', 'x', hexArr[b>>4], hexArr[b&15])
					}
				}
				toWrite = append(toWrite, '"', '+', '`')
				_, err = r.f.Write(toWrite)
				n += len(hexes)
				if err != nil {
					break
				}
				hexes = hexes[:0]
			}
			m, err = r.f.Write(p[:s])
			n += m
			if err != nil {
				break
			}
		}
		p = p[s:]
	}
	return n, err
}

func e(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func ne(_ int, err error) {
	e(err)
}

type imports []string

func (im imports) Len() int {
	return len(im)
}

func (im imports) Less(i, j int) bool {
	si := strings.HasPrefix(im[i], "\"github.com")
	sj := strings.HasPrefix(im[j], "\"github.com")
	vi := strings.HasPrefix(im[i], "\"vimagination.zapto.org")
	vj := strings.HasPrefix(im[j], "\"vimagination.zapto.org")
	if si == sj && vi == vj {
		return im[i] < im[j]
	}
	return !si && !vi || si && vj
}

func (im imports) Swap(i, j int) {
	im[i], im[j] = im[j], im[i]
}

type encoding struct {
	Buffer                    []byte
	Compress, Decompress, Ext string
}

type encodings []encoding

func (e encodings) Len() int {
	return len(e)
}

func (e encodings) Less(i, j int) bool {
	return len(e[i].Buffer) < len(e[j].Buffer)
}

func (e encodings) Swap(i, j int) {
	e[i], e[j] = e[j], e[i]
}

func main() {
	flag.Parse()
	if *help {
		flag.Usage()
		return
	}
	var (
		f    *os.File
		err  error
		date int64
	)
	if *in == "-" || *in == "" {
		f = os.Stdin
		date = time.Now().Unix()
	} else {
		f, err = os.Open(*in)
		e(err)
		fi, err := f.Stat()
		e(err)
		date = fi.ModTime().Unix()
	}
	if *odate != "" {
		date, err = strconv.ParseInt(*odate, 10, 64)
		e(err)
	}

	data := make(memio.Buffer, 0, 1<<20)
	_, err = io.Copy(&data, f)
	e(err)
	e(f.Close())

	im := imports{"\"vimagination.zapto.org/httpdir\"", "\"time\""}

	encs := make(encodings, 1, 4)
	encs[0] = encoding{
		Buffer:     data,
		Compress:   identCompress,
		Decompress: identDecompress,
		Ext:        "",
	}

	if *brcomp {
		var b memio.Buffer
		br := cbrotli.NewWriter(&b, cbrotli.WriterOptions{Quality: 11})
		br.Write(data)
		br.Close()
		if *single {
			im = append(im, brotliImport)
		}
		encs = append(encs, encoding{
			Buffer:     b,
			Compress:   brotliCompress,
			Decompress: brotliDecompress,
			Ext:        ".br",
		})
	}
	if *flcomp {
		var b memio.Buffer
		fl, _ := flate.NewWriter(&b, flate.BestCompression)
		fl.Write(data)
		fl.Close()
		if *single {
			im = append(im, strings.Split(flateImport, "\n")...)
		}
		encs = append(encs, encoding{
			Buffer:     b,
			Compress:   flateCompress,
			Decompress: flateDecompress,
			Ext:        ".fl",
		})
	}
	if *gzcomp || *zpfcomp {
		var b memio.Buffer
		if *zpfcomp {
			zopfli.GzipCompress(&zopfli.Options{
				NumIterations:  100,
				BlockSplitting: true,
				BlockType:      2,
			}, data, &b)
		} else {
			gz, _ := gzip.NewWriterLevel(&b, gzip.BestCompression)
			gz.Write(data)
			gz.Close()
		}
		if *single {
			im = append(im, gzipImport)
		}
		encs = append(encs, encoding{
			Buffer:     b,
			Compress:   gzipCompress,
			Decompress: gzipDecompress,
			Ext:        ".gz",
		})
	}

	sort.Sort(encs)

	if *single && (*gzcomp || *brcomp || *flcomp) {
		im = append(im, "\"vimagination.zapto.org/memio\"")
		if encs[0].Ext != "" {
			im = append(im, "\"strings\"")
		}
	}
	if encs[0].Ext == ".fl" {
		im = append(im, "\"io\"")
	}

	sort.Sort(im)
	var (
		imports string
		ext     bool
	)
	for _, i := range im {
		if !ext && (strings.HasPrefix(i, "\"github.com") || strings.HasPrefix(i, "\"vimagination")) {
			imports += "\n"
			ext = true
		}
		imports += "	" + i + "\n"
	}
	if *out == "-" || *out == "" {
		f = os.Stdout
	} else {
		f, err = os.Create(*out)
		e(err)
	}
	fmt.Fprintf(f, packageStart, *pkg, imports, date)
	if *single {
		f.WriteString(stringStart)
		if encs[0].Ext == "" {
			ne(f.WriteString("`"))
			ne(tickReplacer{f}.Write(encs[0].Buffer))
			ne(f.WriteString("`"))
		} else {
			ne(f.WriteString("\""))
			ne(replacer{f}.Write(encs[0].Buffer))
			ne(f.WriteString("\""))
		}
		f.WriteString(stringEnd)
		for n, enc := range encs {
			var (
				templ string
				vars  = []interface{}{0, *cvarname, *path + enc.Ext}
			)
			if enc.Ext == "" {
				vars = vars[1:]
				vars[0] = *varname
				if n == 0 {
					templ = identDecompress
				} else {
					templ = identCompress
				}
			} else {
				if n == 0 {
					vars[0] = len(data)
					templ = enc.Decompress
				} else {
					vars[0] = len(enc.Buffer)
					templ = enc.Compress
				}
			}
			fmt.Fprintf(f, templ, vars...)
		}
	} else {
		for _, enc := range encs {
			filename := *path + enc.Ext
			ne(fmt.Fprintf(f, soloStart, *varname, filename))
			if enc.Ext == "" {
				ne(f.WriteString("`"))
				ne(tickReplacer{f}.Write(enc.Buffer))
				ne(f.WriteString("`"))
			} else {
				ne(f.WriteString("\""))
				ne(replacer{f}.Write(enc.Buffer))
				ne(f.WriteString("\""))
			}
			ne(f.WriteString(soloEnd))
		}
	}
	ne(fmt.Fprintf(f, packageEnd))
	e(f.Close())
}

const (
	packageStart = `package %s

import (
%s)

func init() {
	date := time.Unix(%d, 0)
`
	stringStart = `	s := `
	stringEnd   = `
`
	packageEnd = `}
`
	soloStart = `	%s.Create(%q, httpdir.FileString(`
	soloEnd   = `, date))
`
	identDecompress = `	b := []byte(s)
	%s.Create(%q, httpdir.FileString(s, date))
`
	identCompress = `	%s.Create(%q, httpdir.FileBytes(b, date))
`

	brotliImport     = "\"github.com/google/brotli/go/cbrotli\""
	brotliDecompress = `	b := make([]byte, %d)
	br := cbrotli.NewReader(strings.NewReader(s))
	br.Read(b)
	br.Close()
	%s.Create(%q, httpdir.FileString(s, date))
`
	brotliCompress = `	brb := make(memio.Buffer, 0, %d)
	br := cbrotli.NewWriter(&brb, cbrotli.WriterOptions{Quality: 11})
	br.Write(b)
	br.Close()
	%s.Create(%q, httpdir.FileBytes(brb, date))
`
	flateImport     = "\"compress/flate\""
	flateDecompress = `	b := make([]byte, %d)
	fl := flate.NewReader(strings.NewReader(s))
	io.ReadFull(fl, b)
	fl.Close()
	%s.Create(%q, httpdir.FileString(s, date))
`
	flateCompress = `	flb := make(memio.Buffer, 0, %d)
	fl, _ := flate.NewWriter(&flb, flate.BestCompression)
	fl.Write(b)
	fl.Close()
	%s.Create(%q, httpdir.FileBytes(flb, date))
`
	gzipImport     = "\"compress/gzip\""
	gzipDecompress = `	b := make([]byte, %d)
	gz := gzip.NewReader(strings.NewReader(s))
	gz.Read(b)
	gz.Close()
	%s.Create(%q, httpdir.FileString(s, date))
`
	gzipCompress = `	gzb := make(memio.Buffer, 0, %d)
	gz, _ := gzip.NewWriterLevel(&gzb, gzip.BestCompression)
	gz.Write(b)
	gz.Close()
	%s.Create(%q, httpdir.FileBytes(gzb, date))
`
)
<|endoftext|>


<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

// Repo represents the repository under test
// For more information about this structure take a look to the README
type Repo struct {
	// URL is the url of the repository
	URL string

	// MasterBranch is the master branch of this repository
	MasterBranch string

	// PR is the pull request number
	PR int

	// RefreshTime is the time to wait for checking if a pull request needs to be tested
	RefreshTime string

	// Toke is the repository access token
	Token string

	// Setup contains the conmmands needed to setup the environment
	Setup []string

	// Run contains the commands to run the test
	Run []string

	// Teardown contains the commands to be executed once Run ends
	Teardown []string

	// OnSuccess contains the commands to be executed if Setup, Run and Teardown finished correctly
	OnSuccess []string

	// OnFailure contains the commands to be executed if any of Setup, Run or Teardown fail
	OnFailure []string

	// TTY specify whether a tty must be allocate to run the stages
	TTY bool

	// PostOnSuccess is the comment to be posted if the test finished correctly
	PostOnSuccess string

	// PostOnFailure is the comment to be posted if the test fails
	PostOnFailure string

	// LogDir is the logs directory
	LogDir string

	// Language is the language of the repository
	Language RepoLanguage

	// CommentTrigger is the comment that must be present to trigger the test
	CommentTrigger RepoComment

	// LogServer contains the information of the server where the logs must be placed
	LogServer LogServer

	// Whitelist is the list of users whose pull request can be tested
	Whitelist string

	// cvr control version repository
	cvr CVR

	// refresh is RefreshTime once parsed
	refresh time.Duration

	// env contains the environment variables to be used in each stage
	env []string

	// whitelistUsers is the whitelist once parsed
	whitelistUsers []string

	// logger of the repository
	logger *logrus.Entry

	// prConfig is the configuration used to create pull request objects
	prConfig pullRequestConfig
}

const (
	logDirMode    = 0755
	logFileMode   = 0664
	logServerUser = "root"
)

var defaultEnv = []string{"CI=true", "LOCALCI=true"}

var runTestsInParallel bool

var testLock sync.Mutex

func (r *Repo) setupCvr() error {
	var err error

	// validate url
	r.URL = strings.TrimSpace(r.URL)
	if len(r.URL) == 0 {
		return fmt.Errorf("missing repository url")
	}

	// set repository logger
	r.logger = ciLog.WithFields(logrus.Fields{
		"Repo": r.URL,
	})

	// get the control version repository
	r.cvr, err = newCVR(r.URL, r.Token)
	r.logger.Debugf("control version repository: %#v", r.cvr)

	return err
}

func (r *Repo) setupLogServer() error {
	if reflect.DeepEqual(r.LogServer, LogServer{}) {
		return nil
	}

	if len(r.LogServer.IP) == 0 {
		return fmt.Errorf("missing server ip")
	}

	if len(r.LogServer.User) == 0 {
		r.LogServer.User = logServerUser
	}

	if len(r.LogServer.Dir) == 0 {
		r.LogServer.Dir = defaultLogDir
	}

	return nil
}

func (r *Repo) setupLogDir() error {
	// create log directory
	if err := os.MkdirAll(r.LogDir, logDirMode); err != nil {
		return err
	}

	return nil
}

func (r *Repo) setupRefreshTime() error {
	var err error

	// validate refresh time
	r.refresh, err = time.ParseDuration(r.RefreshTime)
	if err != nil {
		return fmt.Errorf("failed to parse refresh time '%s' %s", r.RefreshTime, err)
	}

	return nil
}

func (r *Repo) setupCommentTrigger() error {
	if reflect.DeepEqual(r.CommentTrigger, RepoComment{}) {
		return nil
	}

	if len(r.CommentTrigger.Comment) == 0 {
		return fmt.Errorf("missing comment trigger")
	}

	return nil
}

func (r *Repo) setupLanguage() error {
	return r.Language.setup()
}

func (r *Repo) setupStages() error {
	if len(r.Run) == 0 {
		return fmt.Errorf("missing run commands")
	}

	return nil
}

func (r *Repo) setupWhitelist() error {
	// get the list of users
	r.whitelistUsers = strings.Split(r.Whitelist, ",")
	return nil
}

func (r *Repo) setupEnvars() error {
	// add environment variables
	r.env = os.Environ()
	r.env = append(r.env, defaultEnv...)
	repoSlug := fmt.Sprintf("LOCALCI_REPO_SLUG=%s", r.cvr.getRepoSlug())
	r.env = append(r.env, repoSlug)

	return nil
}

// setup the repository. This method MUST BE called before use any other
func (r *Repo) setup() error {
	var err error

	setupFuncs := []func() error{
		r.setupCvr,
		r.setupRefreshTime,
		r.setupLogDir,
		r.setupLogServer,
		r.setupCommentTrigger,
		r.setupLanguage,
		r.setupStages,
		r.setupWhitelist,
		r.setupEnvars,
	}

	for _, setupFunc := range setupFuncs {
		if err = setupFunc(); err != nil {
			return err
		}
	}

	r.prConfig = pullRequestConfig{
		cvr:            r.cvr,
		logger:         r.logger,
		commentTrigger: r.CommentTrigger,
		postOnFailure:  r.PostOnFailure,
		postOnSuccess:  r.PostOnSuccess,
	}

	r.logger.Debugf("control version repository: %#v", r.cvr)

	return nil
}

// loop to monitor the repository
func (r *Repo) loop() {
	revisionsTested := make(map[string]revision)

	r.logger.Debugf("monitoring in a loop the repository: %+v", *r)

	appendPullRequests := func(revisions *[]revision, prs []int) error {
		for _, prNumber := range prs {
			r.logger.Debugf("requesting pull request %d", prNumber)
			pr, err := newPullRequest(prNumber, r.prConfig)
			if err != nil {
				return fmt.Errorf("failed to get pull request '%d' %s", prNumber, err)
			}
			*revisions = append(*revisions, pr)
		}
		return nil
	}

	for {
		var revisionsToTest []revision

		// append master branch
		r.logger.Debugf("requesting master branch: %s", r.MasterBranch)
		branch, err := newRepoBranch(r.MasterBranch, r.cvr, r.logger)
		if err != nil {
			r.logger.Warnf("failed to get master branch %s: %s", r.MasterBranch, err)
		} else {
			revisionsToTest = append(revisionsToTest, branch)
		}

		// append pull requests
		if r.PR != 0 {
			// if PR is not 0 then we have to monitor just one PR
			if err = appendPullRequests(&revisionsToTest, []int{r.PR}); err != nil {
				r.logger.Warnf("failed to append pull request %d", r.PR, err)
			}
		} else {
			// append open pull request
			r.logger.Debugf("requesting open pull requests")
			prs, err := r.cvr.getOpenPullRequests()
			if err != nil {
				r.logger.Warnf("failed to get open pull requests: %s", err)
			} else if err = appendPullRequests(&revisionsToTest, prs); err != nil {
				r.logger.Warnf("failed to append pull requests %+v: %s", prs, err)
			}
		}

		r.logger.Debugf("testing revisions: %#v", revisionsToTest)
		r.testRevisions(revisionsToTest, &revisionsTested)

		r.logger.Debugf("going to sleep: %s", r.RefreshTime)
		time.Sleep(r.refresh)
	}
}

func (r *Repo) testRevisions(revisions []revision, revisionsTested *map[string]revision) {
	// remove revisions that are not in the list of open pull request and already tested
	for k, v := range *revisionsTested {
		found := false
		// iterate over open pull requests and master branch
		for _, r := range revisions {
			if r.id() == k {
				found = true
				break
			}
		}

		if !found && !v.isBeingTested() {
			delete((*revisionsTested), k)
		}
	}

	for _, revision := range revisions {
		tested, ok := (*revisionsTested)[revision.id()]
		if ok {
			// checking if the old version of the PR is being tested
			if tested.isBeingTested() {
				r.logger.Debugf("revision is being tested: %#v", tested)
				continue
			}
			if revision.equal(tested) {
				r.logger.Debugf("revision was already tested: %#v", revision)
				continue
			}
		}

		// setup revision
		langEnv, err := r.setupRevision(revision)
		if err != nil {
			r.logger.Errorf("failed to setup revision %#v: %s", revision, err)
			continue
		}

		// cleanup revision
		defer func() {
			err = langEnv.cleanup()
			if err != nil {
				r.logger.Error(err)
			}
		}()

		// test revision
		if runTestsInParallel {
			go func() {
				if err := r.testRevision(revision, langEnv); err != nil {
					r.logger.Errorf("failed to test revision %#v %s", revision, err)
				}
			}()
		} else {
			testLock.Lock()
			if err := r.testRevision(revision, langEnv); err != nil {
				r.logger.Errorf("failed to test revision %#v %s", revision, err)
			}
			testLock.Unlock()
		}

		// copy the PR that was tested
		(*revisionsTested)[revision.id()] = revision
	}
}

// test the pull request specified in the configuration file
// if pr does not exist an error is returned
func (r *Repo) test() error {
	if r.PR == 0 {
		return fmt.Errorf("Missing pull request number in configuration file")
	}

	rev, err := newPullRequest(r.PR, r.prConfig)
	if err != nil {
		return fmt.Errorf("failed to get pull request %d %s", r.PR, err)
	}

	// run tests in parallel does not make sense when
	// we are just testing one pull request
	runTestsInParallel = false

	// setup revision
	langEnv, err := r.setupRevision(rev)
	if err != nil {
		return fmt.Errorf("failed to setup revision %#v: %s", rev, err)
	}

	// cleanup revision
	defer func() {
		err = langEnv.cleanup()
		if err != nil {
			r.logger.Error(err)
		}
	}()

	// test revision
	return r.testRevision(rev, langEnv)
}

// setupRevision generates a language environment, downloads the revision
// and creates the logs directory
func (r *Repo) setupRevision(rev revision) (languageConfig, error) {
	// generate a new environment to run the stages
	langEnv, err := r.Language.generateEnvironment(r.cvr.getProjectSlug())
	if err != nil {
		return languageConfig{}, err
	}

	// download the revision
	if err = rev.download(langEnv.workingDir); err != nil {
		return languageConfig{}, err
	}

	// cleanup and set the log directory of the pull request
	logDir := filepath.Join(r.LogDir, rev.logDirName())
	_ = os.RemoveAll(logDir)
	if err = os.MkdirAll(logDir, logDirMode); err != nil {
		return languageConfig{}, err
	}

	return langEnv, nil
}

// testRevision tests a specific revision
// returns an error if the test fail
func (r *Repo) testRevision(rev revision, langEnv languageConfig) error {
	config := stageConfig{
		logger:     r.logger,
		workingDir: langEnv.workingDir,
		tty:        r.TTY,
		logDir:     filepath.Join(r.LogDir, rev.logDirName()),
	}

	// set environment variables
	config.env = r.env

	// appends language environment variables
	if len(langEnv.env) > 0 {
		config.env = append(config.env, langEnv.env...)
	}

	// copy logs to server if we have an IP address
	if len(r.LogServer.IP) != 0 {
		defer func() {
			if err := r.LogServer.copy(config.logDir); err != nil {
				r.logger.Errorf("failed to copy log dir %s to server %+v", config.logDir, r.LogServer)
			}
		}()
	}

	r.logger.Debugf("stage config: %+v", config)

	stages := map[string]stage{
		"setup":     stage{name: "setup", commands: r.Setup},
		"run":       stage{name: "run", commands: r.Run},
		"teardown":  stage{name: "teardown", commands: r.Teardown},
		"onSuccess": stage{name: "onSuccess", commands: r.OnSuccess},
		"onFailure": stage{name: "onFailure", commands: r.OnFailure},
	}

	// run test
	return rev.test(config, stages)
}
<commit_msg>localCI: honour whitelist<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

// Repo represents the repository under test
// For more information about this structure take a look to the README
type Repo struct {
	// URL is the url of the repository
	URL string

	// MasterBranch is the master branch of this repository
	MasterBranch string

	// PR is the pull request number
	PR int

	// RefreshTime is the time to wait for checking if a pull request needs to be tested
	RefreshTime string

	// Toke is the repository access token
	Token string

	// Setup contains the conmmands needed to setup the environment
	Setup []string

	// Run contains the commands to run the test
	Run []string

	// Teardown contains the commands to be executed once Run ends
	Teardown []string

	// OnSuccess contains the commands to be executed if Setup, Run and Teardown finished correctly
	OnSuccess []string

	// OnFailure contains the commands to be executed if any of Setup, Run or Teardown fail
	OnFailure []string

	// TTY specify whether a tty must be allocate to run the stages
	TTY bool

	// PostOnSuccess is the comment to be posted if the test finished correctly
	PostOnSuccess string

	// PostOnFailure is the comment to be posted if the test fails
	PostOnFailure string

	// LogDir is the logs directory
	LogDir string

	// Language is the language of the repository
	Language RepoLanguage

	// CommentTrigger is the comment that must be present to trigger the test
	CommentTrigger RepoComment

	// LogServer contains the information of the server where the logs must be placed
	LogServer LogServer

	// Whitelist is the list of users whose pull request can be tested
	Whitelist string

	// cvr control version repository
	cvr CVR

	// refresh is RefreshTime once parsed
	refresh time.Duration

	// env contains the environment variables to be used in each stage
	env []string

	// whitelistUsers is the whitelist once parsed
	whitelistUsers []string

	// logger of the repository
	logger *logrus.Entry

	// prConfig is the configuration used to create pull request objects
	prConfig pullRequestConfig
}

const (
	logDirMode    = 0755
	logFileMode   = 0664
	logServerUser = "root"
)

var defaultEnv = []string{"CI=true", "LOCALCI=true"}

var runTestsInParallel bool

var testLock sync.Mutex

func (r *Repo) setupCvr() error {
	var err error

	// validate url
	r.URL = strings.TrimSpace(r.URL)
	if len(r.URL) == 0 {
		return fmt.Errorf("missing repository url")
	}

	// set repository logger
	r.logger = ciLog.WithFields(logrus.Fields{
		"Repo": r.URL,
	})

	// get the control version repository
	r.cvr, err = newCVR(r.URL, r.Token)
	r.logger.Debugf("control version repository: %#v", r.cvr)

	return err
}

func (r *Repo) setupLogServer() error {
	if reflect.DeepEqual(r.LogServer, LogServer{}) {
		return nil
	}

	if len(r.LogServer.IP) == 0 {
		return fmt.Errorf("missing server ip")
	}

	if len(r.LogServer.User) == 0 {
		r.LogServer.User = logServerUser
	}

	if len(r.LogServer.Dir) == 0 {
		r.LogServer.Dir = defaultLogDir
	}

	return nil
}

func (r *Repo) setupLogDir() error {
	// create log directory
	if err := os.MkdirAll(r.LogDir, logDirMode); err != nil {
		return err
	}

	return nil
}

func (r *Repo) setupRefreshTime() error {
	var err error

	// validate refresh time
	r.refresh, err = time.ParseDuration(r.RefreshTime)
	if err != nil {
		return fmt.Errorf("failed to parse refresh time '%s' %s", r.RefreshTime, err)
	}

	return nil
}

func (r *Repo) setupCommentTrigger() error {
	if reflect.DeepEqual(r.CommentTrigger, RepoComment{}) {
		return nil
	}

	if len(r.CommentTrigger.Comment) == 0 {
		return fmt.Errorf("missing comment trigger")
	}

	return nil
}

func (r *Repo) setupLanguage() error {
	return r.Language.setup()
}

func (r *Repo) setupStages() error {
	if len(r.Run) == 0 {
		return fmt.Errorf("missing run commands")
	}

	return nil
}

func (r *Repo) setupWhitelist() error {
	// get the list of users
	r.whitelistUsers = strings.Split(r.Whitelist, ",")
	return nil
}

func (r *Repo) setupEnvars() error {
	// add environment variables
	r.env = os.Environ()
	r.env = append(r.env, defaultEnv...)
	repoSlug := fmt.Sprintf("LOCALCI_REPO_SLUG=%s", r.cvr.getRepoSlug())
	r.env = append(r.env, repoSlug)

	return nil
}

// setup the repository. This method MUST BE called before use any other
func (r *Repo) setup() error {
	var err error

	setupFuncs := []func() error{
		r.setupCvr,
		r.setupRefreshTime,
		r.setupLogDir,
		r.setupLogServer,
		r.setupCommentTrigger,
		r.setupLanguage,
		r.setupStages,
		r.setupWhitelist,
		r.setupEnvars,
	}

	for _, setupFunc := range setupFuncs {
		if err = setupFunc(); err != nil {
			return err
		}
	}

	r.prConfig = pullRequestConfig{
		cvr:            r.cvr,
		logger:         r.logger,
		commentTrigger: r.CommentTrigger,
		postOnFailure:  r.PostOnFailure,
		postOnSuccess:  r.PostOnSuccess,
		whitelist:      r.Whitelist,
	}

	r.logger.Debugf("control version repository: %#v", r.cvr)

	return nil
}

// loop to monitor the repository
func (r *Repo) loop() {
	revisionsTested := make(map[string]revision)

	r.logger.Debugf("monitoring in a loop the repository: %+v", *r)

	appendPullRequests := func(revisions *[]revision, prs []int) error {
		for _, prNumber := range prs {
			r.logger.Debugf("requesting pull request %d", prNumber)
			pr, err := newPullRequest(prNumber, r.prConfig)
			if err != nil {
				return fmt.Errorf("failed to get pull request '%d' %s", prNumber, err)
			}
			*revisions = append(*revisions, pr)
		}
		return nil
	}

	for {
		var revisionsToTest []revision

		// append master branch
		r.logger.Debugf("requesting master branch: %s", r.MasterBranch)
		branch, err := newRepoBranch(r.MasterBranch, r.cvr, r.logger)
		if err != nil {
			r.logger.Warnf("failed to get master branch %s: %s", r.MasterBranch, err)
		} else {
			revisionsToTest = append(revisionsToTest, branch)
		}

		// append pull requests
		if r.PR != 0 {
			// if PR is not 0 then we have to monitor just one PR
			if err = appendPullRequests(&revisionsToTest, []int{r.PR}); err != nil {
				r.logger.Warnf("failed to append pull request %d", r.PR, err)
			}
		} else {
			// append open pull request
			r.logger.Debugf("requesting open pull requests")
			prs, err := r.cvr.getOpenPullRequests()
			if err != nil {
				r.logger.Warnf("failed to get open pull requests: %s", err)
			} else if err = appendPullRequests(&revisionsToTest, prs); err != nil {
				r.logger.Warnf("failed to append pull requests %+v: %s", prs, err)
			}
		}

		r.logger.Debugf("testing revisions: %#v", revisionsToTest)
		r.testRevisions(revisionsToTest, &revisionsTested)

		r.logger.Debugf("going to sleep: %s", r.RefreshTime)
		time.Sleep(r.refresh)
	}
}

func (r *Repo) testRevisions(revisions []revision, revisionsTested *map[string]revision) {
	// remove revisions that are not in the list of open pull request and already tested
	for k, v := range *revisionsTested {
		found := false
		// iterate over open pull requests and master branch
		for _, r := range revisions {
			if r.id() == k {
				found = true
				break
			}
		}

		if !found && !v.isBeingTested() {
			delete((*revisionsTested), k)
		}
	}

	for _, revision := range revisions {
		tested, ok := (*revisionsTested)[revision.id()]
		if ok {
			// checking if the old version of the PR is being tested
			if tested.isBeingTested() {
				r.logger.Debugf("revision is being tested: %#v", tested)
				continue
			}
			if revision.equal(tested) {
				r.logger.Debugf("revision was already tested: %#v", revision)
				continue
			}
		}

		// check if the revision can be tested
		if err := revision.canBeTested(); err != nil {
			r.logger.Debugf("revision %s cannot be tested: %s", revision.id(), err)
			continue
		}

		// setup revision
		langEnv, err := r.setupRevision(revision)
		if err != nil {
			r.logger.Errorf("failed to setup revision %#v: %s", revision, err)
			continue
		}

		// cleanup revision
		defer func() {
			err = langEnv.cleanup()
			if err != nil {
				r.logger.Error(err)
			}
		}()

		// test revision
		if runTestsInParallel {
			go func() {
				if err := r.testRevision(revision, langEnv); err != nil {
					r.logger.Errorf("failed to test revision %#v %s", revision, err)
				}
			}()
		} else {
			testLock.Lock()
			if err := r.testRevision(revision, langEnv); err != nil {
				r.logger.Errorf("failed to test revision %#v %s", revision, err)
			}
			testLock.Unlock()
		}

		// copy the PR that was tested
		(*revisionsTested)[revision.id()] = revision
	}
}

// test the pull request specified in the configuration file
// if pr does not exist an error is returned
func (r *Repo) test() error {
	if r.PR == 0 {
		return fmt.Errorf("Missing pull request number in configuration file")
	}

	rev, err := newPullRequest(r.PR, r.prConfig)
	if err != nil {
		return fmt.Errorf("failed to get pull request %d %s", r.PR, err)
	}

	// run tests in parallel does not make sense when
	// we are just testing one pull request
	runTestsInParallel = false

	// setup revision
	langEnv, err := r.setupRevision(rev)
	if err != nil {
		return fmt.Errorf("failed to setup revision %#v: %s", rev, err)
	}

	// cleanup revision
	defer func() {
		err = langEnv.cleanup()
		if err != nil {
			r.logger.Error(err)
		}
	}()

	// test revision
	return r.testRevision(rev, langEnv)
}

// setupRevision generates a language environment, downloads the revision
// and creates the logs directory
func (r *Repo) setupRevision(rev revision) (languageConfig, error) {
	// generate a new environment to run the stages
	langEnv, err := r.Language.generateEnvironment(r.cvr.getProjectSlug())
	if err != nil {
		return languageConfig{}, err
	}

	// download the revision
	if err = rev.download(langEnv.workingDir); err != nil {
		return languageConfig{}, err
	}

	// cleanup and set the log directory of the pull request
	logDir := filepath.Join(r.LogDir, rev.logDirName())
	_ = os.RemoveAll(logDir)
	if err = os.MkdirAll(logDir, logDirMode); err != nil {
		return languageConfig{}, err
	}

	return langEnv, nil
}

// testRevision tests a specific revision
// returns an error if the test fail
func (r *Repo) testRevision(rev revision, langEnv languageConfig) error {
	config := stageConfig{
		logger:     r.logger,
		workingDir: langEnv.workingDir,
		tty:        r.TTY,
		logDir:     filepath.Join(r.LogDir, rev.logDirName()),
	}

	// set environment variables
	config.env = r.env

	// appends language environment variables
	if len(langEnv.env) > 0 {
		config.env = append(config.env, langEnv.env...)
	}

	// copy logs to server if we have an IP address
	if len(r.LogServer.IP) != 0 {
		defer func() {
			if err := r.LogServer.copy(config.logDir); err != nil {
				r.logger.Errorf("failed to copy log dir %s to server %+v", config.logDir, r.LogServer)
			}
		}()
	}

	r.logger.Debugf("stage config: %+v", config)

	stages := map[string]stage{
		"setup":     stage{name: "setup", commands: r.Setup},
		"run":       stage{name: "run", commands: r.Run},
		"teardown":  stage{name: "teardown", commands: r.Teardown},
		"onSuccess": stage{name: "onSuccess", commands: r.OnSuccess},
		"onFailure": stage{name: "onFailure", commands: r.OnFailure},
	}

	// run test
	return rev.test(config, stages)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/awsutil"
	libjson "github.com/Symantec/Dominator/lib/json"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
)

type awsGeneratorType struct {
	targets        awsutil.TargetList
	filterTagsFile string
}

type resultType struct {
	mdb *mdb.Mdb
	err error
}

type tagFilterType struct {
	Key    string
	Values []string
}

func newAwsGenerator(args []string) (generator, error) {
	return &awsGeneratorType{
			targets: awsutil.TargetList{awsutil.Target{args[1], args[0]}}},
		nil
}

func newAwsFilteredGenerator(args []string) (generator, error) {
	gen := awsGeneratorType{
		filterTagsFile: args[1],
	}
	if err := gen.targets.Set(args[0]); err != nil {
		return nil, err
	}
	return &gen, nil
}

func newAwsLocalGenerator(args []string) (generator, error) {
	region, err := awsutil.GetLocalRegion()
	if err != nil {
		return nil, err
	}
	return &awsGeneratorType{
			targets: awsutil.TargetList{awsutil.Target{"", region}}},
		nil
}

func (g *awsGeneratorType) Generate(unused_datacentre string,
	logger log.Logger) (*mdb.Mdb, error) {
	resultsChannel := make(chan resultType, 1)
	numTargets, err := awsutil.ForEachTarget(g.targets, nil,
		func(awsService *ec2.EC2, account, region string, logger log.Logger) {
			var result resultType
			result.mdb, result.err = g.generateForTarget(awsService, logger)
			resultsChannel <- result
		},
		logger)
	// Collect results.
	var newMdb mdb.Mdb
	hostnames := make(map[string]struct{})
	for i := 0; i < numTargets; i++ {
		result := <-resultsChannel
		if result.err != nil {
			if err == nil {
				err = result.err
				logger.Println(err)
			}
			continue
		}
		for _, machine := range result.mdb.Machines {
			if _, ok := hostnames[machine.Hostname]; ok {
				txt := "duplicate hostname: " + machine.Hostname
				logger.Println(txt)
				if err == nil {
					err = errors.New(txt)
				}
				break
			}
			newMdb.Machines = append(newMdb.Machines, machine)
		}
	}
	return &newMdb, err
}

func (g *awsGeneratorType) generateForTarget(svc *ec2.EC2, logger log.Logger) (
	*mdb.Mdb, error) {
	filters, err := g.makeFilters()
	if err != nil {
		return nil, err
	}
	resp, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{
		Filters: filters,
	})
	if err != nil {
		return nil, err
	}
	return extractMdb(resp), nil
}

func (g *awsGeneratorType) makeFilters() ([]*ec2.Filter, error) {
	filters := make([]*ec2.Filter, 1, 1)
	filters[0] = &ec2.Filter{
		Name:   aws.String("instance-state-name"),
		Values: []*string{aws.String(ec2.InstanceStateNameRunning)},
	}
	if g.filterTagsFile == "" {
		return filters, nil
	}
	var tags []tagFilterType
	if err := libjson.ReadFromFile(g.filterTagsFile, &tags); err != nil {
		return nil, fmt.Errorf("error loading tags file: %s", err)
	}
	for _, tag := range tags {
		filters = append(filters, &ec2.Filter{
			Name:   aws.String("tag:" + tag.Key),
			Values: aws.StringSlice(tag.Values),
		})
	}
	return filters, nil
}

func extractMdb(output *ec2.DescribeInstancesOutput) *mdb.Mdb {
	var result mdb.Mdb
	for _, reservation := range output.Reservations {
		for _, instance := range reservation.Instances {
			if instance.PrivateDnsName != nil {
				machine := mdb.Machine{
					Hostname: *instance.PrivateDnsName,
					AwsMetadata: &mdb.AwsMetadata{
						InstanceId: *instance.InstanceId,
						Tags:       make(map[string]string),
					},
				}
				if instance.PrivateIpAddress != nil {
					machine.IpAddress = *instance.PrivateIpAddress
				}
				extractTags(instance.Tags, &machine)
				result.Machines = append(result.Machines, machine)
			}
		}
	}
	return &result
}

func extractTags(tags []*ec2.Tag, machine *mdb.Machine) {
	for _, tag := range tags {
		if tag.Key == nil || tag.Value == nil {
			continue
		}
		machine.AwsMetadata.Tags[*tag.Key] = *tag.Value
		switch *tag.Key {
		case "RequiredImage":
			machine.RequiredImage = *tag.Value
		case "PlannedImage":
			machine.PlannedImage = *tag.Value
		case "DisableUpdates":
			machine.DisableUpdates = true
		case "OwnerGroup":
			machine.OwnerGroup = *tag.Value
		}
	}
}
<commit_msg>Populate lib/mdb.AwsMetadata.{AcocuntId,AccountName,Region} fields in mdbd.<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/awsutil"
	libjson "github.com/Symantec/Dominator/lib/json"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
)

type awsGeneratorType struct {
	targets        awsutil.TargetList
	filterTagsFile string
}

type resultType struct {
	mdb *mdb.Mdb
	err error
}

type tagFilterType struct {
	Key    string
	Values []string
}

func newAwsGenerator(args []string) (generator, error) {
	return &awsGeneratorType{
			targets: awsutil.TargetList{awsutil.Target{args[1], args[0]}}},
		nil
}

func newAwsFilteredGenerator(args []string) (generator, error) {
	gen := awsGeneratorType{
		filterTagsFile: args[1],
	}
	if err := gen.targets.Set(args[0]); err != nil {
		return nil, err
	}
	return &gen, nil
}

func newAwsLocalGenerator(args []string) (generator, error) {
	region, err := awsutil.GetLocalRegion()
	if err != nil {
		return nil, err
	}
	return &awsGeneratorType{
			targets: awsutil.TargetList{awsutil.Target{"", region}}},
		nil
}

func (g *awsGeneratorType) Generate(unused_datacentre string,
	logger log.Logger) (*mdb.Mdb, error) {
	resultsChannel := make(chan resultType, 1)
	numTargets, err := awsutil.ForEachTarget(g.targets, nil,
		func(awsService *ec2.EC2, account, region string, logger log.Logger) {
			var result resultType
			result.mdb, result.err = g.generateForTarget(awsService, account,
				region, logger)
			resultsChannel <- result
		},
		logger)
	// Collect results.
	var newMdb mdb.Mdb
	hostnames := make(map[string]struct{})
	for i := 0; i < numTargets; i++ {
		result := <-resultsChannel
		if result.err != nil {
			if err == nil {
				err = result.err
				logger.Println(err)
			}
			continue
		}
		for _, machine := range result.mdb.Machines {
			if _, ok := hostnames[machine.Hostname]; ok {
				txt := "duplicate hostname: " + machine.Hostname
				logger.Println(txt)
				if err == nil {
					err = errors.New(txt)
				}
				break
			}
			newMdb.Machines = append(newMdb.Machines, machine)
		}
	}
	return &newMdb, err
}

func (g *awsGeneratorType) generateForTarget(svc *ec2.EC2, accountName string,
	region string, logger log.Logger) (
	*mdb.Mdb, error) {
	filters, err := g.makeFilters()
	if err != nil {
		return nil, err
	}
	resp, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{
		Filters: filters,
	})
	if err != nil {
		return nil, err
	}
	return extractMdb(resp, accountName, region), nil
}

func (g *awsGeneratorType) makeFilters() ([]*ec2.Filter, error) {
	filters := make([]*ec2.Filter, 1, 1)
	filters[0] = &ec2.Filter{
		Name:   aws.String("instance-state-name"),
		Values: []*string{aws.String(ec2.InstanceStateNameRunning)},
	}
	if g.filterTagsFile == "" {
		return filters, nil
	}
	var tags []tagFilterType
	if err := libjson.ReadFromFile(g.filterTagsFile, &tags); err != nil {
		return nil, fmt.Errorf("error loading tags file: %s", err)
	}
	for _, tag := range tags {
		filters = append(filters, &ec2.Filter{
			Name:   aws.String("tag:" + tag.Key),
			Values: aws.StringSlice(tag.Values),
		})
	}
	return filters, nil
}

func extractMdb(output *ec2.DescribeInstancesOutput, accountName string,
	region string) *mdb.Mdb {
	var result mdb.Mdb
	for _, reservation := range output.Reservations {
		accountId := aws.StringValue(reservation.OwnerId)
		for _, instance := range reservation.Instances {
			if instance.PrivateDnsName != nil {
				machine := mdb.Machine{
					Hostname: *instance.PrivateDnsName,
					AwsMetadata: &mdb.AwsMetadata{
						AccountId:   accountId,
						AccountName: accountName,
						InstanceId:  *instance.InstanceId,
						Region:      region,
						Tags:        make(map[string]string),
					},
				}
				if instance.PrivateIpAddress != nil {
					machine.IpAddress = *instance.PrivateIpAddress
				}
				extractTags(instance.Tags, &machine)
				result.Machines = append(result.Machines, machine)
			}
		}
	}
	return &result
}

func extractTags(tags []*ec2.Tag, machine *mdb.Machine) {
	for _, tag := range tags {
		if tag.Key == nil || tag.Value == nil {
			continue
		}
		machine.AwsMetadata.Tags[*tag.Key] = *tag.Value
		switch *tag.Key {
		case "RequiredImage":
			machine.RequiredImage = *tag.Value
		case "PlannedImage":
			machine.PlannedImage = *tag.Value
		case "DisableUpdates":
			machine.DisableUpdates = true
		case "OwnerGroup":
			machine.OwnerGroup = *tag.Value
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"regexp"
	"sort"

	"github.com/goulash/pacman"
	"github.com/goulash/pacman/meta"
	"github.com/goulash/pr"
	"github.com/spf13/cobra"
)

var (
	// Versioned causes packages to be printed with version information.
	listVersioned bool
	// Mode can be either "count", "filter", or "mark" (which is the default
	// if no match is found.
	listMode string
	// Pending marks packages that need to be added to the database,
	// as well as packages that are in the database but are not available.
	listPending bool
	// Duplicates marks the number of obsolete packages for each package.
	listDuplicates bool
	// Installed marks whether packages are locally installed or not.
	listInstalled bool
	// Synchronize marks which packages have newer versions on AUR.
	listSynchronize bool
	// Same as all of the above.
	listAllOptions bool

	searchPOSIX bool
)

func init() {
	MainCmd.AddCommand(listCmd)

	listCmd.Flags().BoolVarP(&listVersioned, "versioned", "v", false, "show package versions along with name")
	listCmd.Flags().BoolVarP(&listPending, "pending", "p", false, "mark pending changes to the database")
	listCmd.Flags().BoolVarP(&listDuplicates, "duplicates", "d", false, "mark packages with duplicate package files")
	listCmd.Flags().BoolVarP(&listInstalled, "installed", "l", false, "mark packages that are locally installed")
	listCmd.Flags().BoolVarP(&listSynchronize, "outdated", "o", false, "mark packages that are newer in AUR")
	listCmd.Flags().BoolVarP(&listAllOptions, "all", "a", false, "all information; same as -vpdlo")
	listCmd.Flags().BoolVar(&searchPOSIX, "posix", false, "use POSIX-style regular expressions")
}

var listCmd = &cobra.Command{
	Use:     "list [regex]",
	Aliases: []string{"ls"},
	Short:   "list packages that belong to the managed repository",
	Long: `List packages that belong to the managed repository.

  All packages that are in the managed repository are listed,
  whether or not they are registered with the database.
  If you want to filter packages, see the filter command.

  When marking entries, the following symbols are used:

    -package-           package will be deleted
    package <?>         no AUR information could be found
    package <!>         local package is out-of-date
    package <*>         local package is newer than AUR package
    package (n)         there are n extra versions of package

  When versions are shown, local version is adjacent to package name:

    package 1.0 -> 2.0  local package is out-of-date
    package 2.0 <- 1.0  local package is newer than AUR package

  If a valid regular expression is supplied, only packages that match
  the expression will be listed.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) > 1 {
			return &UsageError{"list", "list command takes at most one argument", cmd.Usage}
		}

		if listAllOptions {
			listVersioned = true
			listPending = true
			listDuplicates = true
			listInstalled = true
			listSynchronize = true
		}

		var regex *regexp.Regexp
		if len(args) == 1 {
			var err error
			if searchPOSIX {
				regex, err = regexp.Compile(args[0])
			} else {
				regex, err = regexp.CompilePOSIX(args[0])
			}
			if err != nil {
				return err
			}
		}

		pkgs, err := Repo.ListMeta(nil, listSynchronize, func(mp pacman.AnyPackage) string {
			p := mp.(*meta.Package)
			if regex != nil && !regex.MatchString(p.PkgName()) {
				return ""
			}

			if listPending && !p.HasFiles() {
				return fmt.Sprintf("-%s-", p.Name)
			}

			buf := bytes.NewBufferString(p.Name)
			if listPending && p.HasUpdate() {
				buf.WriteRune('*')
			}
			if listVersioned {
				buf.WriteRune(' ')
				buf.WriteString(p.Version())
			}
			if listSynchronize {
				ap := p.AUR
				if ap == nil {
					buf.WriteString(" <?>") // no aur info
				} else if pacman.PkgNewer(ap, p) {
					if listVersioned {
						buf.WriteString(" -> ") // new version
						buf.WriteString(ap.Version)
					} else {
						buf.WriteString(" <!>") // local version older than aur
					}
				} else if pacman.PkgOlder(ap, p) {
					if listVersioned {
						buf.WriteString(" <- ") // old version
						buf.WriteString(ap.Version)
					} else {
						buf.WriteString(" <*>") // local version newer than aur
					}
				}
			}
			if listDuplicates && len(p.Files)-1 > 0 {
				buf.WriteString(fmt.Sprintf(" (%v)", len(p.Files)-1))
			}

			return buf.String()
		})
		if err != nil {
			return err
		}

		// Print packages to stdout
		printSet(pkgs, "", Conf.Columnate)
		return nil
	},
}

// printSet prints a set of items and optionally a header.
func printSet(list []string, h string, cols bool) {
	sort.Strings(list)
	if h != "" {
		fmt.Printf("\n%s\n", h)
	}
	if cols {
		pr.PrintFlex(list)
	} else if h != "" {
		for _, j := range list {
			fmt.Println(" ", j)
		}
	} else {
		for _, j := range list {
			fmt.Println(j)
		}
	}
}
<commit_msg>Add -r, --registered flag to list command<commit_after>// Copyright (c) 2016, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"regexp"
	"sort"

	"github.com/goulash/pacman"
	"github.com/goulash/pacman/meta"
	"github.com/goulash/pr"
	"github.com/spf13/cobra"
)

var (
	// Versioned causes packages to be printed with version information.
	listVersioned bool
	// Mode can be either "count", "filter", or "mark" (which is the default
	// if no match is found.
	listMode string
	// Pending marks packages that need to be added to the database,
	// as well as packages that are in the database but are not available.
	listPending bool
	// Duplicates marks the number of obsolete packages for each package.
	listDuplicates bool
	// Installed marks whether packages are locally installed or not.
	listInstalled bool
	// Synchronize marks which packages have newer versions on AUR.
	listSynchronize bool
	// Same as all of the above.
	listAllOptions bool
	// Only show registered packages.
	filterRegistered bool

	searchPOSIX bool
)

func init() {
	MainCmd.AddCommand(listCmd)

	listCmd.Flags().BoolVarP(&filterRegistered, "registered", "r", false, "only show packages that are in the database")
	listCmd.Flags().BoolVarP(&listVersioned, "versioned", "v", false, "show package versions along with name")
	listCmd.Flags().BoolVarP(&listPending, "pending", "p", false, "mark pending changes to the database")
	listCmd.Flags().BoolVarP(&listDuplicates, "duplicates", "d", false, "mark packages with duplicate package files")
	listCmd.Flags().BoolVarP(&listInstalled, "installed", "l", false, "mark packages that are locally installed")
	listCmd.Flags().BoolVarP(&listSynchronize, "outdated", "o", false, "mark packages that are newer in AUR")
	listCmd.Flags().BoolVarP(&listAllOptions, "all", "a", false, "all information; same as -vpdlo")
	listCmd.Flags().BoolVar(&searchPOSIX, "posix", false, "use POSIX-style regular expressions")
}

var listCmd = &cobra.Command{
	Use:     "list [regex]",
	Aliases: []string{"ls"},
	Short:   "list packages that belong to the managed repository",
	Long: `List packages that belong to the managed repository.

  All packages that are in the managed repository are listed,
  whether or not they are registered with the database.
  If you only want to show registered packages, use the -r flag.

  When marking entries, the following symbols are used:

    -package-           package will be deleted
    package <?>         no AUR information could be found
    package <!>         local package is out-of-date
    package <*>         local package is newer than AUR package
    package (n)         there are n extra versions of package

  When versions are shown, local version is adjacent to package name:

    package 1.0 -> 2.0  local package is out-of-date
    package 2.0 <- 1.0  local package is newer than AUR package

  If a valid regular expression is supplied, only packages that match
  the expression will be listed.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) > 1 {
			return &UsageError{"list", "list command takes at most one argument", cmd.Usage}
		}

		if listAllOptions {
			listVersioned = true
			listPending = true
			listDuplicates = true
			listInstalled = true
			listSynchronize = true
		}

		var regex *regexp.Regexp
		if len(args) == 1 {
			var err error
			if searchPOSIX {
				regex, err = regexp.Compile(args[0])
			} else {
				regex, err = regexp.CompilePOSIX(args[0])
			}
			if err != nil {
				return err
			}
		}

		pkgs, err := Repo.ListMeta(nil, listSynchronize, func(mp pacman.AnyPackage) string {
			p := mp.(*meta.Package)
			if regex != nil && !regex.MatchString(p.PkgName()) {
				return ""
			}

			if filterRegistered && !p.IsRegistered() {
				return ""
			}

			if listPending && !p.HasFiles() {
				return fmt.Sprintf("-%s-", p.Name)
			}

			buf := bytes.NewBufferString(p.Name)
			if listPending && p.HasUpdate() {
				buf.WriteRune('*')
			}
			if listVersioned {
				buf.WriteRune(' ')
				buf.WriteString(p.Version())
			}
			if listSynchronize {
				ap := p.AUR
				if ap == nil {
					buf.WriteString(" <?>") // no aur info
				} else if pacman.PkgNewer(ap, p) {
					if listVersioned {
						buf.WriteString(" -> ") // new version
						buf.WriteString(ap.Version)
					} else {
						buf.WriteString(" <!>") // local version older than aur
					}
				} else if pacman.PkgOlder(ap, p) {
					if listVersioned {
						buf.WriteString(" <- ") // old version
						buf.WriteString(ap.Version)
					} else {
						buf.WriteString(" <*>") // local version newer than aur
					}
				}
			}
			if listDuplicates && len(p.Files)-1 > 0 {
				buf.WriteString(fmt.Sprintf(" (%v)", len(p.Files)-1))
			}

			return buf.String()
		})
		if err != nil {
			return err
		}

		// Print packages to stdout
		printSet(pkgs, "", Conf.Columnate)
		return nil
	},
}

// printSet prints a set of items and optionally a header.
func printSet(list []string, h string, cols bool) {
	sort.Strings(list)
	if h != "" {
		fmt.Printf("\n%s\n", h)
	}
	if cols {
		pr.PrintFlex(list)
	} else if h != "" {
		for _, j := range list {
			fmt.Println(" ", j)
		}
	} else {
		for _, j := range list {
			fmt.Println(j)
		}
	}
}
<|endoftext|>