<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"strconv"
	"strings"
	"syscall"

	"github.com/subgraph/oz"

	"github.com/codegangsta/cli"
)

var PathDpkgDivert string
var PathDpkg string
var OzConfig *oz.Config
var OzProfiles *oz.Profiles
var OzProfile *oz.Profile

func init() {
	checkRoot()
	PathDpkgDivert = checkDpkgDivert()
	PathDpkg = checkDpkg()
}

func main() {
	app := cli.NewApp()

	app.Name = "oz-utils"
	app.Usage = "command line interface to install, remove, and create Oz sandboxes\nYou can specify a package name, a binary path, or a Oz profile file "
	app.Author = "Subgraph"
	app.Email = "info@subgraph.com"
	app.Version = oz.OzVersion
	app.EnableBashCompletion = true

	flagsHookMode := []cli.Flag{
		cli.BoolFlag{
			Name:  "hook",
			Usage: "Run in hook mode, not normally used by the end user",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "config",
			Usage: "check and show Oz configurations",
			Subcommands: []cli.Command{
				{
					Name:   "check",
					Usage:  "check oz configuration and profiles for errors",
					Action: handleConfigcheck,
				},
				{
					Name:   "show",
					Usage:  "prints ouf oz configuration",
					Action: handleConfigshow,
				},
			},
		},
		{
			Name:   "install",
			Usage:  "install binary diversion for a program",
			Action: handleInstall,
			Flags:  flagsHookMode,
		},
		{
			Name:   "remove",
			Usage:  "remove a binary diversion for a program",
			Action: handleRemove,
			Flags:  flagsHookMode,
		},
		{
			Name:   "status",
			Usage:  "show the status of a binary diversion for a program",
			Action: handleStatus,
		},
		{
			Name:   "create",
			Usage:  "create a new sandbox profile",
			Action: handleCreate,
		},
	}

	app.Run(os.Args)
}

func handleConfigcheck(c *cli.Context) {
	fmt.Println("Here be dragons!")
	os.Exit(1)
}

func handleConfigshow(c *cli.Context) {
	config, err := oz.LoadConfig(oz.DefaultConfigPath)
	useDefaults := false
	if err != nil {
		if os.IsNotExist(err) {
			config = oz.NewDefaultConfig()
			useDefaults = true
		} else {
			fmt.Fprintf(os.Stderr, "Could not load configuration: %s", oz.DefaultConfigPath, err)
			os.Exit(1)
		}
	}

	v := reflect.ValueOf(*config)
	vt := reflect.TypeOf(*config)
	maxFieldLength := 0
	for i := 0; i < v.NumField(); i++ {
		flen := len(vt.Field(i).Tag.Get("json"))
		if flen > maxFieldLength {
			maxFieldLength = flen
		}
	}
	maxValueLength := 0
	for i := 0; i < v.NumField(); i++ {
		sval := fmt.Sprintf("%v", v.Field(i).Interface())
		flen := len(sval)
		if flen > maxValueLength {
			maxValueLength = flen
		}
	}

	sfmt := "%-" + strconv.Itoa(maxFieldLength) + "s: %-" + strconv.Itoa(maxValueLength) + "v"
	hfmt := "%-" + strconv.Itoa(maxFieldLength) + "s: %s\n"

	if !useDefaults {
		fmt.Printf(hfmt, "Config file", oz.DefaultConfigPath)
	} else {
		fmt.Printf(hfmt, "Config file", "Not found - using defaults")
	}

	for i := 0; i < len(fmt.Sprintf(sfmt, "", "")); i++ {
		fmt.Print("=")
	}
	fmt.Println("")

	for i := 0; i < v.NumField(); i++ {
		fval := fmt.Sprintf("%v", v.Field(i).Interface())
		fmt.Printf(sfmt, vt.Field(i).Tag.Get("json"), fval)
		desc := vt.Field(i).Tag.Get("desc")
		if desc != "" {
			fmt.Printf(" # %s", desc)
		}

		fmt.Println("")
	}

	os.Exit(0)
}

func handleInstall(c *cli.Context) {
	OzConfig = loadConfig()
	pname := c.Args()[0]
	OzProfile, err := loadProfile(pname, OzConfig.ProfileDir)
	if err != nil || OzProfile == nil {
		installExit(c.Bool("hook"), fmt.Errorf("Unable to load profiles for %s (%v).\n", pname, err))
		return // For clarity
	}

	if OzConfig.DivertSuffix == "" {
		installExit(c.Bool("hook"), fmt.Errorf("Divert requires a suffix to be set.\n"))
		return // For clarity
	}

	isInstalled, err := isDivertInstalled(OzProfile.Path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unknown error: %+v\n", err)
		os.Exit(1)
	}
	if isInstalled == true {
		fmt.Println("Divert already installed for ", OzProfile.Path)
		os.Exit(0)
	}

	dpkgArgs := []string{
		"--add",
		"--package",
		"oz",
		"--rename",
		"--divert",
		getBinaryPath(OzProfile.Path),
		OzProfile.Path,
	}

	_, err = exec.Command(PathDpkgDivert, dpkgArgs...).Output()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Dpkg divert command `%s %+s` failed: %s", PathDpkgDivert, dpkgArgs, err)
		os.Exit(1)
	}

	err = syscall.Symlink(OzConfig.ClientPath, OzProfile.Path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create symlink %s", err)
		os.Exit(1)
	}

	fmt.Printf("Successfully installed Oz sandbox for: %s.\n", OzProfile.Path)
}

func handleRemove(c *cli.Context) {
	OzConfig = loadConfig()
	pname := c.Args()[0]
	OzProfile, err := loadProfile(pname, OzConfig.ProfileDir)
	if err != nil || OzProfile == nil {
		installExit(c.Bool("hook"), fmt.Errorf("Unable to load profiles for %s.\n", pname))
		return // For clarity
	}

	if OzConfig.DivertSuffix == "" {
		installExit(c.Bool("hook"), fmt.Errorf("Divert requires a suffix to be set.\n"))
		return // For clarity
	}

	isInstalled, err := isDivertInstalled(OzProfile.Path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unknown error: %+v\n", err)
		os.Exit(1)
	}
	if isInstalled == false {
		fmt.Println("Divert is not installed for ", OzProfile.Path)
		os.Exit(0)
	}

	os.Remove(OzProfile.Path)

	dpkgArgs := []string{
		"--rename",
		"--package",
		"oz",
		"--remove",
		OzProfile.Path,
	}

	_, err = exec.Command(PathDpkgDivert, dpkgArgs...).Output()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Dpkg divert command `%s %+s` failed: %s", PathDpkgDivert, dpkgArgs, err)
		os.Exit(1)
	}

	fmt.Printf("Successfully remove jail for: %s.\n", OzProfile.Path)
}

func handleStatus(c *cli.Context) {
	OzConfig = loadConfig()
	pname := c.Args()[0]
	OzProfile, err := loadProfile(pname, OzConfig.ProfileDir)
	if err != nil || OzProfile == nil {
		fmt.Fprintf(os.Stderr, "Unable to load profiles (%s).\n", err)
		os.Exit(1)
	}

	if OzConfig.DivertSuffix == "" {
		fmt.Fprintf(os.Stderr, "Divert requires a suffix to be set.\n")
		os.Exit(1)
	}

	isInstalled, err := isDivertInstalled(OzProfile.Path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unknown error: %+v\n", err)
		os.Exit(1)
	}
	if isInstalled {
		fmt.Println("Package divert is \033[0;32minstalled\033[0m for: ", OzProfile.Path)
	} else {
		fmt.Println("Package divert is \033[0;31mnot installed\033[0m for: ", OzProfile.Path)
	}

}

func handleCreate(c *cli.Context) {
	OzConfig = loadConfig()

	fmt.Println("The weasels ran off with this command... please come back later!")
	os.Exit(1)
}

/*
* UTILITIES
 */

func checkRoot() {
	if os.Getuid() != 0 {
		fmt.Fprintf(os.Stderr, "%s should be used as root.\n", os.Args[0])
		os.Exit(1)
	}
}

func checkDpkgDivert() string {
	ddpath, err := exec.LookPath("dpkg-divert")
	if err != nil {
		fmt.Fprintln(os.Stderr, "You do not appear to have dpkg-divert, are you not running Debian/Ubuntu?")
		os.Exit(1)
	}
	return ddpath
}

func checkDpkg() string {
	dpath, err := exec.LookPath("dpkg")
	if err != nil {
		fmt.Fprintln(os.Stderr, "You do not appear to have dpkg, are you not running Debian/Ubuntu?")
		os.Exit(1)
	}
	return dpath
}

func isDivertInstalled(bpath string) (bool, error) {
	outp, err := exec.Command(PathDpkgDivert, "--truename", bpath).Output()
	if err != nil {
		return false, err
	}
	dpath := strings.TrimSpace(string(outp))

	isInstalled := (dpath == getBinaryPath(string(bpath)))
	if isInstalled {
		_, err := os.Readlink(bpath)
		if err != nil {
			return false, fmt.Errorf("`%s` appears to be diverted but is not installed", dpath)
		}
	}
	return isInstalled, nil
}

func getBinaryPath(bpath string) string {
	bpath = strings.TrimSpace(string(bpath))

	if strings.HasSuffix(bpath, "."+OzConfig.DivertSuffix) == false {
		bpath += "." + OzConfig.DivertSuffix
	}

	return bpath
}

func loadConfig() *oz.Config {
	config, err := oz.LoadConfig(oz.DefaultConfigPath)
	if err != nil {
		if os.IsNotExist(err) {
			fmt.Fprintln(os.Stderr, "Configuration file (%s) is missing, using defaults.", oz.DefaultConfigPath)
			config = oz.NewDefaultConfig()
		} else {
			fmt.Fprintln(os.Stderr, "Could not load configuration: %s", oz.DefaultConfigPath, err)
			os.Exit(1)
		}
	}

	return config
}

func loadProfile(name, profileDir string) (*oz.Profile, error) {
	ps, err := oz.LoadProfiles(profileDir)
	if err != nil {
		return nil, err
	}

	return ps.GetProfileByName(name)

}

func installExit(hook bool, err error) {
	if hook {
		os.Exit(0)
	} else {
		fmt.Fprintf(os.Stderr, "%s", err)
		os.Exit(1)
	}
}
<commit_msg>Added 'config check' to oz-setup<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"strconv"
	"strings"
	"syscall"

	"github.com/subgraph/oz"

	"github.com/codegangsta/cli"
)

var PathDpkgDivert string
var PathDpkg string
var OzConfig *oz.Config
var OzProfiles *oz.Profiles
var OzProfile *oz.Profile

func init() {
	checkRoot()
	PathDpkgDivert = checkDpkgDivert()
	PathDpkg = checkDpkg()
}

func main() {
	app := cli.NewApp()

	app.Name = "oz-utils"
	app.Usage = "command line interface to install, remove, and create Oz sandboxes\nYou can specify a package name, a binary path, or a Oz profile file "
	app.Author = "Subgraph"
	app.Email = "info@subgraph.com"
	app.Version = oz.OzVersion
	app.EnableBashCompletion = true

	flagsHookMode := []cli.Flag{
		cli.BoolFlag{
			Name:  "hook",
			Usage: "Run in hook mode, not normally used by the end user",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "config",
			Usage: "check and show Oz configurations",
			Subcommands: []cli.Command{
				{
					Name:   "check",
					Usage:  "check oz configuration and profiles for errors",
					Action: handleConfigcheck,
				},
				{
					Name:   "show",
					Usage:  "prints ouf oz configuration",
					Action: handleConfigshow,
				},
			},
		},
		{
			Name:   "install",
			Usage:  "install binary diversion for a program",
			Action: handleInstall,
			Flags:  flagsHookMode,
		},
		{
			Name:   "remove",
			Usage:  "remove a binary diversion for a program",
			Action: handleRemove,
			Flags:  flagsHookMode,
		},
		{
			Name:   "status",
			Usage:  "show the status of a binary diversion for a program",
			Action: handleStatus,
		},
		{
			Name:   "create",
			Usage:  "create a new sandbox profile",
			Action: handleCreate,
		},
	}

	app.Run(os.Args)
}

func handleConfigcheck(c *cli.Context) {
	_, err := oz.LoadConfig(oz.DefaultConfigPath)
	if err != nil {
		if !os.IsNotExist(err) {
			fmt.Fprintf(os.Stderr, "Could not load configuration `%s`: %v\n", oz.DefaultConfigPath, err)
			os.Exit(1)
		}
	}

	OzConfig = loadConfig()
	_, err = oz.LoadProfiles(OzConfig.ProfileDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to load profiles from `%s`: %v\n", OzConfig.ProfileDir, err)
		os.Exit(1)
	}

	fmt.Println("Configurations and profiles ok!")
	os.Exit(0)
}

func handleConfigshow(c *cli.Context) {
	config, err := oz.LoadConfig(oz.DefaultConfigPath)
	useDefaults := false
	if err != nil {
		if os.IsNotExist(err) {
			config = oz.NewDefaultConfig()
			useDefaults = true
		} else {
			fmt.Fprintf(os.Stderr, "Could not load configuration `%s`: %v\n", oz.DefaultConfigPath, err)
			os.Exit(1)
		}
	}

	v := reflect.ValueOf(*config)
	vt := reflect.TypeOf(*config)
	maxFieldLength := 0
	for i := 0; i < v.NumField(); i++ {
		flen := len(vt.Field(i).Tag.Get("json"))
		if flen > maxFieldLength {
			maxFieldLength = flen
		}
	}
	maxValueLength := 0
	for i := 0; i < v.NumField(); i++ {
		sval := fmt.Sprintf("%v", v.Field(i).Interface())
		flen := len(sval)
		if flen > maxValueLength {
			maxValueLength = flen
		}
	}

	sfmt := "%-" + strconv.Itoa(maxFieldLength) + "s: %-" + strconv.Itoa(maxValueLength) + "v"
	hfmt := "%-" + strconv.Itoa(maxFieldLength) + "s: %s\n"

	if !useDefaults {
		fmt.Printf(hfmt, "Config file", oz.DefaultConfigPath)
	} else {
		fmt.Printf(hfmt, "Config file", "Not found - using defaults")
	}

	for i := 0; i < len(fmt.Sprintf(sfmt, "", "")); i++ {
		fmt.Print("=")
	}
	fmt.Println("")

	for i := 0; i < v.NumField(); i++ {
		fval := fmt.Sprintf("%v", v.Field(i).Interface())
		fmt.Printf(sfmt, vt.Field(i).Tag.Get("json"), fval)
		desc := vt.Field(i).Tag.Get("desc")
		if desc != "" {
			fmt.Printf(" # %s", desc)
		}

		fmt.Println("")
	}

	os.Exit(0)
}

func handleInstall(c *cli.Context) {
	OzConfig = loadConfig()
	pname := c.Args()[0]
	OzProfile, err := loadProfile(pname, OzConfig.ProfileDir)
	if err != nil || OzProfile == nil {
		installExit(c.Bool("hook"), fmt.Errorf("Unable to load profiles for %s (%v).\n", pname, err))
		return // For clarity
	}

	if OzConfig.DivertSuffix == "" {
		installExit(c.Bool("hook"), fmt.Errorf("Divert requires a suffix to be set.\n"))
		return // For clarity
	}

	isInstalled, err := isDivertInstalled(OzProfile.Path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unknown error: %+v\n", err)
		os.Exit(1)
	}
	if isInstalled == true {
		fmt.Println("Divert already installed for ", OzProfile.Path)
		os.Exit(0)
	}

	dpkgArgs := []string{
		"--add",
		"--package",
		"oz",
		"--rename",
		"--divert",
		getBinaryPath(OzProfile.Path),
		OzProfile.Path,
	}

	_, err = exec.Command(PathDpkgDivert, dpkgArgs...).Output()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Dpkg divert command `%s %+s` failed: %s", PathDpkgDivert, dpkgArgs, err)
		os.Exit(1)
	}

	err = syscall.Symlink(OzConfig.ClientPath, OzProfile.Path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create symlink %s", err)
		os.Exit(1)
	}

	fmt.Printf("Successfully installed Oz sandbox for: %s.\n", OzProfile.Path)
}

func handleRemove(c *cli.Context) {
	OzConfig = loadConfig()
	pname := c.Args()[0]
	OzProfile, err := loadProfile(pname, OzConfig.ProfileDir)
	if err != nil || OzProfile == nil {
		installExit(c.Bool("hook"), fmt.Errorf("Unable to load profiles for %s.\n", pname))
		return // For clarity
	}

	if OzConfig.DivertSuffix == "" {
		installExit(c.Bool("hook"), fmt.Errorf("Divert requires a suffix to be set.\n"))
		return // For clarity
	}

	isInstalled, err := isDivertInstalled(OzProfile.Path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unknown error: %+v\n", err)
		os.Exit(1)
	}
	if isInstalled == false {
		fmt.Println("Divert is not installed for ", OzProfile.Path)
		os.Exit(0)
	}

	os.Remove(OzProfile.Path)

	dpkgArgs := []string{
		"--rename",
		"--package",
		"oz",
		"--remove",
		OzProfile.Path,
	}

	_, err = exec.Command(PathDpkgDivert, dpkgArgs...).Output()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Dpkg divert command `%s %+s` failed: %s", PathDpkgDivert, dpkgArgs, err)
		os.Exit(1)
	}

	fmt.Printf("Successfully remove jail for: %s.\n", OzProfile.Path)
}

func handleStatus(c *cli.Context) {
	OzConfig = loadConfig()
	pname := c.Args()[0]
	OzProfile, err := loadProfile(pname, OzConfig.ProfileDir)
	if err != nil || OzProfile == nil {
		fmt.Fprintf(os.Stderr, "Unable to load profiles (%s).\n", err)
		os.Exit(1)
	}

	if OzConfig.DivertSuffix == "" {
		fmt.Fprintf(os.Stderr, "Divert requires a suffix to be set.\n")
		os.Exit(1)
	}

	isInstalled, err := isDivertInstalled(OzProfile.Path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unknown error: %+v\n", err)
		os.Exit(1)
	}
	if isInstalled {
		fmt.Println("Package divert is \033[0;32minstalled\033[0m for: ", OzProfile.Path)
	} else {
		fmt.Println("Package divert is \033[0;31mnot installed\033[0m for: ", OzProfile.Path)
	}

}

func handleCreate(c *cli.Context) {
	OzConfig = loadConfig()

	fmt.Println("The weasels ran off with this command... please come back later!")
	os.Exit(1)
}

/*
* UTILITIES
 */

func checkRoot() {
	if os.Getuid() != 0 {
		fmt.Fprintf(os.Stderr, "%s should be used as root.\n", os.Args[0])
		os.Exit(1)
	}
}

func checkDpkgDivert() string {
	ddpath, err := exec.LookPath("dpkg-divert")
	if err != nil {
		fmt.Fprintln(os.Stderr, "You do not appear to have dpkg-divert, are you not running Debian/Ubuntu?")
		os.Exit(1)
	}
	return ddpath
}

func checkDpkg() string {
	dpath, err := exec.LookPath("dpkg")
	if err != nil {
		fmt.Fprintln(os.Stderr, "You do not appear to have dpkg, are you not running Debian/Ubuntu?")
		os.Exit(1)
	}
	return dpath
}

func isDivertInstalled(bpath string) (bool, error) {
	outp, err := exec.Command(PathDpkgDivert, "--truename", bpath).Output()
	if err != nil {
		return false, err
	}
	dpath := strings.TrimSpace(string(outp))

	isInstalled := (dpath == getBinaryPath(string(bpath)))
	if isInstalled {
		_, err := os.Readlink(bpath)
		if err != nil {
			return false, fmt.Errorf("`%s` appears to be diverted but is not installed", dpath)
		}
	}
	return isInstalled, nil
}

func getBinaryPath(bpath string) string {
	bpath = strings.TrimSpace(string(bpath))

	if strings.HasSuffix(bpath, "."+OzConfig.DivertSuffix) == false {
		bpath += "." + OzConfig.DivertSuffix
	}

	return bpath
}

func loadConfig() *oz.Config {
	config, err := oz.LoadConfig(oz.DefaultConfigPath)
	if err != nil {
		if os.IsNotExist(err) {
			fmt.Fprintln(os.Stderr, "Configuration file (%s) is missing, using defaults.", oz.DefaultConfigPath)
			config = oz.NewDefaultConfig()
		} else {
			fmt.Fprintln(os.Stderr, "Could not load configuration: %s", oz.DefaultConfigPath, err)
			os.Exit(1)
		}
	}

	return config
}

func loadProfile(name, profileDir string) (*oz.Profile, error) {
	ps, err := oz.LoadProfiles(profileDir)
	if err != nil {
		return nil, err
	}

	return ps.GetProfileByName(name)

}

func installExit(hook bool, err error) {
	if hook {
		os.Exit(0)
	} else {
		fmt.Fprintf(os.Stderr, "%s", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Command line interface to ping ports
package main

import (
	"flag"
	"fmt"
	"os"
	"github.com/janosgyerik/portping"
	"net"
	"time"
)

// TODO
// flags: --tcp, --udp; default is tcp
// flag: -v verbose; default=false
// drop default count, print forever, until cancel with Control-C, and print stats

const (
	defaultCount = 5
	defaultTimeoutSeconds = 10
	defaultNetwork = "tcp"
)

func exit() {
	flag.Usage()
	os.Exit(1)
}

type Params struct {
	host    string
	port    string
	count   int
	timeout time.Duration
}

func parseArgs() Params {
	flag.Usage = func() {
		fmt.Printf("Usage: %s [options] host port\n\n", os.Args[0])
		flag.PrintDefaults()
	}

	countPtr := flag.Int("c", defaultCount, "stop after count connections")
	timeoutPtr := flag.Int("W", defaultTimeoutSeconds, "time in seconds to wait for connections")
	flag.Parse()

	if len(flag.Args()) < 2 {
		exit()
	}

	host := flag.Args()[0]
	port := flag.Args()[1]

	return Params{
		host: host,
		port: port,
		count: *countPtr,
		timeout: time.Duration(*timeoutPtr) * time.Second,
	}
}

// FormatResult converts the result returned by Ping to string.
func formatResult(err error) string {
	if err == nil {
		return "success"
	}
	switch err := err.(type) {
	case *net.OpError:
		return err.Err.Error()
	default:
		return err.Error()
	}
}

func main() {
	params := parseArgs()

	addr := net.JoinHostPort(params.host, params.port)
	fmt.Printf("Starting to ping %s ...\n", addr)

	c := make(chan error)
	go portping.PingN(defaultNetwork, addr, params.timeout, params.count, c)

	allSuccessful := true

	for i := 0; i < params.count; i++ {
		// TODO add time
		err := <-c
		if err != nil {
			allSuccessful = false
		}
		fmt.Printf("%s [%d] -> %s\n", addr, i + 1, formatResult(err))
	}

	// TODO print summary
	// --- host:port ping statistics ---
	// n connections attempted, m successful, x% failed
	// round-trip min/avg/max/stddev = a/b/c/d ms

	if !allSuccessful {
		os.Exit(1)
	}
}
<commit_msg>added network option<commit_after>// Command line interface to ping ports
package main

import (
	"flag"
	"fmt"
	"os"
	"github.com/janosgyerik/portping"
	"net"
	"time"
)

// TODO
// drop default count, print forever, until cancel with Control-C, and print stats

const (
	defaultCount = 5
	defaultTimeoutSeconds = 10
	defaultNetwork = "tcp"
)

func exit() {
	flag.Usage()
	os.Exit(1)
}

type Params struct {
	host    string
	port    string
	count   int
	timeout time.Duration
	network string
}

func parseArgs() Params {
	flag.Usage = func() {
		fmt.Printf("Usage: %s [options] host port\n\n", os.Args[0])
		flag.PrintDefaults()
	}

	countPtr := flag.Int("c", defaultCount, "stop after count connections")
	timeoutPtr := flag.Int("W", defaultTimeoutSeconds, "time in seconds to wait for connections")
	network := flag.String("net", defaultNetwork, "the network to use")
	flag.Parse()

	if len(flag.Args()) < 2 {
		exit()
	}

	host := flag.Args()[0]
	port := flag.Args()[1]

	return Params{
		host: host,
		port: port,
		count: *countPtr,
		timeout: time.Duration(*timeoutPtr) * time.Second,
		network: *network,
	}
}

// FormatResult converts the result returned by Ping to string.
func formatResult(err error) string {
	if err == nil {
		return "success"
	}
	switch err := err.(type) {
	case *net.OpError:
		return err.Err.Error()
	default:
		return err.Error()
	}
}

func main() {
	params := parseArgs()

	addr := net.JoinHostPort(params.host, params.port)
	fmt.Printf("Starting to ping %s ...\n", addr)

	c := make(chan error)
	go portping.PingN(params.network, addr, params.timeout, params.count, c)

	allSuccessful := true

	for i := 0; i < params.count; i++ {
		// TODO add time
		err := <-c
		if err != nil {
			allSuccessful = false
		}
		fmt.Printf("%s [%d] -> %s\n", addr, i + 1, formatResult(err))
	}

	// TODO print summary
	// --- host:port ping statistics ---
	// n connections attempted, m successful, x% failed
	// round-trip min/avg/max/stddev = a/b/c/d ms

	if !allSuccessful {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// root2npy converts the content of a ROOT TTree to a NumPy data file.
//
//  Usage of root2npy:
//   -f string
//     	path to input ROOT file name
//   -o string
//     	path to output npz file name (default "output.npz")
//   -t string
//     	name of the tree to convert (default "tree")
//
package main

import (
	"archive/zip"
	"bytes"
	"flag"
	"io"
	"log"
	"os"
	"reflect"

	"github.com/sbinet/npyio"

	"go-hep.org/x/hep/rootio"
)

func main() {
	log.SetPrefix("root2npy: ")
	log.SetFlags(0)

	fname := flag.String("f", "", "path to input ROOT file name")
	oname := flag.String("o", "output.npz", "path to output npz file name")
	tname := flag.String("t", "tree", "name of the tree to convert")

	flag.Parse()

	if *fname == "" {
		flag.Usage()
		log.Fatalf("missing input ROOT filename argument")
	}

	f, err := rootio.Open(*fname)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	obj, ok := f.Get(*tname)
	if !ok {
		log.Fatalf("no object named %q in file %q", *tname, *fname)
	}

	tree, ok := obj.(rootio.Tree)
	if !ok {
		log.Fatalf("object %q in file %q is not a rootio.Tree", *tname, *fname)
	}

	var nt = ntuple{n: tree.Entries()}
	log.Printf("scanning leaves...")
	for _, leaf := range tree.Leaves() {
		if leaf.Type().Kind() == reflect.String {
			nt.add(leaf.Name(), leaf.Type())
			continue
		}
		if leaf.LeafCount() != nil {
			log.Printf(">>> %q []%v not supported", leaf.Name(), leaf.TypeName())
			continue
		}
		if leaf.Len() > 1 {
			log.Printf(">>> %q [%d]%v not supported", leaf.Name(), leaf.Len(), leaf.TypeName())
			continue
		}
		nt.add(leaf.Name(), leaf.Type())
	}
	log.Printf("scanning leaves... [done]")

	sc, err := rootio.NewScannerVars(tree, nt.args...)
	if err != nil {
		log.Fatal(err)
	}
	defer sc.Close()

	for sc.Next() {
		err = sc.Scan(nt.vars...)
		if err != nil {
			log.Fatal(err)
		}
		nt.fill()
	}

	out, err := os.Create(*oname)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	npz := zip.NewWriter(out)
	defer npz.Close()

	for _, col := range nt.cols {
		buf := new(bytes.Buffer)
		err = npyio.Write(buf, col.slice.Interface())
		if err != nil {
			log.Fatalf("error writing %q: %v\n", col.name, err)
		}

		wz, err := npz.Create(col.name)
		if err != nil {
			log.Fatalf("error creating %q: %v\n", col.name, err)
		}

		_, err = io.Copy(wz, buf)
		if err != nil {
			log.Fatal(err)
		}
	}

	err = npz.Flush()
	if err != nil {
		log.Fatal(err)
	}

	err = npz.Close()
	if err != nil {
		log.Fatal(err)
	}

	err = out.Close()
	if err != nil {
		log.Fatal(err)
	}
}

type ntuple struct {
	n    int64
	cols []column
	args []rootio.ScanVar
	vars []interface{}
}

func (nt *ntuple) add(name string, rt reflect.Type) {
	n := len(nt.cols)
	nt.cols = append(nt.cols, newColumn(name, rt, nt.n))
	nt.args = append(nt.args, rootio.ScanVar{Name: name, Type: rt})
	nt.vars = append(nt.vars, nt.cols[n].data.Addr().Interface())
}

func (nt *ntuple) fill() {
	for i := range nt.cols {
		col := &nt.cols[i]
		col.fill()
	}
}

type column struct {
	name  string
	i     int64
	data  reflect.Value
	slice reflect.Value
}

func newColumn(name string, rt reflect.Type, n int64) column {
	rtype := reflect.SliceOf(rt)
	return column{
		name:  name,
		i:     0,
		data:  reflect.New(rt).Elem(),
		slice: reflect.MakeSlice(rtype, int(n), int(n)),
	}
}

func (col *column) fill() {
	col.slice.Index(int(col.i)).Set(col.data)
	col.i++
}
<commit_msg>cmd/root2npy: add support for arrays<commit_after>// Copyright 2017 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// root2npy converts the content of a ROOT TTree to a NumPy data file.
//
//  Usage of root2npy:
//   -f string
//     	path to input ROOT file name
//   -o string
//     	path to output npz file name (default "output.npz")
//   -t string
//     	name of the tree to convert (default "tree")
//
package main

import (
	"archive/zip"
	"bytes"
	"flag"
	"io"
	"log"
	"os"
	"reflect"

	"github.com/sbinet/npyio"

	"go-hep.org/x/hep/rootio"
)

func main() {
	log.SetPrefix("root2npy: ")
	log.SetFlags(0)

	fname := flag.String("f", "", "path to input ROOT file name")
	oname := flag.String("o", "output.npz", "path to output npz file name")
	tname := flag.String("t", "tree", "name of the tree to convert")

	flag.Parse()

	if *fname == "" {
		flag.Usage()
		log.Fatalf("missing input ROOT filename argument")
	}

	f, err := rootio.Open(*fname)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	obj, ok := f.Get(*tname)
	if !ok {
		log.Fatalf("no object named %q in file %q", *tname, *fname)
	}

	tree, ok := obj.(rootio.Tree)
	if !ok {
		log.Fatalf("object %q in file %q is not a rootio.Tree", *tname, *fname)
	}

	var nt = ntuple{n: tree.Entries()}
	log.Printf("scanning leaves...")
	for _, leaf := range tree.Leaves() {
		if leaf.Kind() == reflect.String {
			nt.add(leaf.Name(), leaf)
			continue
		}
		if leaf.Class() == "TLeafElement" { // FIXME(sbinet): find a better, type-safe way
			log.Printf(">>> %q %v not supported", leaf.Name(), leaf.Class())
			continue
		}
		if leaf.LeafCount() != nil {
			log.Printf(">>> %q []%v not supported", leaf.Name(), leaf.TypeName())
			continue
		}
		nt.add(leaf.Name(), leaf)
	}
	log.Printf("scanning leaves... [done]")

	sc, err := rootio.NewScannerVars(tree, nt.args...)
	if err != nil {
		log.Fatal(err)
	}
	defer sc.Close()

	for sc.Next() {
		err = sc.Scan(nt.vars...)
		if err != nil {
			log.Fatal(err)
		}
		nt.fill()
	}

	out, err := os.Create(*oname)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	npz := zip.NewWriter(out)
	defer npz.Close()

	for _, col := range nt.cols {
		buf := new(bytes.Buffer)
		err = npyio.Write(buf, col.slice.Interface())
		if err != nil {
			log.Fatalf("error writing %q: %v\n", col.name, err)
		}

		wz, err := npz.Create(col.name)
		if err != nil {
			log.Fatalf("error creating %q: %v\n", col.name, err)
		}

		_, err = io.Copy(wz, buf)
		if err != nil {
			log.Fatal(err)
		}
	}

	err = npz.Flush()
	if err != nil {
		log.Fatal(err)
	}

	err = npz.Close()
	if err != nil {
		log.Fatal(err)
	}

	err = out.Close()
	if err != nil {
		log.Fatal(err)
	}
}

type ntuple struct {
	n    int64
	cols []column
	args []rootio.ScanVar
	vars []interface{}
}

func (nt *ntuple) add(name string, leaf rootio.Leaf) {
	n := len(nt.cols)
	nt.cols = append(nt.cols, newColumn(name, leaf, nt.n))
	col := &nt.cols[n]
	nt.args = append(nt.args, rootio.ScanVar{Name: name, Type: col.etype})
	nt.vars = append(nt.vars, col.data.Addr().Interface())
}

func (nt *ntuple) fill() {
	for i := range nt.cols {
		col := &nt.cols[i]
		col.fill()
	}
}

type column struct {
	name  string
	i     int64
	leaf  rootio.Leaf
	etype reflect.Type
	shape []int
	data  reflect.Value
	slice reflect.Value
}

func newColumn(name string, leaf rootio.Leaf, n int64) column {
	etype := leaf.Type()
	shape := []int{int(n)}
	if leaf.Len() > 1 && leaf.Kind() != reflect.String {
		etype = reflect.ArrayOf(leaf.Len(), etype)
		shape = append(shape, leaf.Len())
	}
	rtype := reflect.SliceOf(etype)
	return column{
		name:  name,
		i:     0,
		leaf:  leaf,
		etype: etype,
		shape: shape,
		data:  reflect.New(etype).Elem(),
		slice: reflect.MakeSlice(rtype, int(n), int(n)),
	}
}

func (col *column) fill() {
	col.slice.Index(int(col.i)).Set(col.data)
	col.i++
}
<|endoftext|>


<commit_before>package clusterconf

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"path"
	"time"

	"github.com/cerana/cerana/acomm"
	"github.com/shirou/gopsutil/load"
)

const (
	nodesPrefix      string = "nodes"
	historicalPrefix string = "historical"
)

// Node is current information about a hardware node.
type Node struct {
	c           *ClusterConf
	ID          string       `json:"id"`
	Heartbeat   time.Time    `json:"heartbeat"`
	MemoryTotal uint64       `json:"memoryTotal"`
	MemoryFree  uint64       `json:"memoryFree"`
	CPUCores    int          `json:"cpuCores"`
	CPULoad     load.AvgStat `json:"cpuLoad"`
	DiskTotal   uint64       `json:"diskTotal"`
	DiskFree    uint64       `json:"diskFree"`
}

// NodeHistory is a set of historical information for a node.
type NodeHistory map[time.Time]*Node

// NodesHistory is the historical information for multiple nodes.
type NodesHistory map[string]NodeHistory

// NodeHistoryArgs are arguments for filtering the historical results for nodes.
type NodeHistoryArgs struct {
	IDs    []string  `json:"ids"`
	Before time.Time `json:"before"`
	After  time.Time `json:"after"`
}

// ListNodesResult is the result of ListNodes.
type ListNodesResult struct {
	Nodes []Node `json:"nodes"`
}

// NodePayload can be used for task args or result when a node object needs to
// be sent.
type NodePayload struct {
	Node *Node `json:"node"`
}

// NodesHistoryResult is the result from the GetNodesHistory handler.
type NodesHistoryResult struct {
	History NodesHistory `json:"history"`
}

type nodeFilter func(Node) bool

// NodeHeartbeat records a new node heartbeat.
func (c *ClusterConf) NodeHeartbeat(req *acomm.Request) (interface{}, *url.URL, error) {
	var args NodePayload
	if err := req.UnmarshalArgs(&args); err != nil {
		return nil, nil, err
	}
	if args.Node == nil {
		return nil, nil, errors.New("missing arg: node")
	}
	args.Node.c = c

	return nil, nil, args.Node.update()
}

// GetNode returns the latest information about a node.
func (c *ClusterConf) GetNode(req *acomm.Request) (interface{}, *url.URL, error) {
	var args IDArgs
	if err := req.UnmarshalArgs(&args); err != nil {
		return nil, nil, err
	}
	if args.ID == "" {
		return nil, nil, errors.New("missing arg: id")
	}

	node, err := c.getNode(args.ID)
	return &NodePayload{node}, nil, err
}

// ListNodes list all current nodes.
func (c *ClusterConf) ListNodes(req *acomm.Request) (interface{}, *url.URL, error) {
	nodes, err := c.getNodes()
	if err != nil {
		return nil, nil, err
	}
	return &ListNodesResult{nodes}, nil, nil
}

// GetNodesHistory gets the heartbeat history for one or more nodes.
func (c *ClusterConf) GetNodesHistory(req *acomm.Request) (interface{}, *url.URL, error) {
	var args NodeHistoryArgs
	if err := req.UnmarshalArgs(args); err != nil {
		return nil, nil, err
	}

	history, err := c.getNodesHistory(
		nodeFilterID(args.IDs...),
		nodeFilterHeartbeat(args.Before, args.After),
	)
	if err != nil {
		return nil, nil, err
	}
	return &NodesHistoryResult{*history}, nil, nil

}

func (c *ClusterConf) getNode(id string) (*Node, error) {
	node := &Node{}
	key := path.Join(nodesPrefix, id)
	value, err := c.kvGet(key)
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(value.Data, node); err != nil {
		return nil, err
	}
	node.c = c
	return node, nil
}

func (c *ClusterConf) getNodes() ([]Node, error) {
	values, err := c.kvGetAll(nodesPrefix)
	if err != nil {
		return nil, err
	}

	nodes := make([]Node, 0, len(values))
	for key, value := range values {
		if key == nodesPrefix {
			continue
		}
		var node Node
		if err := json.Unmarshal(value.Data, node); err != nil {
			return nil, err
		}
		node.c = c
		nodes = append(nodes, node)
	}

	return nodes, nil
}

func nodeFilterID(ids ...string) nodeFilter {
	return func(n Node) bool {
		for _, id := range ids {
			if n.ID == id {
				return true
			}
		}
		return false
	}
}

func nodeFilterHeartbeat(before, after time.Time) nodeFilter {
	return func(n Node) bool {
		if !before.IsZero() && !n.Heartbeat.Before(before) {
			return false
		}
		if !after.IsZero() && !n.Heartbeat.After(after) {
			return false
		}
		return true
	}
}

func (c *ClusterConf) getNodesHistory(filters ...nodeFilter) (*NodesHistory, error) {
	values, err := c.kvGetAll(historicalPrefix)
	if err != nil {
		return nil, err
	}

	history := make(NodesHistory)
	for _, value := range values {
		var node Node
		if err := json.Unmarshal(value.Data, &node); err != nil {
			return nil, err
		}

		for _, fn := range filters {
			if !fn(node) {
				continue
			}
		}

		nodeHistory, ok := history[node.ID]
		if !ok {
			nodeHistory = make(NodeHistory)
			history[node.ID] = nodeHistory
		}
		nodeHistory[node.Heartbeat] = &node
	}

	return &history, nil
}

func (n *Node) update() error {
	currentKey := path.Join(nodesPrefix, n.ID)
	historicalKey := path.Join(historicalPrefix, n.ID, n.Heartbeat.Format(time.RFC3339))

	multiRequest := acomm.NewMultiRequest(n.c.tracker, 0)

	currentReq, err := acomm.NewRequest(acomm.RequestOptions{
		Task: "kv-ephemeral",
		Args: map[string]interface{}{
			"key":   currentKey,
			"value": n,
			"ttl":   n.c.config.NodeTTL(),
		},
	})
	if err != nil {
		return err
	}
	historicalReq, err := acomm.NewRequest(acomm.RequestOptions{
		Task: "kv-update",
		Args: map[string]interface{}{
			"key":   historicalKey,
			"value": n,
		},
	})
	if err != nil {
		return err
	}

	requests := map[string]*acomm.Request{
		"current":    currentReq,
		"historical": historicalReq,
	}

	for name, req := range requests {
		if err := multiRequest.AddRequest(name, req); err != nil {
			continue
		}
		if err := acomm.Send(n.c.config.CoordinatorURL(), req); err != nil {
			multiRequest.RemoveRequest(req)
			continue
		}
	}

	responses := multiRequest.Responses()
	for name := range requests {
		resp, ok := responses[name]
		if !ok {
			return fmt.Errorf("failed to send request: %s", name)
		}
		if resp.Error != nil {
			return fmt.Errorf("request failed: %s: %s", name, resp.Error)
		}
	}

	return nil
}
<commit_msg>statspusher - Use kv methods for node heartbeats<commit_after>package clusterconf

import (
	"encoding/json"
	"errors"
	"net/url"
	"path"
	"time"

	"github.com/cerana/cerana/acomm"
	"github.com/shirou/gopsutil/load"
)

const (
	nodesPrefix      string = "nodes"
	historicalPrefix string = "historical"
)

// Node is current information about a hardware node.
type Node struct {
	c           *ClusterConf
	ID          string       `json:"id"`
	Heartbeat   time.Time    `json:"heartbeat"`
	MemoryTotal uint64       `json:"memoryTotal"`
	MemoryFree  uint64       `json:"memoryFree"`
	CPUCores    int          `json:"cpuCores"`
	CPULoad     load.AvgStat `json:"cpuLoad"`
	DiskTotal   uint64       `json:"diskTotal"`
	DiskFree    uint64       `json:"diskFree"`
}

// NodeHistory is a set of historical information for a node.
type NodeHistory map[time.Time]*Node

// NodesHistory is the historical information for multiple nodes.
type NodesHistory map[string]NodeHistory

// NodeHistoryArgs are arguments for filtering the historical results for nodes.
type NodeHistoryArgs struct {
	IDs    []string  `json:"ids"`
	Before time.Time `json:"before"`
	After  time.Time `json:"after"`
}

// ListNodesResult is the result of ListNodes.
type ListNodesResult struct {
	Nodes []Node `json:"nodes"`
}

// NodePayload can be used for task args or result when a node object needs to
// be sent.
type NodePayload struct {
	Node *Node `json:"node"`
}

// NodesHistoryResult is the result from the GetNodesHistory handler.
type NodesHistoryResult struct {
	History NodesHistory `json:"history"`
}

type nodeFilter func(Node) bool

// NodeHeartbeat records a new node heartbeat.
func (c *ClusterConf) NodeHeartbeat(req *acomm.Request) (interface{}, *url.URL, error) {
	var args NodePayload
	if err := req.UnmarshalArgs(&args); err != nil {
		return nil, nil, err
	}
	if args.Node == nil {
		return nil, nil, errors.New("missing arg: node")
	}
	args.Node.c = c

	return nil, nil, args.Node.update()
}

// GetNode returns the latest information about a node.
func (c *ClusterConf) GetNode(req *acomm.Request) (interface{}, *url.URL, error) {
	var args IDArgs
	if err := req.UnmarshalArgs(&args); err != nil {
		return nil, nil, err
	}
	if args.ID == "" {
		return nil, nil, errors.New("missing arg: id")
	}

	node, err := c.getNode(args.ID)
	return &NodePayload{node}, nil, err
}

// ListNodes list all current nodes.
func (c *ClusterConf) ListNodes(req *acomm.Request) (interface{}, *url.URL, error) {
	nodes, err := c.getNodes()
	if err != nil {
		return nil, nil, err
	}
	return &ListNodesResult{nodes}, nil, nil
}

// GetNodesHistory gets the heartbeat history for one or more nodes.
func (c *ClusterConf) GetNodesHistory(req *acomm.Request) (interface{}, *url.URL, error) {
	var args NodeHistoryArgs
	if err := req.UnmarshalArgs(args); err != nil {
		return nil, nil, err
	}

	history, err := c.getNodesHistory(
		nodeFilterID(args.IDs...),
		nodeFilterHeartbeat(args.Before, args.After),
	)
	if err != nil {
		return nil, nil, err
	}
	return &NodesHistoryResult{*history}, nil, nil

}

func (c *ClusterConf) getNode(id string) (*Node, error) {
	node := &Node{}
	key := path.Join(nodesPrefix, id)
	value, err := c.kvGet(key)
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(value.Data, node); err != nil {
		return nil, err
	}
	node.c = c
	return node, nil
}

func (c *ClusterConf) getNodes() ([]Node, error) {
	values, err := c.kvGetAll(nodesPrefix)
	if err != nil {
		return nil, err
	}

	nodes := make([]Node, 0, len(values))
	for key, value := range values {
		if key == nodesPrefix {
			continue
		}
		var node Node
		if err := json.Unmarshal(value.Data, node); err != nil {
			return nil, err
		}
		node.c = c
		nodes = append(nodes, node)
	}

	return nodes, nil
}

func nodeFilterID(ids ...string) nodeFilter {
	return func(n Node) bool {
		for _, id := range ids {
			if n.ID == id {
				return true
			}
		}
		return false
	}
}

func nodeFilterHeartbeat(before, after time.Time) nodeFilter {
	return func(n Node) bool {
		if !before.IsZero() && !n.Heartbeat.Before(before) {
			return false
		}
		if !after.IsZero() && !n.Heartbeat.After(after) {
			return false
		}
		return true
	}
}

func (c *ClusterConf) getNodesHistory(filters ...nodeFilter) (*NodesHistory, error) {
	values, err := c.kvGetAll(historicalPrefix)
	if err != nil {
		return nil, err
	}

	history := make(NodesHistory)
	for _, value := range values {
		var node Node
		if err := json.Unmarshal(value.Data, &node); err != nil {
			return nil, err
		}

		for _, fn := range filters {
			if !fn(node) {
				continue
			}
		}

		nodeHistory, ok := history[node.ID]
		if !ok {
			nodeHistory = make(NodeHistory)
			history[node.ID] = nodeHistory
		}
		nodeHistory[node.Heartbeat] = &node
	}

	return &history, nil
}

func (n *Node) update() error {
	currentKey := path.Join(nodesPrefix, n.ID)
	historicalKey := path.Join(historicalPrefix, n.ID, n.Heartbeat.Format(time.RFC3339))

	if err := n.c.kvEphemeral(currentKey, n, n.c.config.NodeTTL()); err != nil {
		return err
	}

	if _, err := n.c.kvUpdate(historicalKey, n, 0); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"strings"

	"github.com/coreos/torus"
	"github.com/coreos/torus/blockset"
	"github.com/coreos/torus/internal/flagconfig"
	"github.com/coreos/torus/ring"
	"github.com/dustin/go-humanize"
	"github.com/spf13/cobra"

	_ "github.com/coreos/torus/metadata/etcd"
)

var (
	blockSize        uint64
	blockSizeStr     string
	blockSpec        string
	inodeReplication int
	noMakeRing       bool
)

var initCommand = &cobra.Command{
	Use:    "init",
	Short:  "Prepare a new torus cluster by creating the metadata",
	PreRun: initPreRun,
	Run:    initAction,
}

func init() {
	initCommand.Flags().StringVarP(&blockSizeStr, "block-size", "", "512KiB", "size of all data blocks in this storage cluster")
	initCommand.Flags().StringVarP(&blockSpec, "block-spec", "", "crc", "default replication/error correction applied to blocks in this storage cluster")
	initCommand.Flags().IntVarP(&inodeReplication, "inode-replication", "", 3, "default number of times to replicate inodes across the cluster")
	initCommand.Flags().BoolVar(&noMakeRing, "no-ring", false, "do not create the default ring as part of init")
}

func initPreRun(cmd *cobra.Command, args []string) {
	// We *always* need base.
	if !strings.HasSuffix(blockSpec, ",base") {
		blockSpec += ",base"
	}
	var err error
	blockSize, err = humanize.ParseBytes(blockSizeStr)
	if err != nil {
		die("error parsing block-size: %v", err)
	}
}

func initAction(cmd *cobra.Command, args []string) {
	var err error
	md := torus.GlobalMetadata{}
	md.BlockSize = blockSize
	md.DefaultBlockSpec, err = blockset.ParseBlockLayerSpec(blockSpec)
	md.INodeReplication = inodeReplication
	if err != nil {
		die("error parsing block-spec: %v", err)
	}

	cfg := flagconfig.BuildConfigFromFlags()
	ringType := ring.Ketama
	if noMakeRing {
		ringType = ring.Empty
	}
	err = torus.InitMDS("etcd", cfg, md, ringType)
	if err != nil {
		die("error writing metadata: %v", err)
	}
}
<commit_msg>init: tiny: avoid duplicate blockSpec when base is specified at the front<commit_after>package main

import (
	"strings"

	"github.com/coreos/torus"
	"github.com/coreos/torus/blockset"
	"github.com/coreos/torus/internal/flagconfig"
	"github.com/coreos/torus/ring"
	"github.com/dustin/go-humanize"
	"github.com/spf13/cobra"

	_ "github.com/coreos/torus/metadata/etcd"
)

var (
	blockSize        uint64
	blockSizeStr     string
	blockSpec        string
	inodeReplication int
	noMakeRing       bool
)

var initCommand = &cobra.Command{
	Use:    "init",
	Short:  "Prepare a new torus cluster by creating the metadata",
	PreRun: initPreRun,
	Run:    initAction,
}

func init() {
	initCommand.Flags().StringVarP(&blockSizeStr, "block-size", "", "512KiB", "size of all data blocks in this storage cluster")
	initCommand.Flags().StringVarP(&blockSpec, "block-spec", "", "crc", "default replication/error correction applied to blocks in this storage cluster")
	initCommand.Flags().IntVarP(&inodeReplication, "inode-replication", "", 3, "default number of times to replicate inodes across the cluster")
	initCommand.Flags().BoolVar(&noMakeRing, "no-ring", false, "do not create the default ring as part of init")
}

func initPreRun(cmd *cobra.Command, args []string) {
	// We *always* need base.
	if !strings.HasSuffix(blockSpec, ",base") && !strings.HasPrefix(blockSpec, "base") {
		blockSpec += ",base"
	}
	var err error
	blockSize, err = humanize.ParseBytes(blockSizeStr)
	if err != nil {
		die("error parsing block-size: %v", err)
	}
}

func initAction(cmd *cobra.Command, args []string) {
	var err error
	md := torus.GlobalMetadata{}
	md.BlockSize = blockSize
	md.DefaultBlockSpec, err = blockset.ParseBlockLayerSpec(blockSpec)
	md.INodeReplication = inodeReplication
	if err != nil {
		die("error parsing block-spec: %v", err)
	}

	cfg := flagconfig.BuildConfigFromFlags()
	ringType := ring.Ketama
	if noMakeRing {
		ringType = ring.Empty
	}
	err = torus.InitMDS("etcd", cfg, md, ringType)
	if err != nil {
		die("error writing metadata: %v", err)
	}
}
<|endoftext|>


<commit_before>package cradle

import (
	"github.com/spacemonkeygo/errors"

	"polydawn.net/repeatr/def"
)

/*
	Map of policies to the capabilities we give that policy by default.

	May not be implemented/enforced by all executors (e.g. chroot is
	simply not capable of these).
*/
func CapsForPolicy(m def.Policy) []string {
	switch m {
	case def.PolicyRoutine:
		return []string{
			"CAP_AUDIT_WRITE",
			"CAP_KILL",
			"CAP_NET_BIND_SERVICE",
		}
	case def.PolicyUidZero:
		return []string{
			"CAP_AUDIT_WRITE",
			"CAP_KILL",
			"CAP_NET_BIND_SERVICE",
		}
	case def.PolicyGovernor:
		return []string{
			"CAP_AUDIT_WRITE",
			"CAP_CHOWN",
			"CAP_DAC_OVERRIDE",
			"CAP_FSETID",
			"CAP_FOWNER",
			"CAP_KILL",
			"CAP_NET_BIND_SERVICE",
			"CAP_NET_RAW",
			"CAP_SETGID",
			"CAP_SETUID", // fairly terrifying
			"CAP_SETFCAP",
			"CAP_SETPCAP",
			"CAP_SYS_CHROOT",
		}
	case def.PolicySysad:
		return []string{
			"CAP_AUDIT_CONTROL",
			"CAP_AUDIT_READ",
			"CAP_AUDIT_WRITE",
			"CAP_BLOCK_SUSPEND",
			"CAP_CHOWN",
			"CAP_DAC_OVERRIDE",
			"CAP_DAC_READ_SEARCH",
			"CAP_FOWNER",
			"CAP_FSETID",
			"CAP_IPC_LOCK",
			"CAP_IPC_OWNER",
			"CAP_KILL",
			"CAP_LEASE",
			"CAP_LINUX_IMMUTABLE",
			"CAP_MAC_ADMIN",
			"CAP_MAC_OVERRIDE",
			"CAP_MKNOD",
			"CAP_NET_ADMIN",
			"CAP_NET_BIND_SERVICE",
			"CAP_NET_BROADCAST",
			"CAP_NET_RAW",
			"CAP_SETGID",
			"CAP_SETFCAP",
			"CAP_SETPCAP",
			"CAP_SETUID",
			"CAP_SYS_ADMIN",
			"CAP_SYS_BOOT",
			"CAP_SYS_CHROOT",
			"CAP_SYS_MODULE",
			"CAP_SYS_NICE",
			"CAP_SYS_PACCT",
			"CAP_SYS_PTRACE",
			"CAP_SYS_RAWIO",
			"CAP_SYS_RESOURCE",
			"CAP_SYS_TIME",
			"CAP_SYS_TTY_CONFIG",
			"CAP_SYSLOG",
			"CAP_WAKE_ALARM",
		}
	default:
		panic(errors.ProgrammerError.New("missing switch case"))
	}
}
<commit_msg>Drop a cap from Sysad Policy because unsupported.<commit_after>package cradle

import (
	"github.com/spacemonkeygo/errors"

	"polydawn.net/repeatr/def"
)

/*
	Map of policies to the capabilities we give that policy by default.

	May not be implemented/enforced by all executors (e.g. chroot is
	simply not capable of these).
*/
func CapsForPolicy(m def.Policy) []string {
	switch m {
	case def.PolicyRoutine:
		return []string{
			"CAP_AUDIT_WRITE",
			"CAP_KILL",
			"CAP_NET_BIND_SERVICE",
		}
	case def.PolicyUidZero:
		return []string{
			"CAP_AUDIT_WRITE",
			"CAP_KILL",
			"CAP_NET_BIND_SERVICE",
		}
	case def.PolicyGovernor:
		return []string{
			"CAP_AUDIT_WRITE",
			"CAP_CHOWN",
			"CAP_DAC_OVERRIDE",
			"CAP_FSETID",
			"CAP_FOWNER",
			"CAP_KILL",
			"CAP_NET_BIND_SERVICE",
			"CAP_NET_RAW",
			"CAP_SETGID",
			"CAP_SETUID", // fairly terrifying
			"CAP_SETFCAP",
			"CAP_SETPCAP",
			"CAP_SYS_CHROOT",
		}
	case def.PolicySysad:
		return []string{
			"CAP_AUDIT_CONTROL",
			//"CAP_AUDIT_READ", // currently regarded as 'unknown' by runc...?
			"CAP_AUDIT_WRITE",
			"CAP_BLOCK_SUSPEND",
			"CAP_CHOWN",
			"CAP_DAC_OVERRIDE",
			"CAP_DAC_READ_SEARCH",
			"CAP_FOWNER",
			"CAP_FSETID",
			"CAP_IPC_LOCK",
			"CAP_IPC_OWNER",
			"CAP_KILL",
			"CAP_LEASE",
			"CAP_LINUX_IMMUTABLE",
			"CAP_MAC_ADMIN",
			"CAP_MAC_OVERRIDE",
			"CAP_MKNOD",
			"CAP_NET_ADMIN",
			"CAP_NET_BIND_SERVICE",
			"CAP_NET_BROADCAST",
			"CAP_NET_RAW",
			"CAP_SETGID",
			"CAP_SETFCAP",
			"CAP_SETPCAP",
			"CAP_SETUID",
			"CAP_SYS_ADMIN",
			"CAP_SYS_BOOT",
			"CAP_SYS_CHROOT",
			"CAP_SYS_MODULE",
			"CAP_SYS_NICE",
			"CAP_SYS_PACCT",
			"CAP_SYS_PTRACE",
			"CAP_SYS_RAWIO",
			"CAP_SYS_RESOURCE",
			"CAP_SYS_TIME",
			"CAP_SYS_TTY_CONFIG",
			"CAP_SYSLOG",
			"CAP_WAKE_ALARM",
		}
	default:
		panic(errors.ProgrammerError.New("missing switch case"))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
	"runtime"
	"time"

	"github.com/jimmy-go/srest"
	"github.com/jimmy-go/srest/examples/simple/api/friends"
	"github.com/jimmy-go/srest/examples/simple/controllers/home"
	"github.com/jimmy-go/srest/examples/simple/dai"
)

var (
	port    = flag.Int("port", 0, "Listen port")
	dbf     = flag.String("db", "", "Database connection url.")
	tmpls   = flag.String("templates", "", "Templates files dir.")
	static  = flag.String("static", "", "Static dir.")
	workers = flag.Int("workers", 1, "Worker pool size.")
	queue   = flag.Int("queue", 10, "Queue length.")
)

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	log.SetFlags(0)
	log.Printf("templates dir [%v]", *tmpls)
	log.Printf("static dir [%v]", *static)

	// connect to database
	conf := &dai.Options{
		URL:     *dbf,
		Workers: *workers,
		Queue:   *queue,
	}
	now := time.Now()
	err := dai.Connect(conf)
	log.Printf("Database connection time: [%s]", time.Since(now))
	if err != nil {
		log.Fatal(err)
	}

	// load template views only for this project
	err = srest.LoadViews(*tmpls)
	if err != nil {
		log.Fatal(err)
	}

	m := srest.New(nil)
	m.Static("/static", *static)
	m.Use("/v1/api/friends", friends.New())
	m.Use("/v1/api/mid", friends.New(), mwOne, mwTwo)
	m.Use("/home", &home.API{})
	<-m.Run(*port)
	log.Printf("Closing database connections")
	dai.Db.Close()
	log.Printf("Done")
}

func mwOne(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		prev := r.URL.Query().Get("stop")
		if prev == "one" {
			w.Write([]byte("skipped from one"))
			return
		}
		h.ServeHTTP(w, r)
	})
}

func mwTwo(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		prev := r.URL.Query().Get("stop")
		if prev == "two" {
			w.Write([]byte("skipped from two"))
			return
		}
		h.ServeHTTP(w, r)
	})
}
<commit_msg>fixed example.<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	"runtime"
	"time"

	"github.com/jimmy-go/srest"
	"github.com/jimmy-go/srest/examples/simple/api/friends"
	"github.com/jimmy-go/srest/examples/simple/controllers/home"
	"github.com/jimmy-go/srest/examples/simple/dai"
)

var (
	port    = flag.Int("port", 0, "Listen port")
	dbf     = flag.String("db", "", "Database connection url.")
	tmpls   = flag.String("templates", "", "Templates files dir.")
	static  = flag.String("static", "", "Static dir.")
	workers = flag.Int("workers", 1, "Worker pool size.")
	queue   = flag.Int("queue", 10, "Queue length.")
)

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	log.SetFlags(0)
	log.Printf("templates dir [%v]", *tmpls)
	log.Printf("static dir [%v]", *static)

	// connect to database
	conf := &dai.Options{
		URL:     *dbf,
		Workers: *workers,
		Queue:   *queue,
	}
	now := time.Now()
	err := dai.Connect(conf)
	log.Printf("Database connection time: [%s]", time.Since(now))
	if err != nil {
		log.Fatal(err)
	}

	// load template views only for this project
	err = srest.LoadViews(*tmpls, srest.DefaultFuncMap)
	if err != nil {
		log.Fatal(err)
	}

	m := srest.New(nil)
	m.Get("/static", srest.Static("/static", *static))
	m.Use("/v1/api/friends", friends.New())
	m.Use("/v1/api/mid", friends.New(), mwOne, mwTwo)
	m.Use("/home", &home.API{})
	<-m.Run(*port)
	log.Printf("Closing database connections")
	dai.Db.Close()
	log.Printf("Done")
}

func mwOne(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		prev := r.URL.Query().Get("stop")
		if prev == "one" {
			w.Write([]byte("skipped from one"))
			return
		}
		h.ServeHTTP(w, r)
	})
}

func mwTwo(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		prev := r.URL.Query().Get("stop")
		if prev == "two" {
			w.Write([]byte("skipped from two"))
			return
		}
		h.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8schain

import (
	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	credentialprovider "github.com/vdemeester/k8s-pkg-credentialprovider"
	credentialprovidersecrets "github.com/vdemeester/k8s-pkg-credentialprovider/secrets"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// Options holds configuration data for guiding credential resolution.
type Options struct {
	// Namespace holds the namespace inside of which we are resolving the
	// image reference.  If empty, "default" is assumed.
	Namespace string
	// ServiceAccountName holds the serviceaccount as which the container
	// will run (scoped to Namespace).  If empty, "default" is assumed.
	ServiceAccountName string
	// ImagePullSecrets holds the names of the Kubernetes secrets (scoped to
	// Namespace) containing credential data to use for the image pull.
	ImagePullSecrets []string
}

// origKeyRing is a variable so that testing can adjust it.
var origKeyRing = credentialprovider.NewDockerKeyring()

// New returns a new authn.Keychain suitable for resolving image references as
// scoped by the provided Options.  It speaks to Kubernetes through the provided
// client interface.
func New(client kubernetes.Interface, opt Options) (authn.Keychain, error) {
	if opt.Namespace == "" {
		opt.Namespace = "default"
	}
	if opt.ServiceAccountName == "" {
		opt.ServiceAccountName = "default"
	}

	// Implement a Kubernetes-style authentication keychain.
	// This needs to support roughly the following kinds of authentication:
	//  1) The implicit authentication from k8s.io/kubernetes/pkg/credentialprovider
	//  2) The explicit authentication from imagePullSecrets on Pod
	//  3) The semi-implicit authentication where imagePullSecrets are on the
	//    Pod's service account.

	// First, fetch all of the explicitly declared pull secrets
	var pullSecrets []v1.Secret
	if client != nil {
		for _, name := range opt.ImagePullSecrets {
			ps, err := client.CoreV1().Secrets(opt.Namespace).Get(name, metav1.GetOptions{})
			if err != nil {
				return nil, err
			}
			pullSecrets = append(pullSecrets, *ps)
		}

		// Second, fetch all of the pull secrets attached to our service account.
		sa, err := client.CoreV1().ServiceAccounts(opt.Namespace).Get(opt.ServiceAccountName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}
		for _, localObj := range sa.ImagePullSecrets {
			ps, err := client.CoreV1().Secrets(opt.Namespace).Get(localObj.Name, metav1.GetOptions{})
			if err != nil {
				return nil, err
			}
			pullSecrets = append(pullSecrets, *ps)
		}
	}

	// Third, extend the default keyring with the pull secrets.
	kr, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, origKeyRing)
	if err != nil {
		return nil, err
	}
	return &keychain{
		keyring: kr,
	}, nil
}

// NewInCluster returns a new authn.Keychain suitable for resolving image references as
// scoped by the provided Options, constructing a kubernetes.Interface based on in-cluster
// authentication.
func NewInCluster(opt Options) (authn.Keychain, error) {
	clusterConfig, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}

	client, err := kubernetes.NewForConfig(clusterConfig)
	if err != nil {
		return nil, err
	}
	return New(client, opt)
}

// NewNoClient returns a new authn.Keychain that supports the portions of the K8s keychain
// that don't read ImagePullSecrets.  This limits it to roughly the Node-identity-based
// authentication schemes in Kubernetes pkg/credentialprovider.  This version of the
// k8schain drops the requirement that we run as a K8s serviceaccount with access to all
// of the on-cluster secrets.  This drop in fidelity also diminishes its value as a stand-in
// for Kubernetes authentication, but this actually targets a different use-case.  What
// remains is an interesting sweet spot: this variant can serve as a credential provider
// for all of the major public clouds, but in library form (vs. an executable you exec).
func NewNoClient() (authn.Keychain, error) {
	return New(nil, Options{})
}

type lazyProvider credentialprovider.LazyAuthConfiguration

// Authorization implements Authenticator.
func (lp lazyProvider) Authorization() (*authn.AuthConfig, error) {
	authConfig := credentialprovider.LazyProvide(credentialprovider.LazyAuthConfiguration(lp))
	return &authn.AuthConfig{
		Username:      authConfig.Username,
		Password:      authConfig.Password,
		Auth:          authConfig.Auth,
		IdentityToken: authConfig.IdentityToken,
		RegistryToken: authConfig.RegistryToken,
	}, nil
}

type keychain struct {
	keyring credentialprovider.DockerKeyring
}

// Resolve implements authn.Keychain
func (kc *keychain) Resolve(target authn.Resource) (authn.Authenticator, error) {
	var (
		creds []credentialprovider.LazyAuthConfiguration
		found bool
	)
	if repo, ok := target.(name.Repository); ok {
		creds, found = kc.keyring.Lookup(repo.String())
	} else {
		// Lookup expects an image reference and we only have a registry.
		creds, found = kc.keyring.Lookup(target.RegistryStr() + "/foo/bar")
	}
	if !found || len(creds) < 1 {
		return authn.Anonymous, nil
	}
	// TODO(mattmoor): How to support multiple credentials?
	return lazyProvider(creds[0]), nil
}
<commit_msg>Inline NewDockerKeyring (#652)<commit_after>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8schain

import (
	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	credentialprovider "github.com/vdemeester/k8s-pkg-credentialprovider"
	credentialprovidersecrets "github.com/vdemeester/k8s-pkg-credentialprovider/secrets"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// Options holds configuration data for guiding credential resolution.
type Options struct {
	// Namespace holds the namespace inside of which we are resolving the
	// image reference.  If empty, "default" is assumed.
	Namespace string
	// ServiceAccountName holds the serviceaccount as which the container
	// will run (scoped to Namespace).  If empty, "default" is assumed.
	ServiceAccountName string
	// ImagePullSecrets holds the names of the Kubernetes secrets (scoped to
	// Namespace) containing credential data to use for the image pull.
	ImagePullSecrets []string
}

// New returns a new authn.Keychain suitable for resolving image references as
// scoped by the provided Options.  It speaks to Kubernetes through the provided
// client interface.
func New(client kubernetes.Interface, opt Options) (authn.Keychain, error) {
	if opt.Namespace == "" {
		opt.Namespace = "default"
	}
	if opt.ServiceAccountName == "" {
		opt.ServiceAccountName = "default"
	}

	// Implement a Kubernetes-style authentication keychain.
	// This needs to support roughly the following kinds of authentication:
	//  1) The implicit authentication from k8s.io/kubernetes/pkg/credentialprovider
	//  2) The explicit authentication from imagePullSecrets on Pod
	//  3) The semi-implicit authentication where imagePullSecrets are on the
	//    Pod's service account.

	// First, fetch all of the explicitly declared pull secrets
	var pullSecrets []v1.Secret
	if client != nil {
		for _, name := range opt.ImagePullSecrets {
			ps, err := client.CoreV1().Secrets(opt.Namespace).Get(name, metav1.GetOptions{})
			if err != nil {
				return nil, err
			}
			pullSecrets = append(pullSecrets, *ps)
		}

		// Second, fetch all of the pull secrets attached to our service account.
		sa, err := client.CoreV1().ServiceAccounts(opt.Namespace).Get(opt.ServiceAccountName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}
		for _, localObj := range sa.ImagePullSecrets {
			ps, err := client.CoreV1().Secrets(opt.Namespace).Get(localObj.Name, metav1.GetOptions{})
			if err != nil {
				return nil, err
			}
			pullSecrets = append(pullSecrets, *ps)
		}
	}

	// Third, extend the default keyring with the pull secrets.
	kr, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, credentialprovider.NewDockerKeyring())
	if err != nil {
		return nil, err
	}
	return &keychain{
		keyring: kr,
	}, nil
}

// NewInCluster returns a new authn.Keychain suitable for resolving image references as
// scoped by the provided Options, constructing a kubernetes.Interface based on in-cluster
// authentication.
func NewInCluster(opt Options) (authn.Keychain, error) {
	clusterConfig, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}

	client, err := kubernetes.NewForConfig(clusterConfig)
	if err != nil {
		return nil, err
	}
	return New(client, opt)
}

// NewNoClient returns a new authn.Keychain that supports the portions of the K8s keychain
// that don't read ImagePullSecrets.  This limits it to roughly the Node-identity-based
// authentication schemes in Kubernetes pkg/credentialprovider.  This version of the
// k8schain drops the requirement that we run as a K8s serviceaccount with access to all
// of the on-cluster secrets.  This drop in fidelity also diminishes its value as a stand-in
// for Kubernetes authentication, but this actually targets a different use-case.  What
// remains is an interesting sweet spot: this variant can serve as a credential provider
// for all of the major public clouds, but in library form (vs. an executable you exec).
func NewNoClient() (authn.Keychain, error) {
	return New(nil, Options{})
}

type lazyProvider credentialprovider.LazyAuthConfiguration

// Authorization implements Authenticator.
func (lp lazyProvider) Authorization() (*authn.AuthConfig, error) {
	authConfig := credentialprovider.LazyProvide(credentialprovider.LazyAuthConfiguration(lp))
	return &authn.AuthConfig{
		Username:      authConfig.Username,
		Password:      authConfig.Password,
		Auth:          authConfig.Auth,
		IdentityToken: authConfig.IdentityToken,
		RegistryToken: authConfig.RegistryToken,
	}, nil
}

type keychain struct {
	keyring credentialprovider.DockerKeyring
}

// Resolve implements authn.Keychain
func (kc *keychain) Resolve(target authn.Resource) (authn.Authenticator, error) {
	var (
		creds []credentialprovider.LazyAuthConfiguration
		found bool
	)
	if repo, ok := target.(name.Repository); ok {
		creds, found = kc.keyring.Lookup(repo.String())
	} else {
		// Lookup expects an image reference and we only have a registry.
		creds, found = kc.keyring.Lookup(target.RegistryStr() + "/foo/bar")
	}
	if !found || len(creds) < 1 {
		return authn.Anonymous, nil
	}
	// TODO(mattmoor): How to support multiple credentials?
	return lazyProvider(creds[0]), nil
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"

	"gopkg.in/flant/yaml.v2"
)

type rawAnsibleTask struct {
	Block  []rawAnsibleTask       `yaml:"block,omitempty"`
	Rescue []rawAnsibleTask       `yaml:"rescue,omitempty"`
	Always []rawAnsibleTask       `yaml:"always,omitempty"`
	Fields map[string]interface{} `yaml:",inline"`

	rawAnsible *rawAnsible `yaml:"-"` // parent
}

func (c *rawAnsibleTask) UnmarshalYAML(unmarshal func(interface{}) error) error {
	if parent, ok := parentStack.Peek().(*rawAnsible); ok {
		c.rawAnsible = parent
	} else if parent, ok := parentStack.Peek().(*rawAnsibleTask); ok {
		c.rawAnsible = parent.rawAnsible
	}

	parentStack.Push(c)
	type plain rawAnsibleTask
	err := unmarshal((*plain)(c))
	parentStack.Pop()
	if err != nil {
		return err
	}

	if !c.blockDefined() {
		check := false
		for _, supportedModule := range supportedModules() {
			if c.Fields[supportedModule] != nil {
				if check {
					return newDetailedConfigError("invalid ansible task!", c, c.rawAnsible.rawImage.doc)
				} else {
					check = true
				}
			}
		}

		if !check {
			var supportedModulesString string
			for _, supportedModule := range supportedModules() {
				supportedModulesString += fmt.Sprintf("* %s\n", supportedModule)
			}
			return newConfigError(fmt.Sprintf("unsupported ansible task!\n\n%s\nSupported modules list:\n%s\n%s", dumpConfigSection(c), supportedModulesString, dumpConfigDoc(c.rawAnsible.rawImage.doc)))
		}
	}

	return nil
}

func (c *rawAnsibleTask) blockDefined() bool {
	return c.Block != nil || c.Rescue != nil || c.Always != nil
}

func supportedModules() []string {
	var modules []string
	// Commands Modules
	modules = append(modules, []string{"command", "shell", "raw", "script"}...)
	// Files Modules
	modules = append(modules, []string{"assemble", "archive", "unarchive", "blockinfile", "lineinfile", "file", "stat", "find", "tempfile", "copy", "acl", "xattr", "ini_file", "iso_extract"}...)
	// Net Tools Modules
	modules = append(modules, []string{"get_url", "slurp", "uri"}...)

	// Packaging/Language Modules
	modules = append(modules, []string{"bower", "bundler", "composer", "cpanm", "easy_install", "gem", "maven_artifact", "npm", "pear", "pip"}...)

	// Packaging/Os Modules
	modules = append(modules, []string{
		"apk",
		"apt",
		"apt_key",
		"apt_repository",
		"apt_rpm",
		"dnf",
		"dpkg_selections",
		"homebrew",
		"homebrew_cask",
		"homebrew_tap",
		"layman",
		"macports",
		"openbsd_pkg",
		"opkg",
		"package",
		"pacman",
		"pkg5",
		"pkg5_publisher",
		"pkgin",
		"pkgng",
		"pkgutil",
		"portage",
		"portinstall",
		"pulp_repo",
		"redhat_subscription",
		"rhn_channel",
		"rhn_register",
		"rpm_key",
		"slackpkg",
		"sorcery",
		"svr4pkg",
		"swdepot",
		"swupd",
		"urpmi",
		"xbps",
		"yum",
		"yum_repository",
		"zypper",
		"zypper_repository",
	}...)

	// System Modules
	modules = append(modules, []string{"user", "group", "getent", "locale_gen"}...)
	// Utilities Modules
	modules = append(modules, []string{"assert", "debug", "set_fact", "wait_for"}...)
	// Crypto Modules
	modules = append(modules, []string{"openssl_certificate", "openssl_csr", "openssl_privatekey", "openssl_publickey"}...)
	// Other modules
	modules = append(modules, []string{"timezone", "cron"}...)

	return modules
}

func (c *rawAnsibleTask) toDirective() (*AnsibleTask, error) {
	ansibleTask := &AnsibleTask{}

	marshal, err := yaml.Marshal(c)
	if err != nil {
		return nil, err
	}

	var unmarshal map[string]interface{}
	err = yaml.Unmarshal(marshal, &unmarshal)
	if err != nil {
		return nil, err
	}

	ansibleTask.Config = unmarshal
	ansibleTask.raw = c

	return ansibleTask, nil
}
<commit_msg>feat: enable meta and fail modules<commit_after>package config

import (
	"fmt"

	"gopkg.in/flant/yaml.v2"
)

type rawAnsibleTask struct {
	Block  []rawAnsibleTask       `yaml:"block,omitempty"`
	Rescue []rawAnsibleTask       `yaml:"rescue,omitempty"`
	Always []rawAnsibleTask       `yaml:"always,omitempty"`
	Fields map[string]interface{} `yaml:",inline"`

	rawAnsible *rawAnsible `yaml:"-"` // parent
}

func (c *rawAnsibleTask) UnmarshalYAML(unmarshal func(interface{}) error) error {
	if parent, ok := parentStack.Peek().(*rawAnsible); ok {
		c.rawAnsible = parent
	} else if parent, ok := parentStack.Peek().(*rawAnsibleTask); ok {
		c.rawAnsible = parent.rawAnsible
	}

	parentStack.Push(c)
	type plain rawAnsibleTask
	err := unmarshal((*plain)(c))
	parentStack.Pop()
	if err != nil {
		return err
	}

	if !c.blockDefined() {
		check := false
		for _, supportedModule := range supportedModules() {
			if c.Fields[supportedModule] != nil {
				if check {
					return newDetailedConfigError("invalid ansible task!", c, c.rawAnsible.rawImage.doc)
				} else {
					check = true
				}
			}
		}

		if !check {
			var supportedModulesString string
			for _, supportedModule := range supportedModules() {
				supportedModulesString += fmt.Sprintf("* %s\n", supportedModule)
			}
			return newConfigError(fmt.Sprintf("unsupported ansible task!\n\n%s\nSupported modules list:\n%s\n%s", dumpConfigSection(c), supportedModulesString, dumpConfigDoc(c.rawAnsible.rawImage.doc)))
		}
	}

	return nil
}

func (c *rawAnsibleTask) blockDefined() bool {
	return c.Block != nil || c.Rescue != nil || c.Always != nil
}

func supportedModules() []string {
	var modules []string
	// No Cloud modules
	// No Clustering modules
	// Commands Modules (no expect, psexec and telnet)
	modules = append(modules, []string{"command", "shell", "raw", "script"}...)
	// Crypto Modules
	modules = append(modules, []string{"openssl_certificate", "openssl_csr", "openssl_privatekey", "openssl_publickey"}...)
	// No Databases modules
	// Files Modules (no fetch, patch, replace, synchronize, template, xml
	modules = append(modules, []string{
		"acl",
		"archive",
		"assemble",
		"blockinfile",
		"copy",
		"file",
		"find",
		"ini_file",
		"iso_extract",
		"lineinfile",
		"stat",
		"tempfile",
		"unarchive",
		"xattr",
	}...)
	// No Identity modules
	// No Inventory modules
	// No Messaging modules
	// No Monitoring messages
	// Net Tools Modules (only Basics)
	modules = append(modules, []string{"get_url", "slurp", "uri"}...)
    // No Network modules
    // No Notification modules
	// Packaging/Language Modules
	modules = append(modules, []string{"bower", "bundler", "composer", "cpanm", "easy_install", "gem", "maven_artifact", "npm", "pear", "pip"}...)
	// Packaging/Os Modules
	modules = append(modules, []string{
		"apk",
		"apt",
		"apt_key",
		"apt_repository",
		"apt_rpm",
		"dnf",
		"dpkg_selections",
		"homebrew",
		"homebrew_cask",
		"homebrew_tap",
		"layman",
		"macports",
		"openbsd_pkg",
		"opkg",
		"package",
		"pacman",
		"pkg5",
		"pkg5_publisher",
		"pkgin",
		"pkgng",
		"pkgutil",
		"portage",
		"portinstall",
		"pulp_repo",
		"redhat_subscription",
		"rhn_channel",
		"rhn_register",
		"rpm_key",
		"slackpkg",
		"sorcery",
		"svr4pkg",
		"swdepot",
		"swupd",
		"urpmi",
		"xbps",
		"yum",
		"yum_repository",
		"zypper",
		"zypper_repository",
	}...)
    // No Remote Management modules
    // No Source Control modules
    // No Storage modules
	// System Modules (only passwd management and locales)
	modules = append(modules, []string{"cron", "user", "group", "getent", "locale_gen", "timezone"}...)
	// Utilities Modules
	modules = append(modules, []string{"meta", "assert", "debug", "fail", "set_fact", "wait_for"}...)
    // No Web Infrastructure modules
    // No Windows modules

	return modules
}

func (c *rawAnsibleTask) toDirective() (*AnsibleTask, error) {
	ansibleTask := &AnsibleTask{}

	marshal, err := yaml.Marshal(c)
	if err != nil {
		return nil, err
	}

	var unmarshal map[string]interface{}
	err = yaml.Unmarshal(marshal, &unmarshal)
	if err != nil {
		return nil, err
	}

	ansibleTask.Config = unmarshal
	ansibleTask.raw = c

	return ansibleTask, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	cmdutil "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/resource"

	"github.com/daviddengcn/go-colortext"
	"github.com/spf13/cobra"
)

func NewCmdClusterInfo(f *cmdutil.Factory, out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use: "cluster-info",
		// clusterinfo is deprecated.
		Aliases: []string{"clusterinfo"},
		Short:   "Display cluster info",
		Long:    "Display addresses of the master and services with label kubernetes.io/cluster-service=true",
		Run: func(cmd *cobra.Command, args []string) {
			err := RunClusterInfo(f, out, cmd)
			cmdutil.CheckErr(err)
		},
	}
	return cmd
}

func RunClusterInfo(factory *cmdutil.Factory, out io.Writer, cmd *cobra.Command) error {
	if os.Args[1] == "clusterinfo" {
		printDeprecationWarning("cluster-info", "clusterinfo")
	}

	client, err := factory.ClientConfig()
	if err != nil {
		return err
	}
	printService(out, "Kubernetes master", client.Host, false)

	mapper, typer := factory.Object()
	cmdNamespace, err := factory.DefaultNamespace()
	if err != nil {
		return err
	}

	// TODO use generalized labels once they are implemented (#341)
	b := resource.NewBuilder(mapper, typer, factory.ClientMapperForCommand()).
		NamespaceParam(cmdNamespace).DefaultNamespace().
		SelectorParam("kubernetes.io/cluster-service=true").
		ResourceTypeOrNameArgs(false, []string{"services"}...).
		Latest()
	b.Do().Visit(func(r *resource.Info) error {
		services := r.Object.(*api.ServiceList).Items
		for _, service := range services {
			splittedLink := strings.Split(strings.Split(service.ObjectMeta.SelfLink, "?")[0], "/")
			// insert "proxy" into the link
			splittedLink = append(splittedLink, "")
			copy(splittedLink[4:], splittedLink[3:])
			splittedLink[3] = "proxy"
			link := client.Host + strings.Join(splittedLink, "/") + "/"
			printService(out, service.ObjectMeta.Labels["name"], link, true)
		}
		return nil
	})
	return nil

	// TODO consider printing more information about cluster
}

func printService(out io.Writer, name, link string, warn bool) {
	ct.ChangeColor(ct.Green, false, ct.None, false)
	fmt.Fprint(out, name)
	ct.ResetColor()
	fmt.Fprintf(out, " is running at ")
	ct.ChangeColor(ct.Yellow, false, ct.None, false)
	fmt.Fprint(out, link)
	ct.ResetColor()
	// TODO remove this warn once trailing slash is no longer required
	if warn {
		fmt.Fprint(out, " (note the trailing slash)")
	}
	fmt.Fprintln(out, "")
}
<commit_msg>Updated kubectl cluster-info to show v1beta3 addresses<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"os"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	cmdutil "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/resource"

	"github.com/daviddengcn/go-colortext"
	"github.com/spf13/cobra"
)

func NewCmdClusterInfo(f *cmdutil.Factory, out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use: "cluster-info",
		// clusterinfo is deprecated.
		Aliases: []string{"clusterinfo"},
		Short:   "Display cluster info",
		Long:    "Display addresses of the master and services with label kubernetes.io/cluster-service=true",
		Run: func(cmd *cobra.Command, args []string) {
			err := RunClusterInfo(f, out, cmd)
			cmdutil.CheckErr(err)
		},
	}
	return cmd
}

func RunClusterInfo(factory *cmdutil.Factory, out io.Writer, cmd *cobra.Command) error {
	if os.Args[1] == "clusterinfo" {
		printDeprecationWarning("cluster-info", "clusterinfo")
	}

	client, err := factory.ClientConfig()
	if err != nil {
		return err
	}
	printService(out, "Kubernetes master", client.Host)

	mapper, typer := factory.Object()
	cmdNamespace, err := factory.DefaultNamespace()
	if err != nil {
		return err
	}

	// TODO use generalized labels once they are implemented (#341)
	b := resource.NewBuilder(mapper, typer, factory.ClientMapperForCommand()).
		NamespaceParam(cmdNamespace).DefaultNamespace().
		SelectorParam("kubernetes.io/cluster-service=true").
		ResourceTypeOrNameArgs(false, []string{"services"}...).
		Latest()
	b.Do().Visit(func(r *resource.Info) error {
		services := r.Object.(*api.ServiceList).Items
		for _, service := range services {
			link := client.Host + "/api/v1beta3/proxy/namespaces/" + service.ObjectMeta.Namespace + "/services/" + service.ObjectMeta.Name
			printService(out, service.ObjectMeta.Labels["name"], link)
		}
		return nil
	})
	return nil

	// TODO consider printing more information about cluster
}

func printService(out io.Writer, name, link string) {
	ct.ChangeColor(ct.Green, false, ct.None, false)
	fmt.Fprint(out, name)
	ct.ResetColor()
	fmt.Fprintf(out, " is running at ")
	ct.ChangeColor(ct.Yellow, false, ct.None, false)
	fmt.Fprint(out, link)
	ct.ResetColor()
	fmt.Fprintln(out, "")
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package pluginhelp defines structures that represent plugin help information.
// These structs are used by sub-packages 'hook' and 'externalplugins'.
package pluginhelp

// Command is a serializable representation of the command information for a single command.
type Command struct {
	// Usage is a usage string for the command.
	Usage string
	// Featured is a flag for featured/highlight plugins.
	Featured bool
	// Description is a short description about what does the command do.
	Description string
	// Examples is a list of usage example for the command.
	Examples []string
	// WhoCanUse is a description of the permissions/role/authorization required to use the command.
	// This is usually specified as a github permissions, but it can also be a github team, an
	// OWNERS file alias, etc.
	// This field may include HTML.
	WhoCanUse string
}

// PluginHelp is a serializable representation of the help information for a single plugin.
// This includes repo specific configuration for every repo that the plugin is enabled for.
type PluginHelp struct {
	// Description is a description of what the plugin does and what purpose it achieves.
	// This field may include HTML.
	Description string
	// WhoCanUse is a description of the permissions/role/authorization required to use the plugin.
	// This is usually specified as a github permissions, but it can also be a github team, an
	// OWNERS file alias, etc.
	// This field may include HTML.
	WhoCanUse string
	// Usage is a usage string for the plugin. Leave empty if not applicable.
	Usage string
	// Examples is a list of usage examples for the plugin. Leave empty if not applicable.
	Examples []string
	// Config is a map from org/repo strings to a string describing the configuration for that repo.
	// The key "" should map to a string describing configuration that applies to all repos if any.
	// This configuration strings may include HTML.
	Config map[string]string

	// Events is a slice containing the events that are handled by the plugin.
	// NOTE: Plugins do not need to populate this. Hook populates it on their behalf.
	Events []string
	// Commands is a list of available commands of the plugin.
	Commands []Command
}

// Help is a serializable representation of all plugin help information.
type Help struct {
	// AllRepos is a flatten (all org/repo, no org strings) list of the repos that use plugins.
	AllRepos []string
	// RepoPlugins maps org and org/repo strings to the plugins configured for that scope.
	// NOTE: The key "" maps to the list of all existing plugins (including failed help providers).
	// A mix of org or org/repo strings is desirable over the flattened form (all org/repo) that
	// 'AllRepos' uses because it matches the plugin configuration and is more human readable.
	RepoPlugins         map[string][]string
	RepoExternalPlugins map[string][]string
	// PluginHelp is maps plugin names to their help info.
	PluginHelp         map[string]PluginHelp
	ExternalPluginHelp map[string]PluginHelp
}

func (pluginHelp *PluginHelp) AddCommand(command Command) {
	pluginHelp.Commands = append(pluginHelp.Commands, command)
}
<commit_msg>Add TODO comments to deprecated fields.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package pluginhelp defines structures that represent plugin help information.
// These structs are used by sub-packages 'hook' and 'externalplugins'.
package pluginhelp

// Command is a serializable representation of the command information for a single command.
type Command struct {
	// Usage is a usage string for the command.
	Usage string
	// Featured is a flag for featured/highlight plugins.
	Featured bool
	// Description is a short description about what does the command do.
	Description string
	// Examples is a list of usage example for the command.
	Examples []string
	// WhoCanUse is a description of the permissions/role/authorization required to use the command.
	// This is usually specified as a github permissions, but it can also be a github team, an
	// OWNERS file alias, etc.
	// This field may include HTML.
	WhoCanUse string
}

// PluginHelp is a serializable representation of the help information for a single plugin.
// This includes repo specific configuration for every repo that the plugin is enabled for.
type PluginHelp struct {
	// Description is a description of what the plugin does and what purpose it achieves.
	// This field may include HTML.
	Description string
	// WhoCanUse is a description of the permissions/role/authorization required to use the plugin.
	// This is usually specified as a github permissions, but it can also be a github team, an
	// OWNERS file alias, etc.
	// This field may include HTML.
	// TODO(qhuynh96): This field is about to be deprecated. WhoCanUse is now under the scope of Command.
	WhoCanUse string
	// Usage is a usage string for the plugin. Leave empty if not applicable.
	// TODO(qhuynh96): This field is about to be deprecated. Usage is now under the scope of Command.
	Usage string
	// Examples is a list of usage examples for the plugin. Leave empty if not applicable.
	// TODO(qhuynh96): This field is about to be deprecated. Examples is now under the scope of Command
	Examples []string
	// Config is a map from org/repo strings to a string describing the configuration for that repo.
	// The key "" should map to a string describing configuration that applies to all repos if any.
	// This configuration strings may include HTML.
	Config map[string]string

	// Events is a slice containing the events that are handled by the plugin.
	// NOTE: Plugins do not need to populate this. Hook populates it on their behalf.
	Events []string
	// Commands is a list of available commands of the plugin.
	Commands []Command
}

// Help is a serializable representation of all plugin help information.
type Help struct {
	// AllRepos is a flatten (all org/repo, no org strings) list of the repos that use plugins.
	AllRepos []string
	// RepoPlugins maps org and org/repo strings to the plugins configured for that scope.
	// NOTE: The key "" maps to the list of all existing plugins (including failed help providers).
	// A mix of org or org/repo strings is desirable over the flattened form (all org/repo) that
	// 'AllRepos' uses because it matches the plugin configuration and is more human readable.
	RepoPlugins         map[string][]string
	RepoExternalPlugins map[string][]string
	// PluginHelp is maps plugin names to their help info.
	PluginHelp         map[string]PluginHelp
	ExternalPluginHelp map[string]PluginHelp
}

func (pluginHelp *PluginHelp) AddCommand(command Command) {
	pluginHelp.Commands = append(pluginHelp.Commands, command)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	restclient "k8s.io/client-go/rest"
)

const defaultRetry int = 5

// LogAggregator aggregates the logs for all the deployed pods.
type LogAggregator struct {
	Muter

	creationTime time.Time
	output       io.Writer
	retries      int
}

// NewLogAggregator creates a new LogAggregator for a given output.
func NewLogAggregator(out io.Writer) *LogAggregator {
	return &LogAggregator{
		creationTime: time.Now(),
		output:       out,
		retries:      defaultRetry,
	}
}

const streamRetryDelay = 1 * time.Second

// TODO(@r2d4): Figure out how to mock this out. fake.NewSimpleClient
// won't mock out restclient.Request and will just return a nil stream.
var getStream = func(r *restclient.Request) (io.ReadCloser, error) {
	return r.Stream()
}

func (a *LogAggregator) StreamLogs(client corev1.CoreV1Interface, image string) {
	for i := 0; i < a.retries; i++ {
		if err := a.streamLogs(client, image); err != nil {
			logrus.Infof("Error getting logs %s", err)
		}
		time.Sleep(streamRetryDelay)
	}
}

// nolint: interfacer
func (a *LogAggregator) streamLogs(client corev1.CoreV1Interface, image string) error {
	pods, err := client.Pods("").List(meta_v1.ListOptions{
		IncludeUninitialized: true,
	})
	if err != nil {
		return errors.Wrap(err, "getting pods")
	}

	logrus.Infof("Looking for logs to stream for %s", image)
	for _, p := range pods.Items {
		for _, c := range p.Spec.Containers {
			logrus.Debugf("Found container %s with image %s", c.Name, c.Image)
			if c.Image != image {
				continue
			}

			logrus.Infof("Trying to stream logs from pod: %s container: %s", p.Name, c.Name)
			pods := client.Pods(p.Namespace)
			if err := WaitForPodReady(pods, p.Name); err != nil {
				return errors.Wrap(err, "waiting for pod ready")
			}
			req := pods.GetLogs(p.Name, &v1.PodLogOptions{
				Follow:    true,
				Container: c.Name,
				SinceTime: &meta_v1.Time{
					Time: a.creationTime,
				},
			})
			rc, err := getStream(req)
			if err != nil {
				return errors.Wrap(err, "setting up container log stream")
			}
			defer rc.Close()
			header := fmt.Sprintf("[%s %s]", p.Name, c.Name)
			if err := a.streamRequest(header, rc); err != nil {
				return errors.Wrap(err, "streaming request")
			}

			return nil
		}
	}

	return fmt.Errorf("Image %s not found", image)
}

func (a *LogAggregator) streamRequest(header string, rc io.Reader) error {
	r := bufio.NewReader(rc)
	for {
		// Read up to newline
		line, err := r.ReadBytes('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return errors.Wrap(err, "reading bytes from log stream")
		}

		if a.IsMuted() {
			continue
		}

		if _, err := fmt.Fprintf(a.output, "%s %s", header, line); err != nil {
			return errors.Wrap(err, "writing to out")
		}
	}
	logrus.Infof("%s exited", header)
	return nil
}

// Muter can be used to mute/unmute logs.
// It's safe to use in multiple go routines.
type Muter struct {
	muted int32
}

// Mute mutes the logs.
func (m *Muter) Mute() {
	atomic.StoreInt32(&m.muted, 1)
}

// Unmute unmute the logs.
func (m *Muter) Unmute() {
	atomic.StoreInt32(&m.muted, 0)
}

// IsMuted says if the logs are to be muted.
func (m *Muter) IsMuted() bool {
	return atomic.LoadInt32(&m.muted) == 1
}
<commit_msg>Add colors to the logs<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"bufio"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	restclient "k8s.io/client-go/rest"
)

const defaultRetry int = 5

var colors = []int{
	31, // red
	32, // green
	33, // yellow
	34, // blue
	35, // magenta
	36, // cyan
	37, // lightGray
	90, // darkGray
	91, // lightRed
	92, // lightGreen
	93, // lightYellow
	94, // lightBlue
	95, // lightPurple
	96, // lightCyan
	97, // white
}

// LogAggregator aggregates the logs for all the deployed pods.
type LogAggregator struct {
	Muter

	creationTime   time.Time
	output         io.Writer
	retries        int
	nextColorIndex int
	lockColor      sync.Mutex
}

// NewLogAggregator creates a new LogAggregator for a given output.
func NewLogAggregator(out io.Writer) *LogAggregator {
	return &LogAggregator{
		creationTime: time.Now(),
		output:       out,
		retries:      defaultRetry,
	}
}

const streamRetryDelay = 1 * time.Second

// TODO(@r2d4): Figure out how to mock this out. fake.NewSimpleClient
// won't mock out restclient.Request and will just return a nil stream.
var getStream = func(r *restclient.Request) (io.ReadCloser, error) {
	return r.Stream()
}

func (a *LogAggregator) StreamLogs(client corev1.CoreV1Interface, image string) {
	for i := 0; i < a.retries; i++ {
		if err := a.streamLogs(client, image); err != nil {
			logrus.Infof("Error getting logs %s", err)
		}
		time.Sleep(streamRetryDelay)
	}
}

// nolint: interfacer
func (a *LogAggregator) streamLogs(client corev1.CoreV1Interface, image string) error {
	pods, err := client.Pods("").List(meta_v1.ListOptions{
		IncludeUninitialized: true,
	})
	if err != nil {
		return errors.Wrap(err, "getting pods")
	}

	logrus.Infof("Looking for logs to stream for %s", image)
	for _, p := range pods.Items {
		for _, c := range p.Spec.Containers {
			logrus.Debugf("Found container %s with image %s", c.Name, c.Image)
			if c.Image != image {
				continue
			}

			logrus.Infof("Trying to stream logs from pod: %s container: %s", p.Name, c.Name)
			pods := client.Pods(p.Namespace)
			if err := WaitForPodReady(pods, p.Name); err != nil {
				return errors.Wrap(err, "waiting for pod ready")
			}
			req := pods.GetLogs(p.Name, &v1.PodLogOptions{
				Follow:    true,
				Container: c.Name,
				SinceTime: &meta_v1.Time{
					Time: a.creationTime,
				},
			})
			rc, err := getStream(req)
			if err != nil {
				return errors.Wrap(err, "setting up container log stream")
			}
			defer rc.Close()

			color := a.nextColor()

			header := fmt.Sprintf("\033[1;%dm[%s %s]\033[0m", color, p.Name, c.Name)
			if err := a.streamRequest(header, rc); err != nil {
				return errors.Wrap(err, "streaming request")
			}

			return nil
		}
	}

	return fmt.Errorf("Image %s not found", image)
}

func (a *LogAggregator) nextColor() int {
	a.lockColor.Lock()
	color := colors[a.nextColorIndex]
	a.nextColorIndex++
	a.lockColor.Unlock()

	return color
}

func (a *LogAggregator) streamRequest(header string, rc io.Reader) error {
	r := bufio.NewReader(rc)
	for {
		// Read up to newline
		line, err := r.ReadBytes('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return errors.Wrap(err, "reading bytes from log stream")
		}

		if a.IsMuted() {
			continue
		}

		if _, err := fmt.Fprintf(a.output, "%s %s", header, line); err != nil {
			return errors.Wrap(err, "writing to out")
		}
	}
	logrus.Infof("%s exited", header)
	return nil
}

// Muter can be used to mute/unmute logs.
// It's safe to use in multiple go routines.
type Muter struct {
	muted int32
}

// Mute mutes the logs.
func (m *Muter) Mute() {
	atomic.StoreInt32(&m.muted, 1)
}

// Unmute unmute the logs.
func (m *Muter) Unmute() {
	atomic.StoreInt32(&m.muted, 0)
}

// IsMuted says if the logs are to be muted.
func (m *Muter) IsMuted() bool {
	return atomic.LoadInt32(&m.muted) == 1
}
<|endoftext|>


<commit_before>package abkleveldb

import (
  "fmt"

  "github.com/jmhodges/levigo"
)

func boohoo(errstring string, rise bool){
  fmt.Println(errstring)
  if rise == true{ panic(errstring) }
}

func CreateDB(dbname string) (*levigo.DB) {
  opts := levigo.NewOptions()
  opts.SetCache(levigo.NewLRUCache(1<<10))
  opts.SetCreateIfMissing(true)
  db, err := levigo.Open(dbname, opts)
  if err != nil { boohoo("DB " + dbname + " Creation failed.", true) }
  return db
}

/* Push KeyVal */
func PushKeyVal(key string, val string, db *levigo.DB) bool{
  writer := levigo.NewWriteOptions()
  defer writer.Close()

  keyname := []byte(key)
  value := []byte(val)
  err := db.Put(writer, keyname, value)
  if err != nil {
    boohoo("Key " + key + " insertion failed. It's value was " + val, false)
    return false
  }
  return true
}

/* Get Key */
func GetVal(key string, db *levigo.DB) string {
  reader := levigo.NewReadOptions()
  defer reader.Close()

  data, err := db.Get(reader, []byte(key))
  if err != nil { boohoo("Key " + key + " query failed.", false) }
  return string(data)
}

/* Del Key */
func DelKey(key string, db *levigo.DB) bool {
  writer := levigo.NewWriteOptions()
  defer writer.Close()

  err := db.Delete(writer, []byte(key))
  if err != nil { boohoo("Key " + key + " query failed.", false) }
  return true
}
<commit_msg>abkleveldb updated with comments<commit_after>package abkleveldb

import (
  "fmt"

  "github.com/jmhodges/levigo"
)

/*
Prints provided error message and panics if rise value is True.
*/
func boohoo(errstring string, rise bool){
  fmt.Println(errstring)
  if rise == true{ panic(errstring) }
}

/*
Creates a db at provided pathname.
*/
func CreateDB(dbname string) (*levigo.DB) {
  opts := levigo.NewOptions()
  opts.SetCache(levigo.NewLRUCache(1<<10))
  opts.SetCreateIfMissing(true)
  db, err := levigo.Open(dbname, opts)
  if err != nil { boohoo("DB " + dbname + " Creation failed.", true) }
  return db
}

/*
Push KeyVal in provided DB handle.
*/
func PushKeyVal(key string, val string, db *levigo.DB) bool{
  writer := levigo.NewWriteOptions()
  defer writer.Close()

  keyname := []byte(key)
  value := []byte(val)
  err := db.Put(writer, keyname, value)
  if err != nil {
    boohoo("Key " + key + " insertion failed. It's value was " + val, false)
    return false
  }
  return true
}

/*
Get Value of Key from provided db handle.
*/
func GetVal(key string, db *levigo.DB) string {
  reader := levigo.NewReadOptions()
  defer reader.Close()

  data, err := db.Get(reader, []byte(key))
  if err != nil { boohoo("Key " + key + " query failed.", false) }
  return string(data)
}

/*
Del Key from provided DB handle.
*/
func DelKey(key string, db *levigo.DB) bool {
  writer := levigo.NewWriteOptions()
  defer writer.Close()

  err := db.Delete(writer, []byte(key))
  if err != nil { boohoo("Key " + key + " query failed.", false) }
  return true
}
<|endoftext|>