<commit_before>// Copyright 2014 Hari haran. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package extract

import (
	"net/http"
	"net/url"

	"code.google.com/p/cascadia"
	"golang.org/x/net/html"
)

// A selection contains the required elements for extraction.
type selection struct {
	Selector string // CSS Selector
	URL      string
}

// newSelection is a constructor for selection type.
// It takes the selector string and the url as inputs, in that order.
func newSelection(s, u string) *selection {
	return &selection{s, u}
}

// Links returns the absolute URLs of all references from an URL of a webpage.
func Links(u string) ([]string, error) {
	s := newSelection("a[href]", u)
	link, err := url.Parse(s.URL)
	if err != nil {
		return nil, err
	}
	r, err := http.Get(link.String())
	if err != nil {
		return nil, err
	}
	doc, err := html.Parse(r.Body)
	if err != nil {
		return nil, err
	}
	sel, err := cascadia.Compile(s.Selector)
	if err != nil {
		return nil, err
	}
	matches := sel.MatchAll(doc)
	var result []string
	for _, m := range matches {
		r, err := resolveURL(hrefString(m), link)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

// hrefString takes an *html.Node as input and
// returns the value of attribute href.
func hrefString(n *html.Node) string {
	switch n.Type {
	case html.TextNode:
		return ""
	case html.ElementNode:
		return attribute(
			html.Token{
				Type: html.StartTagToken,
				Data: n.Data,
				Attr: n.Attr,
			}, "href")
	}
	return ""
}

// imageString takes an *html.Node as input and
// returns the value of attribute src.
func imageString(n *html.Node) string {
	switch n.Type {
	case html.TextNode:
		return ""
	case html.ElementNode:
		return attribute(
			html.Token{
				Type: html.StartTagToken,
				Data: n.Data,
				Attr: n.Attr,
			}, "src")
	}
	return ""
}

// attribute takes an html Token and the attribute key as inputs
// and returns the value of the attribute.
func attribute(t html.Token, a string) string {
	for _, x := range t.Attr {
		if x.Key == a {
			return x.Val
		}
	}
	return ""
}

// Images returns the absolute URLs of all the images from an URL of a webpage.
func Images(u string) ([]string, error) {
	s := newSelection("img[src]", u)
	link, err := url.Parse(s.URL)
	if err != nil {
		return nil, err
	}
	r, err := http.Get(link.String())
	if err != nil {
		return nil, err
	}
	doc, err := html.Parse(r.Body)
	if err != nil {
		return nil, err
	}
	sel, err := cascadia.Compile(s.Selector)
	if err != nil {
		return nil, err
	}
	matches := sel.MatchAll(doc)
	var result []string
	for _, m := range matches {
		r, err := resolveURL(imageString(m), link)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

// resolveURL converts all input URLs into absolute URLs.
func resolveURL(s string, link *url.URL) (string, error) {
	// x may or may not be an absolute URL.
	x, err := url.Parse(s)
	if err != nil {
		return "", err
	}
	// y is guaranteed to be an absolute URL.
	y := link.ResolveReference(x)
	return y.String(), nil
}
<commit_msg>Rename arguments and label fields explicitly in the composite literal<commit_after>// Copyright 2014 Hari haran. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package extract

import (
	"net/http"
	"net/url"

	"code.google.com/p/cascadia"
	"golang.org/x/net/html"
)

// A selection contains the required elements for extraction.
type selection struct {
	Selector string // CSS Selector
	URL      string
}

// newSelection is a constructor for selection type.
// selector implies a CSS selector string.
func newSelection(selector, url string) *selection {
	return &selection{
		Selector: selector,
		URL:      url,
	}
}

// Links returns the absolute URLs of all references from an URL of a webpage.
func Links(u string) ([]string, error) {
	s := newSelection("a[href]", u)
	link, err := url.Parse(s.URL)
	if err != nil {
		return nil, err
	}
	r, err := http.Get(link.String())
	if err != nil {
		return nil, err
	}
	doc, err := html.Parse(r.Body)
	if err != nil {
		return nil, err
	}
	sel, err := cascadia.Compile(s.Selector)
	if err != nil {
		return nil, err
	}
	matches := sel.MatchAll(doc)
	var result []string
	for _, m := range matches {
		r, err := resolveURL(hrefString(m), link)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

// hrefString takes an *html.Node as input and
// returns the value of attribute href.
func hrefString(n *html.Node) string {
	switch n.Type {
	case html.TextNode:
		return ""
	case html.ElementNode:
		return attribute(
			html.Token{
				Type: html.StartTagToken,
				Data: n.Data,
				Attr: n.Attr,
			}, "href")
	}
	return ""
}

// imageString takes an *html.Node as input and
// returns the value of attribute src.
func imageString(n *html.Node) string {
	switch n.Type {
	case html.TextNode:
		return ""
	case html.ElementNode:
		return attribute(
			html.Token{
				Type: html.StartTagToken,
				Data: n.Data,
				Attr: n.Attr,
			}, "src")
	}
	return ""
}

// attribute takes an html Token and the attribute key as inputs
// and returns the value of the attribute.
func attribute(t html.Token, a string) string {
	for _, x := range t.Attr {
		if x.Key == a {
			return x.Val
		}
	}
	return ""
}

// Images returns the absolute URLs of all the images from an URL of a webpage.
func Images(u string) ([]string, error) {
	s := newSelection("img[src]", u)
	link, err := url.Parse(s.URL)
	if err != nil {
		return nil, err
	}
	r, err := http.Get(link.String())
	if err != nil {
		return nil, err
	}
	doc, err := html.Parse(r.Body)
	if err != nil {
		return nil, err
	}
	sel, err := cascadia.Compile(s.Selector)
	if err != nil {
		return nil, err
	}
	matches := sel.MatchAll(doc)
	var result []string
	for _, m := range matches {
		r, err := resolveURL(imageString(m), link)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

// resolveURL converts all input URLs into absolute URLs.
func resolveURL(s string, link *url.URL) (string, error) {
	// x may or may not be an absolute URL.
	x, err := url.Parse(s)
	if err != nil {
		return "", err
	}
	// y is guaranteed to be an absolute URL.
	y := link.ResolveReference(x)
	return y.String(), nil
}
<|endoftext|>


<commit_before>package dockerlaunch

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/resolvconf"
	"github.com/rancher/docker-from-scratch/util"
	"github.com/rancher/netconf"
)

const defaultPrefix = "/usr"

var (
	mounts [][]string = [][]string{
		{"devtmpfs", "/dev", "devtmpfs", ""},
		{"none", "/dev/pts", "devpts", ""},
		{"none", "/proc", "proc", ""},
		{"none", "/run", "tmpfs", ""},
		{"none", "/sys", "sysfs", ""},
		{"none", "/sys/fs/cgroup", "tmpfs", ""},
	}
)

type Config struct {
	Fork            bool
	CommandName     string
	DnsConfig       netconf.DnsConfig
	BridgeName      string
	BridgeAddress   string
	BridgeMtu       int
	CgroupHierarchy map[string]string
	LogFile         string
	NoLog           bool
}

func createMounts(mounts ...[]string) error {
	for _, mount := range mounts {
		log.Debugf("Mounting %s %s %s %s", mount[0], mount[1], mount[2], mount[3])
		err := util.Mount(mount[0], mount[1], mount[2], mount[3])
		if err != nil {
			return err
		}
	}

	return nil
}

func createDirs(dirs ...string) error {
	for _, dir := range dirs {
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			log.Debugf("Creating %s", dir)
			err = os.MkdirAll(dir, 0755)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func mountCgroups(hierarchyConfig map[string]string) error {
	f, err := os.Open("/proc/cgroups")
	if err != nil {
		return err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)

	hierarchies := make(map[string][]string)

	for scanner.Scan() {
		text := scanner.Text()
		log.Debugf("/proc/cgroups: %s", text)
		fields := strings.SplitN(text, "\t", 3)
		cgroup := fields[0]
		if cgroup == "" || cgroup[0] == '#' || len(fields) < 3 {
			continue
		}

		hierarchy := hierarchyConfig[cgroup]
		if hierarchy == "" {
			hierarchy = fields[1]
		}

		if hierarchy == "0" {
			hierarchy = cgroup
		}

		hierarchies[hierarchy] = append(hierarchies[hierarchy], cgroup)
	}

	for _, hierarchy := range hierarchies {
		if err := mountCgroup(strings.Join(hierarchy, ",")); err != nil {
			return err
		}
	}

	if err = scanner.Err(); err != nil {
		return err
	}

	log.Debug("Done mouting cgroupfs")
	return nil
}

func CreateSymlinks(pathSets [][]string) error {
	for _, paths := range pathSets {
		if err := CreateSymlink(paths[0], paths[1]); err != nil {
			return err
		}
	}

	return nil
}

func CreateSymlink(src, dest string) error {
	if _, err := os.Lstat(dest); os.IsNotExist(err) {
		log.Debugf("Symlinking %s => %s", src, dest)
		if err = os.Symlink(src, dest); err != nil {
			return err
		}
	}

	return nil
}

func mountCgroup(cgroup string) error {
	if err := createDirs("/sys/fs/cgroup/" + cgroup); err != nil {
		return err
	}

	if err := createMounts([][]string{{"none", "/sys/fs/cgroup/" + cgroup, "cgroup", cgroup}}...); err != nil {
		return err
	}

	parts := strings.Split(cgroup, ",")
	if len(parts) > 1 {
		for _, part := range parts {
			if err := CreateSymlink("/sys/fs/cgroup/"+cgroup, "/sys/fs/cgroup/"+part); err != nil {
				return err
			}
		}
	}

	return nil
}

func execDocker(config *Config, docker, cmd string, args []string) (*exec.Cmd, error) {
	if len(args) > 0 && args[0] == "docker" {
		args = args[1:]
	}
	log.Debugf("Launching Docker %s %s %v", docker, cmd, args)

	if config.Fork {
		cmd := exec.Command(docker, args...)
		if !config.NoLog {
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
		}
		err := cmd.Start()
		return cmd, err
	} else {
		err := syscall.Exec(docker, append([]string{cmd}, args...), os.Environ())
		return nil, err
	}
}

func copyDefault(folder, name string) error {
	defaultFile := path.Join(defaultPrefix, folder, name)
	if err := CopyFile(defaultFile, folder, name); err != nil {
		return err
	}

	return nil
}

func defaultFiles(files ...string) error {
	for _, file := range files {
		dir := path.Dir(file)
		name := path.Base(file)
		if err := copyDefault(dir, name); err != nil {
			return err
		}
	}

	return nil
}

func CopyFile(src, folder, name string) error {
	if _, err := os.Stat(src); os.IsNotExist(err) {
		return nil
	}

	dst := path.Join(folder, name)
	if _, err := os.Stat(dst); err == nil {
		return nil
	}

	if err := createDirs(folder); err != nil {
		return err
	}

	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	dstFile, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer dstFile.Close()

	_, err = io.Copy(dstFile, srcFile)
	return err
}

func tryCreateFile(name, content string) error {
	if _, err := os.Stat(name); err == nil {
		return nil
	}

	if err := createDirs(path.Dir(name)); err != nil {
		return err
	}

	return ioutil.WriteFile(name, []byte(content), 0644)
}

func createPasswd() error {
	return tryCreateFile("/etc/passwd", "root:x:0:0:root:/root:/bin/sh\n")
}

func createGroup() error {
	return tryCreateFile("/etc/group", "root:x:0:\n")
}

func setupNetworking(config *Config) error {
	if config == nil {
		return nil
	}

	if len(config.DnsConfig.Nameservers) != 0 {
		if err := resolvconf.Build("/etc/resolv.conf", config.DnsConfig.Nameservers, config.DnsConfig.Search); err != nil {
			return err
		}
	}

	if config.BridgeName != "" {
		log.Debugf("Creating bridge %s (%s)", config.BridgeName, config.BridgeAddress)
		if err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
			Interfaces: map[string]netconf.InterfaceConfig{
				config.BridgeName: {
					Address: config.BridgeAddress,
					MTU:     config.BridgeMtu,
					Bridge:  true,
				},
			},
		}); err != nil {
			return err
		}
	}

	return nil
}

func getValue(index int, args []string) string {
	val := args[index]
	parts := strings.SplitN(val, "=", 2)
	if len(parts) == 1 {
		if len(args) > index+1 {
			return args[index+1]
		} else {
			return ""
		}
	} else {
		return parts[2]
	}
}

func ParseConfig(config *Config, args ...string) []string {
	for i, arg := range args {
		if strings.HasPrefix(arg, "--bip") {
			config.BridgeAddress = getValue(i, args)
		} else if strings.HasPrefix(arg, "--fixed-cidr") {
			config.BridgeAddress = getValue(i, args)
		} else if strings.HasPrefix(arg, "-b") || strings.HasPrefix(arg, "--bridge") {
			config.BridgeName = getValue(i, args)
		} else if strings.HasPrefix(arg, "--mtu") {
			mtu, err := strconv.Atoi(getValue(i, args))
			if err != nil {
				config.BridgeMtu = mtu
			}
		}
	}

	if config.BridgeName != "" && config.BridgeAddress != "" {
		newArgs := []string{}
		skip := false
		for _, arg := range args {
			if skip {
				skip = false
				continue
			}

			if arg == "--bip" {
				skip = true
				continue
			} else if strings.HasPrefix(arg, "--bip=") {
				continue
			}

			newArgs = append(newArgs, arg)
		}

		args = newArgs
	}

	return args
}

func PrepareFs(config *Config) error {
	if err := createMounts(mounts...); err != nil {
		return err
	}

	if err := mountCgroups(config.CgroupHierarchy); err != nil {
		return err
	}

	if err := createLayout(); err != nil {
		return err
	}

	return nil
}

func touchSocket(path string) error {
	if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) {
		return err
	}
	return ioutil.WriteFile(path, []byte{}, 0700)
}

func touchSockets(args ...string) error {
	touched := false

	for i, arg := range args {
		if strings.HasPrefix(arg, "-H") {
			val := getValue(i, args)
			if strings.HasPrefix(val, "unix://") {
				val = val[len("unix://"):]
				log.Debugf("Creating temp file at %s", val)
				if err := touchSocket(val); err != nil {
					return err
				}
				touched = true
			}
		}
	}

	if !touched {
		return touchSocket("/var/run/docker.sock")
	}

	return nil
}

func createLayout() error {
	if err := createDirs("/tmp", "/root/.ssh", "/var"); err != nil {
		return err
	}

	return CreateSymlinks([][]string{
		{"usr/lib", "/lib"},
		{"usr/sbin", "/sbin"},
		{"../run", "/var/run"},
	})
}

func prepare(config *Config, docker string, args ...string) error {
	os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin")

	if err := defaultFiles(
		"/etc/ssl/certs/ca-certificates.crt",
		"/etc/passwd",
		"/etc/group",
	); err != nil {
		return err
	}

	if err := createPasswd(); err != nil {
		return err
	}

	if err := createGroup(); err != nil {
		return err
	}

	if err := setupNetworking(config); err != nil {
		return err
	}

	if err := touchSockets(args...); err != nil {
		return err
	}

	if err := setupLogging(config); err != nil {
		return err
	}

	if err := setupBin(config, docker); err != nil {
		return err
	}

	return nil
}

func setupBin(config *Config, docker string) error {
	if _, err := os.Stat(docker); os.IsNotExist(err) {
		dist := docker + ".dist"
		if _, err := os.Stat(dist); err == nil {
			return os.Symlink(dist, docker)
		}
	}

	return nil
}

func setupLogging(config *Config) error {
	if config.LogFile == "" {
		return nil
	}

	if err := createDirs(path.Dir(config.LogFile)); err != nil {
		return err
	}

	output, err := os.Create(config.LogFile)
	if err != nil {
		return err
	}

	syscall.Dup2(int(output.Fd()), int(os.Stdout.Fd()))
	syscall.Dup2(int(output.Fd()), int(os.Stderr.Fd()))

	return nil
}

func runOrExec(config *Config, docker string, args ...string) (*exec.Cmd, error) {
	if err := prepare(config, docker, args...); err != nil {
		return nil, err
	}

	cmd := "docker"
	if config != nil && config.CommandName != "" {
		cmd = config.CommandName
	}

	return execDocker(config, docker, cmd, args)
}

func LaunchDocker(config *Config, docker string, args ...string) (*exec.Cmd, error) {
	if err := PrepareFs(config); err != nil {
		return nil, err
	}

	return runOrExec(config, docker, args...)
}
<commit_msg>Emulate systemd, support environment, and set ulimit -n<commit_after>package dockerlaunch

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/resolvconf"
	"github.com/rancher/docker-from-scratch/util"
	"github.com/rancher/netconf"
)

const (
	defaultPrefix = "/usr"
	iptables      = "/sbin/iptables"
	modprobe      = "/sbin/modprobe"
	systemdRoot   = "/sys/fs/cgroup/systemd/user.slice"
)

var (
	mounts = [][]string{
		{"devtmpfs", "/dev", "devtmpfs", ""},
		{"none", "/dev/pts", "devpts", ""},
		{"none", "/proc", "proc", ""},
		{"none", "/run", "tmpfs", ""},
		{"none", "/sys", "sysfs", ""},
		{"none", "/sys/fs/cgroup", "tmpfs", ""},
	}
	systemdMounts = [][]string{
		{"systemd", "/sys/fs/cgroup/systemd", "cgroup", "none,name=systemd"},
	}
)

type Config struct {
	Fork            bool
	CommandName     string
	DnsConfig       netconf.DnsConfig
	BridgeName      string
	BridgeAddress   string
	BridgeMtu       int
	CgroupHierarchy map[string]string
	LogFile         string
	NoLog           bool
	EmulateSystemd  bool
	NoFiles         uint64
	Environment     []string
}

func createMounts(mounts ...[]string) error {
	for _, mount := range mounts {
		log.Debugf("Mounting %s %s %s %s", mount[0], mount[1], mount[2], mount[3])
		err := util.Mount(mount[0], mount[1], mount[2], mount[3])
		if err != nil {
			return err
		}
	}

	return nil
}

func createDirs(dirs ...string) error {
	for _, dir := range dirs {
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			log.Debugf("Creating %s", dir)
			err = os.MkdirAll(dir, 0755)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func mountCgroups(hierarchyConfig map[string]string) error {
	f, err := os.Open("/proc/cgroups")
	if err != nil {
		return err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)

	hierarchies := make(map[string][]string)

	for scanner.Scan() {
		text := scanner.Text()
		log.Debugf("/proc/cgroups: %s", text)
		fields := strings.SplitN(text, "\t", 3)
		cgroup := fields[0]
		if cgroup == "" || cgroup[0] == '#' || len(fields) < 3 {
			continue
		}

		hierarchy := hierarchyConfig[cgroup]
		if hierarchy == "" {
			hierarchy = fields[1]
		}

		if hierarchy == "0" {
			hierarchy = cgroup
		}

		hierarchies[hierarchy] = append(hierarchies[hierarchy], cgroup)
	}

	for _, hierarchy := range hierarchies {
		if err := mountCgroup(strings.Join(hierarchy, ",")); err != nil {
			return err
		}
	}

	if err = scanner.Err(); err != nil {
		return err
	}

	log.Debug("Done mouting cgroupfs")
	return nil
}

func CreateSymlinks(pathSets [][]string) error {
	for _, paths := range pathSets {
		if err := CreateSymlink(paths[0], paths[1]); err != nil {
			return err
		}
	}

	return nil
}

func CreateSymlink(src, dest string) error {
	if _, err := os.Lstat(dest); os.IsNotExist(err) {
		log.Debugf("Symlinking %s => %s", src, dest)
		if err = os.Symlink(src, dest); err != nil {
			return err
		}
	}

	return nil
}

func mountCgroup(cgroup string) error {
	if err := createDirs("/sys/fs/cgroup/" + cgroup); err != nil {
		return err
	}

	if err := createMounts([][]string{{"none", "/sys/fs/cgroup/" + cgroup, "cgroup", cgroup}}...); err != nil {
		return err
	}

	parts := strings.Split(cgroup, ",")
	if len(parts) > 1 {
		for _, part := range parts {
			if err := CreateSymlink("/sys/fs/cgroup/"+cgroup, "/sys/fs/cgroup/"+part); err != nil {
				return err
			}
		}
	}

	return nil
}

func execDocker(config *Config, docker, cmd string, args []string) (*exec.Cmd, error) {
	if len(args) > 0 && args[0] == "docker" {
		args = args[1:]
	}
	log.Debugf("Launching Docker %s %s %v", docker, cmd, args)

	env := os.Environ()
	if len(config.Environment) != 0 {
		env = append(env, config.Environment...)
	}

	if config.Fork {
		cmd := exec.Command(docker, args...)
		if !config.NoLog {
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
		}
		cmd.Env = env
		err := cmd.Start()
		return cmd, err
	} else {
		err := syscall.Exec(docker, append([]string{cmd}, args...), env)
		return nil, err
	}
}

func copyDefault(folder, name string) error {
	defaultFile := path.Join(defaultPrefix, folder, name)
	if err := CopyFile(defaultFile, folder, name); err != nil {
		return err
	}

	return nil
}

func defaultFiles(files ...string) error {
	for _, file := range files {
		dir := path.Dir(file)
		name := path.Base(file)
		if err := copyDefault(dir, name); err != nil {
			return err
		}
	}

	return nil
}

func CopyFile(src, folder, name string) error {
	if _, err := os.Stat(src); os.IsNotExist(err) {
		return nil
	}

	dst := path.Join(folder, name)
	if _, err := os.Stat(dst); err == nil {
		return nil
	}

	if err := createDirs(folder); err != nil {
		return err
	}

	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	dstFile, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer dstFile.Close()

	_, err = io.Copy(dstFile, srcFile)
	return err
}

func tryCreateFile(name, content string) error {
	if _, err := os.Stat(name); err == nil {
		return nil
	}

	if err := createDirs(path.Dir(name)); err != nil {
		return err
	}

	return ioutil.WriteFile(name, []byte(content), 0644)
}

func createPasswd() error {
	return tryCreateFile("/etc/passwd", "root:x:0:0:root:/root:/bin/sh\n")
}

func createGroup() error {
	return tryCreateFile("/etc/group", "root:x:0:\n")
}

func setupNetworking(config *Config) error {
	if config == nil {
		return nil
	}

	if len(config.DnsConfig.Nameservers) != 0 {
		if err := resolvconf.Build("/etc/resolv.conf", config.DnsConfig.Nameservers, config.DnsConfig.Search); err != nil {
			return err
		}
	}

	if config.BridgeName != "" {
		log.Debugf("Creating bridge %s (%s)", config.BridgeName, config.BridgeAddress)
		if err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
			Interfaces: map[string]netconf.InterfaceConfig{
				config.BridgeName: {
					Address: config.BridgeAddress,
					MTU:     config.BridgeMtu,
					Bridge:  true,
				},
			},
		}); err != nil {
			return err
		}
	}

	return nil
}

func getValue(index int, args []string) string {
	val := args[index]
	parts := strings.SplitN(val, "=", 2)
	if len(parts) == 1 {
		if len(args) > index+1 {
			return args[index+1]
		} else {
			return ""
		}
	} else {
		return parts[2]
	}
}

func ParseConfig(config *Config, args ...string) []string {
	for i, arg := range args {
		if strings.HasPrefix(arg, "--bip") {
			config.BridgeAddress = getValue(i, args)
		} else if strings.HasPrefix(arg, "--fixed-cidr") {
			config.BridgeAddress = getValue(i, args)
		} else if strings.HasPrefix(arg, "-b") || strings.HasPrefix(arg, "--bridge") {
			config.BridgeName = getValue(i, args)
		} else if strings.HasPrefix(arg, "--mtu") {
			mtu, err := strconv.Atoi(getValue(i, args))
			if err != nil {
				config.BridgeMtu = mtu
			}
		}
	}

	if config.BridgeName != "" && config.BridgeAddress != "" {
		newArgs := []string{}
		skip := false
		for _, arg := range args {
			if skip {
				skip = false
				continue
			}

			if arg == "--bip" {
				skip = true
				continue
			} else if strings.HasPrefix(arg, "--bip=") {
				continue
			}

			newArgs = append(newArgs, arg)
		}

		args = newArgs
	}

	return args
}

func PrepareFs(config *Config) error {
	if err := createMounts(mounts...); err != nil {
		return err
	}

	if err := mountCgroups(config.CgroupHierarchy); err != nil {
		return err
	}

	if err := createLayout(); err != nil {
		return err
	}

	return nil
}

func touchSocket(path string) error {
	if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) {
		return err
	}
	return ioutil.WriteFile(path, []byte{}, 0700)
}

func touchSockets(args ...string) error {
	touched := false

	for i, arg := range args {
		if strings.HasPrefix(arg, "-H") {
			val := getValue(i, args)
			if strings.HasPrefix(val, "unix://") {
				val = val[len("unix://"):]
				log.Debugf("Creating temp file at %s", val)
				if err := touchSocket(val); err != nil {
					return err
				}
				touched = true
			}
		}
	}

	if !touched {
		return touchSocket("/var/run/docker.sock")
	}

	return nil
}

func createLayout() error {
	if err := createDirs("/tmp", "/root/.ssh", "/var"); err != nil {
		return err
	}

	return CreateSymlinks([][]string{
		{"usr/lib", "/lib"},
		{"usr/sbin", "/sbin"},
		{"../run", "/var/run"},
	})
}

func prepare(config *Config, docker string, args ...string) error {
	os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin")

	if err := defaultFiles(
		"/etc/ssl/certs/ca-certificates.crt",
		"/etc/passwd",
		"/etc/group",
	); err != nil {
		return err
	}

	if err := createPasswd(); err != nil {
		return err
	}

	if err := createGroup(); err != nil {
		return err
	}

	if err := setupNetworking(config); err != nil {
		return err
	}

	if err := touchSockets(args...); err != nil {
		return err
	}

	if err := setupLogging(config); err != nil {
		return err
	}

	for _, i := range []string{docker, iptables, modprobe} {
		if err := setupBin(config, i); err != nil {
			return err
		}
	}

	if err := setUlimit(config); err != nil {
		return err
	}

	if err := setupSystemd(config); err != nil {
		return err
	}

	return nil
}

func setupSystemd(config *Config) error {
	if !config.EmulateSystemd {
		cgroups, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cgroup", os.Getpid()))
		if err != nil {
			return err
		}

		if !strings.Contains(string(cgroups), "name=systemd") {
			return nil
		}
	}

	if err := createMounts(systemdMounts...); err != nil {
		return err
	}

	if err := os.Mkdir(systemdRoot, 0755); err != nil {
		return err
	}

	return ioutil.WriteFile(path.Join(systemdRoot, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())+"\n"), 0644)
}

func setupBin(config *Config, bin string) error {
	if _, err := os.Stat(bin); os.IsNotExist(err) {
		dist := bin + ".dist"
		if _, err := os.Stat(dist); err == nil {
			return os.Symlink(dist, bin)
		}
	}

	return nil
}

func setupLogging(config *Config) error {
	if config.LogFile == "" {
		return nil
	}

	if err := createDirs(path.Dir(config.LogFile)); err != nil {
		return err
	}

	output, err := os.Create(config.LogFile)
	if err != nil {
		return err
	}

	syscall.Dup2(int(output.Fd()), int(os.Stdout.Fd()))
	syscall.Dup2(int(output.Fd()), int(os.Stderr.Fd()))

	return nil
}

func setUlimit(cfg *Config) error {
	var rLimit syscall.Rlimit
	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
		return err
	}
	if cfg.NoFiles == 0 {
		rLimit.Max = 1000000
	} else {
		rLimit.Max = cfg.NoFiles
	}
	rLimit.Cur = rLimit.Max
	return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
}

func runOrExec(config *Config, docker string, args ...string) (*exec.Cmd, error) {
	if err := prepare(config, docker, args...); err != nil {
		return nil, err
	}

	cmd := "docker"
	if config != nil && config.CommandName != "" {
		cmd = config.CommandName
	}

	return execDocker(config, docker, cmd, args)
}

func LaunchDocker(config *Config, docker string, args ...string) (*exec.Cmd, error) {
	if err := PrepareFs(config); err != nil {
		return nil, err
	}

	return runOrExec(config, docker, args...)
}
<|endoftext|>


<commit_before>package main

import (
	// "fmt"
	"log"
	"net/http"

	"github.com/codykrieger/jeeves"
)

func main() {
	j := jeeves.New()
	j.RegisterSkill(&jeeves.Skill{
		Name:          "Hello",
		Endpoint:      "/skills/hello",
		ApplicationID: "amzn1.echo-sdk-ams.app.000000-d0ed-0000-ad00-000000d00ebe",
		Handler:       helloHandler,
	})
	log.Fatal(http.ListenAndServe(":3000", j))
}

func helloHandler(skill *jeeves.Skill, req *jeeves.ASKRequest) *jeeves.ASKResponse {
	resp := jeeves.NewASKResponse(req)

	if req.IsLaunchRequest() {
		resp.Body.OutputSpeech = jeeves.NewASKOutputSpeech("Hello there!")
	} else if req.IsIntentRequest() {
	} else if req.IsSessionEndedRequest() {
	}

	return resp
}
<commit_msg>Make it easier to see when the example app is listening.<commit_after>package main

import (
	// "fmt"
	"log"
	"net/http"

	"github.com/codykrieger/jeeves"
)

func main() {
	j := jeeves.New()
	j.RegisterSkill(&jeeves.Skill{
		Name:          "Hello",
		Endpoint:      "/skills/hello",
		ApplicationID: "amzn1.echo-sdk-ams.app.000000-d0ed-0000-ad00-000000d00ebe",
		Handler:       helloHandler,
	})
	log.Println("Listening...")
	log.Fatal(http.ListenAndServe(":3000", j))
}

func helloHandler(skill *jeeves.Skill, req *jeeves.ASKRequest) *jeeves.ASKResponse {
	resp := jeeves.NewASKResponse(req)

	if req.IsLaunchRequest() {
		resp.Body.OutputSpeech = jeeves.NewASKOutputSpeech("Hello there!")
	} else if req.IsIntentRequest() {
	} else if req.IsSessionEndedRequest() {
	}

	return resp
}
<|endoftext|>


<commit_before>// Copyright 2014 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package spinner is a simple pacakge to add a spinner to your application.
package spinner

import (
	"fmt"
	//"github.com/fatih/color"
	"strconv"
	"time"
)

// CharSets is a map of slices containing the available character sets
var CharSets = map[int][]string{
	1:  []string{"←", "↖", "↑", "↗", "→", "↘", "↓", "↙"},
	2:  []string{"▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"},
	3:  []string{"▖", "▘", "▝", "▗"},
	4:  []string{"┤", "┘", "┴", "└", "├", "┌", "┬", "┐"},
	5:  []string{"◢", "◣", "◤", "◥"},
	6:  []string{"◰", "◳", "◲", "◱"},
	7:  []string{"◴", "◷", "◶", "◵"},
	8:  []string{"◐", "◓", "◑", "◒"},
	9:  []string{".", "o", "O", "@", "*"},
	10: []string{"|", "/", "-", "\\"},
	11: []string{"◡◡", "⊙⊙", "◠◠"},
	12: []string{"⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"},
	13: []string{">))'>", " >))'>", "  >))'>", "   >))'>", "    >))'>", "   <'((<", "  <'((<", " <'((<"},
	14: []string{"⠁", "⠂", "⠄", "⡀", "⢀", "⠠", "⠐", "⠈"},
	15: []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
	16: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"},
	17: []string{"▉", "▊", "▋", "▌", "▍", "▎", "▏", "▎", "▍", "▌", "▋", "▊", "▉"},
	18: []string{"■", "□", "▪", "▫"},
	19: []string{"←", "↑", "→", "↓"},
	20: []string{"╫", "╪"},
	21: []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
	22: []string{"⇐", "⇖", "⇑", "⇗", "⇒", "⇘", "⇓", "⇙"},
	23: []string{"⠁", "⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈", "⠈"},
	24: []string{"⠈", "⠉", "⠋", "⠓", "⠒", "⠐", "⠐", "⠒", "⠖", "⠦", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈"},
	25: []string{"⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠴", "⠲", "⠒", "⠂", "⠂", "⠒", "⠚", "⠙", "⠉", "⠁"},
	26: []string{"⠋", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋"},
}

var (
	// StopChan is a bool typed channel used to stop the spinner
	StopChan = make(chan bool, 1)
)

// Spinner struct to hold the provided options
type Spinner struct {
	Chars  []string
	Delay  time.Duration
	Offset int
}

// New provides a pointer to an instance of Spinner with the supplied options
func New(c []string, t time.Duration) *Spinner {
	return &Spinner{
		Chars:  c,
		Delay:  t,
		Offset: len(c) - 1,
	}
}

// Start will start the spinner
func (s *Spinner) Start() {
	count := 0
	go func() {
		for {
			select {
			case <-StopChan:
				return
			default:
				fmt.Printf("\r%s ", s.Chars[count])
				time.Sleep(s.Delay)
				if count != s.Offset {
					count++
				} else {
					count = 0
				}
			}
		}
	}()
}

// Stop stops the spinner
func (s *Spinner) Stop() { StopChan <- true }

// Restart will stop and start the spinner
func (s *Spinner) Restart() {
	s.Stop()
	s.Start()
}

// UpdateSpeed is a convenience function to not have to make you
//create a new instance of the Spinner
func (s *Spinner) UpdateSpeed(delay time.Duration) { s.Delay = delay }

// UpdateCharSet will change the previously select character set to
// the provided one
func (s *Spinner) UpdateCharSet(chars []string) { s.Chars = chars }

// GenerateNumberSequence will generate a slice of integers at the
// provided length and convert them each to a string
func GenerateNumberSequence(length int) []string {
	numSeq := make([]string, 0)
	for i := 0; i < length; i++ {
		numSeq = append(numSeq, strconv.Itoa(i))
	}
	return numSeq
}
<commit_msg>removed unused but commented package ref<commit_after>// Copyright 2014 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package spinner is a simple pacakge to add a spinner to your application.
package spinner

import (
	"fmt"
	"strconv"
	"time"
)

// CharSets is a map of slices containing the available character sets
var CharSets = map[int][]string{
	1:  []string{"←", "↖", "↑", "↗", "→", "↘", "↓", "↙"},
	2:  []string{"▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"},
	3:  []string{"▖", "▘", "▝", "▗"},
	4:  []string{"┤", "┘", "┴", "└", "├", "┌", "┬", "┐"},
	5:  []string{"◢", "◣", "◤", "◥"},
	6:  []string{"◰", "◳", "◲", "◱"},
	7:  []string{"◴", "◷", "◶", "◵"},
	8:  []string{"◐", "◓", "◑", "◒"},
	9:  []string{".", "o", "O", "@", "*"},
	10: []string{"|", "/", "-", "\\"},
	11: []string{"◡◡", "⊙⊙", "◠◠"},
	12: []string{"⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"},
	13: []string{">))'>", " >))'>", "  >))'>", "   >))'>", "    >))'>", "   <'((<", "  <'((<", " <'((<"},
	14: []string{"⠁", "⠂", "⠄", "⡀", "⢀", "⠠", "⠐", "⠈"},
	15: []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
	16: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"},
	17: []string{"▉", "▊", "▋", "▌", "▍", "▎", "▏", "▎", "▍", "▌", "▋", "▊", "▉"},
	18: []string{"■", "□", "▪", "▫"},
	19: []string{"←", "↑", "→", "↓"},
	20: []string{"╫", "╪"},
	21: []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
	22: []string{"⇐", "⇖", "⇑", "⇗", "⇒", "⇘", "⇓", "⇙"},
	23: []string{"⠁", "⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈", "⠈"},
	24: []string{"⠈", "⠉", "⠋", "⠓", "⠒", "⠐", "⠐", "⠒", "⠖", "⠦", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈"},
	25: []string{"⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠴", "⠲", "⠒", "⠂", "⠂", "⠒", "⠚", "⠙", "⠉", "⠁"},
	26: []string{"⠋", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋"},
}

var (
	// StopChan is a bool typed channel used to stop the spinner
	StopChan = make(chan bool, 1)
)

// Spinner struct to hold the provided options
type Spinner struct {
	Chars  []string
	Delay  time.Duration
	Offset int
}

// New provides a pointer to an instance of Spinner with the supplied options
func New(c []string, t time.Duration) *Spinner {
	return &Spinner{
		Chars:  c,
		Delay:  t,
		Offset: len(c) - 1,
	}
}

// Start will start the spinner
func (s *Spinner) Start() {
	count := 0
	go func() {
		for {
			select {
			case <-StopChan:
				return
			default:
				fmt.Printf("\r%s ", s.Chars[count])
				time.Sleep(s.Delay)
				if count != s.Offset {
					count++
				} else {
					count = 0
				}
			}
		}
	}()
}

// Stop stops the spinner
func (s *Spinner) Stop() { StopChan <- true }

// Restart will stop and start the spinner
func (s *Spinner) Restart() {
	s.Stop()
	s.Start()
}

// UpdateSpeed is a convenience function to not have to make you
//create a new instance of the Spinner
func (s *Spinner) UpdateSpeed(delay time.Duration) { s.Delay = delay }

// UpdateCharSet will change the previously select character set to
// the provided one
func (s *Spinner) UpdateCharSet(chars []string) { s.Chars = chars }

// GenerateNumberSequence will generate a slice of integers at the
// provided length and convert them each to a string
func GenerateNumberSequence(length int) []string {
	numSeq := make([]string, 0)
	for i := 0; i < length; i++ {
		numSeq = append(numSeq, strconv.Itoa(i))
	}
	return numSeq
}
<|endoftext|>


<commit_before>package proxy

import (
	"../util"
	"log"
	"math/big"
	"strconv"
	"strings"

	"github.com/ethereum/go-ethereum/common"
)

const maxBacklog = 8

type heightDiffPair struct {
	diff   *big.Int
	height uint64
}

type BlockTemplate struct {
	Header     string
	Seed       string
	Target     string
	Difficulty *big.Int
	Height     uint64
	headers    map[string]heightDiffPair
}

type Block struct {
	difficulty  *big.Int
	hashNoNonce common.Hash
	nonce       uint64
	mixDigest   common.Hash
	number      uint64
}

func (b Block) Difficulty() *big.Int     { return b.difficulty }
func (b Block) HashNoNonce() common.Hash { return b.hashNoNonce }
func (b Block) Nonce() uint64            { return b.nonce }
func (b Block) MixDigest() common.Hash   { return b.mixDigest }
func (b Block) NumberU64() uint64        { return b.number }

func (s *ProxyServer) fetchBlockTemplate() {
	rpc := s.rpc()
	reply, err := rpc.GetWork()
	if err != nil {
		log.Printf("Error while refreshing block template on %s: %s", rpc.Name, err)
		return
	}

	t := s.currentBlockTemplate()

	height, diff, err := s.fetchPendingBlock()
	if err != nil {
		log.Printf("Error while refreshing pending block on %s: %s", rpc.Name, err)
		return
	}
	newTemplate := BlockTemplate{
		Header:     reply[0],
		Seed:       reply[1],
		Target:     reply[2],
		Height:     height,
		Difficulty: diff,
		headers:    make(map[string]heightDiffPair),
	}
	// Copy headers backlog and add current one
	newTemplate.headers[reply[0]] = heightDiffPair{diff: util.TargetHexToDiff(reply[2]), height: height}
	if t != nil {
		for k, v := range t.headers {
			if v.height >= height-maxBacklog {
				newTemplate.headers[k] = v
			}
		}
	}
	s.blockTemplate.Store(&newTemplate)

	if newTemplate.Header != t.Header {
		log.Printf("New block to mine on %s at height: %d / %s", rpc.Name, height, reply[0][0:10])
	}
	return
}

func (s *ProxyServer) fetchPendingBlock() (uint64, *big.Int, error) {
	rpc := s.rpc()
	reply, err := rpc.GetPendingBlock()
	if err != nil {
		return 0, nil, err
	}
	blockNumber, err := strconv.ParseUint(strings.Replace(reply.Number, "0x", "", -1), 16, 64)
	if err != nil {
		log.Println("Can't parse pending block number")
		return 0, nil, err
	}
	blockDiff, err := strconv.ParseInt(strings.Replace(reply.Difficulty, "0x", "", -1), 16, 64)
	if err != nil {
		log.Println("Can't parse pending block difficulty")
		return 0, nil, err
	}

	return blockNumber, big.NewInt(blockDiff), nil
}
<commit_msg>Update job cache only on new job<commit_after>package proxy

import (
	"../util"
	"log"
	"math/big"
	"strconv"
	"strings"

	"github.com/ethereum/go-ethereum/common"
)

const maxBacklog = 8

type heightDiffPair struct {
	diff   *big.Int
	height uint64
}

type BlockTemplate struct {
	Header     string
	Seed       string
	Target     string
	Difficulty *big.Int
	Height     uint64
	headers    map[string]heightDiffPair
}

type Block struct {
	difficulty  *big.Int
	hashNoNonce common.Hash
	nonce       uint64
	mixDigest   common.Hash
	number      uint64
}

func (b Block) Difficulty() *big.Int     { return b.difficulty }
func (b Block) HashNoNonce() common.Hash { return b.hashNoNonce }
func (b Block) Nonce() uint64            { return b.nonce }
func (b Block) MixDigest() common.Hash   { return b.mixDigest }
func (b Block) NumberU64() uint64        { return b.number }

func (s *ProxyServer) fetchBlockTemplate() {
	rpc := s.rpc()
	reply, err := rpc.GetWork()
	if err != nil {
		log.Printf("Error while refreshing block template on %s: %s", rpc.Name, err)
		return
	}
	t := s.currentBlockTemplate()
	// No need to update, we have fresh job
	if t != nil && t.Header == reply[0] {
		return
	}
	height, diff, err := s.fetchPendingBlock()
	if err != nil {
		log.Printf("Error while refreshing pending block on %s: %s", rpc.Name, err)
		return
	}

	newTemplate := BlockTemplate{
		Header:     reply[0],
		Seed:       reply[1],
		Target:     reply[2],
		Height:     height,
		Difficulty: diff,
		headers:    make(map[string]heightDiffPair),
	}
	// Copy headers backlog and add current one
	newTemplate.headers[reply[0]] = heightDiffPair{diff: util.TargetHexToDiff(reply[2]), height: height}
	if t != nil {
		for k, v := range t.headers {
			if v.height > height-maxBacklog {
				newTemplate.headers[k] = v
			}
		}
	}
	s.blockTemplate.Store(&newTemplate)
	log.Printf("New block to mine on %s at height %d / %s", rpc.Name, height, reply[0][0:10])
}

func (s *ProxyServer) fetchPendingBlock() (uint64, *big.Int, error) {
	rpc := s.rpc()
	reply, err := rpc.GetPendingBlock()
	if err != nil {
		return 0, nil, err
	}
	blockNumber, err := strconv.ParseUint(strings.Replace(reply.Number, "0x", "", -1), 16, 64)
	if err != nil {
		log.Println("Can't parse pending block number")
		return 0, nil, err
	}
	blockDiff, err := strconv.ParseInt(strings.Replace(reply.Difficulty, "0x", "", -1), 16, 64)
	if err != nil {
		log.Println("Can't parse pending block difficulty")
		return 0, nil, err
	}

	return blockNumber, big.NewInt(blockDiff), nil
}
<|endoftext|>


<commit_before>package operation

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type UploadResult struct {
	Name      string `json:"name,omitempty"`
	Size      uint32 `json:"size,omitempty"`
	Error     string `json:"error,omitempty"`
	ETag      string `json:"eTag,omitempty"`
	CipherKey []byte `json:"cipherKey,omitempty"`
	Mime      string `json:"mime,omitempty"`
	Gzip      uint32 `json:"gzip,omitempty"`
	Md5       string `json:"md5,omitempty"`
}

func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk {
	return &filer_pb.FileChunk{
		FileId:       fileId,
		Offset:       offset,
		Size:         uint64(uploadResult.Size),
		Mtime:        time.Now().UnixNano(),
		ETag:         uploadResult.ETag,
		CipherKey:    uploadResult.CipherKey,
		IsCompressed: uploadResult.Gzip > 0,
	}
}

// HTTPClient interface for testing
type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

var (
	HttpClient HTTPClient
)

func init() {
	HttpClient = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	if uploadResult != nil {
		uploadResult.Md5 = util.Md5(data)
	}
	return
}

// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	hash := md5.New()
	reader = io.TeeReader(reader, hash)
	uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt)
	if uploadResult != nil {
		uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil))
	}
	return
}

func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	data, err = ioutil.ReadAll(reader)
	if err != nil {
		err = fmt.Errorf("read input: %v", err)
		return
	}
	uploadResult, uploadErr := doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return uploadResult, uploadErr, data
}

func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputCompressed
	shouldGzipNow := false
	if !isInputCompressed {
		if mtype == "" {
			mtype = http.DetectContentType(data)
			// println("detect1 mimetype to", mtype)
			if mtype == "application/octet-stream" {
				mtype = ""
			}
		}
		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
			shouldGzipNow = true
		} else if !iAmSure && mtype == "" && len(data) > 128 {
			var compressed []byte
			compressed, err = util.GzipData(data[0:128])
			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	clearDataLen = len(data)
	if shouldGzipNow {
		compressed, compressErr := util.GzipData(data)
		// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
		if compressErr == nil {
			data = compressed
			contentIsGzipped = true
		}
	} else if isInputCompressed {
		// just to get the clear data length
		clearData, err := util.DecompressData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, len(encryptedData), "", nil, jwt)
		if uploadResult != nil {
			uploadResult.Name = filename
			uploadResult.Mime = mtype
			uploadResult.CipherKey = cipherKey
		}
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, 0, mtype, pairMap, jwt)
	}

	if uploadResult == nil {
		return
	}

	uploadResult.Size = uint32(clearDataLen)
	if contentIsGzipped {
		uploadResult.Gzip = 1
	}

	return uploadResult, err
}

func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	body_buf := bytes.NewBufferString("")
	body_writer := multipart.NewWriter(body_buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, body_buf)
	if postErr != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, postErr.Error())
		return nil, postErr
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	if jwt != "" {
		req.Header.Set("Authorization", "BEARER "+string(jwt))
	}
	resp, post_err := HttpClient.Do(req)
	if post_err != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
		return nil, post_err
	}
	defer resp.Body.Close()

	var ret UploadResult
	etag := getEtag(resp)
	if resp.StatusCode == http.StatusNoContent {
		ret.ETag = etag
		return &ret, nil
	}
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, ra_err
	}
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload response", uploadUrl, string(resp_body))
		return nil, unmarshal_err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	ret.ETag = etag
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<commit_msg>report errors<commit_after>package operation

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type UploadResult struct {
	Name      string `json:"name,omitempty"`
	Size      uint32 `json:"size,omitempty"`
	Error     string `json:"error,omitempty"`
	ETag      string `json:"eTag,omitempty"`
	CipherKey []byte `json:"cipherKey,omitempty"`
	Mime      string `json:"mime,omitempty"`
	Gzip      uint32 `json:"gzip,omitempty"`
	Md5       string `json:"md5,omitempty"`
}

func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk {
	return &filer_pb.FileChunk{
		FileId:       fileId,
		Offset:       offset,
		Size:         uint64(uploadResult.Size),
		Mtime:        time.Now().UnixNano(),
		ETag:         uploadResult.ETag,
		CipherKey:    uploadResult.CipherKey,
		IsCompressed: uploadResult.Gzip > 0,
	}
}

// HTTPClient interface for testing
type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

var (
	HttpClient HTTPClient
)

func init() {
	HttpClient = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	if uploadResult != nil {
		uploadResult.Md5 = util.Md5(data)
	}
	return
}

// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	hash := md5.New()
	reader = io.TeeReader(reader, hash)
	uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt)
	if uploadResult != nil {
		uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil))
	}
	return
}

func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
	data, err = ioutil.ReadAll(reader)
	if err != nil {
		err = fmt.Errorf("read input: %v", err)
		return
	}
	uploadResult, uploadErr := doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
	return uploadResult, uploadErr, data
}

func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputCompressed
	shouldGzipNow := false
	if !isInputCompressed {
		if mtype == "" {
			mtype = http.DetectContentType(data)
			// println("detect1 mimetype to", mtype)
			if mtype == "application/octet-stream" {
				mtype = ""
			}
		}
		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
			shouldGzipNow = true
		} else if !iAmSure && mtype == "" && len(data) > 128 {
			var compressed []byte
			compressed, err = util.GzipData(data[0:128])
			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	clearDataLen = len(data)
	if shouldGzipNow {
		compressed, compressErr := util.GzipData(data)
		// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
		if compressErr == nil {
			data = compressed
			contentIsGzipped = true
		}
	} else if isInputCompressed {
		// just to get the clear data length
		clearData, err := util.DecompressData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, len(encryptedData), "", nil, jwt)
		if uploadResult != nil {
			uploadResult.Name = filename
			uploadResult.Mime = mtype
			uploadResult.CipherKey = cipherKey
		}
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, 0, mtype, pairMap, jwt)
	}

	if uploadResult == nil {
		return
	}

	uploadResult.Size = uint32(clearDataLen)
	if contentIsGzipped {
		uploadResult.Gzip = 1
	}

	return uploadResult, err
}

func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	body_buf := bytes.NewBufferString("")
	body_writer := multipart.NewWriter(body_buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, body_buf)
	if postErr != nil {
		glog.V(1).Infof("failing to upload to %s: %v", uploadUrl, postErr)
		return nil, fmt.Errorf("failing to upload to %s: %v", uploadUrl, postErr)
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	if jwt != "" {
		req.Header.Set("Authorization", "BEARER "+string(jwt))
	}
	resp, post_err := HttpClient.Do(req)
	if post_err != nil {
		glog.V(1).Infof("failing to upload to %v: %v", uploadUrl, post_err)
		return nil, fmt.Errorf("failing to upload to %v: %v", uploadUrl, post_err)
	}
	defer resp.Body.Close()

	var ret UploadResult
	etag := getEtag(resp)
	if resp.StatusCode == http.StatusNoContent {
		ret.ETag = etag
		return &ret, nil
	}
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, ra_err
	}
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload response", uploadUrl, string(resp_body))
		return nil, unmarshal_err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	ret.ETag = etag
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<|endoftext|>


<commit_before><commit_msg>Added fetch by Container(ID)<commit_after><|endoftext|>


<commit_before>package app

import (
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"sync"

	"github.com/gorilla/mux"
	"golang.org/x/net/context"

	"github.com/weaveworks/scope/probe/kubernetes"
	"github.com/weaveworks/scope/render"
	"github.com/weaveworks/scope/report"
)

const apiTopologyURL = "/api/topology/"

var (
	topologyRegistry = &registry{
		items: map[string]APITopologyDesc{},
	}
)

func init() {
	containerFilters := []APITopologyOptionGroup{
		{
			ID:      "system",
			Default: "application",
			Options: []APITopologyOption{
				{"system", "System containers", render.IsSystem},
				{"application", "Application containers", render.IsApplication},
				{"both", "Both", nil},
			},
		},
		{
			ID:      "stopped",
			Default: "running",
			Options: []APITopologyOption{
				{"stopped", "Stopped containers", render.IsStopped},
				{"running", "Running containers", render.IsRunning},
				{"both", "Both", nil},
			},
		},
	}

	unconnectedFilter := []APITopologyOptionGroup{
		{
			ID:      "unconnected",
			Default: "hide",
			Options: []APITopologyOption{
				// Show the user why there are filtered nodes in this view.
				// Don't give them the option to show those nodes.
				{"hide", "Unconnected nodes hidden", nil},
			},
		},
	}

	// Topology option labels should tell the current state. The first item must
	// be the verb to get to that state
	topologyRegistry.add(
		APITopologyDesc{
			id:       "processes",
			renderer: render.FilterUnconnected(render.ProcessWithContainerNameRenderer),
			Name:     "Processes",
			Rank:     1,
			Options:  unconnectedFilter,
		},
		APITopologyDesc{
			id:       "processes-by-name",
			parent:   "processes",
			renderer: render.FilterUnconnected(render.ProcessNameRenderer),
			Name:     "by name",
			Options:  unconnectedFilter,
		},
		APITopologyDesc{
			id:       "containers",
			renderer: render.ContainerWithImageNameRenderer,
			Name:     "Containers",
			Rank:     2,
			Options:  containerFilters,
		},
		APITopologyDesc{
			id:       "containers-by-image",
			parent:   "containers",
			renderer: render.ContainerImageRenderer,
			Name:     "by image",
			Options:  containerFilters,
		},
		APITopologyDesc{
			id:       "containers-by-hostname",
			parent:   "containers",
			renderer: render.ContainerHostnameRenderer,
			Name:     "by DNS name",
			Options:  containerFilters,
		},
		APITopologyDesc{
			id:          "pods",
			renderer:    render.PodRenderer,
			Name:        "Pods",
			Rank:        3,
			HideIfEmpty: true,
		},
		APITopologyDesc{
			id:          "pods-by-service",
			parent:      "pods",
			renderer:    render.PodServiceRenderer,
			Name:        "by service",
			HideIfEmpty: true,
		},
		APITopologyDesc{
			id:       "hosts",
			renderer: render.HostRenderer,
			Name:     "Hosts",
			Rank:     4,
		},
	)
}

// kubernetesFilters generates the current kubernetes filters based on the
// available k8s topologies.
func kubernetesFilters(namespaces ...string) APITopologyOptionGroup {
	options := APITopologyOptionGroup{ID: "namespace", Default: "all"}
	for _, namespace := range namespaces {
		options.Options = append(options.Options, APITopologyOption{namespace, namespace, render.IsNamespace(namespace)})
	}
	options.Options = append(options.Options, APITopologyOption{"all", "All Namespaces", nil})
	return options
}

// updateFilters updates the available filters based on the current report.
// Currently only kubernetes changes.
func updateFilters(rpt report.Report, topologies []APITopologyDesc) []APITopologyDesc {
	namespaces := map[string]struct{}{}
	for _, t := range []report.Topology{rpt.Pod, rpt.Service} {
		for _, n := range t.Nodes {
			if state, ok := n.Latest.Lookup(kubernetes.PodState); ok && state == kubernetes.StateDeleted {
				continue
			}
			if namespace, ok := n.Latest.Lookup(kubernetes.Namespace); ok {
				namespaces[namespace] = struct{}{}
			}
		}
	}
	var ns []string
	for namespace := range namespaces {
		ns = append(ns, namespace)
	}
	sort.Strings(ns)
	for i, t := range topologies {
		if t.id == "pods" || t.id == "pods-by-service" {
			topologies[i] = updateTopologyFilters(t, []APITopologyOptionGroup{kubernetesFilters(ns...)})
		}
	}
	return topologies
}

// updateTopologyFilters recursively sets the options on a topology description
func updateTopologyFilters(t APITopologyDesc, options []APITopologyOptionGroup) APITopologyDesc {
	t.Options = options
	for i, sub := range t.SubTopologies {
		t.SubTopologies[i] = updateTopologyFilters(sub, options)
	}
	return t
}

// registry is a threadsafe store of the available topologies
type registry struct {
	sync.RWMutex
	items map[string]APITopologyDesc
}

// APITopologyDesc is returned in a list by the /api/topology handler.
type APITopologyDesc struct {
	id       string
	parent   string
	renderer render.Renderer

	Name        string                   `json:"name"`
	Rank        int                      `json:"rank"`
	HideIfEmpty bool                     `json:"hide_if_empty"`
	Options     []APITopologyOptionGroup `json:"options"`

	URL           string            `json:"url"`
	SubTopologies []APITopologyDesc `json:"sub_topologies,omitempty"`
	Stats         topologyStats     `json:"stats,omitempty"`
}

type byName []APITopologyDesc

func (a byName) Len() int           { return len(a) }
func (a byName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byName) Less(i, j int) bool { return a[i].Name < a[j].Name }

// APITopologyOptionGroup describes a group of APITopologyOptions
type APITopologyOptionGroup struct {
	ID      string              `json:"id"`
	Default string              `json:"defaultValue,omitempty"`
	Options []APITopologyOption `json:"options,omitempty"`
}

// APITopologyOption describes a &param=value to a given topology.
type APITopologyOption struct {
	Value string `json:"value"`
	Label string `json:"label"`

	filter render.FilterFunc
}

type topologyStats struct {
	NodeCount          int `json:"node_count"`
	NonpseudoNodeCount int `json:"nonpseudo_node_count"`
	EdgeCount          int `json:"edge_count"`
	FilteredNodes      int `json:"filtered_nodes"`
}

func (r *registry) add(ts ...APITopologyDesc) {
	r.Lock()
	defer r.Unlock()
	for _, t := range ts {
		t.URL = apiTopologyURL + t.id

		if t.parent != "" {
			parent := r.items[t.parent]
			parent.SubTopologies = append(parent.SubTopologies, t)
			sort.Sort(byName(parent.SubTopologies))
			r.items[t.parent] = parent
		}

		r.items[t.id] = t
	}
}

func (r *registry) get(name string) (APITopologyDesc, bool) {
	r.RLock()
	defer r.RUnlock()
	t, ok := r.items[name]
	return t, ok
}

func (r *registry) walk(f func(APITopologyDesc)) {
	r.RLock()
	defer r.RUnlock()
	descs := []APITopologyDesc{}
	for _, desc := range r.items {
		if desc.parent != "" {
			continue
		}
		descs = append(descs, desc)
	}
	sort.Sort(byName(descs))
	for _, desc := range descs {
		f(desc)
	}
}

// makeTopologyList returns a handler that yields an APITopologyList.
func (r *registry) makeTopologyList(rep Reporter) CtxHandlerFunc {
	return func(ctx context.Context, w http.ResponseWriter, req *http.Request) {
		report, err := rep.Report(ctx)
		if err != nil {
			respondWith(w, http.StatusInternalServerError, err.Error())
			return
		}
		respondWith(w, http.StatusOK, r.renderTopologies(report, req))
	}
}

func (r *registry) renderTopologies(rpt report.Report, req *http.Request) []APITopologyDesc {
	topologies := []APITopologyDesc{}
	req.ParseForm()
	r.walk(func(desc APITopologyDesc) {
		renderer, decorator, _ := r.rendererForTopology(desc.id, req.Form, rpt)
		desc.Stats = decorateWithStats(rpt, renderer, decorator)
		for i, sub := range desc.SubTopologies {
			renderer, decorator, _ := r.rendererForTopology(sub.id, req.Form, rpt)
			desc.SubTopologies[i].Stats = decorateWithStats(rpt, renderer, decorator)
		}
		topologies = append(topologies, desc)
	})
	return updateFilters(rpt, topologies)
}

func decorateWithStats(rpt report.Report, renderer render.Renderer, decorator render.Decorator) topologyStats {
	var (
		nodes     int
		realNodes int
		edges     int
	)
	for _, n := range renderer.Render(rpt, decorator) {
		nodes++
		if n.Topology != render.Pseudo {
			realNodes++
		}
		edges += len(n.Adjacency)
	}
	renderStats := renderer.Stats(rpt, decorator)
	return topologyStats{
		NodeCount:          nodes,
		NonpseudoNodeCount: realNodes,
		EdgeCount:          edges,
		FilteredNodes:      renderStats.FilteredNodes,
	}
}

func (r *registry) rendererForTopology(topologyID string, values url.Values, rpt report.Report) (render.Renderer, render.Decorator, error) {
	topology, ok := r.get(topologyID)
	if !ok {
		return nil, nil, fmt.Errorf("topology not found: %s", topologyID)
	}
	topology = updateFilters(rpt, []APITopologyDesc{topology})[0]

	var filters []render.FilterFunc
	for _, group := range topology.Options {
		value := values.Get(group.ID)
		for _, opt := range group.Options {
			if opt.filter == nil {
				continue
			}
			if (value == "" && group.Default == opt.Value) || (opt.Value != "" && opt.Value == value) {
				filters = append(filters, opt.filter)
			}
		}
	}
	var decorator render.Decorator
	if len(filters) > 0 {
		decorator = func(renderer render.Renderer) render.Renderer {
			return render.MakeFilter(render.ComposeFilterFuncs(filters...), renderer)
		}
	}
	return topology.renderer, decorator, nil
}

type reporterHandler func(context.Context, Reporter, http.ResponseWriter, *http.Request)

func captureReporter(rep Reporter, f reporterHandler) CtxHandlerFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		f(ctx, rep, w, r)
	}
}

type rendererHandler func(context.Context, render.Renderer, render.Decorator, report.Report, http.ResponseWriter, *http.Request)

func (r *registry) captureRenderer(rep Reporter, f rendererHandler) CtxHandlerFunc {
	return func(ctx context.Context, w http.ResponseWriter, req *http.Request) {
		topologyID := mux.Vars(req)["topology"]
		if _, ok := r.get(topologyID); !ok {
			http.NotFound(w, req)
			return
		}
		rpt, err := rep.Report(ctx)
		if err != nil {
			respondWith(w, http.StatusInternalServerError, err.Error())
			return
		}
		req.ParseForm()
		renderer, decorator, err := r.rendererForTopology(topologyID, req.Form, rpt)
		if err != nil {
			respondWith(w, http.StatusInternalServerError, err.Error())
			return
		}
		f(ctx, renderer, decorator, rpt, w, req)
	}
}
<commit_msg>set 'default' as the default namespace filter instead of 'all' (#1445)<commit_after>package app

import (
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"sync"

	"github.com/gorilla/mux"
	"golang.org/x/net/context"

	"github.com/weaveworks/scope/probe/kubernetes"
	"github.com/weaveworks/scope/render"
	"github.com/weaveworks/scope/report"
)

const apiTopologyURL = "/api/topology/"

var (
	topologyRegistry = &registry{
		items: map[string]APITopologyDesc{},
	}
)

func init() {
	containerFilters := []APITopologyOptionGroup{
		{
			ID:      "system",
			Default: "application",
			Options: []APITopologyOption{
				{"system", "System containers", render.IsSystem},
				{"application", "Application containers", render.IsApplication},
				{"both", "Both", nil},
			},
		},
		{
			ID:      "stopped",
			Default: "running",
			Options: []APITopologyOption{
				{"stopped", "Stopped containers", render.IsStopped},
				{"running", "Running containers", render.IsRunning},
				{"both", "Both", nil},
			},
		},
	}

	unconnectedFilter := []APITopologyOptionGroup{
		{
			ID:      "unconnected",
			Default: "hide",
			Options: []APITopologyOption{
				// Show the user why there are filtered nodes in this view.
				// Don't give them the option to show those nodes.
				{"hide", "Unconnected nodes hidden", nil},
			},
		},
	}

	// Topology option labels should tell the current state. The first item must
	// be the verb to get to that state
	topologyRegistry.add(
		APITopologyDesc{
			id:       "processes",
			renderer: render.FilterUnconnected(render.ProcessWithContainerNameRenderer),
			Name:     "Processes",
			Rank:     1,
			Options:  unconnectedFilter,
		},
		APITopologyDesc{
			id:       "processes-by-name",
			parent:   "processes",
			renderer: render.FilterUnconnected(render.ProcessNameRenderer),
			Name:     "by name",
			Options:  unconnectedFilter,
		},
		APITopologyDesc{
			id:       "containers",
			renderer: render.ContainerWithImageNameRenderer,
			Name:     "Containers",
			Rank:     2,
			Options:  containerFilters,
		},
		APITopologyDesc{
			id:       "containers-by-image",
			parent:   "containers",
			renderer: render.ContainerImageRenderer,
			Name:     "by image",
			Options:  containerFilters,
		},
		APITopologyDesc{
			id:       "containers-by-hostname",
			parent:   "containers",
			renderer: render.ContainerHostnameRenderer,
			Name:     "by DNS name",
			Options:  containerFilters,
		},
		APITopologyDesc{
			id:          "pods",
			renderer:    render.PodRenderer,
			Name:        "Pods",
			Rank:        3,
			HideIfEmpty: true,
		},
		APITopologyDesc{
			id:          "pods-by-service",
			parent:      "pods",
			renderer:    render.PodServiceRenderer,
			Name:        "by service",
			HideIfEmpty: true,
		},
		APITopologyDesc{
			id:       "hosts",
			renderer: render.HostRenderer,
			Name:     "Hosts",
			Rank:     4,
		},
	)
}

// kubernetesFilters generates the current kubernetes filters based on the
// available k8s topologies.
func kubernetesFilters(namespaces ...string) APITopologyOptionGroup {
	options := APITopologyOptionGroup{ID: "namespace", Default: "all"}
	for _, namespace := range namespaces {
		if namespace == "default" {
			options.Default = namespace
		}
		options.Options = append(options.Options, APITopologyOption{namespace, namespace, render.IsNamespace(namespace)})
	}
	options.Options = append(options.Options, APITopologyOption{"all", "All Namespaces", nil})
	return options
}

// updateFilters updates the available filters based on the current report.
// Currently only kubernetes changes.
func updateFilters(rpt report.Report, topologies []APITopologyDesc) []APITopologyDesc {
	namespaces := map[string]struct{}{}
	for _, t := range []report.Topology{rpt.Pod, rpt.Service} {
		for _, n := range t.Nodes {
			if state, ok := n.Latest.Lookup(kubernetes.PodState); ok && state == kubernetes.StateDeleted {
				continue
			}
			if namespace, ok := n.Latest.Lookup(kubernetes.Namespace); ok {
				namespaces[namespace] = struct{}{}
			}
		}
	}
	var ns []string
	for namespace := range namespaces {
		ns = append(ns, namespace)
	}
	sort.Strings(ns)
	for i, t := range topologies {
		if t.id == "pods" || t.id == "pods-by-service" {
			topologies[i] = updateTopologyFilters(t, []APITopologyOptionGroup{kubernetesFilters(ns...)})
		}
	}
	return topologies
}

// updateTopologyFilters recursively sets the options on a topology description
func updateTopologyFilters(t APITopologyDesc, options []APITopologyOptionGroup) APITopologyDesc {
	t.Options = options
	for i, sub := range t.SubTopologies {
		t.SubTopologies[i] = updateTopologyFilters(sub, options)
	}
	return t
}

// registry is a threadsafe store of the available topologies
type registry struct {
	sync.RWMutex
	items map[string]APITopologyDesc
}

// APITopologyDesc is returned in a list by the /api/topology handler.
type APITopologyDesc struct {
	id       string
	parent   string
	renderer render.Renderer

	Name        string                   `json:"name"`
	Rank        int                      `json:"rank"`
	HideIfEmpty bool                     `json:"hide_if_empty"`
	Options     []APITopologyOptionGroup `json:"options"`

	URL           string            `json:"url"`
	SubTopologies []APITopologyDesc `json:"sub_topologies,omitempty"`
	Stats         topologyStats     `json:"stats,omitempty"`
}

type byName []APITopologyDesc

func (a byName) Len() int           { return len(a) }
func (a byName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byName) Less(i, j int) bool { return a[i].Name < a[j].Name }

// APITopologyOptionGroup describes a group of APITopologyOptions
type APITopologyOptionGroup struct {
	ID      string              `json:"id"`
	Default string              `json:"defaultValue,omitempty"`
	Options []APITopologyOption `json:"options,omitempty"`
}

// APITopologyOption describes a &param=value to a given topology.
type APITopologyOption struct {
	Value string `json:"value"`
	Label string `json:"label"`

	filter render.FilterFunc
}

type topologyStats struct {
	NodeCount          int `json:"node_count"`
	NonpseudoNodeCount int `json:"nonpseudo_node_count"`
	EdgeCount          int `json:"edge_count"`
	FilteredNodes      int `json:"filtered_nodes"`
}

func (r *registry) add(ts ...APITopologyDesc) {
	r.Lock()
	defer r.Unlock()
	for _, t := range ts {
		t.URL = apiTopologyURL + t.id

		if t.parent != "" {
			parent := r.items[t.parent]
			parent.SubTopologies = append(parent.SubTopologies, t)
			sort.Sort(byName(parent.SubTopologies))
			r.items[t.parent] = parent
		}

		r.items[t.id] = t
	}
}

func (r *registry) get(name string) (APITopologyDesc, bool) {
	r.RLock()
	defer r.RUnlock()
	t, ok := r.items[name]
	return t, ok
}

func (r *registry) walk(f func(APITopologyDesc)) {
	r.RLock()
	defer r.RUnlock()
	descs := []APITopologyDesc{}
	for _, desc := range r.items {
		if desc.parent != "" {
			continue
		}
		descs = append(descs, desc)
	}
	sort.Sort(byName(descs))
	for _, desc := range descs {
		f(desc)
	}
}

// makeTopologyList returns a handler that yields an APITopologyList.
func (r *registry) makeTopologyList(rep Reporter) CtxHandlerFunc {
	return func(ctx context.Context, w http.ResponseWriter, req *http.Request) {
		report, err := rep.Report(ctx)
		if err != nil {
			respondWith(w, http.StatusInternalServerError, err.Error())
			return
		}
		respondWith(w, http.StatusOK, r.renderTopologies(report, req))
	}
}

func (r *registry) renderTopologies(rpt report.Report, req *http.Request) []APITopologyDesc {
	topologies := []APITopologyDesc{}
	req.ParseForm()
	r.walk(func(desc APITopologyDesc) {
		renderer, decorator, _ := r.rendererForTopology(desc.id, req.Form, rpt)
		desc.Stats = decorateWithStats(rpt, renderer, decorator)
		for i, sub := range desc.SubTopologies {
			renderer, decorator, _ := r.rendererForTopology(sub.id, req.Form, rpt)
			desc.SubTopologies[i].Stats = decorateWithStats(rpt, renderer, decorator)
		}
		topologies = append(topologies, desc)
	})
	return updateFilters(rpt, topologies)
}

func decorateWithStats(rpt report.Report, renderer render.Renderer, decorator render.Decorator) topologyStats {
	var (
		nodes     int
		realNodes int
		edges     int
	)
	for _, n := range renderer.Render(rpt, decorator) {
		nodes++
		if n.Topology != render.Pseudo {
			realNodes++
		}
		edges += len(n.Adjacency)
	}
	renderStats := renderer.Stats(rpt, decorator)
	return topologyStats{
		NodeCount:          nodes,
		NonpseudoNodeCount: realNodes,
		EdgeCount:          edges,
		FilteredNodes:      renderStats.FilteredNodes,
	}
}

func (r *registry) rendererForTopology(topologyID string, values url.Values, rpt report.Report) (render.Renderer, render.Decorator, error) {
	topology, ok := r.get(topologyID)
	if !ok {
		return nil, nil, fmt.Errorf("topology not found: %s", topologyID)
	}
	topology = updateFilters(rpt, []APITopologyDesc{topology})[0]

	var filters []render.FilterFunc
	for _, group := range topology.Options {
		value := values.Get(group.ID)
		for _, opt := range group.Options {
			if opt.filter == nil {
				continue
			}
			if (value == "" && group.Default == opt.Value) || (opt.Value != "" && opt.Value == value) {
				filters = append(filters, opt.filter)
			}
		}
	}
	var decorator render.Decorator
	if len(filters) > 0 {
		decorator = func(renderer render.Renderer) render.Renderer {
			return render.MakeFilter(render.ComposeFilterFuncs(filters...), renderer)
		}
	}
	return topology.renderer, decorator, nil
}

type reporterHandler func(context.Context, Reporter, http.ResponseWriter, *http.Request)

func captureReporter(rep Reporter, f reporterHandler) CtxHandlerFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		f(ctx, rep, w, r)
	}
}

type rendererHandler func(context.Context, render.Renderer, render.Decorator, report.Report, http.ResponseWriter, *http.Request)

func (r *registry) captureRenderer(rep Reporter, f rendererHandler) CtxHandlerFunc {
	return func(ctx context.Context, w http.ResponseWriter, req *http.Request) {
		topologyID := mux.Vars(req)["topology"]
		if _, ok := r.get(topologyID); !ok {
			http.NotFound(w, req)
			return
		}
		rpt, err := rep.Report(ctx)
		if err != nil {
			respondWith(w, http.StatusInternalServerError, err.Error())
			return
		}
		req.ParseForm()
		renderer, decorator, err := r.rendererForTopology(topologyID, req.Form, rpt)
		if err != nil {
			respondWith(w, http.StatusInternalServerError, err.Error())
			return
		}
		f(ctx, renderer, decorator, rpt, w, req)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"log"

	"github.com/go-ego/riot"
	"github.com/go-ego/riot/types"
)

var (
	// searcher is coroutine safe
	searcher = riot.Engine{}
)

func main() {
	// Init searcher
	searcher.Init(types.EngineInitOptions{
		Using:         4,
		SegmenterDict: "./dict/dictionary.txt"})
	defer searcher.Close()

	// Add the document to the index, docId starts at 1
	searcher.IndexDocument(1, types.DocIndexData{Content: "Google Is Experimenting With Virtual Reality Advertising"}, false)
	searcher.IndexDocument(2, types.DocIndexData{Content: "Google accidentally pushed Bluetooth update for Home speaker early"}, false)
	searcher.IndexDocument(3, types.DocIndexData{Content: "Google is testing another Search results layout with rounded cards, new colors, and the 4 mysterious colored dots again"}, false)

	// Wait for the index to refresh
	searcher.FlushIndex()

	// The search output format is found in the types.SearchResponse structure
	log.Print(searcher.Search(types.SearchRequest{Text: "google testing"}))
}
<commit_msg>Update examples<commit_after>// Copyright 2017 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"log"

	"github.com/go-ego/riot"
	"github.com/go-ego/riot/types"
)

var (
	// searcher is coroutine safe
	searcher = riot.Engine{}
)

func main() {
	// Init searcher
	searcher.Init(types.EngineInitOptions{
		Using:             5,
		NotUsingSegmenter: true})
	defer searcher.Close()

	// Add the document to the index, docId starts at 1
	searcher.IndexDocument(1, types.DocIndexData{Content: "Google Is Experimenting With Virtual Reality Advertising"}, false)
	searcher.IndexDocument(2, types.DocIndexData{Content: "Google accidentally pushed Bluetooth update for Home speaker early"}, false)
	searcher.IndexDocument(3, types.DocIndexData{Content: "Google is testing another Search results layout with rounded cards, new colors, and the 4 mysterious colored dots again"}, false)

	// Wait for the index to refresh
	searcher.FlushIndex()

	// The search output format is found in the types.SearchResponse structure
	log.Print(searcher.Search(types.SearchRequest{Text: "google testing"}))
}
<|endoftext|>


<commit_before>package harness

import (
	"fmt"
	"github.com/robfig/revel"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"text/template"
)

var importErrorPattern = regexp.MustCompile("cannot find package \"([^\"]+)\"")

// Build the app:
// 1. Generate the the main.go file.
// 2. Run the appropriate "go build" command.
// Requires that revel.Init has been called previously.
// Returns the path to the built binary, and an error if there was a problem building it.
func Build() (app *App, compileError *revel.Error) {
	// First, clear the generated files (to avoid them messing with ProcessSource).
	cleanSource("tmp", "routes")

	sourceInfo, compileError := ProcessSource(revel.CodePaths)
	if compileError != nil {
		return nil, compileError
	}

	// Add the db.import to the import paths.
	if dbImportPath, found := revel.Config.String("db.import"); found {
		sourceInfo.InitImportPaths = append(sourceInfo.InitImportPaths, dbImportPath)
	}

	// Generate two source files.
	templateArgs := map[string]interface{}{
		"Controllers":    sourceInfo.ControllerSpecs(),
		"ValidationKeys": sourceInfo.ValidationKeys,
		"ImportPaths":    calcImportAliases(sourceInfo),
		"TestSuites":     sourceInfo.TestSuites(),
	}
	genSource("tmp", "main.go", MAIN, templateArgs)
	genSource("routes", "routes.go", ROUTES, templateArgs)

	// Read build config.
	buildTags := revel.Config.StringDefault("build.tags", "")

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		revel.ERROR.Fatalf("Go executable not found in PATH.")
	}

	pkg, err := build.Default.Import(revel.ImportPath, "", build.FindOnly)
	if err != nil {
		revel.ERROR.Fatalln("Failure importing", revel.ImportPath)
	}
	binName := path.Join(pkg.BinDir, path.Base(revel.BasePath))
	if runtime.GOOS == "windows" {
		binName += ".exe"
	}

	gotten := make(map[string]struct{})
	for {
		buildCmd := exec.Command(goPath, "build",
			"-tags", buildTags,
			"-o", binName, path.Join(revel.ImportPath, "app", "tmp"))
		revel.TRACE.Println("Exec:", buildCmd.Args)
		output, err := buildCmd.CombinedOutput()

		// If the build succeeded, we're done.
		if err == nil {
			return NewApp(binName), nil
		}
		revel.ERROR.Println(string(output))

		// See if it was an import error that we can go get.
		matches := importErrorPattern.FindStringSubmatch(string(output))
		if matches == nil {
			return nil, newCompileError(output)
		}

		// Ensure we haven't already tried to go get it.
		pkgName := matches[1]
		if _, alreadyTried := gotten[pkgName]; alreadyTried {
			return nil, newCompileError(output)
		}
		gotten[pkgName] = struct{}{}

		// Execute "go get <pkg>"
		getCmd := exec.Command(goPath, "get", pkgName)
		revel.TRACE.Println("Exec:", getCmd.Args)
		getOutput, err := getCmd.CombinedOutput()
		if err != nil {
			revel.ERROR.Println(string(getOutput))
			return nil, newCompileError(output)
		}

		// Success getting the import, attempt to build again.
	}
	revel.ERROR.Fatalf("Not reachable")
	return nil, nil
}

func cleanSource(dirs ...string) {
	for _, dir := range dirs {
		tmpPath := path.Join(revel.AppPath, dir)
		err := os.RemoveAll(tmpPath)
		if err != nil {
			revel.ERROR.Println("Failed to remove dir:", err)
		}
	}
}

// getSource renders the given template to produce source code, which it writes
// to the given directory and file.
func genSource(dir, filename, templateSource string, args map[string]interface{}) {
	sourceCode := revel.ExecuteTemplate(
		template.Must(template.New("").Parse(templateSource)),
		args)

	// Create a fresh dir.
	tmpPath := path.Join(revel.AppPath, dir)
	err := os.RemoveAll(tmpPath)
	if err != nil {
		revel.ERROR.Println("Failed to remove dir:", err)
	}
	err = os.Mkdir(tmpPath, 0777)
	if err != nil {
		revel.ERROR.Fatalf("Failed to make tmp directory: %v", err)
	}

	// Create the file
	file, err := os.Create(path.Join(tmpPath, filename))
	defer file.Close()
	if err != nil {
		revel.ERROR.Fatalf("Failed to create file: %v", err)
	}
	_, err = file.WriteString(sourceCode)
	if err != nil {
		revel.ERROR.Fatalf("Failed to write to file: %v", err)
	}
}

// Looks through all the method args and returns a set of unique import paths
// that cover all the method arg types.
// Additionally, assign package aliases when necessary to resolve ambiguity.
func calcImportAliases(src *SourceInfo) map[string]string {
	aliases := make(map[string]string)
	typeArrays := [][]*TypeInfo{src.ControllerSpecs(), src.TestSuites()}
	for _, specs := range typeArrays {
		for _, spec := range specs {
			addAlias(aliases, spec.ImportPath, spec.PackageName)

			for _, methSpec := range spec.MethodSpecs {
				for _, methArg := range methSpec.Args {
					if methArg.ImportPath == "" {
						continue
					}

					addAlias(aliases, methArg.ImportPath, methArg.TypeExpr.PkgName)
				}
			}
		}
	}

	// Add the "InitImportPaths", with alias "_"
	for _, importPath := range src.InitImportPaths {
		if _, ok := aliases[importPath]; !ok {
			aliases[importPath] = "_"
		}
	}

	return aliases
}

func addAlias(aliases map[string]string, importPath, pkgName string) {
	alias, ok := aliases[importPath]
	if ok {
		return
	}
	alias = makePackageAlias(aliases, pkgName)
	aliases[importPath] = alias
}

func makePackageAlias(aliases map[string]string, pkgName string) string {
	i := 0
	alias := pkgName
	for containsValue(aliases, alias) {
		alias = fmt.Sprintf("%s%d", pkgName, i)
		i++
	}
	return alias
}

func containsValue(m map[string]string, val string) bool {
	for _, v := range m {
		if v == val {
			return true
		}
	}
	return false
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(output []byte) *revel.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		revel.ERROR.Println("Failed to parse build errors:\n", string(output))
		return &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Description: "See console for build error.",
		}
	}

	// Read the source for the offending file.
	var (
		relFilename    = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename, _ = filepath.Abs(relFilename)
		line, _        = strconv.Atoi(string(errorMatch[2]))
		description    = string(errorMatch[4])
		compileError   = &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	fileStr, err := revel.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		revel.ERROR.Println(compileError.MetaError)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}

const MAIN = `// GENERATED CODE - DO NOT EDIT
package main

import (
	"flag"
	"reflect"
	"github.com/robfig/revel"{{range $k, $v := $.ImportPaths}}
	{{$v}} "{{$k}}"{{end}}
)

var (
	runMode    *string = flag.String("runMode", "", "Run mode.")
	port       *int    = flag.Int("port", 0, "By default, read from app.conf")
	importPath *string = flag.String("importPath", "", "Go Import Path for the app.")
	srcPath    *string = flag.String("srcPath", "", "Path to the source root.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	flag.Parse()
	revel.Init(*runMode, *importPath, *srcPath)
	revel.INFO.Println("Running revel server")
	{{range $i, $c := .Controllers}}
	revel.RegisterController((*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),
		[]*revel.MethodType{
			{{range .MethodSpecs}}&revel.MethodType{
				Name: "{{.Name}}",
				Args: []*revel.MethodArg{ {{range .Args}}
					&revel.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{index $.ImportPaths .ImportPath | .TypeExpr.TypeName}})(nil)) },{{end}}
				},
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	revel.DefaultValidationKeys = map[string]map[int]string{ {{range $path, $lines := .ValidationKeys}}
		"{{$path}}": { {{range $line, $key := $lines}}
			{{$line}}: "{{$key}}",{{end}}
		},{{end}}
	}
	revel.TestSuites = []interface{}{ {{range .TestSuites}}
		(*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),{{end}}
	}

	revel.Run(*port)
}
`
const ROUTES = `// GENERATED CODE - DO NOT EDIT
package routes

import "github.com/robfig/revel"

{{range $i, $c := .Controllers}}
type t{{.StructName}} struct {}
var {{.StructName}} t{{.StructName}}

{{range .MethodSpecs}}
func (p t{{$c.StructName}}) {{.Name}}({{range .Args}}
		{{.Name}} {{if .ImportPath}}interface{}{{else}}{{.TypeExpr.TypeName ""}}{{end}},{{end}}
		) string {
	args := make(map[string]string)
	{{range .Args}}
	revel.Unbind(args, "{{.Name}}", {{.Name}}){{end}}
	return revel.MainRouter.Reverse("{{$c.StructName}}.{{.Name}}", args).Url
}
{{end}}
{{end}}
`
<commit_msg>Re #160: anonymous generated routes receiver, fix comment typo<commit_after>package harness

import (
	"fmt"
	"github.com/robfig/revel"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"text/template"
)

var importErrorPattern = regexp.MustCompile("cannot find package \"([^\"]+)\"")

// Build the app:
// 1. Generate the the main.go file.
// 2. Run the appropriate "go build" command.
// Requires that revel.Init has been called previously.
// Returns the path to the built binary, and an error if there was a problem building it.
func Build() (app *App, compileError *revel.Error) {
	// First, clear the generated files (to avoid them messing with ProcessSource).
	cleanSource("tmp", "routes")

	sourceInfo, compileError := ProcessSource(revel.CodePaths)
	if compileError != nil {
		return nil, compileError
	}

	// Add the db.import to the import paths.
	if dbImportPath, found := revel.Config.String("db.import"); found {
		sourceInfo.InitImportPaths = append(sourceInfo.InitImportPaths, dbImportPath)
	}

	// Generate two source files.
	templateArgs := map[string]interface{}{
		"Controllers":    sourceInfo.ControllerSpecs(),
		"ValidationKeys": sourceInfo.ValidationKeys,
		"ImportPaths":    calcImportAliases(sourceInfo),
		"TestSuites":     sourceInfo.TestSuites(),
	}
	genSource("tmp", "main.go", MAIN, templateArgs)
	genSource("routes", "routes.go", ROUTES, templateArgs)

	// Read build config.
	buildTags := revel.Config.StringDefault("build.tags", "")

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		revel.ERROR.Fatalf("Go executable not found in PATH.")
	}

	pkg, err := build.Default.Import(revel.ImportPath, "", build.FindOnly)
	if err != nil {
		revel.ERROR.Fatalln("Failure importing", revel.ImportPath)
	}
	binName := path.Join(pkg.BinDir, path.Base(revel.BasePath))
	if runtime.GOOS == "windows" {
		binName += ".exe"
	}

	gotten := make(map[string]struct{})
	for {
		buildCmd := exec.Command(goPath, "build",
			"-tags", buildTags,
			"-o", binName, path.Join(revel.ImportPath, "app", "tmp"))
		revel.TRACE.Println("Exec:", buildCmd.Args)
		output, err := buildCmd.CombinedOutput()

		// If the build succeeded, we're done.
		if err == nil {
			return NewApp(binName), nil
		}
		revel.ERROR.Println(string(output))

		// See if it was an import error that we can go get.
		matches := importErrorPattern.FindStringSubmatch(string(output))
		if matches == nil {
			return nil, newCompileError(output)
		}

		// Ensure we haven't already tried to go get it.
		pkgName := matches[1]
		if _, alreadyTried := gotten[pkgName]; alreadyTried {
			return nil, newCompileError(output)
		}
		gotten[pkgName] = struct{}{}

		// Execute "go get <pkg>"
		getCmd := exec.Command(goPath, "get", pkgName)
		revel.TRACE.Println("Exec:", getCmd.Args)
		getOutput, err := getCmd.CombinedOutput()
		if err != nil {
			revel.ERROR.Println(string(getOutput))
			return nil, newCompileError(output)
		}

		// Success getting the import, attempt to build again.
	}
	revel.ERROR.Fatalf("Not reachable")
	return nil, nil
}

func cleanSource(dirs ...string) {
	for _, dir := range dirs {
		tmpPath := path.Join(revel.AppPath, dir)
		err := os.RemoveAll(tmpPath)
		if err != nil {
			revel.ERROR.Println("Failed to remove dir:", err)
		}
	}
}

// genSource renders the given template to produce source code, which it writes
// to the given directory and file.
func genSource(dir, filename, templateSource string, args map[string]interface{}) {
	sourceCode := revel.ExecuteTemplate(
		template.Must(template.New("").Parse(templateSource)),
		args)

	// Create a fresh dir.
	tmpPath := path.Join(revel.AppPath, dir)
	err := os.RemoveAll(tmpPath)
	if err != nil {
		revel.ERROR.Println("Failed to remove dir:", err)
	}
	err = os.Mkdir(tmpPath, 0777)
	if err != nil {
		revel.ERROR.Fatalf("Failed to make tmp directory: %v", err)
	}

	// Create the file
	file, err := os.Create(path.Join(tmpPath, filename))
	defer file.Close()
	if err != nil {
		revel.ERROR.Fatalf("Failed to create file: %v", err)
	}
	_, err = file.WriteString(sourceCode)
	if err != nil {
		revel.ERROR.Fatalf("Failed to write to file: %v", err)
	}
}

// Looks through all the method args and returns a set of unique import paths
// that cover all the method arg types.
// Additionally, assign package aliases when necessary to resolve ambiguity.
func calcImportAliases(src *SourceInfo) map[string]string {
	aliases := make(map[string]string)
	typeArrays := [][]*TypeInfo{src.ControllerSpecs(), src.TestSuites()}
	for _, specs := range typeArrays {
		for _, spec := range specs {
			addAlias(aliases, spec.ImportPath, spec.PackageName)

			for _, methSpec := range spec.MethodSpecs {
				for _, methArg := range methSpec.Args {
					if methArg.ImportPath == "" {
						continue
					}

					addAlias(aliases, methArg.ImportPath, methArg.TypeExpr.PkgName)
				}
			}
		}
	}

	// Add the "InitImportPaths", with alias "_"
	for _, importPath := range src.InitImportPaths {
		if _, ok := aliases[importPath]; !ok {
			aliases[importPath] = "_"
		}
	}

	return aliases
}

func addAlias(aliases map[string]string, importPath, pkgName string) {
	alias, ok := aliases[importPath]
	if ok {
		return
	}
	alias = makePackageAlias(aliases, pkgName)
	aliases[importPath] = alias
}

func makePackageAlias(aliases map[string]string, pkgName string) string {
	i := 0
	alias := pkgName
	for containsValue(aliases, alias) {
		alias = fmt.Sprintf("%s%d", pkgName, i)
		i++
	}
	return alias
}

func containsValue(m map[string]string, val string) bool {
	for _, v := range m {
		if v == val {
			return true
		}
	}
	return false
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(output []byte) *revel.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		revel.ERROR.Println("Failed to parse build errors:\n", string(output))
		return &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Description: "See console for build error.",
		}
	}

	// Read the source for the offending file.
	var (
		relFilename    = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename, _ = filepath.Abs(relFilename)
		line, _        = strconv.Atoi(string(errorMatch[2]))
		description    = string(errorMatch[4])
		compileError   = &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	fileStr, err := revel.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		revel.ERROR.Println(compileError.MetaError)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}

const MAIN = `// GENERATED CODE - DO NOT EDIT
package main

import (
	"flag"
	"reflect"
	"github.com/robfig/revel"{{range $k, $v := $.ImportPaths}}
	{{$v}} "{{$k}}"{{end}}
)

var (
	runMode    *string = flag.String("runMode", "", "Run mode.")
	port       *int    = flag.Int("port", 0, "By default, read from app.conf")
	importPath *string = flag.String("importPath", "", "Go Import Path for the app.")
	srcPath    *string = flag.String("srcPath", "", "Path to the source root.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	flag.Parse()
	revel.Init(*runMode, *importPath, *srcPath)
	revel.INFO.Println("Running revel server")
	{{range $i, $c := .Controllers}}
	revel.RegisterController((*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),
		[]*revel.MethodType{
			{{range .MethodSpecs}}&revel.MethodType{
				Name: "{{.Name}}",
				Args: []*revel.MethodArg{ {{range .Args}}
					&revel.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{index $.ImportPaths .ImportPath | .TypeExpr.TypeName}})(nil)) },{{end}}
				},
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	revel.DefaultValidationKeys = map[string]map[int]string{ {{range $path, $lines := .ValidationKeys}}
		"{{$path}}": { {{range $line, $key := $lines}}
			{{$line}}: "{{$key}}",{{end}}
		},{{end}}
	}
	revel.TestSuites = []interface{}{ {{range .TestSuites}}
		(*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),{{end}}
	}

	revel.Run(*port)
}
`
const ROUTES = `// GENERATED CODE - DO NOT EDIT
package routes

import "github.com/robfig/revel"

{{range $i, $c := .Controllers}}
type t{{.StructName}} struct {}
var {{.StructName}} t{{.StructName}}

{{range .MethodSpecs}}
func (_ t{{$c.StructName}}) {{.Name}}({{range .Args}}
		{{.Name}} {{if .ImportPath}}interface{}{{else}}{{.TypeExpr.TypeName ""}}{{end}},{{end}}
		) string {
	args := make(map[string]string)
	{{range .Args}}
	revel.Unbind(args, "{{.Name}}", {{.Name}}){{end}}
	return revel.MainRouter.Reverse("{{$c.StructName}}.{{.Name}}", args).Url
}
{{end}}
{{end}}
`
<|endoftext|>