<commit_before>package config

func NewConfig() *Config {
	return &Config{
		Debug: DEBUG,
		Dns: []string{
			"8.8.8.8",
			"8.8.4.4",
		},
		State: ConfigState{
			Required: false,
			Dev:      "LABEL=RANCHER_STATE",
			FsType:   "auto",
		},
		SystemDockerArgs: []string{"docker", "-d", "-s", "overlay", "-b", "none", "--restart=false", "-H", DOCKER_SYSTEM_HOST},
		Modules:          []string{},
		Userdocker: UserDockerInfo{
			UseTLS: true,
		},
		Network: NetworkConfig{
			Interfaces: []InterfaceConfig{
				{
					Match: "eth*",
					DHCP:  true,
				},
				{
					Match:   "lo",
					Address: "127.0.0.1/8",
				},
			},
		},
		CloudInit: CloudInit{
			Datasources: []string{"configdrive:/media/config-2"},
		},
		SystemContainers: []ContainerConfig{
			{
				Id: "system-volumes",
				Cmd: "--name=system-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/var/lib/rancher/conf:/var/lib/rancher/conf " +
					"-v=/lib/modules:/lib/modules:ro " +
					"-v=/var/run:/var/run " +
					"-v=/var/log:/var/log " +
					"state",
			},
			{
				Id: "command-volumes",
				Cmd: "--name=command-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/init:/sbin/halt:ro " +
					"-v=/init:/sbin/poweroff:ro " +
					"-v=/init:/sbin/reboot:ro " +
					"-v=/init:/sbin/shutdown:ro " +
					"-v=/init:/sbin/netconf:ro " +
					"-v=/init:/usr/bin/cloud-init:ro " +
					"-v=/init:/usr/bin/rancherctl:ro " +
					"-v=/init:/usr/bin/respawn:ro " +
					"-v=/init:/usr/bin/system-docker:ro " +
					"-v=/lib/modules:/lib/modules:ro " +
					"-v=/usr/bin/docker:/usr/bin/docker:ro " +
					"state",
			},
			{
				Id: "user-volumes",
				Cmd: "--name=user-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/var/lib/rancher/state/home:/home " +
					"-v=/var/lib/rancher/state/opt:/opt " +
					"state",
			},
			{
				Id: "udev",
				Cmd: "--name=udev " +
					"--net=none " +
					"--privileged " +
					"--rm " +
					"-v=/dev:/host/dev " +
					"-v=/lib/modules:/lib/modules:ro " +
					"udev",
			},
			{
				Id: "network",
				Cmd: "--name=network " +
					"--rm " +
					"--cap-add=NET_ADMIN " +
					"--net=host " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"network",
			},
			{
				Id: "cloud-init",
				Cmd: "--name=cloud-init " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"cloudinit",
				ReloadConfig: true,
			},
			{
				Id: "ntp",
				Cmd: "--name=ntp " +
					"--rm " +
					"-d " +
					"--privileged " +
					"--net=host " +
					"ntp",
			},
			{
				Id: "syslog",
				Cmd: "--name=syslog " +
					"-d " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"--ipc=host " +
					"--pid=host " +
					"--volumes-from=system-volumes " +
					"syslog",
			},
			{
				Id: "userdocker",
				Cmd: "--name=userdocker " +
					"-d " +
					"--rm " +
					"--restart=always " +
					"--ipc=host " +
					"--pid=host " +
					"--net=host " +
					"--privileged " +
					"--volumes-from=command-volumes " +
					"--volumes-from=user-volumes " +
					"--volumes-from=system-volumes " +
					"-v=/var/lib/rancher/state/docker:/var/lib/docker " +
					"userdocker",
			},
			{
				Id: "console",
				Cmd: "--name=console " +
					"-d " +
					"--rm " +
					"--privileged " +
					"--volumes-from=command-volumes " +
					"--volumes-from=user-volumes " +
					"--volumes-from=system-volumes " +
					"--restart=always " +
					"--ipc=host " +
					"--net=host " +
					"--pid=host " +
					"console",
			},
		},
		EnabledAddons: []string{/*"ubuntu-console", "etcd", "flannel-conf", "flannel", "k8s-docker", "k8s-master", "k8s-minion"*/},
		Addons: map[string]Config{
			"ubuntu-console": {
				SystemContainers: []ContainerConfig{
					{
						Id: "console",
						Cmd: "--name=ubuntu-console " +
							"-d " +
							"--rm " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=user-volumes " +
							"--volumes-from=system-volumes " +
							"--restart=always " +
							"--ipc=host " +
							"--net=host " +
							"--pid=host " +
							"rancher/ubuntuconsole:v0.0.2",
					},
				},
			},
			"etcd": {
				SystemContainers: []ContainerConfig{
					{
						Id: "etcd",
						Cmd: "--name=etcd " +
							"-d " +
							"--restart=always " +
							"--net=host " +
							"quay.io/coreos/etcd:v2.0.4 " +
							"--listen-client-urls 'http://0.0.0.0:2379,http://0.0.0.0:4001'",
					},
				},
			},
			"flannel-conf": {
				SystemContainers: []ContainerConfig{
					{
						Id: "flannel-conf",
						Cmd: "--name=flannel-conf " +
							"--rm " +
							"--net=host " +
							"--volumes-from=command-volumes " +
							"--volumes-from=system-volumes " +
							"etcdctl " +
							"mk /coreos.com/network/config '{\"Network\":\"10.244.0.0/16\", \"Backend\": {\"Type\": \"vxlan\"}}'",
					},
				},
			},
			"flannel": {
				SystemContainers: []ContainerConfig{
					{
						Id: "flannel",
						Cmd: "--name=flannel " +
							"-d " +
							"--rm " +
							"--restart=always " +
							"--ipc=host " +
							"--pid=host " +
							"--net=host " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=system-volumes " +
							"flannel",
					},
				},
			},
			"k8s-docker": {
				SystemContainers: []ContainerConfig{
					{
						Id: "stopuserdocker",
						Cmd: "--rm " +
							"--ipc=host " +
							"--pid=host " +
							"--net=host " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=system-volumes " +
							"stopuserdocker",
					},
					{
						Id: "k8s-docker",
						Cmd: "--name=k8s-docker " +
							"-d " +
							"--restart=always " +
							"--ipc=host " +
							"--pid=host " +
							"--net=host " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=user-volumes " +
							"--volumes-from=system-volumes " +
							"-v=/var/lib/rancher/state/docker:/var/lib/docker " +
							"k8s-docker",
					},
				},
			},
			"k8s-master": {
				SystemContainers: []ContainerConfig{
					{
						Id: "kube-apiserver",
						Cmd: "--name=kube-apiserver " +
							"-d " +
							"--restart=always " +
							"--ipc=host " +
							"--pid=host " +
							"--net=host " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=system-volumes " +
							"kubernetes " +
							"/kube-apiserver " +
							"--address=0.0.0.0 --port=8080 " +
							"--portal_net=10.100.0.0/16 " +
							"--etcd_servers=http://127.0.0.1:4001 " +
							"--public_address_override=172.17.7.101 " +
							"--logtostderr=true",
					},
					{
						Id: "kube-controller-manager",
						Cmd: "--name=kube-controller-manager " +
						"-d " +
						"--restart=always " +
						"--ipc=host " +
						"--pid=host " +
						"--net=host " +
						"--privileged " +
						"--volumes-from=command-volumes " +
						"--volumes-from=system-volumes " +
						"kubernetes " +
						"/kube-controller-manager " +
						"--master=127.0.0.1:8080 " +
						"--logtostderr=true",
					},
					{
						Id: "kube-scheduler",
						Cmd: "--name=kube-scheduler " +
						"-d " +
						"--restart=always " +
						"--ipc=host " +
						"--pid=host " +
						"--net=host " +
						"--privileged " +
						"--volumes-from=command-volumes " +
						"--volumes-from=system-volumes " +
						"kubernetes " +
						"/kube-scheduler " +
						"--master=127.0.0.1:8080 " +
						"--logtostderr=true",
					},
				},
			},
			"k8s-minion": {
				SystemContainers: []ContainerConfig{
					{
						Id: "kube-proxy",
						Cmd: "--name=kube-proxy " +
						"-d " +
						"--restart=always " +
						"--ipc=host " +
						"--pid=host " +
						"--net=host " +
						"--privileged " +
						"--volumes-from=command-volumes " +
						"--volumes-from=system-volumes " +
						"kubernetes " +
						"/kube-proxy " +
						"--etcd_servers=http://172.17.7.101:4001 " +
						"--logtostderr=true",
					},
					{
						Id: "kubelet",
						Cmd: "--name=kubelet " +
						"-d " +
						"--restart=always " +
						"--ipc=host " +
						"--pid=host " +
						"--net=host " +
						"--privileged " +
						"--volumes-from=command-volumes " +
						"--volumes-from=system-volumes " +
						"kubernetes " +
						"/kubelet.sh",
					},
				},
			},
		},
		RescueContainer: &ContainerConfig{
			Id: "console",
			Cmd: "--name=rescue " +
				"-d " +
				"--rm " +
				"--privileged " +
				"--volumes-from=console-volumes " +
				"--volumes-from=user-volumes " +
				"--volumes-from=system-volumes " +
				"--restart=always " +
				"--ipc=host " +
				"--net=host " +
				"--pid=host " +
				"rescue",
		},
	}
}
<commit_msg>flannel-conf addon: use set instead of mk, so that it doesn't fail after reboot<commit_after>package config

func NewConfig() *Config {
	return &Config{
		Debug: DEBUG,
		Dns: []string{
			"8.8.8.8",
			"8.8.4.4",
		},
		State: ConfigState{
			Required: false,
			Dev:      "LABEL=RANCHER_STATE",
			FsType:   "auto",
		},
		SystemDockerArgs: []string{"docker", "-d", "-s", "overlay", "-b", "none", "--restart=false", "-H", DOCKER_SYSTEM_HOST},
		Modules:          []string{},
		Userdocker: UserDockerInfo{
			UseTLS: true,
		},
		Network: NetworkConfig{
			Interfaces: []InterfaceConfig{
				{
					Match: "eth*",
					DHCP:  true,
				},
				{
					Match:   "lo",
					Address: "127.0.0.1/8",
				},
			},
		},
		CloudInit: CloudInit{
			Datasources: []string{"configdrive:/media/config-2"},
		},
		SystemContainers: []ContainerConfig{
			{
				Id: "system-volumes",
				Cmd: "--name=system-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/var/lib/rancher/conf:/var/lib/rancher/conf " +
					"-v=/lib/modules:/lib/modules:ro " +
					"-v=/var/run:/var/run " +
					"-v=/var/log:/var/log " +
					"state",
			},
			{
				Id: "command-volumes",
				Cmd: "--name=command-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/init:/sbin/halt:ro " +
					"-v=/init:/sbin/poweroff:ro " +
					"-v=/init:/sbin/reboot:ro " +
					"-v=/init:/sbin/shutdown:ro " +
					"-v=/init:/sbin/netconf:ro " +
					"-v=/init:/usr/bin/cloud-init:ro " +
					"-v=/init:/usr/bin/rancherctl:ro " +
					"-v=/init:/usr/bin/respawn:ro " +
					"-v=/init:/usr/bin/system-docker:ro " +
					"-v=/lib/modules:/lib/modules:ro " +
					"-v=/usr/bin/docker:/usr/bin/docker:ro " +
					"state",
			},
			{
				Id: "user-volumes",
				Cmd: "--name=user-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/var/lib/rancher/state/home:/home " +
					"-v=/var/lib/rancher/state/opt:/opt " +
					"state",
			},
			{
				Id: "udev",
				Cmd: "--name=udev " +
					"--net=none " +
					"--privileged " +
					"--rm " +
					"-v=/dev:/host/dev " +
					"-v=/lib/modules:/lib/modules:ro " +
					"udev",
			},
			{
				Id: "network",
				Cmd: "--name=network " +
					"--rm " +
					"--cap-add=NET_ADMIN " +
					"--net=host " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"network",
			},
			{
				Id: "cloud-init",
				Cmd: "--name=cloud-init " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"cloudinit",
				ReloadConfig: true,
			},
			{
				Id: "ntp",
				Cmd: "--name=ntp " +
					"--rm " +
					"-d " +
					"--privileged " +
					"--net=host " +
					"ntp",
			},
			{
				Id: "syslog",
				Cmd: "--name=syslog " +
					"-d " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"--ipc=host " +
					"--pid=host " +
					"--volumes-from=system-volumes " +
					"syslog",
			},
			{
				Id: "userdocker",
				Cmd: "--name=userdocker " +
					"-d " +
					"--rm " +
					"--restart=always " +
					"--ipc=host " +
					"--pid=host " +
					"--net=host " +
					"--privileged " +
					"--volumes-from=command-volumes " +
					"--volumes-from=user-volumes " +
					"--volumes-from=system-volumes " +
					"-v=/var/lib/rancher/state/docker:/var/lib/docker " +
					"userdocker",
			},
			{
				Id: "console",
				Cmd: "--name=console " +
					"-d " +
					"--rm " +
					"--privileged " +
					"--volumes-from=command-volumes " +
					"--volumes-from=user-volumes " +
					"--volumes-from=system-volumes " +
					"--restart=always " +
					"--ipc=host " +
					"--net=host " +
					"--pid=host " +
					"console",
			},
		},
		EnabledAddons: []string{/*"ubuntu-console", "etcd", "flannel-conf", "flannel", "k8s-docker", "k8s-master", "k8s-minion"*/},
		Addons: map[string]Config{
			"ubuntu-console": {
				SystemContainers: []ContainerConfig{
					{
						Id: "console",
						Cmd: "--name=ubuntu-console " +
							"-d " +
							"--rm " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=user-volumes " +
							"--volumes-from=system-volumes " +
							"--restart=always " +
							"--ipc=host " +
							"--net=host " +
							"--pid=host " +
							"rancher/ubuntuconsole:v0.0.2",
					},
				},
			},
			"etcd": {
				SystemContainers: []ContainerConfig{
					{
						Id: "etcd",
						Cmd: "--name=etcd " +
							"-d " +
							"--restart=always " +
							"--net=host " +
							"quay.io/coreos/etcd:v2.0.4 " +
							"--listen-client-urls 'http://0.0.0.0:2379,http://0.0.0.0:4001'",
					},
				},
			},
			"flannel-conf": {
				SystemContainers: []ContainerConfig{
					{
						Id: "flannel-conf",
						Cmd: "--name=flannel-conf " +
							"--rm " +
							"--net=host " +
							"--volumes-from=command-volumes " +
							"--volumes-from=system-volumes " +
							"etcdctl " +
							"set /coreos.com/network/config '{\"Network\":\"10.244.0.0/16\", \"Backend\": {\"Type\": \"vxlan\"}}'",
					},
				},
			},
			"flannel": {
				SystemContainers: []ContainerConfig{
					{
						Id: "flannel",
						Cmd: "--name=flannel " +
							"-d " +
							"--rm " +
							"--restart=always " +
							"--ipc=host " +
							"--pid=host " +
							"--net=host " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=system-volumes " +
							"flannel",
					},
				},
			},
			"k8s-docker": {
				SystemContainers: []ContainerConfig{
					{
						Id: "stopuserdocker",
						Cmd: "--rm " +
							"--ipc=host " +
							"--pid=host " +
							"--net=host " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=system-volumes " +
							"stopuserdocker",
					},
					{
						Id: "k8s-docker",
						Cmd: "--name=k8s-docker " +
							"-d " +
							"--restart=always " +
							"--ipc=host " +
							"--pid=host " +
							"--net=host " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=user-volumes " +
							"--volumes-from=system-volumes " +
							"-v=/var/lib/rancher/state/docker:/var/lib/docker " +
							"k8s-docker",
					},
				},
			},
			"k8s-master": {
				SystemContainers: []ContainerConfig{
					{
						Id: "kube-apiserver",
						Cmd: "--name=kube-apiserver " +
							"-d " +
							"--restart=always " +
							"--ipc=host " +
							"--pid=host " +
							"--net=host " +
							"--privileged " +
							"--volumes-from=command-volumes " +
							"--volumes-from=system-volumes " +
							"kubernetes " +
							"/kube-apiserver " +
							"--address=0.0.0.0 --port=8080 " +
							"--portal_net=10.100.0.0/16 " +
							"--etcd_servers=http://127.0.0.1:4001 " +
							"--public_address_override=172.17.7.101 " +
							"--logtostderr=true",
					},
					{
						Id: "kube-controller-manager",
						Cmd: "--name=kube-controller-manager " +
						"-d " +
						"--restart=always " +
						"--ipc=host " +
						"--pid=host " +
						"--net=host " +
						"--privileged " +
						"--volumes-from=command-volumes " +
						"--volumes-from=system-volumes " +
						"kubernetes " +
						"/kube-controller-manager " +
						"--master=127.0.0.1:8080 " +
						"--logtostderr=true",
					},
					{
						Id: "kube-scheduler",
						Cmd: "--name=kube-scheduler " +
						"-d " +
						"--restart=always " +
						"--ipc=host " +
						"--pid=host " +
						"--net=host " +
						"--privileged " +
						"--volumes-from=command-volumes " +
						"--volumes-from=system-volumes " +
						"kubernetes " +
						"/kube-scheduler " +
						"--master=127.0.0.1:8080 " +
						"--logtostderr=true",
					},
				},
			},
			"k8s-minion": {
				SystemContainers: []ContainerConfig{
					{
						Id: "kube-proxy",
						Cmd: "--name=kube-proxy " +
						"-d " +
						"--restart=always " +
						"--ipc=host " +
						"--pid=host " +
						"--net=host " +
						"--privileged " +
						"--volumes-from=command-volumes " +
						"--volumes-from=system-volumes " +
						"kubernetes " +
						"/kube-proxy " +
						"--etcd_servers=http://172.17.7.101:4001 " +
						"--logtostderr=true",
					},
					{
						Id: "kubelet",
						Cmd: "--name=kubelet " +
						"-d " +
						"--restart=always " +
						"--ipc=host " +
						"--pid=host " +
						"--net=host " +
						"--privileged " +
						"--volumes-from=command-volumes " +
						"--volumes-from=system-volumes " +
						"kubernetes " +
						"/kubelet.sh",
					},
				},
			},
		},
		RescueContainer: &ContainerConfig{
			Id: "console",
			Cmd: "--name=rescue " +
				"-d " +
				"--rm " +
				"--privileged " +
				"--volumes-from=console-volumes " +
				"--volumes-from=user-volumes " +
				"--volumes-from=system-volumes " +
				"--restart=always " +
				"--ipc=host " +
				"--net=host " +
				"--pid=host " +
				"rescue",
		},
	}
}
<|endoftext|>


<commit_before>package hal

import (
	"fmt"
	"regexp"
	"strings"
	"time"
)

// Evt is a generic container for events processed by the bot.
// Event sources are responsible for copying the appropriate data into
// the Evt fields. Routing and most plugins will not work if the body
// isn't copied, at a minimum.
// The original event should usually be attached to the Original
type Evt struct {
	Body     string      `json:"body"`    // body of the event, regardless of source
	Room     string      `json:"room"`    // the room where the event originated
	RoomId   string      `json:"room_id"` // the room id from the source broker
	User     string      `json:"user"`    // the username that created the event
	UserId   string      `json:"user_id"` // the user id from the source broker
	Time     time.Time   `json:"time"`    // timestamp of the event
	Broker   Broker      `json:"broker"`  // the broker the event came from
	Original interface{} // the original message container (e.g. slack.MessageEvent)
	instance *Instance   // used by the broker to provide plugin instance metadata
}

// Clone() returns a copy of the event with the same broker/room/user
// and a current timestamp. Body and Original will be empty.
func (e *Evt) Clone() Evt {
	out := Evt{
		Room:   e.Room,
		RoomId: e.RoomId,
		User:   e.User,
		UserId: e.UserId,
		Time:   time.Now(),
		Broker: e.Broker,
	}

	return out
}

// Reply is a helper that crafts a new event from the provided string
// and initiates the reply on the broker attached to the event.
func (e *Evt) Reply(msg string) {
	out := e.Clone()
	out.Body = msg

	if e.Broker != nil {
		e.Broker.Send(out)
	} else {
		panic("hal.Evt.Reply called with nil Broker!")
	}
}

// Replyf is the same as Reply but allows for string formatting using
// fmt.Sprintf()
func (e *Evt) Replyf(msg string, a ...interface{}) {
	e.Reply(fmt.Sprintf(msg, a...))
}

// Replyf is the same as Reply but allows for string formatting using
// fmt.Sprintf()
func (e *Evt) ReplyTable(hdr []string, rows [][]string) {
	out := e.Clone() // may not be necessary

	if e.Broker != nil {
		e.Broker.SendTable(out, hdr, rows)
	} else {
		panic("hal.Evt.ReplyTable called with nil Broker!")
	}
}

// BrokerName returns the text name of current broker.
func (e *Evt) BrokerName() string {
	return e.Broker.Name()
}

// FindPrefs fetches the union of all matching settings from the database
// for user, broker, room, and plugin.
// Plugins can use the Prefs methods to filter from there.
func (e *Evt) FindPrefs() Prefs {
	broker := e.BrokerName()
	plugin := e.instance.Plugin.Name
	return FindPrefs(e.User, broker, e.RoomId, plugin, "")
}

// InstanceSettings gets all the settings matching the settings defined
// by the plugin's Settings field.
func (e *Evt) InstanceSettings() Prefs {
	broker := e.BrokerName()
	plugin := e.instance.Plugin.Name

	out := make(Prefs, 0)

	for _, stg := range e.instance.Plugin.Settings {
		// ignore room-specific settings for other rooms
		if stg.Room != "" && stg.Room != e.RoomId {
			continue
		}

		pref := GetPref("", broker, e.RoomId, plugin, stg.Key, stg.Default)
		out = append(out, pref)
	}

	return out
}

// NewPref creates a new pref struct with user, room, broker, and plugin
// set using metadata from the event.
func (e *Evt) NewPref() Pref {
	return Pref{
		User:   e.User,
		Room:   e.Room,
		Broker: e.BrokerName(),
		Plugin: e.instance.Plugin.Name,
	}
}

// BodyAsArgv does minimal parsing of the event body, returning an argv-like
// array of strings with quoted strings intact (but with quotes removed).
// The goal is shell-like, and is not a full implementation.
// Leading/trailing whitespace is removed.
// Escaping quotes, etc. is not supported.
func (e *Evt) BodyAsArgv() []string {
	// use a simple RE rather than pulling in a package to do this
	re := regexp.MustCompile(`'[^']*'|"[^"]*"|\S+`)
	body := strings.TrimSpace(e.Body)
	argv := re.FindAllString(body, -1)

	// remove the outer quotes from quoted strings
	for i, val := range argv {
		if strings.HasPrefix(val, `'`) && strings.HasSuffix(val, `'`) {
			tmp := strings.TrimPrefix(val, `'`)
			argv[i] = strings.TrimSuffix(tmp, `'`)
		} else if strings.HasPrefix(val, `"`) && strings.HasSuffix(val, `"`) {
			tmp := strings.TrimPrefix(val, `"`)
			argv[i] = strings.TrimSuffix(tmp, `"`)
		}
	}

	return argv
}

func (e *Evt) String() string {
	return fmt.Sprintf("%s/%s@%s: %s", e.User, e.Room, e.Time.String(), e.Body)
}
<commit_msg>clean up Room/RoomId, add FillPref()<commit_after>package hal

import (
	"fmt"
	"regexp"
	"strings"
	"time"
)

// Evt is a generic container for events processed by the bot.
// Event sources are responsible for copying the appropriate data into
// the Evt fields. Routing and most plugins will not work if the body
// isn't copied, at a minimum.
// The original event should usually be attached to the Original
type Evt struct {
	Body     string      `json:"body"`    // body of the event, regardless of source
	Room     string      `json:"room"`    // the room where the event originated
	RoomId   string      `json:"room_id"` // the room id from the source broker
	User     string      `json:"user"`    // the username that created the event
	UserId   string      `json:"user_id"` // the user id from the source broker
	Time     time.Time   `json:"time"`    // timestamp of the event
	Broker   Broker      `json:"broker"`  // the broker the event came from
	Original interface{} // the original message container (e.g. slack.MessageEvent)
	instance *Instance   // used by the broker to provide plugin instance metadata
}

// Clone() returns a copy of the event with the same broker/room/user
// and a current timestamp. Body and Original will be empty.
func (e *Evt) Clone() Evt {
	out := Evt{
		Room:   e.Room,
		RoomId: e.RoomId,
		User:   e.User,
		UserId: e.UserId,
		Time:   time.Now(),
		Broker: e.Broker,
	}

	return out
}

// Reply is a helper that crafts a new event from the provided string
// and initiates the reply on the broker attached to the event.
func (e *Evt) Reply(msg string) {
	out := e.Clone()
	out.Body = msg

	if e.Broker != nil {
		e.Broker.Send(out)
	} else {
		panic("hal.Evt.Reply called with nil Broker!")
	}
}

// Replyf is the same as Reply but allows for string formatting using
// fmt.Sprintf()
func (e *Evt) Replyf(msg string, a ...interface{}) {
	e.Reply(fmt.Sprintf(msg, a...))
}

// Replyf is the same as Reply but allows for string formatting using
// fmt.Sprintf()
func (e *Evt) ReplyTable(hdr []string, rows [][]string) {
	out := e.Clone() // may not be necessary

	if e.Broker != nil {
		e.Broker.SendTable(out, hdr, rows)
	} else {
		panic("hal.Evt.ReplyTable called with nil Broker!")
	}
}

// BrokerName returns the text name of current broker.
func (e *Evt) BrokerName() string {
	return e.Broker.Name()
}

// FindPrefs fetches the union of all matching settings from the database
// for user, broker, room, and plugin.
// Plugins can use the Prefs methods to filter from there.
func (e *Evt) FindPrefs() Prefs {
	broker := e.BrokerName()
	plugin := e.instance.Plugin.Name
	return FindPrefs(e.User, broker, e.RoomId, plugin, "")
}

// InstanceSettings gets all the settings matching the settings defined
// by the plugin's Settings field.
func (e *Evt) InstanceSettings() Prefs {
	broker := e.BrokerName()
	plugin := e.instance.Plugin.Name

	out := make(Prefs, 0)

	for _, stg := range e.instance.Plugin.Settings {
		// ignore room-specific settings for other rooms
		if stg.Room != "" && stg.Room != e.RoomId {
			continue
		}

		pref := GetPref("", broker, e.RoomId, plugin, stg.Key, stg.Default)
		out = append(out, pref)
	}

	return out
}

// NewPref creates a new pref struct with user, room, broker, and plugin
// set using metadata from the event.
func (e *Evt) NewPref() Pref {
	return Pref{
		User:   e.User,
		Room:   e.RoomId,
		Broker: e.BrokerName(),
		Plugin: e.instance.Plugin.Name,
	}
}

// FillPref returns a copy of the provided pref with user, room, broker,
// and plugin set using data from the event handle for any of those fields
// that don't already have a value. e.g. if the input has a room set it will
// be left alone and the other fields will be set.
func (e *Evt) FillPref(p Pref) Pref {
	if p.User == "" {
		p.User = e.User
	}

	if p.Room == "" {
		p.Room = e.RoomId
	}

	if p.Broker == "" {
		p.Broker = e.BrokerName()
	}

	if p.Plugin == "" {
		p.Plugin = e.instance.Plugin.Name
	}

	return p
}

// BodyAsArgv does minimal parsing of the event body, returning an argv-like
// array of strings with quoted strings intact (but with quotes removed).
// The goal is shell-like, and is not a full implementation.
// Leading/trailing whitespace is removed.
// Escaping quotes, etc. is not supported.
func (e *Evt) BodyAsArgv() []string {
	// use a simple RE rather than pulling in a package to do this
	re := regexp.MustCompile(`'[^']*'|"[^"]*"|\S+`)
	body := strings.TrimSpace(e.Body)
	argv := re.FindAllString(body, -1)

	// remove the outer quotes from quoted strings
	for i, val := range argv {
		if strings.HasPrefix(val, `'`) && strings.HasSuffix(val, `'`) {
			tmp := strings.TrimPrefix(val, `'`)
			argv[i] = strings.TrimSuffix(tmp, `'`)
		} else if strings.HasPrefix(val, `"`) && strings.HasSuffix(val, `"`) {
			tmp := strings.TrimPrefix(val, `"`)
			argv[i] = strings.TrimSuffix(tmp, `"`)
		}
	}

	return argv
}

func (e *Evt) String() string {
	return fmt.Sprintf("%s/%s@%s: %s", e.User, e.Room, e.Time.String(), e.Body)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package http

import (
	"errors"
	"fmt"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/mirrors"
	"strings"
	"sync"
	"time"
)

/*
	Total (all files, all mirrors):
	STATS_TOTAL

	List of hashes for a file:
	STATS_FILE							= path -> value		All time
	STATS_FILE_[year]					= path -> value		By year
	STATS_FILE_[year]_[month]			= path -> value		By month
	STATS_FILE_[year]_[month]_[day]		= path -> value		By day

	List of hashes for a mirror:
	STATS_MIRROR						= mirror -> value	All time
	STATS_MIRROR_[year]					= mirror -> value	By year
	STATS_MIRROR_[year]_[month]			= mirror -> value	By month
	STATS_MIRROR_[year]_[month]_[day]	= mirror -> value	By day
*/

var (
	emptyFileError = errors.New("stats: file parameter is empty")
	unknownMirror  = errors.New("stats: unknown mirror")
)

type Stats struct {
	r         *database.Redis
	countChan chan CountItem
	mapStats  map[string]int64
	stop      chan bool
	wg        sync.WaitGroup
}

type CountItem struct {
	mirrorID string
	filepath string
	size     int64
	time     time.Time
}

func NewStats(redis *database.Redis) *Stats {
	s := &Stats{
		r:         redis,
		countChan: make(chan CountItem, 1000),
		mapStats:  make(map[string]int64),
		stop:      make(chan bool),
	}
	go s.processCountDownload()
	return s
}

func (s *Stats) Terminate() {
	close(s.stop)
	log.Notice("Saving stats")
	s.wg.Wait()
}

// Lightweight method used to count a new download for a specific file and mirror
func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo) error {
	if m.ID == "" {
		return unknownMirror
	}
	if fileinfo.Path == "" {
		return emptyFileError
	}

	s.countChan <- CountItem{m.ID, fileinfo.Path, fileinfo.Size, time.Now()}
	return nil
}

// Process all stacked download messages
func (s *Stats) processCountDownload() {
	s.wg.Add(1)
	pushTicker := time.NewTicker(500 * time.Millisecond)

	for {
		select {
		case <-s.stop:
			s.pushStats()
			s.wg.Done()
			return
		case c := <-s.countChan:
			date := c.time.Format("2006_01_02|") // Includes separator
			s.mapStats["f"+date+c.filepath] += 1
			s.mapStats["m"+date+c.mirrorID] += 1
			s.mapStats["s"+date+c.mirrorID] += c.size
		case <-pushTicker.C:
			s.pushStats()
		}
	}
}

// Push the resulting stats on redis
func (s *Stats) pushStats() {
	if len(s.mapStats) <= 0 {
		return
	}

	rconn := s.r.Get()
	defer rconn.Close()
	rconn.Send("MULTI")

	for k, v := range s.mapStats {
		if v == 0 {
			continue
		}

		separator := strings.Index(k, "|")
		if separator <= 0 {
			log.Critical("Stats: separator not found")
			continue
		}
		typ := k[:1]
		date := k[1:separator]
		object := k[separator+1:]

		if typ == "f" {
			// File

			fkey := fmt.Sprintf("STATS_FILE_%s", date)

			for i := 0; i < 4; i++ {
				rconn.Send("HINCRBY", fkey, object, v)
				fkey = fkey[:strings.LastIndex(fkey, "_")]
			}

			// Increase the total too
			rconn.Send("INCRBY", "STATS_TOTAL", v)
		} else if typ == "m" {
			// Mirror

			mkey := fmt.Sprintf("STATS_MIRROR_%s", date)

			for i := 0; i < 4; i++ {
				rconn.Send("HINCRBY", mkey, object, v)
				mkey = mkey[:strings.LastIndex(mkey, "_")]
			}
		} else if typ == "s" {
			// Bytes

			mkey := fmt.Sprintf("STATS_MIRROR_BYTES_%s", date)

			for i := 0; i < 4; i++ {
				rconn.Send("HINCRBY", mkey, object, v)
				mkey = mkey[:strings.LastIndex(mkey, "_")]
			}
		} else {
			log.Warning("Stats: unknown type", typ)
		}
	}

	_, err := rconn.Do("EXEC")

	if err != nil {
		log.Errorf("Stats: could not save stats to redis: %s", err.Error())
		return
	}

	// Clear the map
	s.mapStats = make(map[string]int64)
}
<commit_msg>stats: use UTC instead of local time<commit_after>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package http

import (
	"errors"
	"fmt"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/mirrors"
	"strings"
	"sync"
	"time"
)

/*
	Total (all files, all mirrors):
	STATS_TOTAL

	List of hashes for a file:
	STATS_FILE							= path -> value		All time
	STATS_FILE_[year]					= path -> value		By year
	STATS_FILE_[year]_[month]			= path -> value		By month
	STATS_FILE_[year]_[month]_[day]		= path -> value		By day

	List of hashes for a mirror:
	STATS_MIRROR						= mirror -> value	All time
	STATS_MIRROR_[year]					= mirror -> value	By year
	STATS_MIRROR_[year]_[month]			= mirror -> value	By month
	STATS_MIRROR_[year]_[month]_[day]	= mirror -> value	By day
*/

var (
	emptyFileError = errors.New("stats: file parameter is empty")
	unknownMirror  = errors.New("stats: unknown mirror")
)

type Stats struct {
	r         *database.Redis
	countChan chan CountItem
	mapStats  map[string]int64
	stop      chan bool
	wg        sync.WaitGroup
}

type CountItem struct {
	mirrorID string
	filepath string
	size     int64
	time     time.Time
}

func NewStats(redis *database.Redis) *Stats {
	s := &Stats{
		r:         redis,
		countChan: make(chan CountItem, 1000),
		mapStats:  make(map[string]int64),
		stop:      make(chan bool),
	}
	go s.processCountDownload()
	return s
}

func (s *Stats) Terminate() {
	close(s.stop)
	log.Notice("Saving stats")
	s.wg.Wait()
}

// Lightweight method used to count a new download for a specific file and mirror
func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo) error {
	if m.ID == "" {
		return unknownMirror
	}
	if fileinfo.Path == "" {
		return emptyFileError
	}

	s.countChan <- CountItem{m.ID, fileinfo.Path, fileinfo.Size, time.Now().UTC()}
	return nil
}

// Process all stacked download messages
func (s *Stats) processCountDownload() {
	s.wg.Add(1)
	pushTicker := time.NewTicker(500 * time.Millisecond)

	for {
		select {
		case <-s.stop:
			s.pushStats()
			s.wg.Done()
			return
		case c := <-s.countChan:
			date := c.time.Format("2006_01_02|") // Includes separator
			s.mapStats["f"+date+c.filepath] += 1
			s.mapStats["m"+date+c.mirrorID] += 1
			s.mapStats["s"+date+c.mirrorID] += c.size
		case <-pushTicker.C:
			s.pushStats()
		}
	}
}

// Push the resulting stats on redis
func (s *Stats) pushStats() {
	if len(s.mapStats) <= 0 {
		return
	}

	rconn := s.r.Get()
	defer rconn.Close()
	rconn.Send("MULTI")

	for k, v := range s.mapStats {
		if v == 0 {
			continue
		}

		separator := strings.Index(k, "|")
		if separator <= 0 {
			log.Critical("Stats: separator not found")
			continue
		}
		typ := k[:1]
		date := k[1:separator]
		object := k[separator+1:]

		if typ == "f" {
			// File

			fkey := fmt.Sprintf("STATS_FILE_%s", date)

			for i := 0; i < 4; i++ {
				rconn.Send("HINCRBY", fkey, object, v)
				fkey = fkey[:strings.LastIndex(fkey, "_")]
			}

			// Increase the total too
			rconn.Send("INCRBY", "STATS_TOTAL", v)
		} else if typ == "m" {
			// Mirror

			mkey := fmt.Sprintf("STATS_MIRROR_%s", date)

			for i := 0; i < 4; i++ {
				rconn.Send("HINCRBY", mkey, object, v)
				mkey = mkey[:strings.LastIndex(mkey, "_")]
			}
		} else if typ == "s" {
			// Bytes

			mkey := fmt.Sprintf("STATS_MIRROR_BYTES_%s", date)

			for i := 0; i < 4; i++ {
				rconn.Send("HINCRBY", mkey, object, v)
				mkey = mkey[:strings.LastIndex(mkey, "_")]
			}
		} else {
			log.Warning("Stats: unknown type", typ)
		}
	}

	_, err := rconn.Do("EXEC")

	if err != nil {
		log.Errorf("Stats: could not save stats to redis: %s", err.Error())
		return
	}

	// Clear the map
	s.mapStats = make(map[string]int64)
}
<|endoftext|>


<commit_before>//An easier way to use http.Client
package com

import (
	"bytes"
	"errors"
	"fmt"
	//	"github.com/alecthomas/log4go"
	"github.com/axgle/mahonia"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/http/cookiejar"
	//	"strings"
	"compress/gzip"
	"io"
	"log"
)

type HttpClient struct {
	c       *http.Client
	cookies []*http.Cookie
	jar     *cookiejar.Jar

	Header http.Header

	//编码转换相关处理
	conv    bool //conv between utf-8 and charset
	charset string
	enc     mahonia.Encoder
	dec     mahonia.Decoder

	//链接转向相关处理
	redirect    bool   //是否转向了。每次Get之前置为false
	redirectUrl string //转向后的链接
}

func NewHttpClient() (this *HttpClient) {

	this = &HttpClient{}

	this.cookies = nil
	this.jar, _ = cookiejar.New(nil)

	this.c = &http.Client{Jar: this.jar, CheckRedirect: func(req *http.Request, via []*http.Request) error {
		this.redirect = true
		this.redirectUrl = req.URL.String()
		//return errors.New("Redirected!")
		return nil
	}}

	this.Header = make(http.Header)
	this.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	this.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	this.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	this.Header.Add("Connection", "keep-alive")
	this.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")

	this.conv = false

	return this
}

//you should set it only once!
func (this *HttpClient) SetCharSet(charset string) {
	this.conv = true
	this.charset = charset
	this.enc = mahonia.NewEncoder(charset)
	this.dec = mahonia.NewDecoder(charset)
}

func (this *HttpClient) SetUa(ua string) {
	this.Header.Set("User-Agent", ua)
}

func (this *HttpClient) Enc(in string) string {
	if this.conv {
		return this.enc.ConvertString(in)
	}
	return in
}

func (this *HttpClient) Dec(in string) string {
	if this.conv {
		return this.dec.ConvertString(in)
	}
	return in
}

func (this *HttpClient) Get(url string) (page string, err error) {
	this.clearRedirect()
	//自动转化为GB2312编码
	gbkUrl := this.Enc(url)

	req, err := http.NewRequest("GET", gbkUrl, nil)
	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.Get(%s),NewRequest error:%s", url, err.Error()))
		return
	}

	for k, v := range this.Header {
		req.Header.Add(k, v[0])
	}

	resp, err := this.c.Do(req)

	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Response error:%s", url, err.Error()))
		return
	}

	defer resp.Body.Close()

	var body string

	if resp.StatusCode == 200 {

		switch resp.Header.Get("Content-Encoding") {
		case "gzip":
			reader, e := gzip.NewReader(resp.Body)
			if e != nil {
				err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Read gzip body error:%s", url, e.Error()))
				return
			}
			for {
				buf := make([]byte, 1024)
				n, err := reader.Read(buf)

				if err != nil && err != io.EOF {
					panic(err)
				}

				if n == 0 {
					break
				}
				body += string(buf)
			}
		default:
			bodyByte, e := ioutil.ReadAll(resp.Body)
			if e != nil {
				err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Read body error:%s", url, e.Error()))
				return
			}
			body = string(bodyByte)
		}
	}

	log.Println(body)

	this.cookies = this.jar.Cookies(req.URL)

	page = this.Dec(string(body))
	return
}

func (this *HttpClient) Post(url, postdata string) (page string, err error) {
	this.clearRedirect()
	//自动转化为GB2312编码
	gbkUrl := this.Enc(url)
	gbkPostdata := this.Enc(postdata)

	req, _ := http.NewRequest("POST", gbkUrl, bytes.NewReader([]byte(gbkPostdata)))
	for k, v := range this.Header {
		req.Header.Add(k, v[0])
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	resp, err := this.c.Do(req)

	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.Post(%s,%s),Response error:%s", url, postdata, err.Error()))
		return
	}
	//	log4go.Debug("Response code:%d status:%s", resp.StatusCode, resp.Status)

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.Post(%s,%s),Read body error:%s", url, postdata, err.Error()))
		return
	}

	this.cookies = this.jar.Cookies(req.URL)

	page = this.Dec(string(body))

	//log4go.Finest("HttpClient.Post(%s,%s) returns:", url, postdata)
	//log4go.Finest(page)
	return
}

/*
///尚未测试
func (this *HttpClient) PostValues(surl string, postDict map[string]string) (page string, err error) {
	log4go.Debug("HttpClient.PostValues(%s,%T)", surl, postDict)
	postValues := url.Values{}
	for postKey, PostValue := range postDict {
		postValues.Set(this.Enc(postKey), this.Enc(PostValue))
	}
	postDataStr := postValues.Encode()

	return this.Post(surl, postDataStr)
}
*/
func (this *HttpClient) PostMultipart(u string, w *multipart.Writer, b *bytes.Buffer) (page string, err error) {
	this.clearRedirect()
	//log4go.Debug("HttpClient.PostMultipart(%s,w)", u)
	//自动转化为GB2312编码
	gbkUrl := this.Enc(u)

	req, _ := http.NewRequest("POST", gbkUrl, b)
	for k, v := range this.Header {
		req.Header.Add(k, v[0])
	}
	req.Header.Add("Content-Type", w.FormDataContentType())
	//log4go.Finest("PostMultipart Content-Type: %s", w.FormDataContentType())
	resp, err := this.c.Do(req)

	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.PostMultipart ,Response error:%s", err.Error()))
		return
	}
	//log4go.Debug("Response code:%d status:%s", resp.StatusCode, resp.Status)

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.PostMultipart ,Read body error:%s", err.Error()))
		return
	}

	this.cookies = this.jar.Cookies(req.URL)

	page = this.Dec(string(body))

	//log4go.Finest("HttpClient.PostMultipart to url :%s returns :", u)
	//log4go.Finest(page)
	return

}

func (p *HttpClient) clearRedirect() {
	p.redirect = false
	p.redirectUrl = ""
}
func (p *HttpClient) CheckRedirect() (b bool, url string) {
	return p.redirect, p.redirectUrl
}
<commit_msg>把Post也改进了<commit_after>//An easier way to use http.Client
package com

import (
	"bytes"
	"errors"
	"fmt"
	//	"github.com/alecthomas/log4go"
	"github.com/axgle/mahonia"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/http/cookiejar"
	//	"strings"
	"compress/gzip"
	"io"
	//	"log"
)

type HttpClient struct {
	c       *http.Client
	cookies []*http.Cookie
	jar     *cookiejar.Jar

	Header http.Header

	//编码转换相关处理
	conv    bool //conv between utf-8 and charset
	charset string
	enc     mahonia.Encoder
	dec     mahonia.Decoder

	//链接转向相关处理
	redirect    bool   //是否转向了。每次Get之前置为false
	redirectUrl string //转向后的链接
}

func NewHttpClient() (this *HttpClient) {

	this = &HttpClient{}

	this.cookies = nil
	this.jar, _ = cookiejar.New(nil)

	this.c = &http.Client{Jar: this.jar, CheckRedirect: func(req *http.Request, via []*http.Request) error {
		this.redirect = true
		this.redirectUrl = req.URL.String()
		//return errors.New("Redirected!")
		return nil
	}}

	this.Header = make(http.Header)
	this.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	this.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	this.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	this.Header.Add("Connection", "keep-alive")
	this.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")

	this.conv = false

	return this
}

//you should set it only once!
func (this *HttpClient) SetCharSet(charset string) {
	this.conv = true
	this.charset = charset
	this.enc = mahonia.NewEncoder(charset)
	this.dec = mahonia.NewDecoder(charset)
}

func (this *HttpClient) SetUa(ua string) {
	this.Header.Set("User-Agent", ua)
}

func (this *HttpClient) Enc(in string) string {
	if this.conv {
		return this.enc.ConvertString(in)
	}
	return in
}

func (this *HttpClient) Dec(in string) string {
	if this.conv {
		return this.dec.ConvertString(in)
	}
	return in
}

func (this *HttpClient) Get(url string) (page string, err error) {
	this.clearRedirect()
	//自动转化为GB2312编码
	gbkUrl := this.Enc(url)

	req, err := http.NewRequest("GET", gbkUrl, nil)
	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.Get(%s),NewRequest error:%s", url, err.Error()))
		return
	}

	for k, v := range this.Header {
		req.Header.Add(k, v[0])
	}

	resp, err := this.c.Do(req)

	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Response error:%s", url, err.Error()))
		return
	}

	defer resp.Body.Close()

	var body string

	if resp.StatusCode == 200 {

		switch resp.Header.Get("Content-Encoding") {
		case "gzip":
			reader, e := gzip.NewReader(resp.Body)
			if e != nil {
				err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Read gzip body error:%s", url, e.Error()))
				return
			}
			for {
				buf := make([]byte, 1024)
				n, err := reader.Read(buf)

				if err != nil && err != io.EOF {
					panic(err)
				}

				if n == 0 {
					break
				}
				body += string(buf)
			}
		default:
			bodyByte, e := ioutil.ReadAll(resp.Body)
			if e != nil {
				err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Read body error:%s", url, e.Error()))
				return
			}
			body = string(bodyByte)
		}
	}

	//log.Println(body)

	this.cookies = this.jar.Cookies(req.URL)

	page = this.Dec(body)
	return
}

func (this *HttpClient) Post(url, postdata string) (page string, err error) {
	this.clearRedirect()
	//自动转化为GB2312编码
	gbkUrl := this.Enc(url)
	gbkPostdata := this.Enc(postdata)

	req, _ := http.NewRequest("POST", gbkUrl, bytes.NewReader([]byte(gbkPostdata)))
	for k, v := range this.Header {
		req.Header.Add(k, v[0])
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	resp, err := this.c.Do(req)

	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.Post(%s,%s),Response error:%s", url, postdata, err.Error()))
		return
	}
	//	log4go.Debug("Response code:%d status:%s", resp.StatusCode, resp.Status)

	defer resp.Body.Close()

	var body string

	if resp.StatusCode == 200 {

		switch resp.Header.Get("Content-Encoding") {
		case "gzip":
			reader, e := gzip.NewReader(resp.Body)
			if e != nil {
				err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Read gzip body error:%s", url, e.Error()))
				return
			}
			for {
				buf := make([]byte, 1024)
				n, err := reader.Read(buf)

				if err != nil && err != io.EOF {
					panic(err)
				}

				if n == 0 {
					break
				}
				body += string(buf)
			}
		default:
			bodyByte, e := ioutil.ReadAll(resp.Body)
			if e != nil {
				err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Read body error:%s", url, e.Error()))
				return
			}
			body = string(bodyByte)
		}
	}

	this.cookies = this.jar.Cookies(req.URL)

	page = this.Dec(body)

	//log4go.Finest("HttpClient.Post(%s,%s) returns:", url, postdata)
	//log4go.Finest(page)
	return
}

/*
///尚未测试
func (this *HttpClient) PostValues(surl string, postDict map[string]string) (page string, err error) {
	log4go.Debug("HttpClient.PostValues(%s,%T)", surl, postDict)
	postValues := url.Values{}
	for postKey, PostValue := range postDict {
		postValues.Set(this.Enc(postKey), this.Enc(PostValue))
	}
	postDataStr := postValues.Encode()

	return this.Post(surl, postDataStr)
}
*/
func (this *HttpClient) PostMultipart(u string, w *multipart.Writer, b *bytes.Buffer) (page string, err error) {
	this.clearRedirect()
	//log4go.Debug("HttpClient.PostMultipart(%s,w)", u)
	//自动转化为GB2312编码
	gbkUrl := this.Enc(u)

	req, _ := http.NewRequest("POST", gbkUrl, b)
	for k, v := range this.Header {
		req.Header.Add(k, v[0])
	}
	req.Header.Add("Content-Type", w.FormDataContentType())
	//log4go.Finest("PostMultipart Content-Type: %s", w.FormDataContentType())
	resp, err := this.c.Do(req)

	if err != nil {
		err = errors.New(fmt.Sprintf("HttpClient.PostMultipart ,Response error:%s", err.Error()))
		return
	}
	//log4go.Debug("Response code:%d status:%s", resp.StatusCode, resp.Status)

	defer resp.Body.Close()

	var body string

	if resp.StatusCode == 200 {

		switch resp.Header.Get("Content-Encoding") {
		case "gzip":
			reader, e := gzip.NewReader(resp.Body)
			if e != nil {
				err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Read gzip body error:%s", u, e.Error()))
				return
			}
			for {
				buf := make([]byte, 1024)
				n, err := reader.Read(buf)

				if err != nil && err != io.EOF {
					panic(err)
				}

				if n == 0 {
					break
				}
				body += string(buf)
			}
		default:
			bodyByte, e := ioutil.ReadAll(resp.Body)
			if e != nil {
				err = errors.New(fmt.Sprintf("HttpClient.Get(%s),Read body error:%s", u, e.Error()))
				return
			}
			body = string(bodyByte)
		}
	}

	this.cookies = this.jar.Cookies(req.URL)

	page = this.Dec(body)

	//log4go.Finest("HttpClient.PostMultipart to url :%s returns :", u)
	//log4go.Finest(page)
	return

}

func (p *HttpClient) clearRedirect() {
	p.redirect = false
	p.redirectUrl = ""
}
func (p *HttpClient) CheckRedirect() (b bool, url string) {
	return p.redirect, p.redirectUrl
}
<|endoftext|>


<commit_before>package bridge

import (
	"errors"
	"log"
	"net"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"

	dockerapi "github.com/fsouza/go-dockerclient"
)

type Bridge struct {
	sync.Mutex
	registry       RegistryAdapter
	docker         *dockerapi.Client
	services       map[string][]*Service
	deadContainers map[string]*DeadContainer
	config         Config
}

func New(docker *dockerapi.Client, adapterUri string, config Config) (*Bridge, error) {
	uri, err := url.Parse(adapterUri)
	if err != nil {
		return nil, errors.New("bad adapter uri: " + adapterUri)
	}
	factory, found := AdapterFactories.Lookup(uri.Scheme)
	if !found {
		return nil, errors.New("unrecognized adapter: " + adapterUri)
	}

	log.Println("Using", uri.Scheme, "adapter:", uri)
	return &Bridge{
		docker:         docker,
		config:         config,
		registry:       factory.New(uri),
		services:       make(map[string][]*Service),
		deadContainers: make(map[string]*DeadContainer),
	}, nil
}

func (b *Bridge) Ping() error {
	return b.registry.Ping()
}

func (b *Bridge) Add(containerId string) {
	b.Lock()
	defer b.Unlock()
	b.add(containerId, false)
}

func (b *Bridge) Remove(containerId string) {
	b.remove(containerId, true)
}

func (b *Bridge) RemoveOnExit(containerId string) {
	b.remove(containerId, b.config.DeregisterCheck == "always" || b.didExitCleanly(containerId))
}

func (b *Bridge) Refresh() {
	b.Lock()
	defer b.Unlock()

	for containerId, deadContainer := range b.deadContainers {
		deadContainer.TTL -= b.config.RefreshInterval
		if deadContainer.TTL <= 0 {
			delete(b.deadContainers, containerId)
		}
	}

	for containerId, services := range b.services {
		for _, service := range services {
			err := b.registry.Refresh(service)
			if err != nil {
				log.Println("refresh failed:", service.ID, err)
				continue
			}
			log.Println("refreshed:", containerId[:12], service.ID)
		}
	}
}

func (b *Bridge) Sync(quiet bool) {
	b.Lock()
	defer b.Unlock()

	containers, err := b.docker.ListContainers(dockerapi.ListContainersOptions{})
	if err != nil && quiet {
		log.Println("error listing containers, skipping sync")
		return
	} else if err != nil && !quiet {
		log.Fatal(err)
	}

	log.Printf("Syncing services on %d containers", len(containers))

	// NOTE: This assumes reregistering will do the right thing, i.e. nothing.
	// NOTE: This will NOT remove services.
	for _, listing := range containers {
		services := b.services[listing.ID]
		if services == nil {
			b.add(listing.ID, quiet)
		} else {
			for _, service := range services {
				err := b.registry.Register(service)
				if err != nil {
					log.Println("sync register failed:", service, err)
				}
			}
		}
	}
}

func (b *Bridge) add(containerId string, quiet bool) {
	if d := b.deadContainers[containerId]; d != nil {
		b.services[containerId] = d.Services
		delete(b.deadContainers, containerId)
	}

	if b.services[containerId] != nil {
		log.Println("container, ", containerId[:12], ", already exists, ignoring")
		// Alternatively, remove and readd or resubmit.
		return
	}

	container, err := b.docker.InspectContainer(containerId)
	if err != nil {
		log.Println("unable to inspect container:", containerId[:12], err)
		return
	}

	ports := make(map[string]ServicePort)

	// Extract configured host port mappings, relevant when using --net=host
	for port, published := range container.HostConfig.PortBindings {
		ports[string(port)] = servicePort(container, port, published)
	}

	// Extract runtime port mappings, relevant when using --net=bridge
	for port, published := range container.NetworkSettings.Ports {
		ports[string(port)] = servicePort(container, port, published)
	}

	if len(ports) == 0 && !quiet {
		log.Println("ignored:", container.ID[:12], "no published ports")
		return
	}

	for _, port := range ports {
		if b.isInternal(container) != true && port.HostPort == "" {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "port", port.ExposedPort, "not published on host")
			}
			continue
		}
		service := b.newService(port, len(ports) > 1)
		if service == nil {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "service on port", port.ExposedPort)
			}
			continue
		}
		err := b.registry.Register(service)
		if err != nil {
			log.Println("register failed:", service, err)
			continue
		}
		b.services[container.ID] = append(b.services[container.ID], service)
		log.Println("added:", container.ID[:12], service.ID)
	}
}

func (b *Bridge) newService(port ServicePort, isgroup bool) *Service {
	container := port.container
	defaultName := strings.Split(path.Base(container.Config.Image), ":")[0]

	// not sure about this logic. kind of want to remove it.
	hostname, err := os.Hostname()
	if err != nil {
		hostname = port.HostIP
	} else {
		if port.HostIP == "0.0.0.0" {
			ip, err := net.ResolveIPAddr("ip", hostname)
			if err == nil {
				port.HostIP = ip.String()
			}
		}
	}

	if b.config.HostIp != "" {
		port.HostIP = b.config.HostIp
	}

	metadata := serviceMetaData(container.Config, port.ExposedPort)

	ignore := mapDefault(metadata, "ignore", "")
	if ignore != "" {
		return nil
	}

	service := new(Service)
	service.Origin = port
	service.ID = hostname + ":" + container.Name[1:] + ":" + port.ExposedPort
	service.Name = mapDefault(metadata, "name", defaultName)
	if isgroup {
		service.Name += "-" + port.ExposedPort
	}

	service.IP, service.Port = b.extractPort(port)

	if port.PortType == "udp" {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags, "udp")
		service.ID = service.ID + ":udp"
	} else {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags)
	}

	id := mapDefault(metadata, "id", "")
	if id != "" {
		service.ID = id
	}

	delete(metadata, "id")
	delete(metadata, "tags")
	delete(metadata, "name")
	service.Attrs = metadata
	service.TTL = b.config.RefreshTtl

	return service
}

func (b *Bridge) remove(containerId string, deregister bool) {
	b.Lock()
	defer b.Unlock()

	if deregister {
		deregisterAll := func(services []*Service) {
			for _, service := range services {
				err := b.registry.Deregister(service)
				if err != nil {
					log.Println("deregister failed:", service.ID, err)
					continue
				}
				log.Println("removed:", containerId[:12], service.ID)
			}
		}
		deregisterAll(b.services[containerId])
		if d := b.deadContainers[containerId]; d != nil {
			deregisterAll(d.Services)
			delete(b.deadContainers, containerId)
		}
	} else if b.config.RefreshTtl != 0 && b.services[containerId] != nil {
		// need to stop the refreshing, but can't delete it yet
		b.deadContainers[containerId] = &DeadContainer{b.config.RefreshTtl, b.services[containerId]}
	}
	delete(b.services, containerId)
}

func (b *Bridge) didExitCleanly(containerId string) bool {
	container, err := b.docker.InspectContainer(containerId)
	if _, ok := err.(*dockerapi.NoSuchContainer); ok {
		// the container has already been removed from Docker
		// e.g. probabably run with "--rm" to remove immediately
		// so its exit code is not accessible
		log.Printf("registrator: container %v was removed, could not fetch exit code", containerId[:12])
		return true
	} else if err != nil {
		log.Printf("registrator: error fetching status for container %v on \"die\" event: %v\n", containerId[:12], err)
		return false
	}
	return !container.State.Running && container.State.ExitCode == 0
}

func (b *Bridge) isInternal(container *dockerapi.Container) bool {

	for _, e := range container.Config.Env {
		if ok, key, value := parseEnv(e); ok && key == "internal" {
			return value == "true"
		}
	}

	return b.config.Internal
}

func (b *Bridge) extractPort(port ServicePort) (string, int) {
	var p int
	var IP string

	container := port.container

	if b.isInternal(container) {
		p, _ := strconv.Atoi(port.ExposedPort)
		IP = port.ExposedIP
	} else {
		p, _ = strconv.Atoi(port.HostPort)
		IP = port.HostIP
	}

	for _, e := range container.Config.Env {
		if ok, key, value := parseEnv(e); ok && key == "ip" {
			IP = value
		}
	}

	return IP, p
}
<commit_msg>p was already defined<commit_after>package bridge

import (
	"errors"
	"log"
	"net"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"

	dockerapi "github.com/fsouza/go-dockerclient"
)

type Bridge struct {
	sync.Mutex
	registry       RegistryAdapter
	docker         *dockerapi.Client
	services       map[string][]*Service
	deadContainers map[string]*DeadContainer
	config         Config
}

func New(docker *dockerapi.Client, adapterUri string, config Config) (*Bridge, error) {
	uri, err := url.Parse(adapterUri)
	if err != nil {
		return nil, errors.New("bad adapter uri: " + adapterUri)
	}
	factory, found := AdapterFactories.Lookup(uri.Scheme)
	if !found {
		return nil, errors.New("unrecognized adapter: " + adapterUri)
	}

	log.Println("Using", uri.Scheme, "adapter:", uri)
	return &Bridge{
		docker:         docker,
		config:         config,
		registry:       factory.New(uri),
		services:       make(map[string][]*Service),
		deadContainers: make(map[string]*DeadContainer),
	}, nil
}

func (b *Bridge) Ping() error {
	return b.registry.Ping()
}

func (b *Bridge) Add(containerId string) {
	b.Lock()
	defer b.Unlock()
	b.add(containerId, false)
}

func (b *Bridge) Remove(containerId string) {
	b.remove(containerId, true)
}

func (b *Bridge) RemoveOnExit(containerId string) {
	b.remove(containerId, b.config.DeregisterCheck == "always" || b.didExitCleanly(containerId))
}

func (b *Bridge) Refresh() {
	b.Lock()
	defer b.Unlock()

	for containerId, deadContainer := range b.deadContainers {
		deadContainer.TTL -= b.config.RefreshInterval
		if deadContainer.TTL <= 0 {
			delete(b.deadContainers, containerId)
		}
	}

	for containerId, services := range b.services {
		for _, service := range services {
			err := b.registry.Refresh(service)
			if err != nil {
				log.Println("refresh failed:", service.ID, err)
				continue
			}
			log.Println("refreshed:", containerId[:12], service.ID)
		}
	}
}

func (b *Bridge) Sync(quiet bool) {
	b.Lock()
	defer b.Unlock()

	containers, err := b.docker.ListContainers(dockerapi.ListContainersOptions{})
	if err != nil && quiet {
		log.Println("error listing containers, skipping sync")
		return
	} else if err != nil && !quiet {
		log.Fatal(err)
	}

	log.Printf("Syncing services on %d containers", len(containers))

	// NOTE: This assumes reregistering will do the right thing, i.e. nothing.
	// NOTE: This will NOT remove services.
	for _, listing := range containers {
		services := b.services[listing.ID]
		if services == nil {
			b.add(listing.ID, quiet)
		} else {
			for _, service := range services {
				err := b.registry.Register(service)
				if err != nil {
					log.Println("sync register failed:", service, err)
				}
			}
		}
	}
}

func (b *Bridge) add(containerId string, quiet bool) {
	if d := b.deadContainers[containerId]; d != nil {
		b.services[containerId] = d.Services
		delete(b.deadContainers, containerId)
	}

	if b.services[containerId] != nil {
		log.Println("container, ", containerId[:12], ", already exists, ignoring")
		// Alternatively, remove and readd or resubmit.
		return
	}

	container, err := b.docker.InspectContainer(containerId)
	if err != nil {
		log.Println("unable to inspect container:", containerId[:12], err)
		return
	}

	ports := make(map[string]ServicePort)

	// Extract configured host port mappings, relevant when using --net=host
	for port, published := range container.HostConfig.PortBindings {
		ports[string(port)] = servicePort(container, port, published)
	}

	// Extract runtime port mappings, relevant when using --net=bridge
	for port, published := range container.NetworkSettings.Ports {
		ports[string(port)] = servicePort(container, port, published)
	}

	if len(ports) == 0 && !quiet {
		log.Println("ignored:", container.ID[:12], "no published ports")
		return
	}

	for _, port := range ports {
		if b.isInternal(container) != true && port.HostPort == "" {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "port", port.ExposedPort, "not published on host")
			}
			continue
		}
		service := b.newService(port, len(ports) > 1)
		if service == nil {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "service on port", port.ExposedPort)
			}
			continue
		}
		err := b.registry.Register(service)
		if err != nil {
			log.Println("register failed:", service, err)
			continue
		}
		b.services[container.ID] = append(b.services[container.ID], service)
		log.Println("added:", container.ID[:12], service.ID)
	}
}

func (b *Bridge) newService(port ServicePort, isgroup bool) *Service {
	container := port.container
	defaultName := strings.Split(path.Base(container.Config.Image), ":")[0]

	// not sure about this logic. kind of want to remove it.
	hostname, err := os.Hostname()
	if err != nil {
		hostname = port.HostIP
	} else {
		if port.HostIP == "0.0.0.0" {
			ip, err := net.ResolveIPAddr("ip", hostname)
			if err == nil {
				port.HostIP = ip.String()
			}
		}
	}

	if b.config.HostIp != "" {
		port.HostIP = b.config.HostIp
	}

	metadata := serviceMetaData(container.Config, port.ExposedPort)

	ignore := mapDefault(metadata, "ignore", "")
	if ignore != "" {
		return nil
	}

	service := new(Service)
	service.Origin = port
	service.ID = hostname + ":" + container.Name[1:] + ":" + port.ExposedPort
	service.Name = mapDefault(metadata, "name", defaultName)
	if isgroup {
		service.Name += "-" + port.ExposedPort
	}

	service.IP, service.Port = b.extractPort(port)

	if port.PortType == "udp" {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags, "udp")
		service.ID = service.ID + ":udp"
	} else {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags)
	}

	id := mapDefault(metadata, "id", "")
	if id != "" {
		service.ID = id
	}

	delete(metadata, "id")
	delete(metadata, "tags")
	delete(metadata, "name")
	service.Attrs = metadata
	service.TTL = b.config.RefreshTtl

	return service
}

func (b *Bridge) remove(containerId string, deregister bool) {
	b.Lock()
	defer b.Unlock()

	if deregister {
		deregisterAll := func(services []*Service) {
			for _, service := range services {
				err := b.registry.Deregister(service)
				if err != nil {
					log.Println("deregister failed:", service.ID, err)
					continue
				}
				log.Println("removed:", containerId[:12], service.ID)
			}
		}
		deregisterAll(b.services[containerId])
		if d := b.deadContainers[containerId]; d != nil {
			deregisterAll(d.Services)
			delete(b.deadContainers, containerId)
		}
	} else if b.config.RefreshTtl != 0 && b.services[containerId] != nil {
		// need to stop the refreshing, but can't delete it yet
		b.deadContainers[containerId] = &DeadContainer{b.config.RefreshTtl, b.services[containerId]}
	}
	delete(b.services, containerId)
}

func (b *Bridge) didExitCleanly(containerId string) bool {
	container, err := b.docker.InspectContainer(containerId)
	if _, ok := err.(*dockerapi.NoSuchContainer); ok {
		// the container has already been removed from Docker
		// e.g. probabably run with "--rm" to remove immediately
		// so its exit code is not accessible
		log.Printf("registrator: container %v was removed, could not fetch exit code", containerId[:12])
		return true
	} else if err != nil {
		log.Printf("registrator: error fetching status for container %v on \"die\" event: %v\n", containerId[:12], err)
		return false
	}
	return !container.State.Running && container.State.ExitCode == 0
}

func (b *Bridge) isInternal(container *dockerapi.Container) bool {

	for _, e := range container.Config.Env {
		if ok, key, value := parseEnv(e); ok && key == "internal" {
			return value == "true"
		}
	}

	return b.config.Internal
}

func (b *Bridge) extractPort(port ServicePort) (string, int) {
	var p int
	var IP string

	container := port.container

	if b.isInternal(container) {
		p, _ = strconv.Atoi(port.ExposedPort)
		IP = port.ExposedIP
	} else {
		p, _ = strconv.Atoi(port.HostPort)
		IP = port.HostIP
	}

	for _, e := range container.Config.Env {
		if ok, key, value := parseEnv(e); ok && key == "ip" {
			IP = value
		}
	}

	return IP, p
}
<|endoftext|>


<commit_before>package chroot

import (
	"context"
	"fmt"
	"os/exec"
	"syscall"

	. "github.com/polydawn/go-errcat"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/repeatr"
	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/repeatr/executor/cradle"
	"go.polydawn.net/repeatr/executor/mixins"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/stitch"
)

type Executor struct {
	workspaceFs   fs.FS             // A working dir per execution will be made in here.
	assemblerTool *stitch.Assembler // Contains: unpackTool, caching cfg, and placer tools.
	packTool      rio.PackFunc
}

func NewExecutor(
	workDir fs.AbsolutePath,
	unpackTool rio.UnpackFunc,
	packTool rio.PackFunc,
) (repeatr.RunFunc, error) {
	asm, err := stitch.NewAssembler(unpackTool)
	if err != nil {
		return nil, repeatr.ReboxRioError(err)
	}
	return Executor{
		osfs.New(workDir),
		asm,
		packTool,
	}.Run, nil
}

var _ repeatr.RunFunc = Executor{}.Run

func (cfg Executor) Run(
	ctx context.Context,
	formula api.Formula,
	formulaCtx api.FormulaContext,
	input repeatr.InputControl,
	mon repeatr.Monitor,
) (*api.RunRecord, error) {
	if mon.Chan != nil {
		defer close(mon.Chan)
	}

	// Workspace setup and params defaulting.
	formula = cradle.FormulaDefaults(formula)                    // Initialize formula default values.
	rr := api.RunRecord{}                                        // Start filling out record keeping!
	mixins.InitRunRecord(&rr, formula)                           // Includes picking a random guid for the job, which we use in all temp files.
	_, chrootFs, err := mixins.MakeWorkDirs(cfg.workspaceFs, rr) // Make work dirs. Including whole workspace dir and parents, if necessary.
	if err != nil {
		return nil, err
	}

	// Shell out to assembler.
	unpackSpecs := stitch.FormulaToUnpackSpecs(formula, formulaCtx, api.Filter_NoMutation)
	wgRioLogs := mixins.ForwardRioUnpackLogs(ctx, mon, unpackSpecs)
	cleanupFunc, err := cfg.assemblerTool.Run(ctx, chrootFs, unpackSpecs, cradle.DirpropsForUserinfo(*formula.Action.Userinfo))
	wgRioLogs.Wait()
	if err != nil {
		return &rr, repeatr.ReboxRioError(err)
	}
	defer func() {
		if err := cleanupFunc(); err != nil {
			// TODO log it
		}
	}()

	// Last bit of filesystem brushup: run cradle fs mutations.
	if err := cradle.TidyFilesystem(formula, chrootFs); err != nil {
		return &rr, err
	}

	// Check that action commands appear to be executable on this filesystem.
	if err := mixins.CheckFSReadyForExec(formula, chrootFs); err != nil {
		return &rr, err
	}

	// Invoke containment and run!
	cmdName := formula.Action.Exec[0]
	cmd := exec.Command(cmdName, formula.Action.Exec[1:]...)
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Chroot: chrootFs.BasePath().String(),
		Credential: &syscall.Credential{
			Uid: uint32(*formula.Action.Userinfo.Uid),
			Gid: uint32(*formula.Action.Userinfo.Gid),
		},
	}
	cmd.Dir = string(formula.Action.Cwd)
	cmd.Env = envToSlice(formula.Action.Env)
	if input.Chan != nil {
		pipe, _ := cmd.StdinPipe()
		mixins.RunInputWriteForwarder(ctx, pipe, input.Chan)
	}
	proxy := mixins.NewOutputEventWriter(ctx, mon.Chan)
	cmd.Stdout = proxy
	cmd.Stderr = proxy
	rr.ExitCode, err = runCmd(cmd)
	if err != nil {
		return &rr, err
	}

	// Pack outputs.
	packSpecs := stitch.FormulaToPackSpecs(formula, formulaCtx, api.Filter_DefaultFlatten)
	rr.Results, err = stitch.PackMulti(ctx, cfg.packTool, chrootFs, packSpecs)
	if err != nil {
		return &rr, err
	}

	// Done!
	return &rr, nil
}

func runCmd(cmd *exec.Cmd) (int, error) {
	if err := cmd.Start(); err != nil {
		return -1, Errorf(repeatr.ErrExecutor, "executor failed to launch: %s", err)
	}
	err := cmd.Wait()
	if err == nil {
		return 0, nil
	}
	exitErr, ok := err.(*exec.ExitError)
	if !ok { // This is basically an "if stdlib isn't what we thought it is" error, so panic-worthy.
		panic(fmt.Errorf("unknown exit reason: %T %s", err, err))
	}
	waitStatus, ok := exitErr.ProcessState.Sys().(syscall.WaitStatus)
	if !ok { // This is basically a "if stdlib[...]" or OS portability issue, so also panic-able.
		panic(fmt.Errorf("unknown process state implementation %T", exitErr.ProcessState.Sys()))
	}
	if waitStatus.Exited() {
		return waitStatus.ExitStatus(), nil
	} else if waitStatus.Signaled() {
		// In bash, when a processs ends from a signal, the $? variable is set to 128+SIG.
		// We follow that same convention here.
		// So, a process terminated by ctrl-C returns 130.  A script that died to kill-9 returns 137.
		return int(waitStatus.Signal()) + 128, nil
	} else {
		return -1, Errorf(repeatr.ErrExecutor, "unknown process wait status (%#v)", waitStatus)
	}

}

func envToSlice(env map[string]string) []string {
	rv := make([]string, len(env))
	i := 0
	for k, v := range env {
		rv[i] = k + "=" + v
		i++
	}
	return rv
}
<commit_msg>executor/impl/chroot: log cleanup errors.<commit_after>package chroot

import (
	"context"
	"fmt"
	"os/exec"
	"syscall"
	"time"

	. "github.com/polydawn/go-errcat"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/repeatr"
	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/repeatr/executor/cradle"
	"go.polydawn.net/repeatr/executor/mixins"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/stitch"
)

type Executor struct {
	workspaceFs   fs.FS             // A working dir per execution will be made in here.
	assemblerTool *stitch.Assembler // Contains: unpackTool, caching cfg, and placer tools.
	packTool      rio.PackFunc
}

func NewExecutor(
	workDir fs.AbsolutePath,
	unpackTool rio.UnpackFunc,
	packTool rio.PackFunc,
) (repeatr.RunFunc, error) {
	asm, err := stitch.NewAssembler(unpackTool)
	if err != nil {
		return nil, repeatr.ReboxRioError(err)
	}
	return Executor{
		osfs.New(workDir),
		asm,
		packTool,
	}.Run, nil
}

var _ repeatr.RunFunc = Executor{}.Run

func (cfg Executor) Run(
	ctx context.Context,
	formula api.Formula,
	formulaCtx api.FormulaContext,
	input repeatr.InputControl,
	mon repeatr.Monitor,
) (*api.RunRecord, error) {
	if mon.Chan != nil {
		defer close(mon.Chan)
	}

	// Workspace setup and params defaulting.
	formula = cradle.FormulaDefaults(formula)                    // Initialize formula default values.
	rr := api.RunRecord{}                                        // Start filling out record keeping!
	mixins.InitRunRecord(&rr, formula)                           // Includes picking a random guid for the job, which we use in all temp files.
	_, chrootFs, err := mixins.MakeWorkDirs(cfg.workspaceFs, rr) // Make work dirs. Including whole workspace dir and parents, if necessary.
	if err != nil {
		return nil, err
	}

	// Shell out to assembler.
	unpackSpecs := stitch.FormulaToUnpackSpecs(formula, formulaCtx, api.Filter_NoMutation)
	wgRioLogs := mixins.ForwardRioUnpackLogs(ctx, mon, unpackSpecs)
	cleanupFunc, err := cfg.assemblerTool.Run(ctx, chrootFs, unpackSpecs, cradle.DirpropsForUserinfo(*formula.Action.Userinfo))
	wgRioLogs.Wait()
	if err != nil {
		return &rr, repeatr.ReboxRioError(err)
	}
	defer func() {
		err := cleanupFunc()
		if err == nil {
			return
		}
		if mon.Chan == nil {
			return // sad, but no one to log it to, and not fatal.
		}
		mon.Chan <- repeatr.Event{
			Log: &repeatr.Event_Log{
				Time:  time.Now(),
				Level: repeatr.LogError,
				Msg:   "error during cleanup",
				Detail: [][2]string{
					{"error", err.Error()},
				},
			}}
	}()

	// Last bit of filesystem brushup: run cradle fs mutations.
	if err := cradle.TidyFilesystem(formula, chrootFs); err != nil {
		return &rr, err
	}

	// Check that action commands appear to be executable on this filesystem.
	if err := mixins.CheckFSReadyForExec(formula, chrootFs); err != nil {
		return &rr, err
	}

	// Invoke containment and run!
	cmdName := formula.Action.Exec[0]
	cmd := exec.Command(cmdName, formula.Action.Exec[1:]...)
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Chroot: chrootFs.BasePath().String(),
		Credential: &syscall.Credential{
			Uid: uint32(*formula.Action.Userinfo.Uid),
			Gid: uint32(*formula.Action.Userinfo.Gid),
		},
	}
	cmd.Dir = string(formula.Action.Cwd)
	cmd.Env = envToSlice(formula.Action.Env)
	if input.Chan != nil {
		pipe, _ := cmd.StdinPipe()
		mixins.RunInputWriteForwarder(ctx, pipe, input.Chan)
	}
	proxy := mixins.NewOutputEventWriter(ctx, mon.Chan)
	cmd.Stdout = proxy
	cmd.Stderr = proxy
	rr.ExitCode, err = runCmd(cmd)
	if err != nil {
		return &rr, err
	}

	// Pack outputs.
	packSpecs := stitch.FormulaToPackSpecs(formula, formulaCtx, api.Filter_DefaultFlatten)
	rr.Results, err = stitch.PackMulti(ctx, cfg.packTool, chrootFs, packSpecs)
	if err != nil {
		return &rr, err
	}

	// Done!
	return &rr, nil
}

func runCmd(cmd *exec.Cmd) (int, error) {
	if err := cmd.Start(); err != nil {
		return -1, Errorf(repeatr.ErrExecutor, "executor failed to launch: %s", err)
	}
	err := cmd.Wait()
	if err == nil {
		return 0, nil
	}
	exitErr, ok := err.(*exec.ExitError)
	if !ok { // This is basically an "if stdlib isn't what we thought it is" error, so panic-worthy.
		panic(fmt.Errorf("unknown exit reason: %T %s", err, err))
	}
	waitStatus, ok := exitErr.ProcessState.Sys().(syscall.WaitStatus)
	if !ok { // This is basically a "if stdlib[...]" or OS portability issue, so also panic-able.
		panic(fmt.Errorf("unknown process state implementation %T", exitErr.ProcessState.Sys()))
	}
	if waitStatus.Exited() {
		return waitStatus.ExitStatus(), nil
	} else if waitStatus.Signaled() {
		// In bash, when a processs ends from a signal, the $? variable is set to 128+SIG.
		// We follow that same convention here.
		// So, a process terminated by ctrl-C returns 130.  A script that died to kill-9 returns 137.
		return int(waitStatus.Signal()) + 128, nil
	} else {
		return -1, Errorf(repeatr.ErrExecutor, "unknown process wait status (%#v)", waitStatus)
	}

}

func envToSlice(env map[string]string) []string {
	rv := make([]string, len(env))
	i := 0
	for k, v := range env {
		rv[i] = k + "=" + v
		i++
	}
	return rv
}
<|endoftext|>


<commit_before>package bucket

import (
	"io"
	"net/http"
	"net/url"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/nabeken/aws-go-s3/bucket/option"
)

// A Bucket is an S3 bucket which holds properties such as bucket name and SSE things for S3 Bucket.
type Bucket struct {
	S3   s3iface.S3API
	Name *string
}

// New returns Bucket instance with bucket name name.
func New(s s3iface.S3API, name string) *Bucket {
	return &Bucket{
		S3:   s,
		Name: aws.String(name),
	}
}

// GetObject returns the s3.GetObjectOutput.
func (b *Bucket) GetObject(key string, opts ...option.GetObjectInput) (*s3.GetObjectOutput, error) {
	req := &s3.GetObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.GetObject(req)
}

// GetObjectReader returns a reader assosiated with body. A caller of this MUST close the reader when it finishes reading.
func (b *Bucket) GetObjectReader(key string, opts ...option.GetObjectInput) (io.ReadCloser, error) {
	resp, err := b.GetObject(key, opts...)
	if err != nil {
		return nil, err
	}

	return resp.Body, nil
}

// GetObjectRequest generates a "aws/request.Request" representing the client's request for the GetObject operation.
func (b *Bucket) GetObjectRequest(key string, opts ...option.GetObjectInput) (*request.Request, *s3.GetObjectOutput) {
	req := &s3.GetObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.GetObjectRequest(req)
}

// HeadObject retrieves an object metadata for key.
func (b *Bucket) HeadObject(key string, opts ...option.HeadObjectInput) (*s3.HeadObjectOutput, error) {
	req := &s3.HeadObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.HeadObject(req)
}

// ExistsObject returns true if key does not exist on bucket.
func (b *Bucket) ExistsObject(key string, opts ...option.HeadObjectInput) (bool, error) {
	_, err := b.HeadObject(key, opts...)
	if err == nil {
		return true, nil
	}

	if s3err, ok := err.(awserr.RequestFailure); ok && s3err.StatusCode() == http.StatusNotFound {
		// actually key does not exist
		return false, nil
	}

	// in some error situation
	return false, err
}

// PutObject puts an object with reading data from reader.
func (b *Bucket) PutObject(key string, rs io.ReadSeeker, opts ...option.PutObjectInput) (*s3.PutObjectOutput, error) {
	req := &s3.PutObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
		Body:   rs,
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.PutObject(req)
}

// DeleteObject deletes an object for key.
func (b *Bucket) DeleteObject(key string) (*s3.DeleteObjectOutput, error) {
	req := &s3.DeleteObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
	}

	return b.S3.DeleteObject(req)
}

// ListObjects lists objects that has prefix.
func (b *Bucket) ListObjects(prefix string, opts ...option.ListObjectsInput) (*s3.ListObjectsOutput, error) {
	req := &s3.ListObjectsInput{
		Bucket: b.Name,
		Prefix: aws.String(prefix),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.ListObjects(req)
}

// CopyObject copies an object within the bucket.
func (b *Bucket) CopyObject(dest, src string, opts ...option.CopyObjectInput) (*s3.CopyObjectOutput, error) {
	req := &s3.CopyObjectInput{
		Bucket:     b.Name,
		Key:        aws.String(dest),
		CopySource: aws.String(aws.StringValue(b.Name) + "/" + url.QueryEscape(src)),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.CopyObject(req)
}
<commit_msg>Create method for bulk deleting S3 objects<commit_after>package bucket

import (
	"io"
	"net/http"
	"net/url"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"

	"github.com/nabeken/aws-go-s3/bucket/option"
)

// A Bucket is an S3 bucket which holds properties such as bucket name and SSE things for S3 Bucket.
type Bucket struct {
	S3   s3iface.S3API
	Name *string
}

// New returns Bucket instance with bucket name name.
func New(s s3iface.S3API, name string) *Bucket {
	return &Bucket{
		S3:   s,
		Name: aws.String(name),
	}
}

// GetObject returns the s3.GetObjectOutput.
func (b *Bucket) GetObject(key string, opts ...option.GetObjectInput) (*s3.GetObjectOutput, error) {
	req := &s3.GetObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.GetObject(req)
}

// GetObjectReader returns a reader assosiated with body. A caller of this MUST close the reader when it finishes reading.
func (b *Bucket) GetObjectReader(key string, opts ...option.GetObjectInput) (io.ReadCloser, error) {
	resp, err := b.GetObject(key, opts...)
	if err != nil {
		return nil, err
	}

	return resp.Body, nil
}

// GetObjectRequest generates a "aws/request.Request" representing the client's request for the GetObject operation.
func (b *Bucket) GetObjectRequest(key string, opts ...option.GetObjectInput) (*request.Request, *s3.GetObjectOutput) {
	req := &s3.GetObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.GetObjectRequest(req)
}

// HeadObject retrieves an object metadata for key.
func (b *Bucket) HeadObject(key string, opts ...option.HeadObjectInput) (*s3.HeadObjectOutput, error) {
	req := &s3.HeadObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.HeadObject(req)
}

// ExistsObject returns true if key does not exist on bucket.
func (b *Bucket) ExistsObject(key string, opts ...option.HeadObjectInput) (bool, error) {
	_, err := b.HeadObject(key, opts...)
	if err == nil {
		return true, nil
	}

	if s3err, ok := err.(awserr.RequestFailure); ok && s3err.StatusCode() == http.StatusNotFound {
		// actually key does not exist
		return false, nil
	}

	// in some error situation
	return false, err
}

// PutObject puts an object with reading data from reader.
func (b *Bucket) PutObject(key string, rs io.ReadSeeker, opts ...option.PutObjectInput) (*s3.PutObjectOutput, error) {
	req := &s3.PutObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
		Body:   rs,
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.PutObject(req)
}

// DeleteObject deletes an object for key.
func (b *Bucket) DeleteObject(key string) (*s3.DeleteObjectOutput, error) {
	req := &s3.DeleteObjectInput{
		Bucket: b.Name,
		Key:    aws.String(key),
	}

	return b.S3.DeleteObject(req)
}

// DeleteObjects deletes each object for the given identifiers.
// A maximum of 1000 objects can be deleted at a time with this method.
func (b *Bucket) DeleteObjects(identifiers []*s3.ObjectIdentifier) (*s3.DeleteObjectsOutput, error) {
	req := &s3.DeleteObjectsInput{
		Bucket: b.Name,
		Delete: &s3.Delete{
			Objects: identifiers,
		},
	}

	return b.S3.DeleteObjects(req)
}

// ListObjects lists objects that has prefix.
func (b *Bucket) ListObjects(prefix string, opts ...option.ListObjectsInput) (*s3.ListObjectsOutput, error) {
	req := &s3.ListObjectsInput{
		Bucket: b.Name,
		Prefix: aws.String(prefix),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.ListObjects(req)
}

// CopyObject copies an object within the bucket.
func (b *Bucket) CopyObject(dest, src string, opts ...option.CopyObjectInput) (*s3.CopyObjectOutput, error) {
	req := &s3.CopyObjectInput{
		Bucket:     b.Name,
		Key:        aws.String(dest),
		CopySource: aws.String(aws.StringValue(b.Name) + "/" + url.QueryEscape(src)),
	}

	for _, f := range opts {
		f(req)
	}

	return b.S3.CopyObject(req)
}
<|endoftext|>


<commit_before>/*
 * constants_test.go
 *
 * Copyright 2013 Krzysztof Wilczynski
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package magic_test

import (
	"testing"

	. "github.com/kwilczynski/go-magic"
)

func TestConstants(t *testing.T) {
	// Any recent version of libmagic have 0x37b000 by default.
	NO_CHECK_BUILTIN_override := NO_CHECK_COMPRESS | NO_CHECK_TAR |
		NO_CHECK_APPTYPE | NO_CHECK_ELF | NO_CHECK_TEXT |
		NO_CHECK_CDF | NO_CHECK_TOKENS | NO_CHECK_ENCODING

	// Older versions of libmagic have 0x3fb000 here historically ...
	if rv, _ := Version(); rv != 0 {
		NO_CHECK_BUILTIN_override ^= 0x080000 // 0x37b000 ^ 0x080000 is 0x3fb000
	}

	// Check if underlaying constants coming from libmagic are sane.
	var constantTests = []struct {
		given    int
		expected int
	}{
		{MIME, MIME_TYPE | MIME_ENCODING},
		{NO_CHECK_BUILTIN, NO_CHECK_BUILTIN_override},
		{NO_CHECK_ASCII, NO_CHECK_TEXT},
	}

	for _, tt := range constantTests {
		if tt.given != tt.expected {
			t.Errorf("value given 0x%x, want 0x%x",
				tt.given, tt.expected)
		}
	}
}
<commit_msg>Fix check as per commit 33d8ee6bdf01a34e4e7f8bd2224a7d018d8d8acb.<commit_after>/*
 * constants_test.go
 *
 * Copyright 2013 Krzysztof Wilczynski
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package magic_test

import (
	"testing"

	. "github.com/kwilczynski/go-magic"
)

func TestConstants(t *testing.T) {
	// Any recent version of libmagic have 0x37b000 by default.
	NO_CHECK_BUILTIN_override := NO_CHECK_COMPRESS | NO_CHECK_TAR |
		NO_CHECK_APPTYPE | NO_CHECK_ELF | NO_CHECK_TEXT |
		NO_CHECK_CDF | NO_CHECK_TOKENS | NO_CHECK_ENCODING

	// Older versions of libmagic have 0x3fb000 here historically ...
	if rv, _ := Version(); rv < 0 {
		NO_CHECK_BUILTIN_override ^= 0x080000 // 0x37b000 ^ 0x080000 is 0x3fb000
	}

	// Check if underlaying constants coming from libmagic are sane.
	var constantTests = []struct {
		given    int
		expected int
	}{
		{MIME, MIME_TYPE | MIME_ENCODING},
		{NO_CHECK_BUILTIN, NO_CHECK_BUILTIN_override},
		{NO_CHECK_ASCII, NO_CHECK_TEXT},
	}

	for _, tt := range constantTests {
		if tt.given != tt.expected {
			t.Errorf("value given 0x%x, want 0x%x",
				tt.given, tt.expected)
		}
	}
}
<|endoftext|>