<commit_before>package main

import (
	"zvr/utils"
	"time"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"encoding/json"
	"github.com/pkg/errors"
	"fmt"
	"zvr/server"
	"strings"
	"os"
)

const (
	VIRTIO_PORT_PATH = "/dev/virtio-ports/applianceVm.vport"
	BOOTSTRAP_INFO_CACHE = "/home/vyos/zvr/bootstrap-info.json"
	TMP_LOCATION_FOR_ESX = "/tmp/bootstrap-info.json"
)

type nic struct {
	mac string
	ip string
	name string
	netmask string
	isDefaultRoute bool
	gateway string
}

var bootstrapInfo map[string]interface{} = make(map[string]interface{})
var nics map[string]*nic = make(map[string]*nic)

func waitIptablesServiceOnline()  {
	bash := utils.Bash{
		Command: "/sbin/iptables-save",
	}

	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		err := bash.Run()
		if err != nil {
			log.Debugf("iptables service seems not ready, %v", err)
		}
		return err == nil
	}, time.Duration(120)*time.Second, time.Duration(500)*time.Millisecond)
}

func waitVirtioPortOnline() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		ok, err := utils.PathExists(VIRTIO_PORT_PATH); utils.PanicOnError(err)
		if !ok {
			log.Debugf("%s doesn't not exist, wait it ...", VIRTIO_PORT_PATH)
		}
		return ok
	}, time.Duration(120)*time.Second, time.Duration(500)*time.Millisecond)
}

func isOnVMwareHypervisor() bool {
	bash := utils.Bash{
		Command: "dmesg | grep -q 'Hypervisor.*VMware'",
	}

	if ret, _, _, err := bash.RunWithReturn(); ret == 0 && err == nil {
		return true
	}

	return false
}

func parseEsxBootInfo() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		if _, err := os.Stat(TMP_LOCATION_FOR_ESX); os.IsNotExist(err) {
			log.Debugf("bootstrap info not ready, waiting ...")
			return false
		}

		content, err := ioutil.ReadFile(TMP_LOCATION_FOR_ESX); utils.PanicOnError(err)
		if err = json.Unmarshal(content, &bootstrapInfo); err != nil {
			panic(errors.Wrap(err, fmt.Sprintf("unable to JSON parse:\n %s", string(content))))
		}

		err = utils.MkdirForFile(BOOTSTRAP_INFO_CACHE, 0666); utils.PanicOnError(err)
		err = os.Rename(TMP_LOCATION_FOR_ESX, BOOTSTRAP_INFO_CACHE); utils.PanicOnError(err)
		err = os.Chmod(BOOTSTRAP_INFO_CACHE, 0777); utils.PanicOnError(err)
		log.Debugf("recieved bootstrap info:\n%s", string(content))
		return true
	}, time.Duration(300)*time.Second, time.Duration(1)*time.Second)
}

func parseKvmBootInfo() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		content, err := ioutil.ReadFile(VIRTIO_PORT_PATH); utils.PanicOnError(err)
		if len(content) == 0 {
			log.Debugf("no content in %s, it may not be ready, wait it ...", VIRTIO_PORT_PATH)
			return false
		}

		if err := json.Unmarshal(content, &bootstrapInfo); err != nil {
			panic(errors.Wrap(err, fmt.Sprintf("unable to JSON parse:\n %s", string(content))))
		}

		err = utils.MkdirForFile(BOOTSTRAP_INFO_CACHE, 0666); utils.PanicOnError(err)
		err = ioutil.WriteFile(BOOTSTRAP_INFO_CACHE, content, 0666); utils.PanicOnError(err)
		err = os.Chmod(BOOTSTRAP_INFO_CACHE, 0777); utils.PanicOnError(err)
		log.Debugf("recieved bootstrap info:\n%s", string(content))
		return true
	}, time.Duration(300)*time.Second, time.Duration(1)*time.Second)
}

func resetVyos()  {
	// clear all configuration in case someone runs 'save' command manually before,
	// to keep the vyos must be stateless

	// delete all interfaces
	tree := server.NewParserFromShowConfiguration().Tree
	tree.Delete("interfaces ethernet")
	tree.Apply(true)

	// reload default configuration
	server.RunVyosScriptAsUserVyos("load /opt/vyatta/etc/config.boot.default\nsave")
}

func configureVyos()  {
	resetVyos()

	mgmtNic := bootstrapInfo["managementNic"].(map[string]interface{})
	if mgmtNic == nil {
		panic(errors.New("no field 'managementNic' in bootstrap info"))
	}

	eth0 := &nic{ name: "eth0" }
	var ok bool
	eth0.mac, ok = mgmtNic["mac"].(string); utils.PanicIfError(ok, errors.New("cannot find 'mac' field for the management nic"))
	eth0.netmask, ok = mgmtNic["netmask"].(string); utils.PanicIfError(ok, errors.New("cannot find 'netmask' field for the management nic"))
	eth0.ip, ok = mgmtNic["ip"].(string); utils.PanicIfError(ok, errors.New("cannot find 'ip' field for the management nic"))
	eth0.isDefaultRoute = mgmtNic["isDefaultRoute"].(bool)
	eth0.gateway = mgmtNic["gateway"].(string)
	nics[eth0.name] = eth0

	otherNics := bootstrapInfo["additionalNics"].([]interface{})
	if otherNics != nil {
		for _, o := range otherNics {
			onic := o.(map[string]interface{})
			n := &nic{}
			n.name, ok = onic["deviceName"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'deviceName' field for the nic"))
			n.mac, ok = onic["mac"].(string); utils.PanicIfError(ok, errors.New("cannot find 'mac' field for the nic"))
			n.netmask, ok = onic["netmask"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'netmask' field for the nic[name:%s]", n.name))
			n.ip, ok = onic["ip"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'ip' field for the nic[name:%s]", n.name))
			n.gateway = onic["gateway"].(string)
			n.isDefaultRoute = onic["isDefaultRoute"].(bool)
			nics[n.name] = n
		}
	}

	type deviceName struct {
		expected string
		actual string
		swap string
	}

	devNames := make([]*deviceName, 0)

	// check integrity of nics
	for _, nic := range nics {
		utils.Assertf(nic.name != "", "name cannot be empty[mac:%s]", nic.mac)
		utils.Assertf(nic.ip != "", "ip cannot be empty[nicname: %s]", nic.name)
		utils.Assertf(nic.gateway != "", "gateway cannot be empty[nicname:%s]", nic.name)
		utils.Assertf(nic.netmask != "", "netmask cannot be empty[nicname:%s]", nic.name)
		utils.Assertf(nic.mac != "", "mac cannot be empty[nicname:%s]", nic.name)

		nicname, err := utils.GetNicNameByMac(nic.mac); utils.PanicOnError(err)
		if nicname != nic.name {
			devNames = append(devNames, &deviceName{
				expected: nic.name,
				actual: nicname,
			})
		}
	}

	if len(devNames) != 0 {
		// shutdown links and change to temporary names
		cmds := make([]string, 0)
		for i, devname := range devNames {
			devnum := 1000 + i

			devname.swap = fmt.Sprintf("eth%v", devnum)
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v down", devname.actual))
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v name %v", devname.actual, devname.swap))
		}

		b := utils.Bash{
			Command: strings.Join(cmds, "\n"),
		}

		b.Run()
		b.PanicIfError()

		// change temporary names to real names and bring up links
		cmds = make([]string, 0)
		for _, devname := range devNames {
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v name %v", devname.swap, devname.expected))
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v up", devname.expected))
		}

		b = utils.Bash{
			Command: strings.Join(cmds, "\n"),
		}

		b.Run()
		b.PanicIfError()
	}

	vyos := server.NewParserFromShowConfiguration()
	tree := vyos.Tree

	sshkey := bootstrapInfo["publicKey"].(string)
	utils.Assert(sshkey != "", "cannot find 'publicKey' in bootstrap info")
	sshkeyparts := strings.Split(sshkey, " ")
	sshtype := sshkeyparts[0]
	key := sshkeyparts[1]
	id := sshkeyparts[2]

	tree.Setf("system login user vyos authentication public-keys %s key %s", id, key)
	tree.Setf("system login user vyos authentication public-keys %s type %s", id, sshtype)

	setNic := func(nic *nic) {
		cidr, err := utils.NetmaskToCIDR(nic.netmask); utils.PanicOnError(err)
		//tree.Setf("interfaces ethernet %s hw-id %s", nic.name, nic.mac)
		tree.Setf("interfaces ethernet %s address %s", nic.name, fmt.Sprintf("%v/%v", nic.ip, cidr))
		tree.Setf("interfaces ethernet %s duplex auto", nic.name)
		tree.Setf("interfaces ethernet %s smp_affinity auto", nic.name)
		tree.Setf("interfaces ethernet %s speed auto", nic.name)
		if nic.isDefaultRoute {
			tree.Setf("system gateway-address %v", nic.gateway)
		}
	}

	sshport := bootstrapInfo["sshPort"].(float64)
	utils.Assert(sshport != 0, "sshport not found in bootstrap info")
	tree.Setf("service ssh port %v", int(sshport))
	tree.Setf("service ssh listen-address %v", eth0.ip)

	// configure firewall
	for _, nic := range nics {
		setNic(nic)

		tree.SetFirewallOnInterface(nic.name, "local",
			"action accept",
			"state established enable",
			"state related enable",
			fmt.Sprintf("destination address %v", nic.ip),
		)
		tree.SetFirewallOnInterface(nic.name, "local",
			"action accept",
			"protocol icmp",
			fmt.Sprintf("destination address %v", nic.ip),
		)

		if !nic.isDefaultRoute && nic.name != "eth0" {
			// for nic connecting to the guest network
			// make its FORWARDING chain open
			// here in = "-A FORWARDING -i nic.name -j ACCEPT"
			tree.SetFirewallOnInterface(nic.name, "in",
				"action accept",
				"state established enable",
				"state related enable",
				"state new enable",
			)
		} else {
			tree.SetFirewallOnInterface(nic.name, "in",
				"action accept",
				"state established enable",
				"state related enable",
			)
		}

		tree.SetFirewallOnInterface(nic.name, "in",
			"action accept",
			"protocol icmp",
		)

		// only allow ssh traffic on eth0, disable on others
		if nic.name == "eth0" {
			tree.SetFirewallOnInterface(nic.name, "local",
				fmt.Sprintf("destination port %v", int(sshport)),
				fmt.Sprintf("destination address %v", nic.ip),
				"protocol tcp",
				"action accept",
			)
		} else {
			tree.SetFirewallOnInterface(nic.name, "local",
				fmt.Sprintf("destination port %v", int(sshport)),
				fmt.Sprintf("destination address %v", nic.ip),
				"protocol tcp",
				"action reject",
			)
		}

		tree.SetFirewallDefaultAction(nic.name, "local", "reject")
		tree.SetFirewallDefaultAction(nic.name, "in", "reject")

		tree.AttachFirewallToInterface(nic.name, "local")
		tree.AttachFirewallToInterface(nic.name, "in")
	}

	tree.Set("system time-zone Asia/Shanghai")

	password := bootstrapInfo["vyosPassword"]; utils.Assert(password != "", "vyosPassword cannot be empty")
	if !isOnVMwareHypervisor() {
		tree.Setf("system login user vyos authentication plaintext-password %v", password)
	}

	tree.Apply(true)

	arping := func(nicname, ip, gateway string) {
		b := utils.Bash{ Command: fmt.Sprintf("arping -A -U -c 1 -I %s -s %s %s", nicname, ip, gateway) }
		b.Run()
	}

	// arping to advocate our mac addresses
	arping("eth0", eth0.ip, eth0.gateway)
	for _, nic := range nics {
		arping(nic.name, nic.ip, nic.gateway)
	}
}

func startZvr()  {
	b := utils.Bash{
		Command: "bash -x /etc/init.d/zstack-virtualrouteragent restart >> /tmp/agentRestart.log",
	}
	b.Run()
	b.PanicIfError()
}

func main() {
	utils.InitLog("/home/vyos/zvr/zvrboot.log", false)
	waitIptablesServiceOnline()
	if isOnVMwareHypervisor() {
		parseEsxBootInfo()
	} else {
		waitVirtioPortOnline()
		parseKvmBootInfo()
	}
	configureVyos()
	startZvr()
	log.Debugf("successfully configured the sysmtem and bootstrap the zstack virtual router agents")
}
<commit_msg>add 2>&1 in execute restart agent<commit_after>package main

import (
	"zvr/utils"
	"time"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"encoding/json"
	"github.com/pkg/errors"
	"fmt"
	"zvr/server"
	"strings"
	"os"
)

const (
	VIRTIO_PORT_PATH = "/dev/virtio-ports/applianceVm.vport"
	BOOTSTRAP_INFO_CACHE = "/home/vyos/zvr/bootstrap-info.json"
	TMP_LOCATION_FOR_ESX = "/tmp/bootstrap-info.json"
)

type nic struct {
	mac string
	ip string
	name string
	netmask string
	isDefaultRoute bool
	gateway string
}

var bootstrapInfo map[string]interface{} = make(map[string]interface{})
var nics map[string]*nic = make(map[string]*nic)

func waitIptablesServiceOnline()  {
	bash := utils.Bash{
		Command: "/sbin/iptables-save",
	}

	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		err := bash.Run()
		if err != nil {
			log.Debugf("iptables service seems not ready, %v", err)
		}
		return err == nil
	}, time.Duration(120)*time.Second, time.Duration(500)*time.Millisecond)
}

func waitVirtioPortOnline() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		ok, err := utils.PathExists(VIRTIO_PORT_PATH); utils.PanicOnError(err)
		if !ok {
			log.Debugf("%s doesn't not exist, wait it ...", VIRTIO_PORT_PATH)
		}
		return ok
	}, time.Duration(120)*time.Second, time.Duration(500)*time.Millisecond)
}

func isOnVMwareHypervisor() bool {
	bash := utils.Bash{
		Command: "dmesg | grep -q 'Hypervisor.*VMware'",
	}

	if ret, _, _, err := bash.RunWithReturn(); ret == 0 && err == nil {
		return true
	}

	return false
}

func parseEsxBootInfo() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		if _, err := os.Stat(TMP_LOCATION_FOR_ESX); os.IsNotExist(err) {
			log.Debugf("bootstrap info not ready, waiting ...")
			return false
		}

		content, err := ioutil.ReadFile(TMP_LOCATION_FOR_ESX); utils.PanicOnError(err)
		if err = json.Unmarshal(content, &bootstrapInfo); err != nil {
			panic(errors.Wrap(err, fmt.Sprintf("unable to JSON parse:\n %s", string(content))))
		}

		err = utils.MkdirForFile(BOOTSTRAP_INFO_CACHE, 0666); utils.PanicOnError(err)
		err = os.Rename(TMP_LOCATION_FOR_ESX, BOOTSTRAP_INFO_CACHE); utils.PanicOnError(err)
		err = os.Chmod(BOOTSTRAP_INFO_CACHE, 0777); utils.PanicOnError(err)
		log.Debugf("recieved bootstrap info:\n%s", string(content))
		return true
	}, time.Duration(300)*time.Second, time.Duration(1)*time.Second)
}

func parseKvmBootInfo() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		content, err := ioutil.ReadFile(VIRTIO_PORT_PATH); utils.PanicOnError(err)
		if len(content) == 0 {
			log.Debugf("no content in %s, it may not be ready, wait it ...", VIRTIO_PORT_PATH)
			return false
		}

		if err := json.Unmarshal(content, &bootstrapInfo); err != nil {
			panic(errors.Wrap(err, fmt.Sprintf("unable to JSON parse:\n %s", string(content))))
		}

		err = utils.MkdirForFile(BOOTSTRAP_INFO_CACHE, 0666); utils.PanicOnError(err)
		err = ioutil.WriteFile(BOOTSTRAP_INFO_CACHE, content, 0666); utils.PanicOnError(err)
		err = os.Chmod(BOOTSTRAP_INFO_CACHE, 0777); utils.PanicOnError(err)
		log.Debugf("recieved bootstrap info:\n%s", string(content))
		return true
	}, time.Duration(300)*time.Second, time.Duration(1)*time.Second)
}

func resetVyos()  {
	// clear all configuration in case someone runs 'save' command manually before,
	// to keep the vyos must be stateless

	// delete all interfaces
	tree := server.NewParserFromShowConfiguration().Tree
	tree.Delete("interfaces ethernet")
	tree.Apply(true)

	// reload default configuration
	server.RunVyosScriptAsUserVyos("load /opt/vyatta/etc/config.boot.default\nsave")
}

func configureVyos()  {
	resetVyos()

	mgmtNic := bootstrapInfo["managementNic"].(map[string]interface{})
	if mgmtNic == nil {
		panic(errors.New("no field 'managementNic' in bootstrap info"))
	}

	eth0 := &nic{ name: "eth0" }
	var ok bool
	eth0.mac, ok = mgmtNic["mac"].(string); utils.PanicIfError(ok, errors.New("cannot find 'mac' field for the management nic"))
	eth0.netmask, ok = mgmtNic["netmask"].(string); utils.PanicIfError(ok, errors.New("cannot find 'netmask' field for the management nic"))
	eth0.ip, ok = mgmtNic["ip"].(string); utils.PanicIfError(ok, errors.New("cannot find 'ip' field for the management nic"))
	eth0.isDefaultRoute = mgmtNic["isDefaultRoute"].(bool)
	eth0.gateway = mgmtNic["gateway"].(string)
	nics[eth0.name] = eth0

	otherNics := bootstrapInfo["additionalNics"].([]interface{})
	if otherNics != nil {
		for _, o := range otherNics {
			onic := o.(map[string]interface{})
			n := &nic{}
			n.name, ok = onic["deviceName"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'deviceName' field for the nic"))
			n.mac, ok = onic["mac"].(string); utils.PanicIfError(ok, errors.New("cannot find 'mac' field for the nic"))
			n.netmask, ok = onic["netmask"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'netmask' field for the nic[name:%s]", n.name))
			n.ip, ok = onic["ip"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'ip' field for the nic[name:%s]", n.name))
			n.gateway = onic["gateway"].(string)
			n.isDefaultRoute = onic["isDefaultRoute"].(bool)
			nics[n.name] = n
		}
	}

	type deviceName struct {
		expected string
		actual string
		swap string
	}

	devNames := make([]*deviceName, 0)

	// check integrity of nics
	for _, nic := range nics {
		utils.Assertf(nic.name != "", "name cannot be empty[mac:%s]", nic.mac)
		utils.Assertf(nic.ip != "", "ip cannot be empty[nicname: %s]", nic.name)
		utils.Assertf(nic.gateway != "", "gateway cannot be empty[nicname:%s]", nic.name)
		utils.Assertf(nic.netmask != "", "netmask cannot be empty[nicname:%s]", nic.name)
		utils.Assertf(nic.mac != "", "mac cannot be empty[nicname:%s]", nic.name)

		nicname, err := utils.GetNicNameByMac(nic.mac); utils.PanicOnError(err)
		if nicname != nic.name {
			devNames = append(devNames, &deviceName{
				expected: nic.name,
				actual: nicname,
			})
		}
	}

	if len(devNames) != 0 {
		// shutdown links and change to temporary names
		cmds := make([]string, 0)
		for i, devname := range devNames {
			devnum := 1000 + i

			devname.swap = fmt.Sprintf("eth%v", devnum)
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v down", devname.actual))
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v name %v", devname.actual, devname.swap))
		}

		b := utils.Bash{
			Command: strings.Join(cmds, "\n"),
		}

		b.Run()
		b.PanicIfError()

		// change temporary names to real names and bring up links
		cmds = make([]string, 0)
		for _, devname := range devNames {
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v name %v", devname.swap, devname.expected))
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v up", devname.expected))
		}

		b = utils.Bash{
			Command: strings.Join(cmds, "\n"),
		}

		b.Run()
		b.PanicIfError()
	}

	vyos := server.NewParserFromShowConfiguration()
	tree := vyos.Tree

	sshkey := bootstrapInfo["publicKey"].(string)
	utils.Assert(sshkey != "", "cannot find 'publicKey' in bootstrap info")
	sshkeyparts := strings.Split(sshkey, " ")
	sshtype := sshkeyparts[0]
	key := sshkeyparts[1]
	id := sshkeyparts[2]

	tree.Setf("system login user vyos authentication public-keys %s key %s", id, key)
	tree.Setf("system login user vyos authentication public-keys %s type %s", id, sshtype)

	setNic := func(nic *nic) {
		cidr, err := utils.NetmaskToCIDR(nic.netmask); utils.PanicOnError(err)
		//tree.Setf("interfaces ethernet %s hw-id %s", nic.name, nic.mac)
		tree.Setf("interfaces ethernet %s address %s", nic.name, fmt.Sprintf("%v/%v", nic.ip, cidr))
		tree.Setf("interfaces ethernet %s duplex auto", nic.name)
		tree.Setf("interfaces ethernet %s smp_affinity auto", nic.name)
		tree.Setf("interfaces ethernet %s speed auto", nic.name)
		if nic.isDefaultRoute {
			tree.Setf("system gateway-address %v", nic.gateway)
		}
	}

	sshport := bootstrapInfo["sshPort"].(float64)
	utils.Assert(sshport != 0, "sshport not found in bootstrap info")
	tree.Setf("service ssh port %v", int(sshport))
	tree.Setf("service ssh listen-address %v", eth0.ip)

	// configure firewall
	for _, nic := range nics {
		setNic(nic)

		tree.SetFirewallOnInterface(nic.name, "local",
			"action accept",
			"state established enable",
			"state related enable",
			fmt.Sprintf("destination address %v", nic.ip),
		)
		tree.SetFirewallOnInterface(nic.name, "local",
			"action accept",
			"protocol icmp",
			fmt.Sprintf("destination address %v", nic.ip),
		)

		if !nic.isDefaultRoute && nic.name != "eth0" {
			// for nic connecting to the guest network
			// make its FORWARDING chain open
			// here in = "-A FORWARDING -i nic.name -j ACCEPT"
			tree.SetFirewallOnInterface(nic.name, "in",
				"action accept",
				"state established enable",
				"state related enable",
				"state new enable",
			)
		} else {
			tree.SetFirewallOnInterface(nic.name, "in",
				"action accept",
				"state established enable",
				"state related enable",
			)
		}

		tree.SetFirewallOnInterface(nic.name, "in",
			"action accept",
			"protocol icmp",
		)

		// only allow ssh traffic on eth0, disable on others
		if nic.name == "eth0" {
			tree.SetFirewallOnInterface(nic.name, "local",
				fmt.Sprintf("destination port %v", int(sshport)),
				fmt.Sprintf("destination address %v", nic.ip),
				"protocol tcp",
				"action accept",
			)
		} else {
			tree.SetFirewallOnInterface(nic.name, "local",
				fmt.Sprintf("destination port %v", int(sshport)),
				fmt.Sprintf("destination address %v", nic.ip),
				"protocol tcp",
				"action reject",
			)
		}

		tree.SetFirewallDefaultAction(nic.name, "local", "reject")
		tree.SetFirewallDefaultAction(nic.name, "in", "reject")

		tree.AttachFirewallToInterface(nic.name, "local")
		tree.AttachFirewallToInterface(nic.name, "in")
	}

	tree.Set("system time-zone Asia/Shanghai")

	password := bootstrapInfo["vyosPassword"]; utils.Assert(password != "", "vyosPassword cannot be empty")
	if !isOnVMwareHypervisor() {
		tree.Setf("system login user vyos authentication plaintext-password %v", password)
	}

	tree.Apply(true)

	arping := func(nicname, ip, gateway string) {
		b := utils.Bash{ Command: fmt.Sprintf("arping -A -U -c 1 -I %s -s %s %s", nicname, ip, gateway) }
		b.Run()
	}

	// arping to advocate our mac addresses
	arping("eth0", eth0.ip, eth0.gateway)
	for _, nic := range nics {
		arping(nic.name, nic.ip, nic.gateway)
	}
}

func startZvr()  {
	b := utils.Bash{
		Command: "bash -x /etc/init.d/zstack-virtualrouteragent restart >> /tmp/agentRestart.log 2>&1",
	}
	b.Run()
	b.PanicIfError()
}

func main() {
	utils.InitLog("/home/vyos/zvr/zvrboot.log", false)
	waitIptablesServiceOnline()
	if isOnVMwareHypervisor() {
		parseEsxBootInfo()
	} else {
		waitVirtioPortOnline()
		parseKvmBootInfo()
	}
	configureVyos()
	startZvr()
	log.Debugf("successfully configured the sysmtem and bootstrap the zstack virtual router agents")
}
<|endoftext|>


<commit_before>package library

import (
	"container/heap"
	"errors"
	"github.com/nytlabs/gojee"
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"
	"time"
)

// specify those channels we're going to use to communicate with streamtools
type Pack struct {
	blocks.Block
	queryrule chan blocks.MsgChan
	inrule    blocks.MsgChan
	in        blocks.MsgChan
	out       blocks.MsgChan
	quit      blocks.MsgChan
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewPack() blocks.BlockInterface {
	return &Pack{}
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *Pack) Setup() {
	b.Kind = "Pack"
	b.Desc = "groups messages together based on a common value, similar to 'group-by' in other languages"
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.quit = b.Quit()
	b.out = b.Broadcast()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Pack) Run() {
	var tree *jee.TokenTree
	var emitAfter, path string
	var err error

	afterDuration := time.Duration(0)
	waitTimer := time.NewTimer(100 * time.Millisecond)
	bunches := make(map[string][]interface{})
	pq := &PriorityQueue{}
	heap.Init(pq)

	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			rule, ok := ruleI.(map[string]interface{})
			if !ok {
				b.Error(errors.New("coudln't assert rule to map"))
				continue
			}
			path, err = util.ParseString(rule, "Path")
			if err != nil {
				b.Error(err)
				continue
			}
			token, err := jee.Lexer(path)
			if err != nil {
				b.Error(err)
				continue
			}
			tree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				continue
			}
			emitAfter, err = util.ParseString(rule, "EmitAfter")
			if err != nil {
				b.Error(err)
				continue
			}
			afterDuration, err = time.ParseDuration(emitAfter)
			if err != nil {
				b.Error(err)
				continue
			}

		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			// deal with inbound data
			if tree == nil {
				break
			}

			id, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				continue
			}
			idStr, ok := id.(string)
			if !ok {
				b.Error(errors.New("could not assert id to string"))
				break
			}
			if len(bunches[idStr]) > 0 {
				bunches[idStr] = append(bunches[idStr], msg)
			} else {
				bunches[idStr] = []interface{}{msg}
			}

			val := map[string]interface{}{
				"id":     idStr,
				"length": len(bunches[idStr]),
			}

			queueMessage := &PQMessage{
				val: val,
				t:   time.Now(),
			}
			heap.Push(pq, queueMessage)
		case c := <-b.queryrule:
			// deal with a query request
			c <- map[string]interface{}{
				"Path":      path,
				"EmitAfter": emitAfter,
			}
		}
		for {
			pqMsg, diff := pq.PeekAndShift(time.Now(), afterDuration)
			if pqMsg == nil {
				// either the queue is empty, or it's not time to emit
				waitTimer.Reset(diff)
				break
			}
			v := pqMsg.(*PQMessage).val.(map[string]interface{})

			l, ok := v["length"]
			if !ok {
				b.Error(errors.New("couldn't find length in message"))
				continue
			}
			lInt := l.(int)
			id, ok := v["id"]
			if !ok {
				b.Error(errors.New("couldn't find id in message"))
				continue
			}
			idStr := id.(string)
			if lInt == len(bunches[idStr]) {
				// we've not seen anything since putting this message in the queue

				msg := map[string]interface{}{
					"pack": bunches[idStr],
				}

				b.out <- msg

				delete(bunches, idStr)
			}
		}
	}
}
<commit_msg>quick fix to pack<commit_after>package library

import (
	"container/heap"
	"errors"
	"github.com/nytlabs/gojee"
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"
	"time"
)

// specify those channels we're going to use to communicate with streamtools
type Pack struct {
	blocks.Block
	queryrule chan blocks.MsgChan
	inrule    blocks.MsgChan
	in        blocks.MsgChan
	out       blocks.MsgChan
	quit      blocks.MsgChan
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewPack() blocks.BlockInterface {
	return &Pack{}
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *Pack) Setup() {
	b.Kind = "Pack"
	b.Desc = "groups messages together based on a common value, similar to 'group-by' in other languages"
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.quit = b.Quit()
	b.out = b.Broadcast()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Pack) Run() {
	var tree *jee.TokenTree
	var emitAfter, path string
	var err error

	afterDuration := time.Duration(0)
	waitTimer := time.NewTimer(100 * time.Millisecond)
	bunches := make(map[string][]interface{})
	pq := &PriorityQueue{}
	heap.Init(pq)

	for {
		select {
		case <-waitTimer.C:
		case ruleI := <-b.inrule:
			// set a parameter of the block
			rule, ok := ruleI.(map[string]interface{})
			if !ok {
				b.Error(errors.New("coudln't assert rule to map"))
				continue
			}
			path, err = util.ParseString(rule, "Path")
			if err != nil {
				b.Error(err)
				continue
			}
			token, err := jee.Lexer(path)
			if err != nil {
				b.Error(err)
				continue
			}
			tree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				continue
			}
			emitAfter, err = util.ParseString(rule, "EmitAfter")
			if err != nil {
				b.Error(err)
				continue
			}
			afterDuration, err = time.ParseDuration(emitAfter)
			if err != nil {
				b.Error(err)
				continue
			}

		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			// deal with inbound data
			if tree == nil {
				break
			}

			id, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				continue
			}
			idStr, ok := id.(string)
			if !ok {
				b.Error(errors.New("could not assert id to string"))
				break
			}
			if len(bunches[idStr]) > 0 {
				bunches[idStr] = append(bunches[idStr], msg)
			} else {
				bunches[idStr] = []interface{}{msg}
			}

			val := map[string]interface{}{
				"id":     idStr,
				"length": len(bunches[idStr]),
			}

			queueMessage := &PQMessage{
				val: val,
				t:   time.Now(),
			}
			heap.Push(pq, queueMessage)
		case c := <-b.queryrule:
			// deal with a query request
			c <- map[string]interface{}{
				"Path":      path,
				"EmitAfter": emitAfter,
			}
		}
		for {
			pqMsg, diff := pq.PeekAndShift(time.Now(), afterDuration)
			if pqMsg == nil {
				// either the queue is empty, or it's not time to emit
				waitTimer.Reset(diff)
				break
			}
			v := pqMsg.(*PQMessage).val.(map[string]interface{})

			l, ok := v["length"]
			if !ok {
				b.Error(errors.New("couldn't find length in message"))
				continue
			}
			lInt := l.(int)
			id, ok := v["id"]
			if !ok {
				b.Error(errors.New("couldn't find id in message"))
				continue
			}
			idStr := id.(string)
			if lInt == len(bunches[idStr]) {
				// we've not seen anything since putting this message in the queue

				msg := map[string]interface{}{
					"pack": bunches[idStr],
				}

				b.out <- msg

				delete(bunches, idStr)
			}
		}
	}
}
<|endoftext|>


<commit_before>package state

import (
	"fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/txn"
	"strings"
)

// annotatorDoc represents the internal state of annotations for an Entity in
// MongoDB. Note that the annotations map is not maintained in local storage
// due to the fact that it is not accessed directly, but through
// Annotations/Annotation below.
type annotatorDoc struct {
	GlobalKey   string `bson:"_id"`
	EntityName  string
	Annotations map[string]string
}

// annotator implements annotation-related methods
// for any entity that wishes to use it.
type annotator struct {
	globalKey  string
	entityName string
	st         *State
}

// SetAnnotations adds key/value pairs to annotations in MongoDB.
func (a *annotator) SetAnnotations(pairs map[string]string) error {
	if len(pairs) == 0 {
		return nil
	}
	// Collect in separate maps pairs to be inserted/updated or removed.
	toRemove := make(map[string]bool)
	toInsert := make(map[string]string)
	toUpdate := make(map[string]string)
	for key, value := range pairs {
		if strings.Contains(key, ".") {
			return fmt.Errorf("invalid key %q", key)
		}
		if value == "" {
			toRemove["annotations."+key] = true
		} else {
			toInsert[key] = value
			toUpdate["annotations."+key] = value
		}
	}
	// Two attempts should be enough to update annotations even with racing
	// clients - if the document does not already exist, one of the clients
	// will create it and the others will fail, then all the rest of the
	// clients should succeed on their second attempt. If the referred-to
	// entity has disappeared, and removed its annotations in the meantime,
	// we consider that worthy of an error (will be fixed when new entities
	// can never share names with old ones).
	for i := 0; i < 2; i++ {
		ops, err := a.setAnnotationsOps(toInsert, toUpdate, toRemove)
		if err != nil {
			return err
		}
		if err := a.st.runner.Run(ops, "", nil); err == nil {
			return nil
		} else if err != txn.ErrAborted {
			return fmt.Errorf("cannot update annotations on %s: %v", a.entityName, err)
		}
		// Check that the annotator entity was not previously destroyed.
		if count, err := a.st.annotations.FindId(a.globalKey).Count(); err != nil {
			return err
		} else if count == 0 {
			return fmt.Errorf("%s no longer exists", a.entityName)
		}
	}
	return fmt.Errorf("cannot update annotations on %s: %v", a.entityName, ErrExcessiveContention)
}

// setAnnotationsOps returns the operations required to insert, update or
// remove annotations in MongoDB.
func (a *annotator) setAnnotationsOps(toInsert, toUpdate map[string]string, toRemove map[string]bool) ([]txn.Op, error) {
	id := a.globalKey
	coll := a.st.annotations.Name
	entityName := a.entityName
	if count, err := a.st.annotations.FindId(id).Count(); err != nil {
		return nil, err
	} else if count == 0 {
		// The document is missing: no need to remove pairs.
		var ops []txn.Op
		// If the entity is not the environment, add a DocExists check on the
		// entity document, in order to avoid possible races between entity
		// removal and annotation creation.
		if !strings.HasPrefix(entityName, "environment-") {
			ecoll, eid, err := a.st.ParseEntityName(entityName)
			if err != nil {
				return nil, err
			}
			entityOp := txn.Op{
				C:      ecoll,
				Id:     eid,
				Assert: txn.DocExists,
			}
			ops = append(ops, entityOp)
		}
		// Insert pairs as required.
		insertOp := txn.Op{
			C:      coll,
			Id:     id,
			Assert: txn.DocMissing,
			Insert: &annotatorDoc{id, a.entityName, toInsert},
		}
		return append(ops, insertOp), nil
	}
	// The document exists: update and remove pairs.
	return []txn.Op{{
		C:      coll,
		Id:     id,
		Assert: txn.DocExists,
		Update: D{{"$set", toUpdate}, {"$unset", toRemove}},
	}}, nil
}

// Annotations returns all the annotations corresponding to an entity.
func (a *annotator) Annotations() (map[string]string, error) {
	doc := new(annotatorDoc)
	err := a.st.annotations.FindId(a.globalKey).One(doc)
	if err == mgo.ErrNotFound {
		// Returning an empty map if there are no annotations.
		return make(map[string]string), nil
	}
	if err != nil {
		return nil, err
	}
	return doc.Annotations, nil
}

// Annotation returns the annotation value corresponding to the given key.
// If the requested annotation is not found, an empty string is returned.
func (a *annotator) Annotation(key string) (string, error) {
	ann, err := a.Annotations()
	if err != nil {
		return "", err
	}
	return ann[key], nil
}

// annotationRemoveOp returns an operation to remove a given annotation
// document from MongoDB.
func annotationRemoveOp(st *State, id string) txn.Op {
	return txn.Op{
		C:      st.annotations.Name,
		Id:     id,
		Remove: true,
	}
}
<commit_msg>Changes as per review.<commit_after>package state

import (
	"fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/txn"
	"strings"
)

// annotatorDoc represents the internal state of annotations for an Entity in
// MongoDB. Note that the annotations map is not maintained in local storage
// due to the fact that it is not accessed directly, but through
// Annotations/Annotation below.
type annotatorDoc struct {
	GlobalKey   string `bson:"_id"`
	EntityName  string
	Annotations map[string]string
}

// annotator implements annotation-related methods
// for any entity that wishes to use it.
type annotator struct {
	globalKey  string
	entityName string
	st         *State
}

// SetAnnotations adds key/value pairs to annotations in MongoDB.
func (a *annotator) SetAnnotations(pairs map[string]string) error {
	if len(pairs) == 0 {
		return nil
	}
	// Collect in separate maps pairs to be inserted/updated or removed.
	toRemove := make(map[string]bool)
	toInsert := make(map[string]string)
	toUpdate := make(map[string]string)
	for key, value := range pairs {
		if strings.Contains(key, ".") {
			return fmt.Errorf("invalid key %q", key)
		}
		if value == "" {
			toRemove["annotations."+key] = true
		} else {
			toInsert[key] = value
			toUpdate["annotations."+key] = value
		}
	}
	// Two attempts should be enough to update annotations even with racing
	// clients - if the document does not already exist, one of the clients
	// will create it and the others will fail, then all the rest of the
	// clients should succeed on their second attempt. If the referred-to
	// entity has disappeared, and removed its annotations in the meantime,
	// we consider that worthy of an error (will be fixed when new entities
	// can never share names with old ones).
	for i := 0; i < 2; i++ {
		var ops []txn.Op
		if count, err := a.st.annotations.FindId(a.globalKey).Count(); err != nil {
			return err
		} else if count == 0 {
			// Check that the annotator entity was not previously destroyed.
			if i != 0 {
				return fmt.Errorf("%s no longer exists", a.entityName)
			}
			ops, err = a.insertOps(toInsert)
			if err != nil {
				return err
			}
		} else {
			ops = a.updateOps(toUpdate, toRemove)
		}
		if err := a.st.runner.Run(ops, "", nil); err == nil {
			return nil
		} else if err != txn.ErrAborted {
			return fmt.Errorf("cannot update annotations on %s: %v", a.entityName, err)
		}
	}
	return fmt.Errorf("cannot update annotations on %s: %v", a.entityName, ErrExcessiveContention)
}

// insertOps returns the operations required to insert annotations in MongoDB.
func (a *annotator) insertOps(toInsert map[string]string) ([]txn.Op, error) {
	var ops []txn.Op
	entityName := a.entityName
	// If the entity is not the environment, add a DocExists check on the
	// entity document, in order to avoid possible races between entity
	// removal and annotation creation.
	if !strings.HasPrefix(entityName, "environment-") {
		coll, id, err := a.st.ParseEntityName(entityName)
		if err != nil {
			return nil, err
		}
		ops = append(
			ops,
			txn.Op{
				C:      coll,
				Id:     id,
				Assert: txn.DocExists,
			},
		)
	}
	return append(
		ops,
		txn.Op{
			C:      a.st.annotations.Name,
			Id:     a.globalKey,
			Assert: txn.DocMissing,
			Insert: &annotatorDoc{a.globalKey, entityName, toInsert},
		},
	), nil
}

// updateOps returns the operations required to update or remove annotations in MongoDB.
func (a *annotator) updateOps(toUpdate map[string]string, toRemove map[string]bool) []txn.Op {
	return []txn.Op{{
		C:      a.st.annotations.Name,
		Id:     a.globalKey,
		Assert: txn.DocExists,
		Update: D{{"$set", toUpdate}, {"$unset", toRemove}},
	}}
}

// Annotations returns all the annotations corresponding to an entity.
func (a *annotator) Annotations() (map[string]string, error) {
	doc := new(annotatorDoc)
	err := a.st.annotations.FindId(a.globalKey).One(doc)
	if err == mgo.ErrNotFound {
		// Returning an empty map if there are no annotations.
		return make(map[string]string), nil
	}
	if err != nil {
		return nil, err
	}
	return doc.Annotations, nil
}

// Annotation returns the annotation value corresponding to the given key.
// If the requested annotation is not found, an empty string is returned.
func (a *annotator) Annotation(key string) (string, error) {
	ann, err := a.Annotations()
	if err != nil {
		return "", err
	}
	return ann[key], nil
}

// annotationRemoveOp returns an operation to remove a given annotation
// document from MongoDB.
func annotationRemoveOp(st *State, id string) txn.Op {
	return txn.Op{
		C:      st.annotations.Name,
		Id:     id,
		Remove: true,
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package gogs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

// Permission represents a API permission.
type Permission struct {
	Admin bool `json:"admin"`
	Push  bool `json:"push"`
	Pull  bool `json:"pull"`
}

// Repository represents a API repository.
type Repository struct {
	Id          int64      `json:"id"`
	Owner       User       `json:"owner"`
	FullName    string     `json:"full_name"`
	Private     bool       `json:"private"`
	Fork        bool       `json:"fork"`
	HtmlUrl     string     `json:"html_url"`
	CloneUrl    string     `json:"clone_url"`
	SshUrl      string     `json:"ssh_url"`
	Permissions Permission `json:"permissions"`
}

// ListMyRepos lists all repositories for the authenticated user that has access to.
func (c *Client) ListMyRepos() ([]*Repository, error) {
	repos := make([]*Repository, 0, 10)
	err := c.getParsedResponse("GET", "/user/repos", nil, nil, &repos)
	return repos, err
}

type CreateRepoOption struct {
	Name        string `json:"name" binding:"Required"`
	Description string `json:"description" binding:"MaxSize(255)"`
	Private     bool   `form:"private"`
	AutoInit    bool   `form:"auto_init"`
	Gitignore   string `form:"gitignore"`
	License     string `form:"license"`
}

// CreateRepo creates a repository for authenticated user.
func (c *Client) CreateRepo(opt CreateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", "/user/repos",
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}

// CreateOrgRepo creates an organization repository for authenticated user.
func (c *Client) CreateOrgRepo(org string, opt CreateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", fmt.Sprintf("/org/%s/repos", org),
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}
<commit_msg>fix json tag<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package gogs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

// Permission represents a API permission.
type Permission struct {
	Admin bool `json:"admin"`
	Push  bool `json:"push"`
	Pull  bool `json:"pull"`
}

// Repository represents a API repository.
type Repository struct {
	Id          int64      `json:"id"`
	Owner       User       `json:"owner"`
	FullName    string     `json:"full_name"`
	Private     bool       `json:"private"`
	Fork        bool       `json:"fork"`
	HtmlUrl     string     `json:"html_url"`
	CloneUrl    string     `json:"clone_url"`
	SshUrl      string     `json:"ssh_url"`
	Permissions Permission `json:"permissions"`
}

// ListMyRepos lists all repositories for the authenticated user that has access to.
func (c *Client) ListMyRepos() ([]*Repository, error) {
	repos := make([]*Repository, 0, 10)
	err := c.getParsedResponse("GET", "/user/repos", nil, nil, &repos)
	return repos, err
}

type CreateRepoOption struct {
	Name        string `json:"name" binding:"Required"`
	Description string `json:"description" binding:"MaxSize(255)"`
	Private     bool   `json:"private"`
	AutoInit    bool   `json:"auto_init"`
	Gitignore   string `json:"gitignore"`
	License     string `json:"license"`
}

// CreateRepo creates a repository for authenticated user.
func (c *Client) CreateRepo(opt CreateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", "/user/repos",
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}

// CreateOrgRepo creates an organization repository for authenticated user.
func (c *Client) CreateOrgRepo(org string, opt CreateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", fmt.Sprintf("/org/%s/repos", org),
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/fsouza/go-dockerclient"
	"github.com/joho/godotenv"
	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
	"github.com/wantedly/risu/shell"
)

const (
	CacheBaseDir         = "/var/risu/cache"
	DefaultDockerEndpoint = "unix:///var/run/docker.sock"
	DefaultSourceBaseDir  = "/var/risu/src/github.com/"
)

var ren = render.New()
var reg = registry.NewRegistry(os.Getenv("REGISTRY_BACKEND"), os.Getenv("REGISTRY_ENDPOINT"))

func loadEnv() {
	err := godotenv.Load()
	if err != nil {
		log.Print("Error loading .env file")
	}
}

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}

	build := schema.NewBuild(opts)
	err = reg.Set(build)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusAccepted, build)

	go func() {
		if err := checkoutGitRepository(build, DefaultSourceBaseDir); err != nil {
			return
		}

		if err := dockerBuild(build); err != nil {
			return
		}

		go dockerPush(build)
		go pushCache(build)
	}()
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
		return
	}
	ren.JSON(w, http.StatusOK, build)
}

// Clone run "git clone <repository_URL>" and "git checkout branch"
func checkoutGitRepository(build schema.Build, dir string) error {
	if _, err := os.Stat(dir); err != nil {
		if err := os.MkdirAll(dir, 0755); err != nil {
			log.Fatal(err)
		}
	}

	// htpps://<token>@github.com/<SourceRepo>.git
	cloneURL := "https://" + os.Getenv("GITHUB_ACCESS_TOKEN") + "@github.com/" + build.SourceRepo + ".git"

	// debug
	fmt.Println(cloneURL)

	clonePath := dir + build.SourceRepo

	// debug
	fmt.Println(clonePath)

	shell.Command("git", "clone", cloneURL, clonePath)
	shell.CommandInDir(clonePath, "git", "fetch", "origin", build.SourceBranch)
	shell.CommandInDir(clonePath, "git", "checkout", "remotes/origin/"+build.SourceBranch, "-f")
	return nil
}
func dockerBuild(build schema.Build) error {
	// TODO (@dtan4)
	return nil
}

func dockerPush(build schema.Build) error {
	// TODO (@koudaii)

	var dockerEndpoint string

	if os.Getenv("DOCKER_HOST") != "" {
		dockerEndpoint = os.Getenv("DOCKER_HOST")
	}

	if dockerEndpoint == "" {
		dockerEndpoint = DefaultDockerEndpoint
	}

	client, err := docker.NewClient(dockerEndpoint)

	if err != nil {
		return err
	}

	nameTag := strings.Split(build.ImageName, ":")
	dockerRegistry := strings.Split(build.ImageName, "/")

	outputbuf := bytes.NewBuffer(nil)
	// push build image
	pushOpts := docker.PushImageOptions{
		Name:         nameTag[0],
		Tag:          nameTag[1],
		Registry:     dockerRegistry[0],
		OutputStream: outputbuf,
	}
	authConfig := docker.AuthConfiguration{
		Username:      os.Getenv("DOCKER_AUTH_USER_NAME"),
		Password:      os.Getenv("DOCKER_AUTH_USER_PASSWORD"),
		Email:         os.Getenv("DOCKER_AUTH_USER_EMAIL"),
		ServerAddress: dockerRegistry[0],
	}
	if err := client.PushImage(pushOpts, authConfig); err != nil {
		return err
	}
	os.Stdout.Write(outputbuf.Bytes())
	return nil
}

func pushCache(build schema.Build) error {
	// TODO (@dtan4)
	return nil
}

func setUpServer() *negroni.Negroni {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	return n
}

func main() {
	loadEnv()
	if os.Getenv("GITHUB_ACCESS_TOKEN") == "" {
		log.Fatal("Please provide 'GITHUB_ACCESS_TOKEN' through environment")
	}
	n := setUpServer()
	n.Run(":8080")
}
<commit_msg>Fix Cache base dir<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/fsouza/go-dockerclient"
	"github.com/joho/godotenv"
	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
	"github.com/wantedly/risu/shell"
)

const (
	CacheBaseDir         = "/var/risu/cache/"
	DefaultDockerEndpoint = "unix:///var/run/docker.sock"
	DefaultSourceBaseDir  = "/var/risu/src/github.com/"
)

var ren = render.New()
var reg = registry.NewRegistry(os.Getenv("REGISTRY_BACKEND"), os.Getenv("REGISTRY_ENDPOINT"))

func loadEnv() {
	err := godotenv.Load()
	if err != nil {
		log.Print("Error loading .env file")
	}
}

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}

	build := schema.NewBuild(opts)
	err = reg.Set(build)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusAccepted, build)

	go func() {
		if err := checkoutGitRepository(build, DefaultSourceBaseDir); err != nil {
			return
		}

		if err := dockerBuild(build); err != nil {
			return
		}

		go dockerPush(build)
		go pushCache(build)
	}()
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
		return
	}
	ren.JSON(w, http.StatusOK, build)
}

// Clone run "git clone <repository_URL>" and "git checkout branch"
func checkoutGitRepository(build schema.Build, dir string) error {
	if _, err := os.Stat(dir); err != nil {
		if err := os.MkdirAll(dir, 0755); err != nil {
			log.Fatal(err)
		}
	}

	// htpps://<token>@github.com/<SourceRepo>.git
	cloneURL := "https://" + os.Getenv("GITHUB_ACCESS_TOKEN") + "@github.com/" + build.SourceRepo + ".git"

	// debug
	fmt.Println(cloneURL)

	clonePath := dir + build.SourceRepo

	// debug
	fmt.Println(clonePath)

	shell.Command("git", "clone", cloneURL, clonePath)
	shell.CommandInDir(clonePath, "git", "fetch", "origin", build.SourceBranch)
	shell.CommandInDir(clonePath, "git", "checkout", "remotes/origin/"+build.SourceBranch, "-f")
	return nil
}
func dockerBuild(build schema.Build) error {
	// TODO (@dtan4)
	return nil
}

func dockerPush(build schema.Build) error {
	// TODO (@koudaii)

	var dockerEndpoint string

	if os.Getenv("DOCKER_HOST") != "" {
		dockerEndpoint = os.Getenv("DOCKER_HOST")
	}

	if dockerEndpoint == "" {
		dockerEndpoint = DefaultDockerEndpoint
	}

	client, err := docker.NewClient(dockerEndpoint)

	if err != nil {
		return err
	}

	nameTag := strings.Split(build.ImageName, ":")
	dockerRegistry := strings.Split(build.ImageName, "/")

	outputbuf := bytes.NewBuffer(nil)
	// push build image
	pushOpts := docker.PushImageOptions{
		Name:         nameTag[0],
		Tag:          nameTag[1],
		Registry:     dockerRegistry[0],
		OutputStream: outputbuf,
	}
	authConfig := docker.AuthConfiguration{
		Username:      os.Getenv("DOCKER_AUTH_USER_NAME"),
		Password:      os.Getenv("DOCKER_AUTH_USER_PASSWORD"),
		Email:         os.Getenv("DOCKER_AUTH_USER_EMAIL"),
		ServerAddress: dockerRegistry[0],
	}
	if err := client.PushImage(pushOpts, authConfig); err != nil {
		return err
	}
	os.Stdout.Write(outputbuf.Bytes())
	return nil
}

func pushCache(build schema.Build) error {
	// TODO (@dtan4)
	return nil
}

func setUpServer() *negroni.Negroni {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	return n
}

func main() {
	loadEnv()
	if os.Getenv("GITHUB_ACCESS_TOKEN") == "" {
		log.Fatal("Please provide 'GITHUB_ACCESS_TOKEN' through environment")
	}
	n := setUpServer()
	n.Run(":8080")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/libgit2/git2go"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

const (
	SourceBasePath = "/var/risu/src/github.com/"
	CacheBasePath  = "/var/risu/cache"
)

var ren = render.New()
var reg = registry.NewRegistry(os.Getenv("REGISTRY_BACKEND"), os.Getenv("REGISTRY_ENDPOINT"))

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}

	build := schema.NewBuild(opts)
	err = reg.Set(build)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusAccepted, build)

	go func() {
		if err := gitClone(build); err != nil {
			return
		}

		if err := dockerBuild(build); err != nil {
			return
		}

		go dockerPush(build)
		go pushCache(build)
	}()
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
		return
	}
	ren.JSON(w, http.StatusOK, build)
}

// Clone run "git clone <repository_URL>" and "git checkout branch"
func gitClone(build schema.Build) error {
	if _, err := os.Stat(SourceBasePath); err != nil {
		os.MkdirAll(SourceBasePath, 0755)
	}

	// htpps://<token>@github.com/<SourceRepo>.git
	cloneURL := "https://" + os.Getenv("GITHUB_ACCESS_TOKEN") + "@github.com/" + build.SourceRepo + ".git"

	// debug
	fmt.Println(cloneURL)

	clonePath := SourceBasePath + build.SourceRepo

	// debug
	fmt.Println(clonePath)

	_, err := git.Clone(cloneURL, clonePath, &git.CloneOptions{CheckoutBranch: build.SourceBranch})
	if err != nil {
		return err
	}
	return nil
}

func dockerBuild(build schema.Build) error {
	// TODO (@dtan4)
	return nil
}

func dockerPush(build schema.Build) error {
	// TODO (@koudaii)
	return nil
}

func pushCache(build schema.Build) error {
	// TODO (@dtan4)
	return nil
}

func setUpServer() *negroni.Negroni {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	return n
}

func main() {
	if os.Getenv("GITHUB_ACCESS_TOKEN") == "" {
		log.Fatal("Please provide 'GITHUB_ACCESS_TOKEN' through environment")
	}
	n := setUpServer()
	n.Run(":8080")
}
<commit_msg>Replace git clone to use shell script instead of libgit2<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
	"github.com/wantedly/risu/shell"
)

const (
	SourceBasePath = "/var/risu/src/github.com/"
	CacheBasePath  = "/var/risu/cache"
)

var ren = render.New()
var reg = registry.NewRegistry(os.Getenv("REGISTRY_BACKEND"), os.Getenv("REGISTRY_ENDPOINT"))

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}

	build := schema.NewBuild(opts)
	err = reg.Set(build)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusAccepted, build)

	go func() {
		if err := gitClone(build); err != nil {
			return
		}

		if err := dockerBuild(build); err != nil {
			return
		}

		go dockerPush(build)
		go pushCache(build)
	}()
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
		return
	}
	ren.JSON(w, http.StatusOK, build)
}

// Clone run "git clone <repository_URL>" and "git checkout branch"
func gitClone(build schema.Build) error {
	if _, err := os.Stat(SourceBasePath); err != nil {
		os.MkdirAll(SourceBasePath, 0755)
	}

	// htpps://<token>@github.com/<SourceRepo>.git
	cloneURL := "https://" + os.Getenv("GITHUB_ACCESS_TOKEN") + "@github.com/" + build.SourceRepo + ".git"

	// debug
	fmt.Println(cloneURL)

	clonePath := SourceBasePath + build.SourceRepo

	// debug
	fmt.Println(clonePath)

	_, err := shell.Command("git", "clone", cloneURL, clonePath)
	if err != nil {
		return err
	}
	return nil
}

func dockerBuild(build schema.Build) error {
	// TODO (@dtan4)
	return nil
}

func dockerPush(build schema.Build) error {
	// TODO (@koudaii)
	return nil
}

func pushCache(build schema.Build) error {
	// TODO (@dtan4)
	return nil
}

func setUpServer() *negroni.Negroni {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	return n
}

func main() {
	if os.Getenv("GITHUB_ACCESS_TOKEN") == "" {
		log.Fatal("Please provide 'GITHUB_ACCESS_TOKEN' through environment")
	}
	n := setUpServer()
	n.Run(":8080")
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"time"
)

type Communicator struct {
	ContainerId  string
	HostDir      string
	ContainerDir string
}

func (c *Communicator) Start(remote *packer.RemoteCmd) error {
	cmd := exec.Command("docker", "attach", c.ContainerId)
	stdin_w, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	// TODO(mitchellh): We need to hijack the command to write the exit
	// code to a temporary file in the shared folder so that we can read it
	// out. Since we're going over a pty, we can't get the exit code another
	// way.

	cmd.Stdout = remote.Stdout
	cmd.Stderr = remote.Stderr

	log.Printf("Executing in container %s: %#v", c.ContainerId, remote.Command)
	if err := cmd.Start(); err != nil {
		return err
	}

	go func() {
		defer stdin_w.Close()

		// This sleep needs to be here because of the issue linked to below.
		// Basically, without it, Docker will hang on reading stdin forever,
		// and won't see what we write, for some reason.
		//
		// https://github.com/dotcloud/docker/issues/2628
		time.Sleep(2 * time.Second)

		stdin_w.Write([]byte(remote.Command + "\n"))
	}()

	var exitStatus int = 0
	err = cmd.Wait()
	if exitErr, ok := err.(*exec.ExitError); ok {
		exitStatus = 1

		// There is no process-independent way to get the REAL
		// exit status so we just try to go deeper.
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		}
	}

	// Say that we ended!
	remote.SetExited(exitStatus)

	return nil
}

func (c *Communicator) Upload(dst string, src io.Reader) error {
	// Create a temporary file to store the upload
	tempfile, err := ioutil.TempFile(c.HostDir, "upload")
	if err != nil {
		return err
	}
	defer os.Remove(tempfile.Name())

	// Copy the contents to the temporary file
	_, err = io.Copy(tempfile, src)
	tempfile.Close()
	if err != nil {
		return err
	}

	// TODO(mitchellh): Copy the file into place
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("cp %s/%s %s", c.ContainerDir,
			filepath.Base(tempfile.Name()), dst),
	}

	if err := c.Start(cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Upload failed with non-zero exit status: %d", cmd.ExitStatus)
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	return nil
}

func (c *Communicator) Download(src string, dst io.Writer) error {
	return nil
}
<commit_msg>builder/docker: command output and exit codes work<commit_after>package docker

import (
	"bytes"
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"
)

type Communicator struct {
	ContainerId  string
	HostDir      string
	ContainerDir string
}

func (c *Communicator) Start(remote *packer.RemoteCmd) error {
	// Create a temporary file to store the output. Because of a bug in
	// Docker, sometimes all the output doesn't properly show up. This
	// file will capture ALL of the output, and we'll read that.
	//
	// https://github.com/dotcloud/docker/issues/2625
	outputFile, err := ioutil.TempFile(c.HostDir, "cmd")
	if err != nil {
		return err
	}
	outputFile.Close()
	defer os.Remove(outputFile.Name())

	// This file will store the exit code of the command once it is complete.
	exitCodePath := outputFile.Name() + "-exit"

	// Modify the remote command so that all the output of the commands
	// go to a single file and so that the exit code is redirected to
	// a single file. This lets us determine both when the command
	// is truly complete (because the file will have data), what the
	// exit status is (because Docker loses it because of the pty, not
	// Docker's fault), and get the output (Docker bug).
	remoteCmd := fmt.Sprintf("(%s) >%s 2>&1; echo $? >%s",
		remote.Command,
		filepath.Join(c.ContainerDir, filepath.Base(outputFile.Name())),
		filepath.Join(c.ContainerDir, filepath.Base(exitCodePath)))

	cmd := exec.Command("docker", "attach", c.ContainerId)
	stdin_w, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	log.Printf("Executing in container %s: %#v", c.ContainerId, remoteCmd)
	if err := cmd.Start(); err != nil {
		return err
	}

	go func() {
		defer stdin_w.Close()

		// This sleep needs to be here because of the issue linked to below.
		// Basically, without it, Docker will hang on reading stdin forever,
		// and won't see what we write, for some reason.
		//
		// https://github.com/dotcloud/docker/issues/2628
		time.Sleep(2 * time.Second)

		stdin_w.Write([]byte(remoteCmd + "\n"))
	}()

	err = cmd.Wait()
	if exitErr, ok := err.(*exec.ExitError); ok {
		exitStatus := 1

		// There is no process-independent way to get the REAL
		// exit status so we just try to go deeper.
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		}

		// Say that we ended, since if Docker itself failed, then
		// the command must've not run, or so we assume
		remote.SetExited(exitStatus)
		return nil
	}

	// Wait for the exit code to appear in our file...
	log.Println("Waiting for exit code to appear for remote command...")
	for {
		fi, err := os.Stat(exitCodePath)
		if err == nil && fi.Size() > 0 {
			break
		}

		time.Sleep(1 * time.Second)
	}

	// Read the exit code
	exitRaw, err := ioutil.ReadFile(exitCodePath)
	if err != nil {
		return err
	}

	exitStatus, err := strconv.ParseInt(string(bytes.TrimSpace(exitRaw)), 10, 0)
	if err != nil {
		return err
	}
	log.Printf("Executed command exit status: %d", exitStatus)

	// Read the output
	f, err := os.Open(outputFile.Name())
	if err != nil {
		return err
	}
	defer f.Close()

	if remote.Stdout != nil {
		io.Copy(remote.Stdout, f)
	} else {
		output, err := ioutil.ReadAll(f)
		if err != nil {
			return err
		}

		log.Printf("Command output: %s", string(output))
	}

	// Finally, we're done
	remote.SetExited(int(exitStatus))

	return nil
}

func (c *Communicator) Upload(dst string, src io.Reader) error {
	// Create a temporary file to store the upload
	tempfile, err := ioutil.TempFile(c.HostDir, "upload")
	if err != nil {
		return err
	}
	defer os.Remove(tempfile.Name())

	// Copy the contents to the temporary file
	_, err = io.Copy(tempfile, src)
	tempfile.Close()
	if err != nil {
		return err
	}

	// TODO(mitchellh): Copy the file into place
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("cp %s/%s %s", c.ContainerDir,
			filepath.Base(tempfile.Name()), dst),
	}

	if err := c.Start(cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Upload failed with non-zero exit status: %d", cmd.ExitStatus)
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	return nil
}

func (c *Communicator) Download(src string, dst io.Writer) error {
	return nil
}
<|endoftext|>


<commit_before>package httpserver

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"strings"
	"time"

	"github.com/mholt/caddy"
	"github.com/mholt/caddy/caddyfile"
	"github.com/mholt/caddy/caddytls"
)

const serverType = "http"

func init() {
	flag.StringVar(&Host, "host", DefaultHost, "Default host")
	flag.StringVar(&Port, "port", DefaultPort, "Default port")
	flag.StringVar(&Root, "root", DefaultRoot, "Root path of default site")
	flag.DurationVar(&GracefulTimeout, "grace", 5*time.Second, "Maximum duration of graceful shutdown") // TODO
	flag.BoolVar(&HTTP2, "http2", true, "Use HTTP/2")
	flag.BoolVar(&QUIC, "quic", false, "Use experimental QUIC")

	caddy.RegisterServerType(serverType, caddy.ServerType{
		Directives: directives,
		DefaultInput: func() caddy.Input {
			if Port == DefaultPort && Host != "" {
				// by leaving the port blank in this case we give auto HTTPS
				// a chance to set the port to 443 for us
				return caddy.CaddyfileInput{
					Contents:       []byte(fmt.Sprintf("%s\nroot %s", Host, Root)),
					ServerTypeName: serverType,
				}
			}
			return caddy.CaddyfileInput{
				Contents:       []byte(fmt.Sprintf("%s:%s\nroot %s", Host, Port, Root)),
				ServerTypeName: serverType,
			}
		},
		NewContext: newContext,
	})
	caddy.RegisterCaddyfileLoader("short", caddy.LoaderFunc(shortCaddyfileLoader))
	caddy.RegisterParsingCallback(serverType, "tls", activateHTTPS)
	caddytls.RegisterConfigGetter(serverType, func(c *caddy.Controller) *caddytls.Config { return GetConfig(c).TLS })
}

func newContext() caddy.Context {
	return &httpContext{keysToSiteConfigs: make(map[string]*SiteConfig)}
}

type httpContext struct {
	// keysToSiteConfigs maps an address at the top of a
	// server block (a "key") to its SiteConfig. Not all
	// SiteConfigs will be represented here, only ones
	// that appeared in the Caddyfile.
	keysToSiteConfigs map[string]*SiteConfig

	// siteConfigs is the master list of all site configs.
	siteConfigs []*SiteConfig
}

func (h *httpContext) saveConfig(key string, cfg *SiteConfig) {
	h.siteConfigs = append(h.siteConfigs, cfg)
	h.keysToSiteConfigs[key] = cfg
}

// InspectServerBlocks make sure that everything checks out before
// executing directives and otherwise prepares the directives to
// be parsed and executed.
func (h *httpContext) InspectServerBlocks(sourceFile string, serverBlocks []caddyfile.ServerBlock) ([]caddyfile.ServerBlock, error) {
	// For each address in each server block, make a new config
	for _, sb := range serverBlocks {
		for _, key := range sb.Keys {
			key = strings.ToLower(key)
			if _, dup := h.keysToSiteConfigs[key]; dup {
				return serverBlocks, fmt.Errorf("duplicate site address: %s", key)
			}
			addr, err := standardizeAddress(key)
			if err != nil {
				return serverBlocks, err
			}

			// Fill in address components from command line so that middleware
			// have access to the correct information during setup
			if addr.Host == "" && Host != DefaultHost {
				addr.Host = Host
			}
			if addr.Port == "" && Port != DefaultPort {
				addr.Port = Port
			}

			// Save the config to our master list, and key it for lookups
			cfg := &SiteConfig{
				Addr:        addr,
				Root:        Root,
				TLS:         &caddytls.Config{Hostname: addr.Host},
				HiddenFiles: []string{sourceFile},
			}
			h.saveConfig(key, cfg)
		}
	}

	// For sites that have gzip (which gets chained in
	// before the error handler) we should ensure that the
	// errors directive also appears so error pages aren't
	// written after the gzip writer is closed. See #616.
	for _, sb := range serverBlocks {
		_, hasGzip := sb.Tokens["gzip"]
		_, hasErrors := sb.Tokens["errors"]
		if hasGzip && !hasErrors {
			sb.Tokens["errors"] = []caddyfile.Token{{Text: "errors"}}
		}
	}

	return serverBlocks, nil
}

// MakeServers uses the newly-created siteConfigs to
// create and return a list of server instances.
func (h *httpContext) MakeServers() ([]caddy.Server, error) {
	// make sure TLS is disabled for explicitly-HTTP sites
	// (necessary when HTTP address shares a block containing tls)
	for _, cfg := range h.siteConfigs {
		if !cfg.TLS.Enabled {
			continue
		}
		if cfg.Addr.Port == "80" || cfg.Addr.Scheme == "http" {
			cfg.TLS.Enabled = false
			log.Printf("[WARNING] TLS disabled for %s", cfg.Addr)
		} else if cfg.Addr.Scheme == "" {
			// set scheme to https ourselves, since TLS is enabled
			// and it was not explicitly set to something else. this
			// makes it appear as "https" when we print the list of
			// running sites; otherwise "http" would be assumed which
			// is incorrect for this site.
			cfg.Addr.Scheme = "https"
		}
		if cfg.Addr.Port == "" {
			// this is vital, otherwise the function call below that
			// sets the listener address will use the default port
			// instead of 443 because it doesn't know about TLS.
			cfg.Addr.Port = "443"
		}
	}

	// we must map (group) each config to a bind address
	groups, err := groupSiteConfigsByListenAddr(h.siteConfigs)
	if err != nil {
		return nil, err
	}

	// then we create a server for each group
	var servers []caddy.Server
	for addr, group := range groups {
		s, err := NewServer(addr, group)
		if err != nil {
			return nil, err
		}
		servers = append(servers, s)
	}

	return servers, nil
}

// GetConfig gets the SiteConfig that corresponds to c.
// If none exist (should only happen in tests), then a
// new, empty one will be created.
func GetConfig(c *caddy.Controller) *SiteConfig {
	ctx := c.Context().(*httpContext)
	if cfg, ok := ctx.keysToSiteConfigs[c.Key]; ok {
		return cfg
	}
	// we should only get here during tests because directive
	// actions typically skip the server blocks where we make
	// the configs
	ctx.saveConfig(c.Key, &SiteConfig{Root: Root, TLS: new(caddytls.Config)})
	return GetConfig(c)
}

// shortCaddyfileLoader loads a Caddyfile if positional arguments are
// detected, or, in other words, if un-named arguments are provided to
// the program. A "short Caddyfile" is one in which each argument
// is a line of the Caddyfile. The default host and port are prepended
// according to the Host and Port values.
func shortCaddyfileLoader(serverType string) (caddy.Input, error) {
	if flag.NArg() > 0 && serverType == "http" {
		confBody := fmt.Sprintf("%s:%s\n%s", Host, Port, strings.Join(flag.Args(), "\n"))
		return caddy.CaddyfileInput{
			Contents:       []byte(confBody),
			Filepath:       "args",
			ServerTypeName: serverType,
		}, nil
	}
	return nil, nil
}

// groupSiteConfigsByListenAddr groups site configs by their listen
// (bind) address, so sites that use the same listener can be served
// on the same server instance. The return value maps the listen
// address (what you pass into net.Listen) to the list of site configs.
// This function does NOT vet the configs to ensure they are compatible.
func groupSiteConfigsByListenAddr(configs []*SiteConfig) (map[string][]*SiteConfig, error) {
	groups := make(map[string][]*SiteConfig)

	for _, conf := range configs {
		if caddy.IsLoopback(conf.Addr.Host) && conf.ListenHost == "" {
			// special case: one would not expect a site served
			// at loopback to be connected to from the outside.
			conf.ListenHost = conf.Addr.Host
		}
		if conf.Addr.Port == "" {
			conf.Addr.Port = Port
		}
		addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(conf.ListenHost, conf.Addr.Port))
		if err != nil {
			return nil, err
		}
		addrstr := addr.String()
		groups[addrstr] = append(groups[addrstr], conf)
	}

	return groups, nil
}

// AddMiddleware adds a middleware to a site's middleware stack.
func (sc *SiteConfig) AddMiddleware(m Middleware) {
	sc.middleware = append(sc.middleware, m)
}

// Address represents a site address. It contains
// the original input value, and the component
// parts of an address. The component parts may be
// updated to the correct values as setup proceeds,
// but the original value should never be changed.
type Address struct {
	Original, Scheme, Host, Port, Path string
}

// String returns a human-friendly print of the address.
func (a Address) String() string {
	if a.Host == "" && a.Port == "" {
		return ""
	}
	scheme := a.Scheme
	if scheme == "" {
		if a.Port == "443" {
			scheme = "https"
		} else {
			scheme = "http"
		}
	}
	s := scheme
	if s != "" {
		s += "://"
	}
	s += a.Host
	if a.Port != "" &&
		((scheme == "https" && a.Port != "443") ||
			(scheme == "http" && a.Port != "80")) {
		s += ":" + a.Port
	}
	if a.Path != "" {
		s += a.Path
	}
	return s
}

// VHost returns a sensible concatenation of Host:Port/Path from a.
// It's basically the a.Original but without the scheme.
func (a Address) VHost() string {
	if idx := strings.Index(a.Original, "://"); idx > -1 {
		return a.Original[idx+3:]
	}
	return a.Original
}

// standardizeAddress parses an address string into a structured format with separate
// scheme, host, and port portions, as well as the original input string.
func standardizeAddress(str string) (Address, error) {
	input := str

	// Split input into components (prepend with // to assert host by default)
	if !strings.Contains(str, "//") {
		str = "//" + str
	}
	u, err := url.Parse(str)
	if err != nil {
		return Address{}, err
	}

	// separate host and port
	host, port, err := net.SplitHostPort(u.Host)
	if err != nil {
		host, port, err = net.SplitHostPort(u.Host + ":")
		if err != nil {
			host = u.Host
		}
	}

	// see if we can set port based off scheme
	if port == "" {
		if u.Scheme == "http" {
			port = "80"
		} else if u.Scheme == "https" {
			port = "443"
		}
	}

	// repeated or conflicting scheme is confusing, so error
	if u.Scheme != "" && (port == "http" || port == "https") {
		return Address{}, fmt.Errorf("[%s] scheme specified twice in address", input)
	}

	// error if scheme and port combination violate convention
	if (u.Scheme == "http" && port == "443") || (u.Scheme == "https" && port == "80") {
		return Address{}, fmt.Errorf("[%s] scheme and port violate convention", input)
	}

	// standardize http and https ports to their respective port numbers
	if port == "http" {
		u.Scheme = "http"
		port = "80"
	} else if port == "https" {
		u.Scheme = "https"
		port = "443"
	}

	return Address{Original: input, Scheme: u.Scheme, Host: host, Port: port, Path: u.Path}, err
}

// directives is the list of all directives known to exist for the
// http server type, including non-standard (3rd-party) directives.
// The ordering of this list is important.
var directives = []string{
	// primitive actions that set up the fundamental vitals of each config
	"root",
	"tls",
	"bind",

	// services/utilities, or other directives that don't necessarily inject handlers
	"startup",
	"shutdown",
	"realip", // github.com/captncraig/caddy-realip
	"git",    // github.com/abiosoft/caddy-git

	// directives that add middleware to the stack
	"log",
	"gzip",
	"errors",
	"minify",   // github.com/hacdias/caddy-minify
	"ipfilter", // github.com/pyed/ipfilter
	"search",   // github.com/pedronasser/caddy-search
	"header",
	"cors", // github.com/captncraig/cors/caddy
	"rewrite",
	"redir",
	"ext",
	"mime",
	"basicauth",
	"jwt",    // github.com/BTBurke/caddy-jwt
	"jsonp",  // github.com/pschlump/caddy-jsonp
	"upload", // blitznote.com/src/caddy.upload
	"internal",
	"pprof",
	"expvar",
	"proxy",
	"fastcgi",
	"websocket",
	"markdown",
	"templates",
	"browse",
	"hugo",       // github.com/hacdias/caddy-hugo
	"mailout",    // github.com/SchumacherFM/mailout
	"prometheus", // github.com/miekg/caddy-prometheus
}

const (
	// DefaultHost is the default host.
	DefaultHost = ""
	// DefaultPort is the default port.
	DefaultPort = "2015"
	// DefaultRoot is the default root folder.
	DefaultRoot = "."
)

// These "soft defaults" are configurable by
// command line flags, etc.
var (
	// Root is the site root
	Root = DefaultRoot

	// Host is the site host
	Host = DefaultHost

	// Port is the site port
	Port = DefaultPort

	// GracefulTimeout is the maximum duration of a graceful shutdown.
	GracefulTimeout time.Duration

	// HTTP2 indicates whether HTTP2 is enabled or not.
	HTTP2 bool

	// QUIC indicates whether QUIC is enabled or not.
	QUIC bool
)
<commit_msg>Add filemanager directive<commit_after>package httpserver

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"strings"
	"time"

	"github.com/mholt/caddy"
	"github.com/mholt/caddy/caddyfile"
	"github.com/mholt/caddy/caddytls"
)

const serverType = "http"

func init() {
	flag.StringVar(&Host, "host", DefaultHost, "Default host")
	flag.StringVar(&Port, "port", DefaultPort, "Default port")
	flag.StringVar(&Root, "root", DefaultRoot, "Root path of default site")
	flag.DurationVar(&GracefulTimeout, "grace", 5*time.Second, "Maximum duration of graceful shutdown") // TODO
	flag.BoolVar(&HTTP2, "http2", true, "Use HTTP/2")
	flag.BoolVar(&QUIC, "quic", false, "Use experimental QUIC")

	caddy.RegisterServerType(serverType, caddy.ServerType{
		Directives: directives,
		DefaultInput: func() caddy.Input {
			if Port == DefaultPort && Host != "" {
				// by leaving the port blank in this case we give auto HTTPS
				// a chance to set the port to 443 for us
				return caddy.CaddyfileInput{
					Contents:       []byte(fmt.Sprintf("%s\nroot %s", Host, Root)),
					ServerTypeName: serverType,
				}
			}
			return caddy.CaddyfileInput{
				Contents:       []byte(fmt.Sprintf("%s:%s\nroot %s", Host, Port, Root)),
				ServerTypeName: serverType,
			}
		},
		NewContext: newContext,
	})
	caddy.RegisterCaddyfileLoader("short", caddy.LoaderFunc(shortCaddyfileLoader))
	caddy.RegisterParsingCallback(serverType, "tls", activateHTTPS)
	caddytls.RegisterConfigGetter(serverType, func(c *caddy.Controller) *caddytls.Config { return GetConfig(c).TLS })
}

func newContext() caddy.Context {
	return &httpContext{keysToSiteConfigs: make(map[string]*SiteConfig)}
}

type httpContext struct {
	// keysToSiteConfigs maps an address at the top of a
	// server block (a "key") to its SiteConfig. Not all
	// SiteConfigs will be represented here, only ones
	// that appeared in the Caddyfile.
	keysToSiteConfigs map[string]*SiteConfig

	// siteConfigs is the master list of all site configs.
	siteConfigs []*SiteConfig
}

func (h *httpContext) saveConfig(key string, cfg *SiteConfig) {
	h.siteConfigs = append(h.siteConfigs, cfg)
	h.keysToSiteConfigs[key] = cfg
}

// InspectServerBlocks make sure that everything checks out before
// executing directives and otherwise prepares the directives to
// be parsed and executed.
func (h *httpContext) InspectServerBlocks(sourceFile string, serverBlocks []caddyfile.ServerBlock) ([]caddyfile.ServerBlock, error) {
	// For each address in each server block, make a new config
	for _, sb := range serverBlocks {
		for _, key := range sb.Keys {
			key = strings.ToLower(key)
			if _, dup := h.keysToSiteConfigs[key]; dup {
				return serverBlocks, fmt.Errorf("duplicate site address: %s", key)
			}
			addr, err := standardizeAddress(key)
			if err != nil {
				return serverBlocks, err
			}

			// Fill in address components from command line so that middleware
			// have access to the correct information during setup
			if addr.Host == "" && Host != DefaultHost {
				addr.Host = Host
			}
			if addr.Port == "" && Port != DefaultPort {
				addr.Port = Port
			}

			// Save the config to our master list, and key it for lookups
			cfg := &SiteConfig{
				Addr:        addr,
				Root:        Root,
				TLS:         &caddytls.Config{Hostname: addr.Host},
				HiddenFiles: []string{sourceFile},
			}
			h.saveConfig(key, cfg)
		}
	}

	// For sites that have gzip (which gets chained in
	// before the error handler) we should ensure that the
	// errors directive also appears so error pages aren't
	// written after the gzip writer is closed. See #616.
	for _, sb := range serverBlocks {
		_, hasGzip := sb.Tokens["gzip"]
		_, hasErrors := sb.Tokens["errors"]
		if hasGzip && !hasErrors {
			sb.Tokens["errors"] = []caddyfile.Token{{Text: "errors"}}
		}
	}

	return serverBlocks, nil
}

// MakeServers uses the newly-created siteConfigs to
// create and return a list of server instances.
func (h *httpContext) MakeServers() ([]caddy.Server, error) {
	// make sure TLS is disabled for explicitly-HTTP sites
	// (necessary when HTTP address shares a block containing tls)
	for _, cfg := range h.siteConfigs {
		if !cfg.TLS.Enabled {
			continue
		}
		if cfg.Addr.Port == "80" || cfg.Addr.Scheme == "http" {
			cfg.TLS.Enabled = false
			log.Printf("[WARNING] TLS disabled for %s", cfg.Addr)
		} else if cfg.Addr.Scheme == "" {
			// set scheme to https ourselves, since TLS is enabled
			// and it was not explicitly set to something else. this
			// makes it appear as "https" when we print the list of
			// running sites; otherwise "http" would be assumed which
			// is incorrect for this site.
			cfg.Addr.Scheme = "https"
		}
		if cfg.Addr.Port == "" {
			// this is vital, otherwise the function call below that
			// sets the listener address will use the default port
			// instead of 443 because it doesn't know about TLS.
			cfg.Addr.Port = "443"
		}
	}

	// we must map (group) each config to a bind address
	groups, err := groupSiteConfigsByListenAddr(h.siteConfigs)
	if err != nil {
		return nil, err
	}

	// then we create a server for each group
	var servers []caddy.Server
	for addr, group := range groups {
		s, err := NewServer(addr, group)
		if err != nil {
			return nil, err
		}
		servers = append(servers, s)
	}

	return servers, nil
}

// GetConfig gets the SiteConfig that corresponds to c.
// If none exist (should only happen in tests), then a
// new, empty one will be created.
func GetConfig(c *caddy.Controller) *SiteConfig {
	ctx := c.Context().(*httpContext)
	if cfg, ok := ctx.keysToSiteConfigs[c.Key]; ok {
		return cfg
	}
	// we should only get here during tests because directive
	// actions typically skip the server blocks where we make
	// the configs
	ctx.saveConfig(c.Key, &SiteConfig{Root: Root, TLS: new(caddytls.Config)})
	return GetConfig(c)
}

// shortCaddyfileLoader loads a Caddyfile if positional arguments are
// detected, or, in other words, if un-named arguments are provided to
// the program. A "short Caddyfile" is one in which each argument
// is a line of the Caddyfile. The default host and port are prepended
// according to the Host and Port values.
func shortCaddyfileLoader(serverType string) (caddy.Input, error) {
	if flag.NArg() > 0 && serverType == "http" {
		confBody := fmt.Sprintf("%s:%s\n%s", Host, Port, strings.Join(flag.Args(), "\n"))
		return caddy.CaddyfileInput{
			Contents:       []byte(confBody),
			Filepath:       "args",
			ServerTypeName: serverType,
		}, nil
	}
	return nil, nil
}

// groupSiteConfigsByListenAddr groups site configs by their listen
// (bind) address, so sites that use the same listener can be served
// on the same server instance. The return value maps the listen
// address (what you pass into net.Listen) to the list of site configs.
// This function does NOT vet the configs to ensure they are compatible.
func groupSiteConfigsByListenAddr(configs []*SiteConfig) (map[string][]*SiteConfig, error) {
	groups := make(map[string][]*SiteConfig)

	for _, conf := range configs {
		if caddy.IsLoopback(conf.Addr.Host) && conf.ListenHost == "" {
			// special case: one would not expect a site served
			// at loopback to be connected to from the outside.
			conf.ListenHost = conf.Addr.Host
		}
		if conf.Addr.Port == "" {
			conf.Addr.Port = Port
		}
		addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(conf.ListenHost, conf.Addr.Port))
		if err != nil {
			return nil, err
		}
		addrstr := addr.String()
		groups[addrstr] = append(groups[addrstr], conf)
	}

	return groups, nil
}

// AddMiddleware adds a middleware to a site's middleware stack.
func (sc *SiteConfig) AddMiddleware(m Middleware) {
	sc.middleware = append(sc.middleware, m)
}

// Address represents a site address. It contains
// the original input value, and the component
// parts of an address. The component parts may be
// updated to the correct values as setup proceeds,
// but the original value should never be changed.
type Address struct {
	Original, Scheme, Host, Port, Path string
}

// String returns a human-friendly print of the address.
func (a Address) String() string {
	if a.Host == "" && a.Port == "" {
		return ""
	}
	scheme := a.Scheme
	if scheme == "" {
		if a.Port == "443" {
			scheme = "https"
		} else {
			scheme = "http"
		}
	}
	s := scheme
	if s != "" {
		s += "://"
	}
	s += a.Host
	if a.Port != "" &&
		((scheme == "https" && a.Port != "443") ||
			(scheme == "http" && a.Port != "80")) {
		s += ":" + a.Port
	}
	if a.Path != "" {
		s += a.Path
	}
	return s
}

// VHost returns a sensible concatenation of Host:Port/Path from a.
// It's basically the a.Original but without the scheme.
func (a Address) VHost() string {
	if idx := strings.Index(a.Original, "://"); idx > -1 {
		return a.Original[idx+3:]
	}
	return a.Original
}

// standardizeAddress parses an address string into a structured format with separate
// scheme, host, and port portions, as well as the original input string.
func standardizeAddress(str string) (Address, error) {
	input := str

	// Split input into components (prepend with // to assert host by default)
	if !strings.Contains(str, "//") {
		str = "//" + str
	}
	u, err := url.Parse(str)
	if err != nil {
		return Address{}, err
	}

	// separate host and port
	host, port, err := net.SplitHostPort(u.Host)
	if err != nil {
		host, port, err = net.SplitHostPort(u.Host + ":")
		if err != nil {
			host = u.Host
		}
	}

	// see if we can set port based off scheme
	if port == "" {
		if u.Scheme == "http" {
			port = "80"
		} else if u.Scheme == "https" {
			port = "443"
		}
	}

	// repeated or conflicting scheme is confusing, so error
	if u.Scheme != "" && (port == "http" || port == "https") {
		return Address{}, fmt.Errorf("[%s] scheme specified twice in address", input)
	}

	// error if scheme and port combination violate convention
	if (u.Scheme == "http" && port == "443") || (u.Scheme == "https" && port == "80") {
		return Address{}, fmt.Errorf("[%s] scheme and port violate convention", input)
	}

	// standardize http and https ports to their respective port numbers
	if port == "http" {
		u.Scheme = "http"
		port = "80"
	} else if port == "https" {
		u.Scheme = "https"
		port = "443"
	}

	return Address{Original: input, Scheme: u.Scheme, Host: host, Port: port, Path: u.Path}, err
}

// directives is the list of all directives known to exist for the
// http server type, including non-standard (3rd-party) directives.
// The ordering of this list is important.
var directives = []string{
	// primitive actions that set up the fundamental vitals of each config
	"root",
	"tls",
	"bind",

	// services/utilities, or other directives that don't necessarily inject handlers
	"startup",
	"shutdown",
	"realip", // github.com/captncraig/caddy-realip
	"git",    // github.com/abiosoft/caddy-git

	// directives that add middleware to the stack
	"log",
	"gzip",
	"errors",
	"minify",   // github.com/hacdias/caddy-minify
	"ipfilter", // github.com/pyed/ipfilter
	"search",   // github.com/pedronasser/caddy-search
	"header",
	"cors", // github.com/captncraig/cors/caddy
	"rewrite",
	"redir",
	"ext",
	"mime",
	"basicauth",
	"jwt",    // github.com/BTBurke/caddy-jwt
	"jsonp",  // github.com/pschlump/caddy-jsonp
	"upload", // blitznote.com/src/caddy.upload
	"internal",
	"pprof",
	"expvar",
	"proxy",
	"fastcgi",
	"websocket",
	"markdown",
	"templates",
	"browse",
	"hugo",        // github.com/hacdias/caddy-hugo
	"filemanager", // github.com/hacdias/caddy-filemanager
	"mailout",     // github.com/SchumacherFM/mailout
	"prometheus",  // github.com/miekg/caddy-prometheus
}

const (
	// DefaultHost is the default host.
	DefaultHost = ""
	// DefaultPort is the default port.
	DefaultPort = "2015"
	// DefaultRoot is the default root folder.
	DefaultRoot = "."
)

// These "soft defaults" are configurable by
// command line flags, etc.
var (
	// Root is the site root
	Root = DefaultRoot

	// Host is the site host
	Host = DefaultHost

	// Port is the site port
	Port = DefaultPort

	// GracefulTimeout is the maximum duration of a graceful shutdown.
	GracefulTimeout time.Duration

	// HTTP2 indicates whether HTTP2 is enabled or not.
	HTTP2 bool

	// QUIC indicates whether QUIC is enabled or not.
	QUIC bool
)
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib-docker/dockerfile"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

const DockerfileUnitType = "Dockerfile"

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Dockerfiles",
		"Scan the directory tree rooted at the current directory for Dockerfiles.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	// We don't need any config yet, but read it anyway so that we
	// don't silently allow people to send us bad JSON.
	var config interface{}
	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil && err != io.EOF {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	var dockerfiles []string
	root := "."
	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsDir() {
			// Don't traverse into VCS dirs.
			if isVCSDir[info.Name()] {
				return filepath.SkipDir
			}

			// Don't traverse into git submodules.
			if path != root {
				fs, err := ioutil.ReadDir(path)
				if err != nil {
					return err
				}
				for _, f := range fs {
					if isVCSDir[f.Name()] {
						return filepath.SkipDir
					}
				}
			}
		}

		// Collect Dockerfiles.
		if info.Mode().IsRegular() && info.Name() == "Dockerfile" {
			dockerfiles = append(dockerfiles, path)
		}

		return nil
	})
	if err != nil {
		return err
	}

	if GlobalOpt.Verbose && len(dockerfiles) > 0 {
		log.Printf("Found %d Dockerfiles: %v", len(dockerfiles), dockerfiles)
	}

	units := make([]*unit.SourceUnit, len(dockerfiles))
	for i, dfpath := range dockerfiles {
		data, err := ioutil.ReadFile(dfpath)
		if err != nil {
			return err
		}

		// Parse Dockerfile.
		var baseImages []interface{}
		df, err := dockerfile.Decode(bytes.NewReader(data))
		if err == nil {
			if df.From != "" {
				baseImages = append(baseImages, df.From)
			}
		} else {
			log.Printf("Error parsing Dockerfile %q: %s. Adding without dependencies.", dfpath, err)
		}

		units[i] = &unit.SourceUnit{
			Name:         filepath.Dir(dfpath),
			Type:         DockerfileUnitType,
			Dir:          filepath.Dir(dfpath),
			Files:        []string{dfpath},
			Data:         string(data),
			Dependencies: baseImages,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	fmt.Println()
	return nil
}

var isVCSDir = map[string]bool{".git": true, ".hg": true}
<commit_msg>ignore perm-denied and not-exists errors<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib-docker/dockerfile"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

const DockerfileUnitType = "Dockerfile"

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Dockerfiles",
		"Scan the directory tree rooted at the current directory for Dockerfiles.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func ignoreWalkError(err error) bool {
	return os.IsPermission(err) || os.IsNotExist(err)
}

func (c *ScanCmd) Execute(args []string) error {
	// We don't need any config yet, but read it anyway so that we
	// don't silently allow people to send us bad JSON.
	var config interface{}
	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil && err != io.EOF {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	var dockerfiles []string
	root := "."
	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil && !ignoreWalkError(err) {
			return err
		}

		if info.Mode().IsDir() {
			// Don't traverse into VCS dirs.
			if isVCSDir[info.Name()] {
				return filepath.SkipDir
			}

			// Don't traverse into git submodules.
			if path != root {
				fs, err := ioutil.ReadDir(path)
				if err != nil && !ignoreWalkError(err) {
					return err
				}
				for _, f := range fs {
					if isVCSDir[f.Name()] {
						return filepath.SkipDir
					}
				}
			}
		}

		// Collect Dockerfiles.
		if info.Mode().IsRegular() && info.Name() == "Dockerfile" {
			dockerfiles = append(dockerfiles, path)
		}

		return nil
	})
	if err != nil {
		return err
	}

	if GlobalOpt.Verbose && len(dockerfiles) > 0 {
		log.Printf("Found %d Dockerfiles: %v", len(dockerfiles), dockerfiles)
	}

	units := make([]*unit.SourceUnit, len(dockerfiles))
	for i, dfpath := range dockerfiles {
		data, err := ioutil.ReadFile(dfpath)
		if err != nil {
			return err
		}

		// Parse Dockerfile.
		var baseImages []interface{}
		df, err := dockerfile.Decode(bytes.NewReader(data))
		if err == nil {
			if df.From != "" {
				baseImages = append(baseImages, df.From)
			}
		} else {
			log.Printf("Error parsing Dockerfile %q: %s. Adding without dependencies.", dfpath, err)
		}

		units[i] = &unit.SourceUnit{
			Name:         filepath.Dir(dfpath),
			Type:         DockerfileUnitType,
			Dir:          filepath.Dir(dfpath),
			Files:        []string{dfpath},
			Data:         string(data),
			Dependencies: baseImages,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	fmt.Println()
	return nil
}

var isVCSDir = map[string]bool{".git": true, ".hg": true}
<|endoftext|>


<commit_before>package sugarcane

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"io"
	"log"
	"os"
)

// Prepare encodes struct into byte.Buffer.
func prepare(p interface{}) bytes.Buffer {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(p)
	if err != nil {
		log.Fatal("encode error:", err)
	}
	return buf
}

// Write writes byte buffer to file f.
func Write(f *os.File, buf bytes.Buffer) {
	w := bufio.NewWriter(f)
	_, err := w.Write(buf.Bytes())
	if err != nil {
		panic(err)
	}
	w.Flush()
}

// Open opens a file for writing.
func Open(filename string) *os.File {
	w, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
	if err != nil {
		panic(err)
	}
	return w
}

// Read reads a file
func Read(filename string) *bytes.Buffer {
	f, err := os.OpenFile(filename, os.O_RDONLY, 0600)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		panic(err)
	}

	buf := make([]byte, fi.Size()) // make the buffer as big as the file
	r := bufio.NewReader(f)
	for {
		n, err := r.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}
		if n == 0 {
			break
		}
	}
	return bytes.NewBuffer(buf)
}

// Insert prepares structure for disk saving.
func Insert(p interface{}, f *os.File) {
	binary := prepare(p)
	Write(f, binary)
}

// ReadOne returns a first structure from byte buffer and decodes it according given structure.
func ReadOne(p interface{}, data *bytes.Buffer) error {
	dec := gob.NewDecoder(data)
	err := dec.Decode(p)
	if err != nil {
		return err
	}
	return err
}
<commit_msg>proper error callbacks<commit_after>package sugarcane

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"io"
	"os"
)

// Prepare encodes struct into byte.Buffer.
func prepare(p interface{}) (bytes.Buffer, error) {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(p)
	if err != nil {
		return buf, err
	}
	return buf, nil
}

// Write writes byte buffer to file f.
func Write(f *os.File, buf bytes.Buffer) error {
	w := bufio.NewWriter(f)
	_, err := w.Write(buf.Bytes())
	if err != nil {
		return err
	}
	w.Flush()
	return nil
}

// Open opens a file for writing.
func Open(filename string) (*os.File, error) {
	w, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
	if err != nil {
		return w, err
	}
	return w, nil
}

// Read reads a file
func Read(filename string) (*bytes.Buffer, error) {
	f, err := os.OpenFile(filename, os.O_RDONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	buf := make([]byte, fi.Size()) // make the buffer as big as the file
	r := bufio.NewReader(f)
	for {
		n, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return nil, err
		}
		if n == 0 {
			break
		}
	}
	return bytes.NewBuffer(buf), nil
}

// Insert prepares structure for disk saving.
func Insert(p interface{}, f *os.File) error {
	binary, err := prepare(p)
	if err != nil {
		return err
	}
	Write(f, binary)
	return nil
}

// ReadOne returns a first structure from byte buffer and decodes it according given structure.
func ReadOne(p interface{}, data *bytes.Buffer) error {
	dec := gob.NewDecoder(data)
	err := dec.Decode(p)
	if err != nil {
		return err
	}
	return err
}
<|endoftext|>