<commit_before>package device

import (
	"fmt"
	"strings"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
)

const nicRoutedIPv4GW = "169.254.0.1"
const nicRoutedIPv6GW = "fe80::1"

type nicRouted struct {
	deviceCommon
}

func (d *nicRouted) CanHotPlug() (bool, []string) {
	return false, []string{}
}

// validateConfig checks the supplied config for correctness.
func (d *nicRouted) validateConfig(instConf instance.ConfigReader) error {
	if !instanceSupported(instConf.Type(), instancetype.Container) {
		return ErrUnsupportedDevType
	}

	requiredFields := []string{}
	optionalFields := []string{
		"name",
		"parent",
		"mtu",
		"hwaddr",
		"host_name",
		"vlan",
		"ipv4.gateway",
		"ipv6.gateway",
		"ipv4.host_address",
		"ipv6.host_address",
	}

	rules := nicValidationRules(requiredFields, optionalFields)
	rules["ipv4.address"] = func(value string) error {
		if value == "" {
			return nil
		}

		return NetworkValidAddressV4List(value)
	}
	rules["ipv6.address"] = func(value string) error {
		if value == "" {
			return nil
		}

		return NetworkValidAddressV6List(value)
	}

	err := d.config.Validate(rules)
	if err != nil {
		return err
	}

	return nil
}

// validateEnvironment checks the runtime environment for correctness.
func (d *nicRouted) validateEnvironment() error {
	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
		return fmt.Errorf("Requires name property to start")
	}

	if d.config["parent"] != "" && !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) {
		return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"])
	}

	if d.config["parent"] == "" && d.config["vlan"] != "" {
		return fmt.Errorf("The vlan setting can only be used when combined with a parent interface")
	}

	extensions := d.state.OS.LXCFeatures
	if !extensions["network_veth_router"] || !extensions["network_l2proxy"] {
		return fmt.Errorf("Requires liblxc has following API extensions: network_veth_router, network_l2proxy")
	}

	// Check necessary sysctls are configured for use with l2proxy parent for routed mode.
	if d.config["parent"] != "" && d.config["ipv4.address"] != "" {
		ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", d.config["parent"])
		sysctlVal, err := util.SysctlGet(ipv4FwdPath)
		if err != nil || sysctlVal != "1\n" {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv4FwdPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv4.conf.%s.forwarding=1", d.config["parent"])
		}
	}

	// Check necessary sysctls are configured for use with l2proxy parent for routed mode.
	if d.config["parent"] != "" && d.config["ipv6.address"] != "" {
		// net.ipv6.conf.all.forwarding=1 is required to enable general packet forwarding for IPv6.
		ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", "all")
		sysctlVal, err := util.SysctlGet(ipv6FwdPath)
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", "all")
		}

		ipv6FwdPath = fmt.Sprintf("net/ipv6/conf/%s/forwarding", d.config["parent"])
		sysctlVal, err = util.SysctlGet(ipv6FwdPath)
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", d.config["parent"])
		}

		// net.ipv6.conf.all.proxy_ndp=1 is needed otherwise unicast neighbour solicitations are rejected.
		// This causes periodic latency spikes every 15-20s as the neighbour has to resort to using
		// multicast NDP resolution and expires the previous neighbour entry.
		ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", "all")
		sysctlVal, err = util.SysctlGet(ipv6ProxyNdpPath)
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", "all")
		}

		ipv6ProxyNdpPath = fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", d.config["parent"])
		sysctlVal, err = util.SysctlGet(ipv6ProxyNdpPath)
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", d.config["parent"])
		}
	}

	return nil
}

// Start is run when the instance is starting up (Routed mode doesn't support hot plugging).
func (d *nicRouted) Start() (*deviceConfig.RunConfig, error) {
	err := d.validateEnvironment()
	if err != nil {
		return nil, err
	}

	// Lock to avoid issues with containers starting in parallel.
	networkCreateSharedDeviceLock.Lock()
	defer networkCreateSharedDeviceLock.Unlock()

	saveData := make(map[string]string)

	// Decide which parent we should use based on VLAN setting.
	parentName := ""
	if d.config["parent"] != "" {
		parentName = network.GetHostDevice(d.config["parent"], d.config["vlan"])

		statusDev, err := networkCreateVlanDeviceIfNeeded(d.state, d.config["parent"], parentName, d.config["vlan"])
		if err != nil {
			return nil, err
		}

		// Record whether we created this device or not so it can be removed on stop.
		saveData["last_state.created"] = fmt.Sprintf("%t", statusDev != "existing")

		// If we created a VLAN interface, we need to setup the sysctls on that interface.
		if statusDev == "created" {
			err := d.setupParentSysctls(parentName)
			if err != nil {
				return nil, err
			}
		}
	}

	hostName := d.config["host_name"]
	if hostName == "" {
		hostName = networkRandomDevName("veth")
	}
	saveData["host_name"] = hostName

	err = d.volatileSet(saveData)
	if err != nil {
		return nil, err
	}

	runConf := deviceConfig.RunConfig{}
	nic := []deviceConfig.RunConfigItem{
		{Key: "name", Value: d.config["name"]},
		{Key: "type", Value: "veth"},
		{Key: "flags", Value: "up"},
		{Key: "veth.mode", Value: "router"},
		{Key: "veth.pair", Value: saveData["host_name"]},
	}

	// If there is a designated parent interface, activate the layer2 proxy mode to advertise
	// the instance's IPs over that interface using proxy APR/NDP.
	if parentName != "" {
		nic = append(nic,
			deviceConfig.RunConfigItem{Key: "l2proxy", Value: "1"},
			deviceConfig.RunConfigItem{Key: "link", Value: parentName},
		)
	}

	if d.config["mtu"] != "" {
		nic = append(nic, deviceConfig.RunConfigItem{Key: "mtu", Value: d.config["mtu"]})
	}

	if d.config["ipv4.address"] != "" {
		for _, addr := range strings.Split(d.config["ipv4.address"], ",") {
			addr = strings.TrimSpace(addr)
			nic = append(nic, deviceConfig.RunConfigItem{Key: "ipv4.address", Value: fmt.Sprintf("%s/32", addr)})
		}

		if nicHasAutoGateway(d.config["ipv4.gateway"]) {
			// Use a fixed link-local address as the next-hop default gateway.
			nic = append(nic, deviceConfig.RunConfigItem{Key: "ipv4.gateway", Value: d.ipv4HostAddress()})
		}
	}

	if d.config["ipv6.address"] != "" {
		for _, addr := range strings.Split(d.config["ipv6.address"], ",") {
			addr = strings.TrimSpace(addr)
			nic = append(nic, deviceConfig.RunConfigItem{Key: "ipv6.address", Value: fmt.Sprintf("%s/128", addr)})
		}

		if nicHasAutoGateway(d.config["ipv6.gateway"]) {
			// Use a fixed link-local address as the next-hop default gateway.
			nic = append(nic, deviceConfig.RunConfigItem{Key: "ipv6.gateway", Value: d.ipv6HostAddress()})
		}
	}

	runConf.NetworkInterface = nic
	runConf.PostHooks = append(runConf.PostHooks, d.postStart)
	return &runConf, nil
}

// setupParentSysctls configures the required sysctls on the parent to allow l2proxy to work.
// Because of our policy not to modify sysctls on existing interfaces, this should only be called
// if we created the parent interface.
func (d *nicRouted) setupParentSysctls(parentName string) error {
	if d.config["ipv4.address"] != "" {
		// Set necessary sysctls for use with l2proxy parent in routed mode.
		ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", parentName)
		err := util.SysctlSet(ipv4FwdPath, "1")
		if err != nil {
			return fmt.Errorf("Error setting net sysctl %s: %v", ipv4FwdPath, err)
		}
	}

	if d.config["ipv6.address"] != "" {
		// Set necessary sysctls use with l2proxy parent in routed mode.
		ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", parentName)
		err := util.SysctlSet(ipv6FwdPath, "1")
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
		}

		ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", parentName)
		err = util.SysctlSet(ipv6ProxyNdpPath, "1")
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
		}
	}

	return nil
}

// postStart is run after the instance is started.
func (d *nicRouted) postStart() error {
	v := d.volatileGet()

	// If host_name is defined (and it should be), then we add the dummy link-local gateway IPs
	// to the host end of the veth pair. This ensures that liveness detection of the gateways
	// inside the instance work and ensure that traffic doesn't periodically halt whilst ARP/NDP
	// is re-detected.
	if v["host_name"] != "" {
		// Attempt to disable IPv6 router advertisement acceptance.
		err := util.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", v["host_name"]), "0")
		if err != nil {
			return err
		}

		if d.config["ipv4.address"] != "" {
			_, err := shared.RunCommand("ip", "-4", "addr", "add", fmt.Sprintf("%s/32", d.ipv4HostAddress()), "dev", v["host_name"])
			if err != nil {
				return err
			}
		}

		if d.config["ipv6.address"] != "" {
			_, err := shared.RunCommand("ip", "-6", "addr", "add", fmt.Sprintf("%s/128", d.ipv6HostAddress()), "dev", v["host_name"])
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Stop is run when the device is removed from the instance.
func (d *nicRouted) Stop() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{
		PostHooks: []func() error{d.postStop},
	}

	return &runConf, nil
}

// postStop is run after the device is removed from the instance.
func (d *nicRouted) postStop() error {
	defer d.volatileSet(map[string]string{
		"last_state.created": "",
		"host_name":          "",
	})

	v := d.volatileGet()

	// This will delete the parent interface if we created it for VLAN parent.
	if shared.IsTrue(v["last_state.created"]) {
		parentName := network.GetHostDevice(d.config["parent"], d.config["vlan"])
		err := networkRemoveInterfaceIfNeeded(d.state, parentName, d.inst, d.config["parent"], d.config["vlan"])
		if err != nil {
			return err
		}
	}

	return nil
}

func (d *nicRouted) ipv4HostAddress() string {
	if d.config["ipv4.host_address"] != "" {
		return d.config["ipv4.host_address"]
	}

	return nicRoutedIPv4GW
}

func (d *nicRouted) ipv6HostAddress() string {
	if d.config["ipv6.host_address"] != "" {
		return d.config["ipv6.host_address"]
	}

	return nicRoutedIPv6GW
}
<commit_msg>lxd/device/nic_routed: Don't fail on missing IPv6<commit_after>package device

import (
	"fmt"
	"os"
	"strings"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
)

const nicRoutedIPv4GW = "169.254.0.1"
const nicRoutedIPv6GW = "fe80::1"

type nicRouted struct {
	deviceCommon
}

func (d *nicRouted) CanHotPlug() (bool, []string) {
	return false, []string{}
}

// validateConfig checks the supplied config for correctness.
func (d *nicRouted) validateConfig(instConf instance.ConfigReader) error {
	if !instanceSupported(instConf.Type(), instancetype.Container) {
		return ErrUnsupportedDevType
	}

	requiredFields := []string{}
	optionalFields := []string{
		"name",
		"parent",
		"mtu",
		"hwaddr",
		"host_name",
		"vlan",
		"ipv4.gateway",
		"ipv6.gateway",
		"ipv4.host_address",
		"ipv6.host_address",
	}

	rules := nicValidationRules(requiredFields, optionalFields)
	rules["ipv4.address"] = func(value string) error {
		if value == "" {
			return nil
		}

		return NetworkValidAddressV4List(value)
	}
	rules["ipv6.address"] = func(value string) error {
		if value == "" {
			return nil
		}

		return NetworkValidAddressV6List(value)
	}

	err := d.config.Validate(rules)
	if err != nil {
		return err
	}

	return nil
}

// validateEnvironment checks the runtime environment for correctness.
func (d *nicRouted) validateEnvironment() error {
	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
		return fmt.Errorf("Requires name property to start")
	}

	if d.config["parent"] != "" && !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) {
		return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"])
	}

	if d.config["parent"] == "" && d.config["vlan"] != "" {
		return fmt.Errorf("The vlan setting can only be used when combined with a parent interface")
	}

	extensions := d.state.OS.LXCFeatures
	if !extensions["network_veth_router"] || !extensions["network_l2proxy"] {
		return fmt.Errorf("Requires liblxc has following API extensions: network_veth_router, network_l2proxy")
	}

	// Check necessary sysctls are configured for use with l2proxy parent for routed mode.
	if d.config["parent"] != "" && d.config["ipv4.address"] != "" {
		ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", d.config["parent"])
		sysctlVal, err := util.SysctlGet(ipv4FwdPath)
		if err != nil || sysctlVal != "1\n" {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv4FwdPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv4.conf.%s.forwarding=1", d.config["parent"])
		}
	}

	// Check necessary sysctls are configured for use with l2proxy parent for routed mode.
	if d.config["parent"] != "" && d.config["ipv6.address"] != "" {
		// net.ipv6.conf.all.forwarding=1 is required to enable general packet forwarding for IPv6.
		ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", "all")
		sysctlVal, err := util.SysctlGet(ipv6FwdPath)
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", "all")
		}

		ipv6FwdPath = fmt.Sprintf("net/ipv6/conf/%s/forwarding", d.config["parent"])
		sysctlVal, err = util.SysctlGet(ipv6FwdPath)
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", d.config["parent"])
		}

		// net.ipv6.conf.all.proxy_ndp=1 is needed otherwise unicast neighbour solicitations are rejected.
		// This causes periodic latency spikes every 15-20s as the neighbour has to resort to using
		// multicast NDP resolution and expires the previous neighbour entry.
		ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", "all")
		sysctlVal, err = util.SysctlGet(ipv6ProxyNdpPath)
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", "all")
		}

		ipv6ProxyNdpPath = fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", d.config["parent"])
		sysctlVal, err = util.SysctlGet(ipv6ProxyNdpPath)
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
		}
		if sysctlVal != "1\n" {
			return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", d.config["parent"])
		}
	}

	return nil
}

// Start is run when the instance is starting up (Routed mode doesn't support hot plugging).
func (d *nicRouted) Start() (*deviceConfig.RunConfig, error) {
	err := d.validateEnvironment()
	if err != nil {
		return nil, err
	}

	// Lock to avoid issues with containers starting in parallel.
	networkCreateSharedDeviceLock.Lock()
	defer networkCreateSharedDeviceLock.Unlock()

	saveData := make(map[string]string)

	// Decide which parent we should use based on VLAN setting.
	parentName := ""
	if d.config["parent"] != "" {
		parentName = network.GetHostDevice(d.config["parent"], d.config["vlan"])

		statusDev, err := networkCreateVlanDeviceIfNeeded(d.state, d.config["parent"], parentName, d.config["vlan"])
		if err != nil {
			return nil, err
		}

		// Record whether we created this device or not so it can be removed on stop.
		saveData["last_state.created"] = fmt.Sprintf("%t", statusDev != "existing")

		// If we created a VLAN interface, we need to setup the sysctls on that interface.
		if statusDev == "created" {
			err := d.setupParentSysctls(parentName)
			if err != nil {
				return nil, err
			}
		}
	}

	hostName := d.config["host_name"]
	if hostName == "" {
		hostName = networkRandomDevName("veth")
	}
	saveData["host_name"] = hostName

	err = d.volatileSet(saveData)
	if err != nil {
		return nil, err
	}

	runConf := deviceConfig.RunConfig{}
	nic := []deviceConfig.RunConfigItem{
		{Key: "name", Value: d.config["name"]},
		{Key: "type", Value: "veth"},
		{Key: "flags", Value: "up"},
		{Key: "veth.mode", Value: "router"},
		{Key: "veth.pair", Value: saveData["host_name"]},
	}

	// If there is a designated parent interface, activate the layer2 proxy mode to advertise
	// the instance's IPs over that interface using proxy APR/NDP.
	if parentName != "" {
		nic = append(nic,
			deviceConfig.RunConfigItem{Key: "l2proxy", Value: "1"},
			deviceConfig.RunConfigItem{Key: "link", Value: parentName},
		)
	}

	if d.config["mtu"] != "" {
		nic = append(nic, deviceConfig.RunConfigItem{Key: "mtu", Value: d.config["mtu"]})
	}

	if d.config["ipv4.address"] != "" {
		for _, addr := range strings.Split(d.config["ipv4.address"], ",") {
			addr = strings.TrimSpace(addr)
			nic = append(nic, deviceConfig.RunConfigItem{Key: "ipv4.address", Value: fmt.Sprintf("%s/32", addr)})
		}

		if nicHasAutoGateway(d.config["ipv4.gateway"]) {
			// Use a fixed link-local address as the next-hop default gateway.
			nic = append(nic, deviceConfig.RunConfigItem{Key: "ipv4.gateway", Value: d.ipv4HostAddress()})
		}
	}

	if d.config["ipv6.address"] != "" {
		for _, addr := range strings.Split(d.config["ipv6.address"], ",") {
			addr = strings.TrimSpace(addr)
			nic = append(nic, deviceConfig.RunConfigItem{Key: "ipv6.address", Value: fmt.Sprintf("%s/128", addr)})
		}

		if nicHasAutoGateway(d.config["ipv6.gateway"]) {
			// Use a fixed link-local address as the next-hop default gateway.
			nic = append(nic, deviceConfig.RunConfigItem{Key: "ipv6.gateway", Value: d.ipv6HostAddress()})
		}
	}

	runConf.NetworkInterface = nic
	runConf.PostHooks = append(runConf.PostHooks, d.postStart)
	return &runConf, nil
}

// setupParentSysctls configures the required sysctls on the parent to allow l2proxy to work.
// Because of our policy not to modify sysctls on existing interfaces, this should only be called
// if we created the parent interface.
func (d *nicRouted) setupParentSysctls(parentName string) error {
	if d.config["ipv4.address"] != "" {
		// Set necessary sysctls for use with l2proxy parent in routed mode.
		ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", parentName)
		err := util.SysctlSet(ipv4FwdPath, "1")
		if err != nil {
			return fmt.Errorf("Error setting net sysctl %s: %v", ipv4FwdPath, err)
		}
	}

	if d.config["ipv6.address"] != "" {
		// Set necessary sysctls use with l2proxy parent in routed mode.
		ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", parentName)
		err := util.SysctlSet(ipv6FwdPath, "1")
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err)
		}

		ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", parentName)
		err = util.SysctlSet(ipv6ProxyNdpPath, "1")
		if err != nil {
			return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err)
		}
	}

	return nil
}

// postStart is run after the instance is started.
func (d *nicRouted) postStart() error {
	v := d.volatileGet()

	// If host_name is defined (and it should be), then we add the dummy link-local gateway IPs
	// to the host end of the veth pair. This ensures that liveness detection of the gateways
	// inside the instance work and ensure that traffic doesn't periodically halt whilst ARP/NDP
	// is re-detected.
	if v["host_name"] != "" {
		// Attempt to disable IPv6 router advertisement acceptance.
		err := util.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", v["host_name"]), "0")
		if err != nil && !os.IsNotExist(err) {
			return err
		}

		if d.config["ipv4.address"] != "" {
			_, err := shared.RunCommand("ip", "-4", "addr", "add", fmt.Sprintf("%s/32", d.ipv4HostAddress()), "dev", v["host_name"])
			if err != nil {
				return err
			}
		}

		if d.config["ipv6.address"] != "" {
			_, err := shared.RunCommand("ip", "-6", "addr", "add", fmt.Sprintf("%s/128", d.ipv6HostAddress()), "dev", v["host_name"])
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Stop is run when the device is removed from the instance.
func (d *nicRouted) Stop() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{
		PostHooks: []func() error{d.postStop},
	}

	return &runConf, nil
}

// postStop is run after the device is removed from the instance.
func (d *nicRouted) postStop() error {
	defer d.volatileSet(map[string]string{
		"last_state.created": "",
		"host_name":          "",
	})

	v := d.volatileGet()

	// This will delete the parent interface if we created it for VLAN parent.
	if shared.IsTrue(v["last_state.created"]) {
		parentName := network.GetHostDevice(d.config["parent"], d.config["vlan"])
		err := networkRemoveInterfaceIfNeeded(d.state, parentName, d.inst, d.config["parent"], d.config["vlan"])
		if err != nil {
			return err
		}
	}

	return nil
}

func (d *nicRouted) ipv4HostAddress() string {
	if d.config["ipv4.host_address"] != "" {
		return d.config["ipv4.host_address"]
	}

	return nicRoutedIPv4GW
}

func (d *nicRouted) ipv6HostAddress() string {
	if d.config["ipv6.host_address"] != "" {
		return d.config["ipv6.host_address"]
	}

	return nicRoutedIPv6GW
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

func containerMetadataGet(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}
	metadataPath := filepath.Join(c.Path(), "metadata.yaml")

	// Start the storage if needed
	ourStart, err := c.StorageStart()
	if err != nil {
		return response.SmartError(err)
	}
	if ourStart {
		defer c.StorageStop()
	}

	// If missing, just return empty result
	if !shared.PathExists(metadataPath) {
		return response.SyncResponse(true, api.ImageMetadata{})
	}

	// Read the metadata
	metadataFile, err := os.Open(metadataPath)
	if err != nil {
		return response.InternalError(err)
	}
	defer metadataFile.Close()

	data, err := ioutil.ReadAll(metadataFile)
	if err != nil {
		return response.InternalError(err)
	}

	// Parse into the API struct
	metadata := api.ImageMetadata{}
	err = yaml.Unmarshal(data, &metadata)
	if err != nil {
		return response.SmartError(err)
	}

	return response.SyncResponse(true, metadata)
}

func containerMetadataPut(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}
	metadataPath := filepath.Join(c.Path(), "metadata.yaml")

	// Start the storage if needed
	ourStart, err := c.StorageStart()
	if err != nil {
		return response.SmartError(err)
	}
	if ourStart {
		defer c.StorageStop()
	}

	// Read the new metadata
	metadata := api.ImageMetadata{}
	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
		return response.BadRequest(err)
	}

	// Write as YAML
	data, err := yaml.Marshal(metadata)
	if err != nil {
		return response.BadRequest(err)
	}

	if err := ioutil.WriteFile(metadataPath, data, 0644); err != nil {
		return response.InternalError(err)
	}

	return response.EmptySyncResponse
}

// Return a list of templates used in a container or the content of a template
func containerMetadataTemplatesGet(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	// Start the storage if needed
	ourStart, err := c.StorageStart()
	if err != nil {
		return response.SmartError(err)
	}
	if ourStart {
		defer c.StorageStop()
	}

	// Look at the request
	templateName := r.FormValue("path")
	if templateName == "" {
		templates := []string{}
		if !shared.PathExists(filepath.Join(c.Path(), "templates")) {
			return response.SyncResponse(true, templates)
		}

		// List templates
		templatesPath := filepath.Join(c.Path(), "templates")
		filesInfo, err := ioutil.ReadDir(templatesPath)
		if err != nil {
			return response.InternalError(err)
		}

		for _, info := range filesInfo {
			if !info.IsDir() {
				templates = append(templates, info.Name())
			}
		}

		return response.SyncResponse(true, templates)
	}

	// Check if the template exists
	templatePath, err := getContainerTemplatePath(c, templateName)
	if err != nil {
		return response.SmartError(err)
	}

	if !shared.PathExists(templatePath) {
		return response.NotFound(fmt.Errorf("Template '%s' not found", templateName))
	}

	// Create a temporary file with the template content (since the container
	// storage might not be available when the file is read from FileResponse)
	template, err := os.Open(templatePath)
	if err != nil {
		return response.SmartError(err)
	}
	defer template.Close()

	tempfile, err := ioutil.TempFile("", "lxd_template")
	if err != nil {
		return response.SmartError(err)
	}
	defer tempfile.Close()

	_, err = io.Copy(tempfile, template)
	if err != nil {
		return response.InternalError(err)
	}

	files := make([]response.FileResponseEntry, 1)
	files[0].Identifier = templateName
	files[0].Path = tempfile.Name()
	files[0].Filename = templateName
	return response.FileResponse(r, files, nil, true)
}

// Add a container template file
func containerMetadataTemplatesPostPut(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	// Start the storage if needed
	ourStart, err := c.StorageStart()
	if err != nil {
		return response.SmartError(err)
	}
	if ourStart {
		defer c.StorageStop()
	}

	// Look at the request
	templateName := r.FormValue("path")
	if templateName == "" {
		return response.BadRequest(fmt.Errorf("missing path argument"))
	}

	if !shared.PathExists(filepath.Join(c.Path(), "templates")) {
		err := os.MkdirAll(filepath.Join(c.Path(), "templates"), 0711)
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Check if the template already exists
	templatePath, err := getContainerTemplatePath(c, templateName)
	if err != nil {
		return response.SmartError(err)
	}

	if r.Method == "POST" && shared.PathExists(templatePath) {
		return response.BadRequest(fmt.Errorf("Template already exists"))
	}

	// Write the new template
	template, err := os.OpenFile(templatePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return response.SmartError(err)
	}
	defer template.Close()

	_, err = io.Copy(template, r.Body)
	if err != nil {
		return response.InternalError(err)
	}

	return response.EmptySyncResponse
}

// Delete a container template
func containerMetadataTemplatesDelete(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)

	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	// Start the storage if needed
	ourStart, err := c.StorageStart()
	if err != nil {
		return response.SmartError(err)
	}
	if ourStart {
		defer c.StorageStop()
	}

	// Look at the request
	templateName := r.FormValue("path")
	if templateName == "" {
		return response.BadRequest(fmt.Errorf("missing path argument"))
	}

	templatePath, err := getContainerTemplatePath(c, templateName)
	if err != nil {
		return response.SmartError(err)
	}

	if !shared.PathExists(templatePath) {
		return response.NotFound(fmt.Errorf("Template '%s' not found", templateName))
	}

	// Delete the template
	err = os.Remove(templatePath)
	if err != nil {
		return response.InternalError(err)
	}

	return response.EmptySyncResponse
}

// Return the full path of a container template.
func getContainerTemplatePath(c instance.Instance, filename string) (string, error) {
	if strings.Contains(filename, "/") {
		return "", fmt.Errorf("Invalid template filename")
	}

	return filepath.Join(c.Path(), "templates", filename), nil
}
<commit_msg>lxd/instance/metadata: Removes use of c.StorageStart and c.StorageStop<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/response"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

func containerMetadataGet(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	// Start the storage if needed
	pool, err := storagePools.GetPoolByInstance(d.State(), c)
	if err != nil {
		return response.SmartError(err)
	}

	_, err = storagePools.InstanceMount(pool, c, nil)
	if err != nil {
		return response.SmartError(err)
	}
	defer storagePools.InstanceUnmount(pool, c, nil)

	// If missing, just return empty result
	metadataPath := filepath.Join(c.Path(), "metadata.yaml")
	if !shared.PathExists(metadataPath) {
		return response.SyncResponse(true, api.ImageMetadata{})
	}

	// Read the metadata
	metadataFile, err := os.Open(metadataPath)
	if err != nil {
		return response.InternalError(err)
	}
	defer metadataFile.Close()

	data, err := ioutil.ReadAll(metadataFile)
	if err != nil {
		return response.InternalError(err)
	}

	// Parse into the API struct
	metadata := api.ImageMetadata{}
	err = yaml.Unmarshal(data, &metadata)
	if err != nil {
		return response.SmartError(err)
	}

	return response.SyncResponse(true, metadata)
}

func containerMetadataPut(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	// Start the storage if needed
	pool, err := storagePools.GetPoolByInstance(d.State(), c)
	if err != nil {
		return response.SmartError(err)
	}

	_, err = storagePools.InstanceMount(pool, c, nil)
	if err != nil {
		return response.SmartError(err)
	}
	defer storagePools.InstanceUnmount(pool, c, nil)

	// Read the new metadata
	metadata := api.ImageMetadata{}
	if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
		return response.BadRequest(err)
	}

	// Write as YAML
	data, err := yaml.Marshal(metadata)
	if err != nil {
		return response.BadRequest(err)
	}

	metadataPath := filepath.Join(c.Path(), "metadata.yaml")
	if err := ioutil.WriteFile(metadataPath, data, 0644); err != nil {
		return response.InternalError(err)
	}

	return response.EmptySyncResponse
}

// Return a list of templates used in a container or the content of a template
func containerMetadataTemplatesGet(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	// Start the storage if needed
	pool, err := storagePools.GetPoolByInstance(d.State(), c)
	if err != nil {
		return response.SmartError(err)
	}

	_, err = storagePools.InstanceMount(pool, c, nil)
	if err != nil {
		return response.SmartError(err)
	}
	defer storagePools.InstanceUnmount(pool, c, nil)

	// Look at the request
	templateName := r.FormValue("path")
	if templateName == "" {
		templates := []string{}
		if !shared.PathExists(filepath.Join(c.Path(), "templates")) {
			return response.SyncResponse(true, templates)
		}

		// List templates
		templatesPath := filepath.Join(c.Path(), "templates")
		filesInfo, err := ioutil.ReadDir(templatesPath)
		if err != nil {
			return response.InternalError(err)
		}

		for _, info := range filesInfo {
			if !info.IsDir() {
				templates = append(templates, info.Name())
			}
		}

		return response.SyncResponse(true, templates)
	}

	// Check if the template exists
	templatePath, err := getContainerTemplatePath(c, templateName)
	if err != nil {
		return response.SmartError(err)
	}

	if !shared.PathExists(templatePath) {
		return response.NotFound(fmt.Errorf("Template '%s' not found", templateName))
	}

	// Create a temporary file with the template content (since the container
	// storage might not be available when the file is read from FileResponse)
	template, err := os.Open(templatePath)
	if err != nil {
		return response.SmartError(err)
	}
	defer template.Close()

	tempfile, err := ioutil.TempFile("", "lxd_template")
	if err != nil {
		return response.SmartError(err)
	}
	defer tempfile.Close()

	_, err = io.Copy(tempfile, template)
	if err != nil {
		return response.InternalError(err)
	}

	files := make([]response.FileResponseEntry, 1)
	files[0].Identifier = templateName
	files[0].Path = tempfile.Name()
	files[0].Filename = templateName
	return response.FileResponse(r, files, nil, true)
}

// Add a container template file
func containerMetadataTemplatesPostPut(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	// Start the storage if needed
	pool, err := storagePools.GetPoolByInstance(d.State(), c)
	if err != nil {
		return response.SmartError(err)
	}

	_, err = storagePools.InstanceMount(pool, c, nil)
	if err != nil {
		return response.SmartError(err)
	}
	defer storagePools.InstanceUnmount(pool, c, nil)

	// Look at the request
	templateName := r.FormValue("path")
	if templateName == "" {
		return response.BadRequest(fmt.Errorf("missing path argument"))
	}

	if !shared.PathExists(filepath.Join(c.Path(), "templates")) {
		err := os.MkdirAll(filepath.Join(c.Path(), "templates"), 0711)
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Check if the template already exists
	templatePath, err := getContainerTemplatePath(c, templateName)
	if err != nil {
		return response.SmartError(err)
	}

	if r.Method == "POST" && shared.PathExists(templatePath) {
		return response.BadRequest(fmt.Errorf("Template already exists"))
	}

	// Write the new template
	template, err := os.OpenFile(templatePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return response.SmartError(err)
	}
	defer template.Close()

	_, err = io.Copy(template, r.Body)
	if err != nil {
		return response.InternalError(err)
	}

	return response.EmptySyncResponse
}

// Delete a container template
func containerMetadataTemplatesDelete(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)

	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	// Load the container
	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	// Start the storage if needed
	pool, err := storagePools.GetPoolByInstance(d.State(), c)
	if err != nil {
		return response.SmartError(err)
	}

	_, err = storagePools.InstanceMount(pool, c, nil)
	if err != nil {
		return response.SmartError(err)
	}
	defer storagePools.InstanceUnmount(pool, c, nil)

	// Look at the request
	templateName := r.FormValue("path")
	if templateName == "" {
		return response.BadRequest(fmt.Errorf("missing path argument"))
	}

	templatePath, err := getContainerTemplatePath(c, templateName)
	if err != nil {
		return response.SmartError(err)
	}

	if !shared.PathExists(templatePath) {
		return response.NotFound(fmt.Errorf("Template '%s' not found", templateName))
	}

	// Delete the template
	err = os.Remove(templatePath)
	if err != nil {
		return response.InternalError(err)
	}

	return response.EmptySyncResponse
}

// Return the full path of a container template.
func getContainerTemplatePath(c instance.Instance, filename string) (string, error) {
	if strings.Contains(filename, "/") {
		return "", fmt.Errorf("Invalid template filename")
	}

	return filepath.Join(c.Path(), "templates", filename), nil
}
<|endoftext|>


<commit_before>package main_test

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"testing"

	strings "github.com/NeowayLabs/nash/stdbin/strings"
)

func TestStrings(t *testing.T) {

	type testcase struct {
		name        string
		input       func([]byte) []byte
		output      []string
		minWordSize uint
	}

	tcases := []testcase{
		{
			name:        "UTF-8With2Bytes",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte("λ"), bin...)
			},
			output: []string{"λ"},
		},
		{
			name:        "UTF-8With3Bytes",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte("€"), bin...)
			},
			output: []string{"€"},
		},
		{
			name:        "UTF-8With4Bytes",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte("𐍈"), bin...)
			},
			output: []string{"𐍈"},
		},
		{
			name:        "NonASCIIWordHasOneLessCharThanMin",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("λ"), bin...)
			},
			output: []string{},
		},
		{
			name:        "NonASCIIWordHasMinWordSize",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("λλ"), bin...)
			},
			output: []string{"λλ"},
		},
		{
			name:        "WordHasOneLessCharThanMin",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("k"), bin...)
			},
			output: []string{},
		},
		{
			name:        "WordHasMinWordSize",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("kz"), bin...)
			},
			output: []string{"kz"},
		},
		{
			name:        "WordHasOneMoreCharThanMinWordSize",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("ktz"), bin...)
			},
			output: []string{"ktz"},
		},
		{
			name:        "StartingWithOneChar",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte("k"), bin...)
			},
			output: []string{"k"},
		},
		{
			name:        "EndWithOneChar",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append(bin, []byte("k")...)
			},
			output: []string{"k"},
		},
		{
			name:        "OneCharInTheMiddle",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				t := append(bin, []byte("k")...)
				t = append(t, bin...)
				return t
			},
			output: []string{"k"},
		},
		{
			name:        "StartingWithText",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "textOnBeggining"
				return append([]byte(expected), bin...)
			},
			output: []string{"textOnBeggining"},
		},
		{
			name:        "TextOnMiddle",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "textOnMiddle"
				return append(bin, append([]byte(expected), bin...)...)
			},
			output: []string{"textOnMiddle"},
		},
		{
			name:        "NonASCIITextOnMiddle",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "λλλ"
				return append(bin, append([]byte(expected), bin...)...)
			},
			output: []string{"λλλ"},
		},
		{
			name:        "ASCIIAndNonASCII",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "(define (λ (x) (+ x a)))"
				return append(bin, append([]byte(expected), bin...)...)
			},
			output: []string{"(define (λ (x) (+ x a)))"},
		},
		{
			name:        "TextOnEnd",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "textOnEnd"
				return append(bin, append([]byte(expected), bin...)...)
			},
			output: []string{"textOnEnd"},
		},
		{
			name:        "JustText",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return []byte("justtext")
			},
			output: []string{"justtext"},
		},
		{
			name:        "JustBinary",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return bin
			},
			output: []string{},
		},
		{
			name:        "TextSeparatedByBinary",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				text := []byte("text")
				t := []byte{}
				t = append(t, bin...)
				t = append(t, text...)
				t = append(t, bin...)
				t = append(t, text...)
				return t
			},
			output: []string{"text", "text"},
		},
		{
			name:        "NonASCIITextSeparatedByBinary",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				text := []byte("awesomeλ=)")
				t := []byte{}
				t = append(t, bin...)
				t = append(t, text...)
				t = append(t, bin...)
				t = append(t, text...)
				return t
			},
			output: []string{"awesomeλ=)", "awesomeλ=)"},
		},
		{
			name:        "WordsAreNotAccumulativeBetweenBinData",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				t := append([]byte("k"), bin...)
				return append(t, byte('t'))
			},
			output: []string{},
		},
		{
			name:        "ASCIISeparatedByByteThatLooksLikeUTF",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte{
					'n',
					runestart,
					'k',
				}, bin...)
			},
			output: []string{"n", "k"},
		},
		{
			name:        "ASCIIAfterPossibleFirstByteOfUTF",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte{
					runestart,
					'k',
				}, bin...)
			},
			output: []string{"k"},
		},
		{
			name:        "ASCIIAfterPossibleSecondByteOfUTF",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte{
					byte(0xE2),
					byte(0x82),
					'k',
				}, bin...)
			},
			output: []string{"k"},
		},
		{
			name:        "ASCIIAfterPossibleThirdByteOfUTF",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte{
					byte(0xF0),
					byte(0x90),
					byte(0x8D),
					'k',
				}, bin...)
			},
			output: []string{"k"},
		},
		{
			name:        "AfterFalseRuneStartRuneStartOnSecondByte",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				i := []byte{byte(0xF0)}
				i = append(i, []byte("λ")...)
				return append(i, bin...)
			},
			output: []string{"λ"},
		},
		{
			name:        "AfterFalseRuneStartRuneStartOnThirdByte",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				i := []byte{byte(0xF0), byte(0x90)}
				i = append(i, []byte("λ")...)
				return append(i, bin...)
			},
			output: []string{"λ"},
		},
		{
			name:        "AfterFalseRuneStartRuneStartOnFourthByte",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				i := []byte{byte(0xF0), byte(0x90), byte(0x8D)}
				i = append(i, []byte("λ")...)
				return append(i, bin...)
			},
			output: []string{"λ"},
		},
		{
			name:        "ASCIIFakeRuneAndThemRune",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				i := []byte{'v'}
				i = append(i, byte(0xF0))
				i = append(i, []byte("λ")...)
				return append(i, bin...)
			},
			output: []string{"v", "λ"},
		},
		{
			name:        "ASCIISplittedByZero",
			minWordSize: 1,
			input: func([]byte) []byte {
				return []byte{'k', 0, 'n', 0, 'v'}
			},
			output: []string{"k", "n", "v"},
		},
		{
			name:        "RunesSplittedByZero",
			minWordSize: 1,
			input: func([]byte) []byte {
				i := []byte("λ")
				i = append(i, 0)
				i = append(i, []byte("λ")...)
				return i
			},
			output: []string{"λ", "λ"},
		},
		{
			name:        "ASCIIAndRunesSplittedByZero",
			minWordSize: 1,
			input: func([]byte) []byte {
				i := []byte("λ")
				i = append(i, 0)
				i = append(i, 's')
				i = append(i, 0)
				i = append(i, []byte("λ")...)
				return i
			},
			output: []string{"λ", "s", "λ"},
		},
	}

	minBinChunkSize := 1
	maxBinChunkSize := 128

	for _, tcase := range tcases {
		for i := minBinChunkSize; i <= maxBinChunkSize; i++ {
			binsize := i
			testname := fmt.Sprintf("%s/binSize%d", tcase.name, binsize)
			t.Run(testname, func(t *testing.T) {
				bin := newBinary(uint(binsize))
				input := tcase.input(bin)
				scanner := strings.Do(bytes.NewBuffer(input), tcase.minWordSize)

				lines := []string{}
				for scanner.Scan() {
					lines = append(lines, scanner.Text())
				}

				if len(lines) != len(tcase.output) {
					t.Errorf("wanted size[%d] got size[%d]", len(tcase.output), len(lines))
					t.Fatalf("wanted[%s] got[%s]", tcase.output, lines)
				}

				for i, want := range tcase.output {
					got := lines[i]
					if want != got {
						t.Errorf("unexpected line at[%d]", i)
						t.Errorf("wanted[%s] got[%s]", want, got)
						t.Errorf("wantedLines[%s] gotLines[%s]", tcase.output, lines)
					}
				}

				if scanner.Err() != nil {
					t.Fatal("unexpected error[%s]", scanner.Err())
				}
			})
		}
	}
}

func TestStringsReadErrorOnFirstByte(t *testing.T) {
	var minWordSize uint = 1
	scanner := strings.Do(newFakeReader(func(d []byte) (int, error) {
		return 0, errors.New("fake injected error")
	}), minWordSize)
	assertScannerFails(t, scanner, 0)
}

func TestStringsReadErrorOnSecondByte(t *testing.T) {
	var minWordSize uint = 1
	sentFirstByte := false
	scanner := strings.Do(newFakeReader(func(d []byte) (int, error) {
		if sentFirstByte {
			return 0, errors.New("fake injected error")
		}
		d[0] = 'k'
		sentFirstByte = true
		return 1, nil
	}), minWordSize)
	assertScannerFails(t, scanner, 1)
}

func TestStringsReadErrorAfterValidUTF8StartingByte(t *testing.T) {
	var minWordSize uint = 1
	sentFirstByte := false
	scanner := strings.Do(newFakeReader(func(d []byte) (int, error) {
		if sentFirstByte {
			return 0, errors.New("fake injected error")
		}
		sentFirstByte = true
		d[0] = runestart
		return 1, nil
	}), minWordSize)
	assertScannerFails(t, scanner, 0)
}

func TestStringsReadCanReturnEOFWithData(t *testing.T) {
	var minWordSize uint = 1
	want := byte('k')

	scanner := strings.Do(newFakeReader(func(d []byte) (int, error) {
		if len(d) == 0 {
			t.Fatal("empty data on Read operation")
		}
		d[0] = want
		return 1, io.EOF
	}), minWordSize)

	if !scanner.Scan() {
		t.Fatal("unexpected Scan failure")
	}
	got := scanner.Text()
	if string(want) != got {
		t.Fatalf("want[%s] != got[%s]", string(want), got)
	}
}

const runestart byte = 0xC2

type FakeReader struct {
	read func([]byte) (int, error)
}

func (f *FakeReader) Read(d []byte) (int, error) {
	if f.read == nil {
		return 0, fmt.Errorf("FakeReader has no Read implementation")
	}
	return f.read(d)
}

func newFakeReader(read func([]byte) (int, error)) *FakeReader {
	return &FakeReader{read: read}
}

func assertScannerFails(t *testing.T, scanner *bufio.Scanner, expectedIter uint) {
	var iterations uint
	for scanner.Scan() {
		iterations += 1
	}

	if iterations != expectedIter {
		t.Fatalf("expected[%d] Scan calls, got [%d]", expectedIter, iterations)
	}

	if scanner.Err() == nil {
		t.Fatal("expected failure on scanner, got none")
	}
}

func newBinary(size uint) []byte {
	// WHY: Starting with the most significant bit as 1 helps to test
	// UTF-8 corner cases. Don't change this without providing
	// testing for this. Not the best way to do this (not explicit)
	// but it is what we have for today =).
	bin := make([]byte, size)
	for i := 0; i < int(size); i++ {
		bin[i] = 0xFF
	}
	return bin
}
<commit_msg>Fix error calling t.Fatal with format<commit_after>package main_test

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"testing"

	strings "github.com/NeowayLabs/nash/stdbin/strings"
)

func TestStrings(t *testing.T) {

	type testcase struct {
		name        string
		input       func([]byte) []byte
		output      []string
		minWordSize uint
	}

	tcases := []testcase{
		{
			name:        "UTF-8With2Bytes",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte("λ"), bin...)
			},
			output: []string{"λ"},
		},
		{
			name:        "UTF-8With3Bytes",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte("€"), bin...)
			},
			output: []string{"€"},
		},
		{
			name:        "UTF-8With4Bytes",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte("𐍈"), bin...)
			},
			output: []string{"𐍈"},
		},
		{
			name:        "NonASCIIWordHasOneLessCharThanMin",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("λ"), bin...)
			},
			output: []string{},
		},
		{
			name:        "NonASCIIWordHasMinWordSize",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("λλ"), bin...)
			},
			output: []string{"λλ"},
		},
		{
			name:        "WordHasOneLessCharThanMin",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("k"), bin...)
			},
			output: []string{},
		},
		{
			name:        "WordHasMinWordSize",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("kz"), bin...)
			},
			output: []string{"kz"},
		},
		{
			name:        "WordHasOneMoreCharThanMinWordSize",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				return append([]byte("ktz"), bin...)
			},
			output: []string{"ktz"},
		},
		{
			name:        "StartingWithOneChar",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte("k"), bin...)
			},
			output: []string{"k"},
		},
		{
			name:        "EndWithOneChar",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append(bin, []byte("k")...)
			},
			output: []string{"k"},
		},
		{
			name:        "OneCharInTheMiddle",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				t := append(bin, []byte("k")...)
				t = append(t, bin...)
				return t
			},
			output: []string{"k"},
		},
		{
			name:        "StartingWithText",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "textOnBeggining"
				return append([]byte(expected), bin...)
			},
			output: []string{"textOnBeggining"},
		},
		{
			name:        "TextOnMiddle",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "textOnMiddle"
				return append(bin, append([]byte(expected), bin...)...)
			},
			output: []string{"textOnMiddle"},
		},
		{
			name:        "NonASCIITextOnMiddle",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "λλλ"
				return append(bin, append([]byte(expected), bin...)...)
			},
			output: []string{"λλλ"},
		},
		{
			name:        "ASCIIAndNonASCII",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "(define (λ (x) (+ x a)))"
				return append(bin, append([]byte(expected), bin...)...)
			},
			output: []string{"(define (λ (x) (+ x a)))"},
		},
		{
			name:        "TextOnEnd",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				expected := "textOnEnd"
				return append(bin, append([]byte(expected), bin...)...)
			},
			output: []string{"textOnEnd"},
		},
		{
			name:        "JustText",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return []byte("justtext")
			},
			output: []string{"justtext"},
		},
		{
			name:        "JustBinary",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return bin
			},
			output: []string{},
		},
		{
			name:        "TextSeparatedByBinary",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				text := []byte("text")
				t := []byte{}
				t = append(t, bin...)
				t = append(t, text...)
				t = append(t, bin...)
				t = append(t, text...)
				return t
			},
			output: []string{"text", "text"},
		},
		{
			name:        "NonASCIITextSeparatedByBinary",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				text := []byte("awesomeλ=)")
				t := []byte{}
				t = append(t, bin...)
				t = append(t, text...)
				t = append(t, bin...)
				t = append(t, text...)
				return t
			},
			output: []string{"awesomeλ=)", "awesomeλ=)"},
		},
		{
			name:        "WordsAreNotAccumulativeBetweenBinData",
			minWordSize: 2,
			input: func(bin []byte) []byte {
				t := append([]byte("k"), bin...)
				return append(t, byte('t'))
			},
			output: []string{},
		},
		{
			name:        "ASCIISeparatedByByteThatLooksLikeUTF",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte{
					'n',
					runestart,
					'k',
				}, bin...)
			},
			output: []string{"n", "k"},
		},
		{
			name:        "ASCIIAfterPossibleFirstByteOfUTF",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte{
					runestart,
					'k',
				}, bin...)
			},
			output: []string{"k"},
		},
		{
			name:        "ASCIIAfterPossibleSecondByteOfUTF",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte{
					byte(0xE2),
					byte(0x82),
					'k',
				}, bin...)
			},
			output: []string{"k"},
		},
		{
			name:        "ASCIIAfterPossibleThirdByteOfUTF",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				return append([]byte{
					byte(0xF0),
					byte(0x90),
					byte(0x8D),
					'k',
				}, bin...)
			},
			output: []string{"k"},
		},
		{
			name:        "AfterFalseRuneStartRuneStartOnSecondByte",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				i := []byte{byte(0xF0)}
				i = append(i, []byte("λ")...)
				return append(i, bin...)
			},
			output: []string{"λ"},
		},
		{
			name:        "AfterFalseRuneStartRuneStartOnThirdByte",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				i := []byte{byte(0xF0), byte(0x90)}
				i = append(i, []byte("λ")...)
				return append(i, bin...)
			},
			output: []string{"λ"},
		},
		{
			name:        "AfterFalseRuneStartRuneStartOnFourthByte",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				i := []byte{byte(0xF0), byte(0x90), byte(0x8D)}
				i = append(i, []byte("λ")...)
				return append(i, bin...)
			},
			output: []string{"λ"},
		},
		{
			name:        "ASCIIFakeRuneAndThemRune",
			minWordSize: 1,
			input: func(bin []byte) []byte {
				i := []byte{'v'}
				i = append(i, byte(0xF0))
				i = append(i, []byte("λ")...)
				return append(i, bin...)
			},
			output: []string{"v", "λ"},
		},
		{
			name:        "ASCIISplittedByZero",
			minWordSize: 1,
			input: func([]byte) []byte {
				return []byte{'k', 0, 'n', 0, 'v'}
			},
			output: []string{"k", "n", "v"},
		},
		{
			name:        "RunesSplittedByZero",
			minWordSize: 1,
			input: func([]byte) []byte {
				i := []byte("λ")
				i = append(i, 0)
				i = append(i, []byte("λ")...)
				return i
			},
			output: []string{"λ", "λ"},
		},
		{
			name:        "ASCIIAndRunesSplittedByZero",
			minWordSize: 1,
			input: func([]byte) []byte {
				i := []byte("λ")
				i = append(i, 0)
				i = append(i, 's')
				i = append(i, 0)
				i = append(i, []byte("λ")...)
				return i
			},
			output: []string{"λ", "s", "λ"},
		},
	}

	minBinChunkSize := 1
	maxBinChunkSize := 128

	for _, tcase := range tcases {
		for i := minBinChunkSize; i <= maxBinChunkSize; i++ {
			binsize := i
			testname := fmt.Sprintf("%s/binSize%d", tcase.name, binsize)
			t.Run(testname, func(t *testing.T) {
				bin := newBinary(uint(binsize))
				input := tcase.input(bin)
				scanner := strings.Do(bytes.NewBuffer(input), tcase.minWordSize)

				lines := []string{}
				for scanner.Scan() {
					lines = append(lines, scanner.Text())
				}

				if len(lines) != len(tcase.output) {
					t.Errorf("wanted size[%d] got size[%d]", len(tcase.output), len(lines))
					t.Fatalf("wanted[%s] got[%s]", tcase.output, lines)
				}

				for i, want := range tcase.output {
					got := lines[i]
					if want != got {
						t.Errorf("unexpected line at[%d]", i)
						t.Errorf("wanted[%s] got[%s]", want, got)
						t.Errorf("wantedLines[%s] gotLines[%s]", tcase.output, lines)
					}
				}

				if scanner.Err() != nil {
					t.Fatalf("unexpected error[%s]", scanner.Err())
				}
			})
		}
	}
}

func TestStringsReadErrorOnFirstByte(t *testing.T) {
	var minWordSize uint = 1
	scanner := strings.Do(newFakeReader(func(d []byte) (int, error) {
		return 0, errors.New("fake injected error")
	}), minWordSize)
	assertScannerFails(t, scanner, 0)
}

func TestStringsReadErrorOnSecondByte(t *testing.T) {
	var minWordSize uint = 1
	sentFirstByte := false
	scanner := strings.Do(newFakeReader(func(d []byte) (int, error) {
		if sentFirstByte {
			return 0, errors.New("fake injected error")
		}
		d[0] = 'k'
		sentFirstByte = true
		return 1, nil
	}), minWordSize)
	assertScannerFails(t, scanner, 1)
}

func TestStringsReadErrorAfterValidUTF8StartingByte(t *testing.T) {
	var minWordSize uint = 1
	sentFirstByte := false
	scanner := strings.Do(newFakeReader(func(d []byte) (int, error) {
		if sentFirstByte {
			return 0, errors.New("fake injected error")
		}
		sentFirstByte = true
		d[0] = runestart
		return 1, nil
	}), minWordSize)
	assertScannerFails(t, scanner, 0)
}

func TestStringsReadCanReturnEOFWithData(t *testing.T) {
	var minWordSize uint = 1
	want := byte('k')

	scanner := strings.Do(newFakeReader(func(d []byte) (int, error) {
		if len(d) == 0 {
			t.Fatal("empty data on Read operation")
		}
		d[0] = want
		return 1, io.EOF
	}), minWordSize)

	if !scanner.Scan() {
		t.Fatal("unexpected Scan failure")
	}
	got := scanner.Text()
	if string(want) != got {
		t.Fatalf("want[%s] != got[%s]", string(want), got)
	}
}

const runestart byte = 0xC2

type FakeReader struct {
	read func([]byte) (int, error)
}

func (f *FakeReader) Read(d []byte) (int, error) {
	if f.read == nil {
		return 0, fmt.Errorf("FakeReader has no Read implementation")
	}
	return f.read(d)
}

func newFakeReader(read func([]byte) (int, error)) *FakeReader {
	return &FakeReader{read: read}
}

func assertScannerFails(t *testing.T, scanner *bufio.Scanner, expectedIter uint) {
	var iterations uint
	for scanner.Scan() {
		iterations += 1
	}

	if iterations != expectedIter {
		t.Fatalf("expected[%d] Scan calls, got [%d]", expectedIter, iterations)
	}

	if scanner.Err() == nil {
		t.Fatal("expected failure on scanner, got none")
	}
}

func newBinary(size uint) []byte {
	// WHY: Starting with the most significant bit as 1 helps to test
	// UTF-8 corner cases. Don't change this without providing
	// testing for this. Not the best way to do this (not explicit)
	// but it is what we have for today =).
	bin := make([]byte, size)
	for i := 0; i < int(size); i++ {
		bin[i] = 0xFF
	}
	return bin
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/binary"
	"os"
	"path/filepath"
	"time"

	"go.etcd.io/bbolt"

	"github.com/anacrolix/torrent/metainfo"
)

const (
	boltDbCompleteValue   = "c"
	boltDbIncompleteValue = "i"
)

var (
	completionBucketKey = []byte("completion")
)

type boltPieceCompletion struct {
	db *bbolt.DB
}

var _ PieceCompletion = (*boltPieceCompletion)(nil)

func NewBoltPieceCompletion(dir string) (ret PieceCompletion, err error) {
	os.MkdirAll(dir, 0770)
	p := filepath.Join(dir, ".torrent.bolt.db")
	db, err := bbolt.Open(p, 0660, &bbolt.Options{
		Timeout: time.Second,
	})
	if err != nil {
		return
	}
	db.NoSync = true
	ret = &boltPieceCompletion{db}
	return
}

func (me boltPieceCompletion) Get(pk metainfo.PieceKey) (cn Completion, err error) {
	err = me.db.View(func(tx *bbolt.Tx) error {
		cb := tx.Bucket(completionBucketKey)
		if cb == nil {
			return nil
		}
		ih := cb.Bucket(pk.InfoHash[:])
		if ih == nil {
			return nil
		}
		var key [4]byte
		binary.BigEndian.PutUint32(key[:], uint32(pk.Index))
		cn.Ok = true
		switch string(ih.Get(key[:])) {
		case boltDbCompleteValue:
			cn.Complete = true
		case boltDbIncompleteValue:
			cn.Complete = false
		default:
			cn.Ok = false
		}
		return nil
	})
	return
}

func (me boltPieceCompletion) Set(pk metainfo.PieceKey, b bool) error {
	return me.db.Update(func(tx *bbolt.Tx) error {
		c, err := tx.CreateBucketIfNotExists(completionBucketKey)
		if err != nil {
			return err
		}
		ih, err := c.CreateBucketIfNotExists(pk.InfoHash[:])
		if err != nil {
			return err
		}
		var key [4]byte
		binary.BigEndian.PutUint32(key[:], uint32(pk.Index))
		return ih.Put(key[:], []byte(func() string {
			if b {
				return boltDbCompleteValue
			} else {
				return boltDbIncompleteValue
			}
		}()))
	})
}

func (me *boltPieceCompletion) Close() error {
	return me.db.Close()
}
<commit_msg>Only write update bolt piece completion if the existing value differs<commit_after>package storage

import (
	"encoding/binary"
	"os"
	"path/filepath"
	"time"

	"go.etcd.io/bbolt"

	"github.com/anacrolix/torrent/metainfo"
)

const (
	boltDbCompleteValue   = "c"
	boltDbIncompleteValue = "i"
)

var (
	completionBucketKey = []byte("completion")
)

type boltPieceCompletion struct {
	db *bbolt.DB
}

var _ PieceCompletion = (*boltPieceCompletion)(nil)

func NewBoltPieceCompletion(dir string) (ret PieceCompletion, err error) {
	os.MkdirAll(dir, 0770)
	p := filepath.Join(dir, ".torrent.bolt.db")
	db, err := bbolt.Open(p, 0660, &bbolt.Options{
		Timeout: time.Second,
	})
	if err != nil {
		return
	}
	db.NoSync = true
	ret = &boltPieceCompletion{db}
	return
}

func (me boltPieceCompletion) Get(pk metainfo.PieceKey) (cn Completion, err error) {
	err = me.db.View(func(tx *bbolt.Tx) error {
		cb := tx.Bucket(completionBucketKey)
		if cb == nil {
			return nil
		}
		ih := cb.Bucket(pk.InfoHash[:])
		if ih == nil {
			return nil
		}
		var key [4]byte
		binary.BigEndian.PutUint32(key[:], uint32(pk.Index))
		cn.Ok = true
		switch string(ih.Get(key[:])) {
		case boltDbCompleteValue:
			cn.Complete = true
		case boltDbIncompleteValue:
			cn.Complete = false
		default:
			cn.Ok = false
		}
		return nil
	})
	return
}

func (me boltPieceCompletion) Set(pk metainfo.PieceKey, b bool) error {
	if c, err := me.Get(pk); err == nil && c.Ok && c.Complete == b {
		return nil
	}
	return me.db.Update(func(tx *bbolt.Tx) error {
		c, err := tx.CreateBucketIfNotExists(completionBucketKey)
		if err != nil {
			return err
		}
		ih, err := c.CreateBucketIfNotExists(pk.InfoHash[:])
		if err != nil {
			return err
		}
		var key [4]byte
		binary.BigEndian.PutUint32(key[:], uint32(pk.Index))
		return ih.Put(key[:], []byte(func() string {
			if b {
				return boltDbCompleteValue
			} else {
				return boltDbIncompleteValue
			}
		}()))
	})
}

func (me *boltPieceCompletion) Close() error {
	return me.db.Close()
}
<|endoftext|>


<commit_before>package meta

import (
	"context"
	"sync"
	"time"

	"github.com/pkg/errors"
	"github.com/projecteru2/core/log"
	"github.com/projecteru2/core/types"
	clientv3 "go.etcd.io/etcd/client/v3"
)

// StartEphemeral starts an empheral kv pair.
func (e *ETCD) StartEphemeral(ctx context.Context, path string, heartbeat time.Duration) (<-chan struct{}, func(), error) {
	lease, err := e.cliv3.Grant(ctx, int64(heartbeat/time.Second))
	if err != nil {
		return nil, nil, errors.WithStack(err)
	}

	switch tx, err := e.cliv3.Txn(ctx).
		If(clientv3.Compare(clientv3.Version(path), "=", 0)).
		Then(clientv3.OpPut(path, "", clientv3.WithLease(lease.ID))).
		Commit(); {
	case err != nil:
		return nil, nil, errors.WithStack(err)
	case !tx.Succeeded:
		return nil, nil, errors.Wrap(types.ErrKeyExists, path)
	}

	ctx, cancel := context.WithCancel(context.Background())
	expiry := make(chan struct{})

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		defer close(expiry)

		tick := time.NewTicker(heartbeat / 3)
		defer tick.Stop()

		revoke := func() {
			cctx, ccancel := context.WithTimeout(context.Background(), time.Minute) // todo minute sucks
			defer ccancel()
			if _, err := e.cliv3.Revoke(cctx, lease.ID); err != nil {
				log.Errorf(ctx, "[StartEphemeral] revoke %d with %s failed: %v", lease.ID, path, err)
			}
		}

		for {
			select {
			case <-tick.C:
				cctx, ccancel := context.WithTimeout(ctx, time.Minute) // todo minute sucks
				if _, err := e.cliv3.KeepAliveOnce(cctx, lease.ID); err != nil {
					log.Errorf(ctx, "[StartEphemeral] keepalive %d with %s failed: %v", lease.ID, path, err)
					ccancel()
					revoke()
					return
				}
				ccancel()

			case <-ctx.Done():
				revoke()
				return
			}
		}
	}()

	return expiry, func() {
		cancel()
		wg.Wait()
	}, nil
}
<commit_msg>uses the upstream ctx (#464)<commit_after>package meta

import (
	"context"
	"sync"
	"time"

	"github.com/pkg/errors"
	"github.com/projecteru2/core/log"
	"github.com/projecteru2/core/types"
	clientv3 "go.etcd.io/etcd/client/v3"
)

// StartEphemeral starts an empheral kv pair.
func (e *ETCD) StartEphemeral(ctx context.Context, path string, heartbeat time.Duration) (<-chan struct{}, func(), error) {
	lease, err := e.cliv3.Grant(ctx, int64(heartbeat/time.Second))
	if err != nil {
		return nil, nil, errors.WithStack(err)
	}

	switch tx, err := e.cliv3.Txn(ctx).
		If(clientv3.Compare(clientv3.Version(path), "=", 0)).
		Then(clientv3.OpPut(path, "", clientv3.WithLease(lease.ID))).
		Commit(); {
	case err != nil:
		return nil, nil, errors.WithStack(err)
	case !tx.Succeeded:
		return nil, nil, errors.Wrap(types.ErrKeyExists, path)
	}

	ctx, cancel := context.WithCancel(ctx)
	expiry := make(chan struct{})

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		defer close(expiry)

		tick := time.NewTicker(heartbeat / 3)
		defer tick.Stop()

		// Revokes the lease.
		defer func() {
			// It shouldn't be inheriting from the ctx.
			ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
			defer cancel()
			if _, err := e.cliv3.Revoke(ctx, lease.ID); err != nil {
				log.Errorf(ctx, "[StartEphemeral] revoke %d with %s failed: %v", lease.ID, path, err)
			}
		}()

		for {
			select {
			case <-tick.C:
				if _, err := e.cliv3.KeepAliveOnce(ctx, lease.ID); err != nil {
					log.Errorf(ctx, "[StartEphemeral] keepalive %d with %s failed: %v", lease.ID, path, err)
					return
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	return expiry, func() {
		cancel()
		wg.Wait()
	}, nil
}
<|endoftext|>


<commit_before>// Package postgresql implements a micro Store backed by sql
package postgresql

import (
	"database/sql"
	"fmt"
	"strings"
	"time"
	"unicode"

	"github.com/lib/pq"
	"github.com/pkg/errors"

	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/store"
)

// DefaultNamespace is the namespace that the sql store
// will use if no namespace is provided.
var (
	DefaultNamespace = "micro"
	DefaultPrefix    = "micro"
)

type sqlStore struct {
	db *sql.DB

	database string
	table    string
	options.Options
}

// List all the known records
func (s *sqlStore) List() ([]*store.Record, error) {
	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s;", s.database, s.table))
	if err != nil {
		return nil, err
	}
	var records []*store.Record
	var timehelper pq.NullTime
	rows, err := q.Query()
	if err != nil {
		if err == sql.ErrNoRows {
			return records, nil
		}
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		record := &store.Record{}
		if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
			return records, err
		}
		if timehelper.Valid {
			if timehelper.Time.Before(time.Now()) {
				// record has expired
				go s.Delete(record.Key)
			} else {
				record.Expiry = time.Until(timehelper.Time)
				records = append(records, record)
			}
		} else {
			records = append(records, record)
		}

	}
	rowErr := rows.Close()
	if rowErr != nil {
		// transaction rollback or something
		return records, rowErr
	}
	if err := rows.Err(); err != nil {
		return records, err
	}
	return records, nil
}

// Read all records with keys
func (s *sqlStore) Read(keys ...string) ([]*store.Record, error) {
	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s WHERE key = $1;", s.database, s.table))
	if err != nil {
		return nil, err
	}
	var records []*store.Record
	var timehelper pq.NullTime
	for _, key := range keys {
		row := q.QueryRow(key)
		record := &store.Record{}
		if err := row.Scan(&record.Key, &record.Value, &timehelper); err != nil {
			if err == sql.ErrNoRows {
				return records, store.ErrNotFound
			}
			return records, err
		}
		if timehelper.Valid {
			if timehelper.Time.Before(time.Now()) {
				// record has expired
				go s.Delete(key)
				return records, store.ErrNotFound
			}
			record.Expiry = time.Until(timehelper.Time)
			records = append(records, record)
		} else {
			records = append(records, record)
		}
	}
	return records, nil
}

// Write records
func (s *sqlStore) Write(rec ...*store.Record) error {
	q, err := s.db.Prepare(fmt.Sprintf(`INSERT INTO %s.%s(key, value, expiry)
		VALUES ($1, $2::bytea, $3)
		ON CONFLICT (key)
		DO UPDATE
		SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;`, s.database, s.table))
	if err != nil {
		return err
	}
	for _, r := range rec {
		var err error
		if r.Expiry != 0 {
			_, err = q.Exec(r.Key, r.Value, time.Now().Add(r.Expiry))
		} else {
			_, err = q.Exec(r.Key, r.Value, nil)
		}
		if err != nil {
			return errors.Wrap(err, "Couldn't insert record "+r.Key)
		}
	}

	return nil
}

// Delete records with keys
func (s *sqlStore) Delete(keys ...string) error {
	q, err := s.db.Prepare(fmt.Sprintf("DELETE FROM %s.%s WHERE key = $1;", s.database, s.table))
	if err != nil {
		return err
	}
	for _, key := range keys {
		result, err := q.Exec(key)
		if err != nil {
			return err
		}
		_, err = result.RowsAffected()
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *sqlStore) initDB(options options.Options) error {
	// Get the store.namespace option, or use sql.DefaultNamespace
	namespaceOpt, found := options.Values().Get("store.namespace")
	if !found {
		s.database = DefaultNamespace
	} else {
		if namespace, ok := namespaceOpt.(string); ok {
			s.database = namespace
		} else {
			return errors.New("store.namespace option must be a string")
		}
	}
	// Get the store.namespace option, or use sql.DefaultNamespace
	prefixOpt, found := options.Values().Get("store.prefix")
	if !found {
		s.table = DefaultPrefix
	} else {
		if prefix, ok := prefixOpt.(string); ok {
			s.table = prefix
		} else {
			return errors.New("store.namespace option must be a string")
		}
	}

	// Create "micro" schema
	schema, err := s.db.Prepare("CREATE SCHEMA IF NOT EXISTS micro ;")
	if err != nil {
		return err
	}
	_, err = schema.Exec()
	if err != nil {
		return errors.Wrap(err, "Couldn't create Schema")
	}

	// Create a table for the Store namespace
	tableq, err := s.db.Prepare(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.%s
	(
		key text COLLATE "default" NOT NULL,
		value bytea,
		expiry timestamp with time zone,
		CONSTRAINT %s_pkey PRIMARY KEY (key)
	);`, s.database, s.table, s.table))
	if err != nil {
		return errors.Wrap(err, "SQL statement preparation failed")
	}
	_, err = tableq.Exec()
	if err != nil {
		return errors.Wrap(err, "Couldn't create table")
	}

	return nil
}

// New returns a new micro Store backed by sql
func New(opts ...options.Option) (store.Store, error) {
	options := options.NewOptions(opts...)
	driver, dataSourceName, err := validateOptions(options)
	if err != nil {
		return nil, err
	}
	if !strings.Contains(dataSourceName, " ") {
		dataSourceName = fmt.Sprintf("host=%s", dataSourceName)
	}
	db, err := sql.Open(driver, dataSourceName)
	if err != nil {
		return nil, err
	}
	if err := db.Ping(); err != nil {
		return nil, err
	}
	s := &sqlStore{
		db: db,
	}

	return s, s.initDB(options)
}

// validateOptions checks whether the provided options are valid, then returns the driver
// and data source name.
func validateOptions(options options.Options) (driver, dataSourceName string, err error) {
	driverOpt, found := options.Values().Get("store.sql.driver")
	if !found {
		return "", "", errors.New("No store.sql.driver option specified")
	}
	nodesOpt, found := options.Values().Get("store.nodes")
	if !found {
		return "", "", errors.New("No store.nodes option specified (expected a database connection string)")
	}
	driver, ok := driverOpt.(string)
	if !ok {
		return "", "", errors.New("store.sql.driver option must be a string")
	}
	nodes, ok := nodesOpt.([]string)
	if !ok {
		return "", "", errors.New("store.nodes option must be a []string")
	}
	if len(nodes) != 1 {
		return "", "", errors.New("expected only 1 store.nodes option")
	}
	namespaceOpt, found := options.Values().Get("store.namespace")
	if found {
		namespace, ok := namespaceOpt.(string)
		if !ok {
			return "", "", errors.New("store.namespace must me a string")
		}
		for _, r := range namespace {
			if !unicode.IsLetter(r) {
				return "", "", errors.New("store.namespace must only contain letters")
			}
		}
	}
	return driver, nodes[0], nil
}
<commit_msg>Create database should take the name of the database<commit_after>// Package postgresql implements a micro Store backed by sql
package postgresql

import (
	"database/sql"
	"fmt"
	"strings"
	"time"
	"unicode"

	"github.com/lib/pq"
	"github.com/pkg/errors"

	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/store"
)

// DefaultNamespace is the namespace that the sql store
// will use if no namespace is provided.
var (
	DefaultNamespace = "micro"
	DefaultPrefix    = "micro"
)

type sqlStore struct {
	db *sql.DB

	database string
	table    string
	options.Options
}

// List all the known records
func (s *sqlStore) List() ([]*store.Record, error) {
	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s;", s.database, s.table))
	if err != nil {
		return nil, err
	}
	var records []*store.Record
	var timehelper pq.NullTime
	rows, err := q.Query()
	if err != nil {
		if err == sql.ErrNoRows {
			return records, nil
		}
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		record := &store.Record{}
		if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
			return records, err
		}
		if timehelper.Valid {
			if timehelper.Time.Before(time.Now()) {
				// record has expired
				go s.Delete(record.Key)
			} else {
				record.Expiry = time.Until(timehelper.Time)
				records = append(records, record)
			}
		} else {
			records = append(records, record)
		}

	}
	rowErr := rows.Close()
	if rowErr != nil {
		// transaction rollback or something
		return records, rowErr
	}
	if err := rows.Err(); err != nil {
		return records, err
	}
	return records, nil
}

// Read all records with keys
func (s *sqlStore) Read(keys ...string) ([]*store.Record, error) {
	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s WHERE key = $1;", s.database, s.table))
	if err != nil {
		return nil, err
	}
	var records []*store.Record
	var timehelper pq.NullTime
	for _, key := range keys {
		row := q.QueryRow(key)
		record := &store.Record{}
		if err := row.Scan(&record.Key, &record.Value, &timehelper); err != nil {
			if err == sql.ErrNoRows {
				return records, store.ErrNotFound
			}
			return records, err
		}
		if timehelper.Valid {
			if timehelper.Time.Before(time.Now()) {
				// record has expired
				go s.Delete(key)
				return records, store.ErrNotFound
			}
			record.Expiry = time.Until(timehelper.Time)
			records = append(records, record)
		} else {
			records = append(records, record)
		}
	}
	return records, nil
}

// Write records
func (s *sqlStore) Write(rec ...*store.Record) error {
	q, err := s.db.Prepare(fmt.Sprintf(`INSERT INTO %s.%s(key, value, expiry)
		VALUES ($1, $2::bytea, $3)
		ON CONFLICT (key)
		DO UPDATE
		SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;`, s.database, s.table))
	if err != nil {
		return err
	}
	for _, r := range rec {
		var err error
		if r.Expiry != 0 {
			_, err = q.Exec(r.Key, r.Value, time.Now().Add(r.Expiry))
		} else {
			_, err = q.Exec(r.Key, r.Value, nil)
		}
		if err != nil {
			return errors.Wrap(err, "Couldn't insert record "+r.Key)
		}
	}

	return nil
}

// Delete records with keys
func (s *sqlStore) Delete(keys ...string) error {
	q, err := s.db.Prepare(fmt.Sprintf("DELETE FROM %s.%s WHERE key = $1;", s.database, s.table))
	if err != nil {
		return err
	}
	for _, key := range keys {
		result, err := q.Exec(key)
		if err != nil {
			return err
		}
		_, err = result.RowsAffected()
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *sqlStore) initDB(options options.Options) error {
	// Get the store.namespace option, or use sql.DefaultNamespace
	namespaceOpt, found := options.Values().Get("store.namespace")
	if !found {
		s.database = DefaultNamespace
	} else {
		if namespace, ok := namespaceOpt.(string); ok {
			s.database = namespace
		} else {
			return errors.New("store.namespace option must be a string")
		}
	}
	// Get the store.namespace option, or use sql.DefaultNamespace
	prefixOpt, found := options.Values().Get("store.prefix")
	if !found {
		s.table = DefaultPrefix
	} else {
		if prefix, ok := prefixOpt.(string); ok {
			s.table = prefix
		} else {
			return errors.New("store.namespace option must be a string")
		}
	}

	// Create "micro" schema
	schema, err := s.db.Prepare(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s ;", s.database))
	if err != nil {
		return err
	}
	_, err = schema.Exec()
	if err != nil {
		return errors.Wrap(err, "Couldn't create database")
	}

	// Create a table for the Store namespace
	tableq, err := s.db.Prepare(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.%s
	(
		key text COLLATE "default" NOT NULL,
		value bytea,
		expiry timestamp with time zone,
		CONSTRAINT %s_pkey PRIMARY KEY (key)
	);`, s.database, s.table, s.table))
	if err != nil {
		return errors.Wrap(err, "SQL statement preparation failed")
	}
	_, err = tableq.Exec()
	if err != nil {
		return errors.Wrap(err, "Couldn't create table")
	}

	return nil
}

// New returns a new micro Store backed by sql
func New(opts ...options.Option) (store.Store, error) {
	options := options.NewOptions(opts...)
	driver, dataSourceName, err := validateOptions(options)
	if err != nil {
		return nil, err
	}
	if !strings.Contains(dataSourceName, " ") {
		dataSourceName = fmt.Sprintf("host=%s", dataSourceName)
	}
	db, err := sql.Open(driver, dataSourceName)
	if err != nil {
		return nil, err
	}
	if err := db.Ping(); err != nil {
		return nil, err
	}
	s := &sqlStore{
		db: db,
	}

	return s, s.initDB(options)
}

// validateOptions checks whether the provided options are valid, then returns the driver
// and data source name.
func validateOptions(options options.Options) (driver, dataSourceName string, err error) {
	driverOpt, found := options.Values().Get("store.sql.driver")
	if !found {
		return "", "", errors.New("No store.sql.driver option specified")
	}
	nodesOpt, found := options.Values().Get("store.nodes")
	if !found {
		return "", "", errors.New("No store.nodes option specified (expected a database connection string)")
	}
	driver, ok := driverOpt.(string)
	if !ok {
		return "", "", errors.New("store.sql.driver option must be a string")
	}
	nodes, ok := nodesOpt.([]string)
	if !ok {
		return "", "", errors.New("store.nodes option must be a []string")
	}
	if len(nodes) != 1 {
		return "", "", errors.New("expected only 1 store.nodes option")
	}
	namespaceOpt, found := options.Values().Get("store.namespace")
	if found {
		namespace, ok := namespaceOpt.(string)
		if !ok {
			return "", "", errors.New("store.namespace must me a string")
		}
		for _, r := range namespace {
			if !unicode.IsLetter(r) {
				return "", "", errors.New("store.namespace must only contain letters")
			}
		}
	}
	return driver, nodes[0], nil
}
<|endoftext|>