<commit_before>package device

import (
	"fmt"
	"net"
	"os"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/cluster"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/dnsmasq/dhcpalloc"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/network/openvswitch"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
)

type nicOVN struct {
	deviceCommon

	network network.Network
}

// getIntegrationBridgeName returns the OVS integration bridge to use.
func (d *nicOVN) getIntegrationBridgeName() (string, error) {
	integrationBridge, err := cluster.ConfigGetString(d.state.Cluster, "network.ovn.integration_bridge")
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get OVN integration bridge name")
	}

	return integrationBridge, nil
}

// validateConfig checks the supplied config for correctness.
func (d *nicOVN) validateConfig(instConf instance.ConfigReader) error {
	if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) {
		return ErrUnsupportedDevType
	}

	requiredFields := []string{
		"network",
	}

	optionalFields := []string{
		"name",
		"hwaddr",
		"host_name",
		"mtu",
		"ipv4.address",
		"ipv6.address",
		"boot.priority",
	}

	// The NIC's network may be a non-default project, so lookup project and get network's project name.
	networkProjectName, _, err := project.NetworkProject(d.state.Cluster, instConf.Project())
	if err != nil {
		return errors.Wrapf(err, "Failed loading network project name")
	}

	// Lookup network settings and apply them to the device's config.
	n, err := network.LoadByName(d.state, networkProjectName, d.config["network"])
	if err != nil {
		return errors.Wrapf(err, "Error loading network config for %q", d.config["network"])
	}

	if n.Status() == api.NetworkStatusPending {
		return fmt.Errorf("Specified network is not fully created")
	}

	if n.Type() != "ovn" {
		return fmt.Errorf("Specified network must be of type ovn")
	}

	bannedKeys := []string{"mtu"}
	for _, bannedKey := range bannedKeys {
		if d.config[bannedKey] != "" {
			return fmt.Errorf("Cannot use %q property in conjunction with %q property", bannedKey, "network")
		}
	}

	d.network = n // Stored loaded instance for use by other functions.
	netConfig := d.network.Config()

	if d.config["ipv4.address"] != "" {
		// Check that DHCPv4 is enabled on parent network (needed to use static assigned IPs).
		if n.DHCPv4Subnet() == nil {
			return fmt.Errorf("Cannot specify %q when DHCP is disabled on network %q", "ipv4.address", d.config["network"])
		}

		_, subnet, err := net.ParseCIDR(netConfig["ipv4.address"])
		if err != nil {
			return errors.Wrapf(err, "Invalid network ipv4.address")
		}

		// Check the static IP supplied is valid for the linked network. It should be part of the
		// network's subnet, but not necessarily part of the dynamic allocation ranges.
		if !dhcpalloc.DHCPValidIP(subnet, nil, net.ParseIP(d.config["ipv4.address"])) {
			return fmt.Errorf("Device IP address %q not within network %q subnet", d.config["ipv4.address"], d.config["network"])
		}
	}

	if d.config["ipv6.address"] != "" {
		// Check that DHCPv6 is enabled on parent network (needed to use static assigned IPs).
		if n.DHCPv6Subnet() == nil || !shared.IsTrue(netConfig["ipv6.dhcp.stateful"]) {
			return fmt.Errorf("Cannot specify %q when DHCP or %q are disabled on network %q", "ipv6.address", "ipv6.dhcp.stateful", d.config["network"])
		}

		_, subnet, err := net.ParseCIDR(netConfig["ipv6.address"])
		if err != nil {
			return errors.Wrapf(err, "Invalid network ipv6.address")
		}

		// Check the static IP supplied is valid for the linked network. It should be part of the
		// network's subnet, but not necessarily part of the dynamic allocation ranges.
		if !dhcpalloc.DHCPValidIP(subnet, nil, net.ParseIP(d.config["ipv6.address"])) {
			return fmt.Errorf("Device IP address %q not within network %q subnet", d.config["ipv6.address"], d.config["network"])
		}
	}

	// Apply network level config options to device config before validation.
	d.config["mtu"] = fmt.Sprintf("%s", netConfig["bridge.mtu"])

	rules := nicValidationRules(requiredFields, optionalFields)

	// Now run normal validation.
	err = d.config.Validate(rules)
	if err != nil {
		return err
	}

	return nil
}

// validateEnvironment checks the runtime environment for correctness.
func (d *nicOVN) validateEnvironment() error {
	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
		return fmt.Errorf("Requires name property to start")
	}

	integrationBridge, err := d.getIntegrationBridgeName()
	if err != nil {
		return err
	}

	if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", integrationBridge)) {
		return fmt.Errorf("OVS integration bridge device %q doesn't exist", integrationBridge)
	}

	return nil
}

// CanHotPlug returns whether the device can be managed whilst the instance is running, it also
// returns a list of fields that can be updated without triggering a device remove & add.
func (d *nicOVN) CanHotPlug() (bool, []string) {
	return true, []string{}
}

// Add is run when a device is added to an instance whether or not the instance is running.
func (d *nicOVN) Add() error {
	return nil
}

// Start is run when the device is added to a running instance or instance is starting up.
func (d *nicOVN) Start() (*deviceConfig.RunConfig, error) {
	err := d.validateEnvironment()
	if err != nil {
		return nil, err
	}

	revert := revert.New()
	defer revert.Fail()

	saveData := make(map[string]string)
	saveData["host_name"] = d.config["host_name"]

	var peerName string

	// Create veth pair and configure the peer end with custom hwaddr and mtu if supplied.
	if d.inst.Type() == instancetype.Container {
		if saveData["host_name"] == "" {
			saveData["host_name"] = network.RandomDevName("veth")
		}
		peerName, err = networkCreateVethPair(saveData["host_name"], d.config)
	} else if d.inst.Type() == instancetype.VM {
		if saveData["host_name"] == "" {
			saveData["host_name"] = network.RandomDevName("tap")
		}
		peerName = saveData["host_name"] // VMs use the host_name to link to the TAP FD.
		err = networkCreateTap(saveData["host_name"], d.config)
	}

	if err != nil {
		return nil, err
	}

	revert.Add(func() { NetworkRemoveInterface(saveData["host_name"]) })

	// Populate device config with volatile fields if needed.
	networkVethFillFromVolatile(d.config, saveData)

	// Apply host-side limits.
	err = networkSetupHostVethLimits(d.config)
	if err != nil {
		return nil, err
	}

	// Disable IPv6 on host-side veth interface (prevents host-side interface getting link-local address)
	// which isn't needed because the host-side interface is connected to a bridge.
	err = util.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", saveData["host_name"]), "1")
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	mac, err := net.ParseMAC(d.config["hwaddr"])
	if err != nil {
		return nil, err
	}

	ips := []net.IP{}
	for _, key := range []string{"ipv4.address", "ipv6.address"} {
		if d.config[key] != "" {
			ip := net.ParseIP(d.config[key])
			if ip == nil {
				return nil, fmt.Errorf("Invalid %s value %q", key, d.config[key])
			}
			ips = append(ips, ip)
		}
	}

	// Add new OVN logical switch port for instance.
	logicalPortName, err := network.OVNInstanceDevicePortAdd(d.network, d.inst.ID(), d.inst.Name(), d.name, mac, ips)
	if err != nil {
		return nil, err
	}

	revert.Add(func() { network.OVNInstanceDevicePortDelete(d.network, d.inst.ID(), d.name) })

	// Attach host side veth interface to bridge.
	integrationBridge, err := d.getIntegrationBridgeName()
	if err != nil {
		return nil, err
	}

	ovs := openvswitch.NewOVS()
	err = ovs.BridgePortAdd(integrationBridge, saveData["host_name"], true)
	if err != nil {
		return nil, err
	}

	revert.Add(func() { ovs.BridgePortDelete(integrationBridge, saveData["host_name"]) })

	// Link OVS port to OVN logical port.
	err = ovs.InterfaceAssociateOVNSwitchPort(saveData["host_name"], logicalPortName)
	if err != nil {
		return nil, err
	}

	// Attempt to disable router advertisement acceptance.
	err = util.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", saveData["host_name"]), "0")
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	// Attempt to disable IPv4 forwarding.
	err = util.SysctlSet(fmt.Sprintf("net/ipv4/conf/%s/forwarding", saveData["host_name"]), "0")
	if err != nil {
		return nil, err
	}

	err = d.volatileSet(saveData)
	if err != nil {
		return nil, err
	}

	runConf := deviceConfig.RunConfig{}
	runConf.NetworkInterface = []deviceConfig.RunConfigItem{
		{Key: "name", Value: d.config["name"]},
		{Key: "type", Value: "phys"},
		{Key: "flags", Value: "up"},
		{Key: "link", Value: peerName},
	}

	if d.inst.Type() == instancetype.VM {
		runConf.NetworkInterface = append(runConf.NetworkInterface,
			[]deviceConfig.RunConfigItem{
				{Key: "devName", Value: d.name},
				{Key: "hwaddr", Value: d.config["hwaddr"]},
			}...)
	}

	revert.Success()
	return &runConf, nil
}

// Update applies configuration changes to a started device.
func (d *nicOVN) Update(oldDevices deviceConfig.Devices, isRunning bool) error {
	oldConfig := oldDevices[d.name]

	v := d.volatileGet()

	// Populate device config with volatile fields if needed.
	networkVethFillFromVolatile(d.config, v)

	// If instance is running, apply host side limits and filters first before rebuilding
	// dnsmasq config below so that existing config can be used as part of the filter removal.
	if isRunning {
		err := d.validateEnvironment()
		if err != nil {
			return err
		}

		// Apply host-side limits.
		err = networkSetupHostVethLimits(d.config)
		if err != nil {
			return err
		}
	}

	// If an IPv6 address has changed, if the instance is running we should bounce the host-side
	// veth interface to give the instance a chance to detect the change and re-apply for an
	// updated lease with new IP address.
	if d.config["ipv6.address"] != oldConfig["ipv6.address"] && d.config["host_name"] != "" && shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["host_name"])) {
		_, err := shared.RunCommand("ip", "link", "set", d.config["host_name"], "down")
		if err != nil {
			return err
		}
		_, err = shared.RunCommand("ip", "link", "set", d.config["host_name"], "up")
		if err != nil {
			return err
		}
	}

	return nil
}

// Stop is run when the device is removed from the instance.
func (d *nicOVN) Stop() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{
		PostHooks: []func() error{d.postStop},
	}

	err := network.OVNInstanceDevicePortDelete(d.network, d.inst.ID(), d.name)
	if err != nil {
		// Don't fail here as we still want the postStop hook to run to clean up the local veth pair.
		d.logger.Error("Failed to remove OVN device port", log.Ctx{"err": err})
	}

	return &runConf, nil
}

// postStop is run after the device is removed from the instance.
func (d *nicOVN) postStop() error {
	defer d.volatileSet(map[string]string{
		"host_name": "",
	})

	v := d.volatileGet()

	networkVethFillFromVolatile(d.config, v)

	if d.config["host_name"] != "" && shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["host_name"])) {
		integrationBridge, err := d.getIntegrationBridgeName()
		if err != nil {
			return err
		}

		ovs := openvswitch.NewOVS()

		// Detach host-side end of veth pair from bridge (required for openvswitch particularly).
		err = ovs.BridgePortDelete(integrationBridge, d.config["host_name"])
		if err != nil {
			return errors.Wrapf(err, "Failed to detach interface %q from %q", d.config["host_name"], integrationBridge)
		}

		// Removing host-side end of veth pair will delete the peer end too.
		err = NetworkRemoveInterface(d.config["host_name"])
		if err != nil {
			return errors.Wrapf(err, "Failed to remove interface %q", d.config["host_name"])
		}
	}

	return nil
}

// Remove is run when the device is removed from the instance or the instance is deleted.
func (d *nicOVN) Remove() error {
	return nil
}
<commit_msg>lxd/device/nic/ovn: Improves error message in Start<commit_after>package device

import (
	"fmt"
	"net"
	"os"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/cluster"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/dnsmasq/dhcpalloc"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/network/openvswitch"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
)

type nicOVN struct {
	deviceCommon

	network network.Network
}

// getIntegrationBridgeName returns the OVS integration bridge to use.
func (d *nicOVN) getIntegrationBridgeName() (string, error) {
	integrationBridge, err := cluster.ConfigGetString(d.state.Cluster, "network.ovn.integration_bridge")
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get OVN integration bridge name")
	}

	return integrationBridge, nil
}

// validateConfig checks the supplied config for correctness.
func (d *nicOVN) validateConfig(instConf instance.ConfigReader) error {
	if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) {
		return ErrUnsupportedDevType
	}

	requiredFields := []string{
		"network",
	}

	optionalFields := []string{
		"name",
		"hwaddr",
		"host_name",
		"mtu",
		"ipv4.address",
		"ipv6.address",
		"boot.priority",
	}

	// The NIC's network may be a non-default project, so lookup project and get network's project name.
	networkProjectName, _, err := project.NetworkProject(d.state.Cluster, instConf.Project())
	if err != nil {
		return errors.Wrapf(err, "Failed loading network project name")
	}

	// Lookup network settings and apply them to the device's config.
	n, err := network.LoadByName(d.state, networkProjectName, d.config["network"])
	if err != nil {
		return errors.Wrapf(err, "Error loading network config for %q", d.config["network"])
	}

	if n.Status() == api.NetworkStatusPending {
		return fmt.Errorf("Specified network is not fully created")
	}

	if n.Type() != "ovn" {
		return fmt.Errorf("Specified network must be of type ovn")
	}

	bannedKeys := []string{"mtu"}
	for _, bannedKey := range bannedKeys {
		if d.config[bannedKey] != "" {
			return fmt.Errorf("Cannot use %q property in conjunction with %q property", bannedKey, "network")
		}
	}

	d.network = n // Stored loaded instance for use by other functions.
	netConfig := d.network.Config()

	if d.config["ipv4.address"] != "" {
		// Check that DHCPv4 is enabled on parent network (needed to use static assigned IPs).
		if n.DHCPv4Subnet() == nil {
			return fmt.Errorf("Cannot specify %q when DHCP is disabled on network %q", "ipv4.address", d.config["network"])
		}

		_, subnet, err := net.ParseCIDR(netConfig["ipv4.address"])
		if err != nil {
			return errors.Wrapf(err, "Invalid network ipv4.address")
		}

		// Check the static IP supplied is valid for the linked network. It should be part of the
		// network's subnet, but not necessarily part of the dynamic allocation ranges.
		if !dhcpalloc.DHCPValidIP(subnet, nil, net.ParseIP(d.config["ipv4.address"])) {
			return fmt.Errorf("Device IP address %q not within network %q subnet", d.config["ipv4.address"], d.config["network"])
		}
	}

	if d.config["ipv6.address"] != "" {
		// Check that DHCPv6 is enabled on parent network (needed to use static assigned IPs).
		if n.DHCPv6Subnet() == nil || !shared.IsTrue(netConfig["ipv6.dhcp.stateful"]) {
			return fmt.Errorf("Cannot specify %q when DHCP or %q are disabled on network %q", "ipv6.address", "ipv6.dhcp.stateful", d.config["network"])
		}

		_, subnet, err := net.ParseCIDR(netConfig["ipv6.address"])
		if err != nil {
			return errors.Wrapf(err, "Invalid network ipv6.address")
		}

		// Check the static IP supplied is valid for the linked network. It should be part of the
		// network's subnet, but not necessarily part of the dynamic allocation ranges.
		if !dhcpalloc.DHCPValidIP(subnet, nil, net.ParseIP(d.config["ipv6.address"])) {
			return fmt.Errorf("Device IP address %q not within network %q subnet", d.config["ipv6.address"], d.config["network"])
		}
	}

	// Apply network level config options to device config before validation.
	d.config["mtu"] = fmt.Sprintf("%s", netConfig["bridge.mtu"])

	rules := nicValidationRules(requiredFields, optionalFields)

	// Now run normal validation.
	err = d.config.Validate(rules)
	if err != nil {
		return err
	}

	return nil
}

// validateEnvironment checks the runtime environment for correctness.
func (d *nicOVN) validateEnvironment() error {
	if d.inst.Type() == instancetype.Container && d.config["name"] == "" {
		return fmt.Errorf("Requires name property to start")
	}

	integrationBridge, err := d.getIntegrationBridgeName()
	if err != nil {
		return err
	}

	if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", integrationBridge)) {
		return fmt.Errorf("OVS integration bridge device %q doesn't exist", integrationBridge)
	}

	return nil
}

// CanHotPlug returns whether the device can be managed whilst the instance is running, it also
// returns a list of fields that can be updated without triggering a device remove & add.
func (d *nicOVN) CanHotPlug() (bool, []string) {
	return true, []string{}
}

// Add is run when a device is added to an instance whether or not the instance is running.
func (d *nicOVN) Add() error {
	return nil
}

// Start is run when the device is added to a running instance or instance is starting up.
func (d *nicOVN) Start() (*deviceConfig.RunConfig, error) {
	err := d.validateEnvironment()
	if err != nil {
		return nil, err
	}

	revert := revert.New()
	defer revert.Fail()

	saveData := make(map[string]string)
	saveData["host_name"] = d.config["host_name"]

	var peerName string

	// Create veth pair and configure the peer end with custom hwaddr and mtu if supplied.
	if d.inst.Type() == instancetype.Container {
		if saveData["host_name"] == "" {
			saveData["host_name"] = network.RandomDevName("veth")
		}
		peerName, err = networkCreateVethPair(saveData["host_name"], d.config)
	} else if d.inst.Type() == instancetype.VM {
		if saveData["host_name"] == "" {
			saveData["host_name"] = network.RandomDevName("tap")
		}
		peerName = saveData["host_name"] // VMs use the host_name to link to the TAP FD.
		err = networkCreateTap(saveData["host_name"], d.config)
	}

	if err != nil {
		return nil, err
	}

	revert.Add(func() { NetworkRemoveInterface(saveData["host_name"]) })

	// Populate device config with volatile fields if needed.
	networkVethFillFromVolatile(d.config, saveData)

	// Apply host-side limits.
	err = networkSetupHostVethLimits(d.config)
	if err != nil {
		return nil, err
	}

	// Disable IPv6 on host-side veth interface (prevents host-side interface getting link-local address)
	// which isn't needed because the host-side interface is connected to a bridge.
	err = util.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", saveData["host_name"]), "1")
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	mac, err := net.ParseMAC(d.config["hwaddr"])
	if err != nil {
		return nil, err
	}

	ips := []net.IP{}
	for _, key := range []string{"ipv4.address", "ipv6.address"} {
		if d.config[key] != "" {
			ip := net.ParseIP(d.config[key])
			if ip == nil {
				return nil, fmt.Errorf("Invalid %s value %q", key, d.config[key])
			}
			ips = append(ips, ip)
		}
	}

	// Add new OVN logical switch port for instance.
	logicalPortName, err := network.OVNInstanceDevicePortAdd(d.network, d.inst.ID(), d.inst.Name(), d.name, mac, ips)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed adding OVN port")
	}

	revert.Add(func() { network.OVNInstanceDevicePortDelete(d.network, d.inst.ID(), d.name) })

	// Attach host side veth interface to bridge.
	integrationBridge, err := d.getIntegrationBridgeName()
	if err != nil {
		return nil, err
	}

	ovs := openvswitch.NewOVS()
	err = ovs.BridgePortAdd(integrationBridge, saveData["host_name"], true)
	if err != nil {
		return nil, err
	}

	revert.Add(func() { ovs.BridgePortDelete(integrationBridge, saveData["host_name"]) })

	// Link OVS port to OVN logical port.
	err = ovs.InterfaceAssociateOVNSwitchPort(saveData["host_name"], logicalPortName)
	if err != nil {
		return nil, err
	}

	// Attempt to disable router advertisement acceptance.
	err = util.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", saveData["host_name"]), "0")
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	// Attempt to disable IPv4 forwarding.
	err = util.SysctlSet(fmt.Sprintf("net/ipv4/conf/%s/forwarding", saveData["host_name"]), "0")
	if err != nil {
		return nil, err
	}

	err = d.volatileSet(saveData)
	if err != nil {
		return nil, err
	}

	runConf := deviceConfig.RunConfig{}
	runConf.NetworkInterface = []deviceConfig.RunConfigItem{
		{Key: "name", Value: d.config["name"]},
		{Key: "type", Value: "phys"},
		{Key: "flags", Value: "up"},
		{Key: "link", Value: peerName},
	}

	if d.inst.Type() == instancetype.VM {
		runConf.NetworkInterface = append(runConf.NetworkInterface,
			[]deviceConfig.RunConfigItem{
				{Key: "devName", Value: d.name},
				{Key: "hwaddr", Value: d.config["hwaddr"]},
			}...)
	}

	revert.Success()
	return &runConf, nil
}

// Update applies configuration changes to a started device.
func (d *nicOVN) Update(oldDevices deviceConfig.Devices, isRunning bool) error {
	oldConfig := oldDevices[d.name]

	v := d.volatileGet()

	// Populate device config with volatile fields if needed.
	networkVethFillFromVolatile(d.config, v)

	// If instance is running, apply host side limits and filters first before rebuilding
	// dnsmasq config below so that existing config can be used as part of the filter removal.
	if isRunning {
		err := d.validateEnvironment()
		if err != nil {
			return err
		}

		// Apply host-side limits.
		err = networkSetupHostVethLimits(d.config)
		if err != nil {
			return err
		}
	}

	// If an IPv6 address has changed, if the instance is running we should bounce the host-side
	// veth interface to give the instance a chance to detect the change and re-apply for an
	// updated lease with new IP address.
	if d.config["ipv6.address"] != oldConfig["ipv6.address"] && d.config["host_name"] != "" && shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["host_name"])) {
		_, err := shared.RunCommand("ip", "link", "set", d.config["host_name"], "down")
		if err != nil {
			return err
		}
		_, err = shared.RunCommand("ip", "link", "set", d.config["host_name"], "up")
		if err != nil {
			return err
		}
	}

	return nil
}

// Stop is run when the device is removed from the instance.
func (d *nicOVN) Stop() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{
		PostHooks: []func() error{d.postStop},
	}

	err := network.OVNInstanceDevicePortDelete(d.network, d.inst.ID(), d.name)
	if err != nil {
		// Don't fail here as we still want the postStop hook to run to clean up the local veth pair.
		d.logger.Error("Failed to remove OVN device port", log.Ctx{"err": err})
	}

	return &runConf, nil
}

// postStop is run after the device is removed from the instance.
func (d *nicOVN) postStop() error {
	defer d.volatileSet(map[string]string{
		"host_name": "",
	})

	v := d.volatileGet()

	networkVethFillFromVolatile(d.config, v)

	if d.config["host_name"] != "" && shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["host_name"])) {
		integrationBridge, err := d.getIntegrationBridgeName()
		if err != nil {
			return err
		}

		ovs := openvswitch.NewOVS()

		// Detach host-side end of veth pair from bridge (required for openvswitch particularly).
		err = ovs.BridgePortDelete(integrationBridge, d.config["host_name"])
		if err != nil {
			return errors.Wrapf(err, "Failed to detach interface %q from %q", d.config["host_name"], integrationBridge)
		}

		// Removing host-side end of veth pair will delete the peer end too.
		err = NetworkRemoveInterface(d.config["host_name"])
		if err != nil {
			return errors.Wrapf(err, "Failed to remove interface %q", d.config["host_name"])
		}
	}

	return nil
}

// Remove is run when the device is removed from the instance or the instance is deleted.
func (d *nicOVN) Remove() error {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

type storageShared struct {
	sType        storageType
	sTypeName    string
	sTypeVersion string

	d *Daemon

	poolID int64
	pool   *api.StoragePool

	volume *api.StorageVolume
}

func (s *storageShared) GetStorageType() storageType {
	return s.sType
}

func (s *storageShared) GetStorageTypeName() string {
	return s.sTypeName
}

func (s *storageShared) GetStorageTypeVersion() string {
	return s.sTypeVersion
}

func (s *storageShared) shiftRootfs(c container) error {
	dpath := c.Path()
	rpath := c.RootfsPath()

	shared.LogDebugf("Shifting root filesystem \"%s\" for \"%s\".", rpath, c.Name())

	idmapset := c.IdmapSet()

	if idmapset == nil {
		return fmt.Errorf("IdmapSet of container '%s' is nil", c.Name())
	}

	err := idmapset.ShiftRootfs(rpath)
	if err != nil {
		shared.LogDebugf("Shift of rootfs %s failed: %s", rpath, err)
		return err
	}

	/* Set an acl so the container root can descend the container dir */
	// TODO: i changed this so it calls s.setUnprivUserAcl, which does
	// the acl change only if the container is not privileged, think thats right.
	return s.setUnprivUserAcl(c, dpath)
}

func (s *storageShared) setUnprivUserAcl(c container, destPath string) error {
	idmapset := c.IdmapSet()

	// Skip for privileged containers
	if idmapset == nil {
		return nil
	}

	// Make sure the map is valid. Skip if container uid 0 == host uid 0
	uid, _ := idmapset.ShiftIntoNs(0, 0)
	switch uid {
	case -1:
		return fmt.Errorf("Container doesn't have a uid 0 in its map")
	case 0:
		return nil
	}

	// Attempt to set a POSIX ACL first.
	acl := fmt.Sprintf("%d:rx", uid)
	err := exec.Command("setfacl", "-m", acl, destPath).Run()
	if err == nil {
		shared.LogDebugf("Failed to set acl permission on container path: %s", err)
		return nil
	}

	// Fallback to chmod if the fs doesn't support it.
	err = exec.Command("chmod", "+x", destPath).Run()
	if err != nil {
		shared.LogDebugf("Failed to set executable bit on the container path: %s", err)
		return err
	}

	return nil
}

func (s *storageShared) createImageDbPoolVolume(fingerprint string) error {
	// Fill in any default volume config.
	volumeConfig := map[string]string{}
	err := storageVolumeFillDefault(s.pool.Name, volumeConfig, s.pool)
	if err != nil {
		return err
	}

	// Create a db entry for the storage volume of the image.
	_, err = dbStoragePoolVolumeCreate(s.d.db, fingerprint, storagePoolVolumeTypeImage, s.poolID, volumeConfig)
	if err != nil {
		// Try to delete the db entry on error.
		s.deleteImageDbPoolVolume(fingerprint)
		return err
	}

	return nil
}

func (s *storageShared) deleteImageDbPoolVolume(fingerprint string) error {
	err := dbStoragePoolVolumeDelete(s.d.db, fingerprint, storagePoolVolumeTypeImage, s.poolID)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Remove wrong error message<commit_after>package main

import (
	"fmt"
	"os/exec"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

type storageShared struct {
	sType        storageType
	sTypeName    string
	sTypeVersion string

	d *Daemon

	poolID int64
	pool   *api.StoragePool

	volume *api.StorageVolume
}

func (s *storageShared) GetStorageType() storageType {
	return s.sType
}

func (s *storageShared) GetStorageTypeName() string {
	return s.sTypeName
}

func (s *storageShared) GetStorageTypeVersion() string {
	return s.sTypeVersion
}

func (s *storageShared) shiftRootfs(c container) error {
	dpath := c.Path()
	rpath := c.RootfsPath()

	shared.LogDebugf("Shifting root filesystem \"%s\" for \"%s\".", rpath, c.Name())

	idmapset := c.IdmapSet()

	if idmapset == nil {
		return fmt.Errorf("IdmapSet of container '%s' is nil", c.Name())
	}

	err := idmapset.ShiftRootfs(rpath)
	if err != nil {
		shared.LogDebugf("Shift of rootfs %s failed: %s", rpath, err)
		return err
	}

	/* Set an acl so the container root can descend the container dir */
	// TODO: i changed this so it calls s.setUnprivUserAcl, which does
	// the acl change only if the container is not privileged, think thats right.
	return s.setUnprivUserAcl(c, dpath)
}

func (s *storageShared) setUnprivUserAcl(c container, destPath string) error {
	idmapset := c.IdmapSet()

	// Skip for privileged containers
	if idmapset == nil {
		return nil
	}

	// Make sure the map is valid. Skip if container uid 0 == host uid 0
	uid, _ := idmapset.ShiftIntoNs(0, 0)
	switch uid {
	case -1:
		return fmt.Errorf("Container doesn't have a uid 0 in its map")
	case 0:
		return nil
	}

	// Attempt to set a POSIX ACL first.
	acl := fmt.Sprintf("%d:rx", uid)
	err := exec.Command("setfacl", "-m", acl, destPath).Run()
	if err == nil {
		return nil
	}

	// Fallback to chmod if the fs doesn't support it.
	err = exec.Command("chmod", "+x", destPath).Run()
	if err != nil {
		shared.LogDebugf("Failed to set executable bit on the container path: %s", err)
		return err
	}

	return nil
}

func (s *storageShared) createImageDbPoolVolume(fingerprint string) error {
	// Fill in any default volume config.
	volumeConfig := map[string]string{}
	err := storageVolumeFillDefault(s.pool.Name, volumeConfig, s.pool)
	if err != nil {
		return err
	}

	// Create a db entry for the storage volume of the image.
	_, err = dbStoragePoolVolumeCreate(s.d.db, fingerprint, storagePoolVolumeTypeImage, s.poolID, volumeConfig)
	if err != nil {
		// Try to delete the db entry on error.
		s.deleteImageDbPoolVolume(fingerprint)
		return err
	}

	return nil
}

func (s *storageShared) deleteImageDbPoolVolume(fingerprint string) error {
	err := dbStoragePoolVolumeDelete(s.d.db, fingerprint, storagePoolVolumeTypeImage, s.poolID)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package views

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"sort"

	"github.com/cswank/kcli/internal/colors"
	"github.com/cswank/kcli/internal/kafka"
)

//feeder feeds the screen the data that it craves
type feeder interface {
	print()
	getRows() ([]string, error)
	page(page int) error
	header() string
	enter(row int) (feeder, error)
	jump(i int64) error
	search(s string, cb func(int64, int64)) (int64, error)
	row() int
}

type root struct {
	cli          *kafka.Client
	width        int
	height       int
	topics       []string
	enteredAt    int
	pg           int
	flashMessage chan<- string
}

func newRoot(cli *kafka.Client, width, height int, flashMessage chan<- string) (*root, error) {
	topics, err := cli.GetTopics()
	if len(topics) == 0 {
		return nil, fmt.Errorf("no topics found in kafka")
	}

	sort.Strings(topics)
	return &root{
		cli:          cli,
		width:        width,
		height:       height,
		topics:       topics,
		flashMessage: flashMessage,
	}, err
}

func (r *root) print() {
	fmt.Println(r.header())
	for _, t := range r.topics {
		fmt.Println(t)
	}
}

func (r *root) page(pg int) error {
	if (r.pg == 0 && pg < 0) || (r.pg+pg)*r.height > len(r.topics) {
		return nil
	}
	r.pg += pg
	return nil
}

func (r *root) getRows() ([]string, error) {
	start := r.pg * r.height
	end := r.pg*r.height + r.height
	if end >= len(r.topics) {
		end = len(r.topics)
	}
	return r.topics[start:end], nil
}

func (r *root) enter(row int) (feeder, error) {
	if row >= len(r.topics) {
		r.flashMessage <- "nothing to see here"
		return nil, errNoData
	}
	r.enteredAt = row
	return newTopic(r.cli, r.topics[row], r.width, r.height, r.flashMessage)
}

func (r *root) jump(_ int64) error                                   { return nil }
func (r *root) search(_ string, _ func(int64, int64)) (int64, error) { return -1, nil }

func (r *root) row() int { return r.enteredAt }

func (r *root) header() string {
	return "topics"
}

type topic struct {
	cli    *kafka.Client
	height int
	width  int
	offset int

	topic        string
	partitions   []kafka.Partition
	fmt          string
	enteredAt    int
	flashMessage chan<- string
}

func newTopic(cli *kafka.Client, t string, width, height int, flashMessage chan<- string) (feeder, error) {
	partitions, err := cli.GetTopic(t)
	return &topic{
		cli:          cli,
		width:        width,
		height:       height,
		topic:        t,
		partitions:   partitions,
		fmt:          c2("%-13d %-22d %-22d %-22d %d"),
		flashMessage: flashMessage,
	}, err
}

func (t *topic) search(s string, cb func(int64, int64)) (int64, error) {
	results, err := t.cli.SearchTopic(t.partitions, s, false, cb)
	if err != nil || len(results) == 0 {
		return -1, err
	}
	t.partitions = results

	return int64(len(results)), nil
}

func (t *topic) jump(i int64) error {
	if int(i) >= len(t.partitions) || int(i) < 0 {
		t.flashMessage <- "nothing to see here"
		return nil
	}
	t.offset = int(i)
	return nil
}

func (t *topic) row() int { return t.enteredAt }

func (t *topic) header() string {
	return "partition     1st offset             current offset         last offset            size"
}

func (t *topic) setOffset(n int64) error {
	for i, part := range t.partitions {
		if n > 0 {
			end := part.Offset + n
			if end >= part.End {
				end = part.End - 1
				if end < 0 {
					end = 0
				}
			}
			part.Offset = end
		} else {
			end := part.End + n
			if end <= part.Start {
				end = part.Start
			}
			part.Offset = end
		}
		t.partitions[i] = part
	}
	return nil
}

func (t *topic) page(pg int) error {
	offset := t.offset + (t.height * pg)
	if offset > len(t.partitions) {
		return nil
	}
	if offset < 0 {
		offset = 0
	}
	t.offset = offset
	return nil
}

func (t *topic) getRows() ([]string, error) {
	end := t.offset + t.height
	if end >= len(t.partitions) {
		end = len(t.partitions)
	}

	chunk := t.partitions[t.offset:end]
	out := make([]string, len(chunk))
	for i, p := range chunk {
		out[i] = fmt.Sprintf(t.fmt, p.Partition, p.Start, p.Offset, p.End, p.End-p.Start)
	}

	return out, nil
}

func (t *topic) enter(row int) (feeder, error) {
	t.enteredAt = row
	row = t.offset + row
	if row >= len(t.partitions) {
		go func() { t.flashMessage <- "nothing to see here" }()
		return nil, errNoData
	}
	p := t.partitions[row]
	if p.End-p.Start == 0 {
		go func() { t.flashMessage <- "nothing to see here" }()
		return nil, errNoData
	}
	return newPartition(t.cli, p, t.width, t.height, t.flashMessage)
}

func (t *topic) print() {
	fmt.Println(t.header())
	f := t.fmt + "\n"
	for _, p := range t.partitions {
		fmt.Printf(f, p.Partition, p.Start, p.Offset, p.End, p.End-p.Start)
	}
}

type partition struct {
	cli          *kafka.Client
	height       int
	width        int
	partition    kafka.Partition
	rows         []kafka.Message
	enteredAt    int
	fmt          string
	pg           int
	flashMessage chan<- string
}

func newPartition(cli *kafka.Client, p kafka.Partition, width, height int, flashMessage chan<- string) (feeder, error) {
	rows, err := cli.GetPartition(p, height, func(_ []byte) bool { return true })
	return &partition{
		cli:          cli,
		width:        width,
		height:       height,
		partition:    p,
		rows:         rows,
		fmt:          "%-12d %s",
		flashMessage: flashMessage,
	}, err
}

func (p *partition) search(s string, cb func(int64, int64)) (int64, error) {
	i, err := p.cli.Search(p.partition, s, cb)
	if err != nil || i == -1 {
		return i, err
	}

	return i, p.jump(i)
}

func (p *partition) jump(i int64) error {
	if i >= p.partition.End {
		return nil
	}

	p.pg = int(i) / p.height
	p.partition.Offset = i
	rows, err := p.cli.GetPartition(p.partition, p.height, func(_ []byte) bool { return true })
	if err != nil {
		return err
	}
	p.rows = rows
	return nil
}

func (p *partition) row() int { return p.enteredAt }

func (p *partition) header() string {
	return fmt.Sprintf(
		"offset       message    topic: %s partition: %d start: %d end: %d",
		p.partition.Topic,
		p.partition.Partition,
		p.partition.Start,
		p.partition.End,
	)
}

func (p *partition) getRows() ([]string, error) {
	out := make([]string, len(p.rows))
	for i, msg := range p.rows {
		end := p.width
		if len(msg.Value) < end {
			end = len(msg.Value)
		}
		out[i] = fmt.Sprintf(p.fmt, p.partition.Offset+int64(i), string(msg.Value[:end]))
	}

	return out, nil
}

func (p *partition) page(pg int) error {
	if p.pg == 0 && pg < 0 && p.partition.Offset == p.partition.Start {
		return nil
	} else if p.pg == 0 && pg < 0 && p.partition.Offset > p.partition.Start {
		pg = 0
	}

	o := int64((p.pg+pg)*p.height) + p.partition.Start
	if o >= p.partition.End {
		return nil
	}
	p.pg += pg
	p.partition.Offset = o
	rows, err := p.cli.GetPartition(p.partition, p.height, func(_ []byte) bool { return true })
	if err != nil {
		return err
	}
	p.rows = rows
	return nil
}

func (p *partition) enter(row int) (feeder, error) {
	if row >= len(p.rows) {
		go func() { p.flashMessage <- "nothing to see here" }()
		return nil, errNoData
	}
	p.enteredAt = row
	return newMessage(p.rows[row], p.width, p.height, p.flashMessage)
}

func (p *partition) print() {
	p.cli.Fetch(p.partition, p.partition.End, func(s string) {
		fmt.Println(s)
	})
}

type message struct {
	height       int
	width        int
	msg          kafka.Message
	enteredAt    int
	body         []string
	pg           int
	flashMessage chan<- string
}

func newMessage(msg kafka.Message, width, height int, flashMessage chan<- string) (feeder, error) {
	buf, err := prettyMessage(msg.Value)
	if err != nil {
		return nil, err
	}

	var body []string
	scanner := bufio.NewScanner(buf)
	for scanner.Scan() {
		body = append(body, scanner.Text())
	}

	return &message{
		width:        width,
		height:       height,
		msg:          msg,
		body:         body,
		flashMessage: flashMessage,
	}, nil
}

func (m *message) print() {
	for _, r := range m.body {
		fmt.Println(r)
	}
}

func (m *message) search(_ string, _ func(int64, int64)) (int64, error) { return -1, nil }

func (m *message) jump(_ int64) error { return nil }

func (m *message) row() int { return m.enteredAt }

func (m *message) header() string {
	return fmt.Sprintf(
		"topic: %s partition: %d offset: %d",
		m.msg.Partition.Topic,
		m.msg.Partition.Partition,
		m.msg.Offset,
	)
}

func (m *message) page(pg int) error {
	if m.pg == 0 && pg < 0 {
		return nil
	}
	if (pg+m.pg)*m.height > len(m.body) {
		return nil
	}
	m.pg += pg
	return nil
}

func (m *message) enter(row int) (feeder, error) {
	m.enteredAt = row
	return nil, errNoData
}

func (m *message) getRows() ([]string, error) {
	start := m.pg * m.height
	end := start + m.height
	if end >= len(m.body) {
		end = len(m.body)
	}
	return m.body[start:end], nil
}

func prettyMessage(val []byte) (io.Reader, error) {
	var i interface{}
	if err := json.Unmarshal(val, &i); err != nil {
		//not json, so return original data
		return bytes.NewBuffer(val), nil
	}

	d, err := colors.Marshal(i)
	buf := bytes.NewBuffer(d)
	return buf, err
}
<commit_msg>Added search and jump to multi-line messages<commit_after>package views

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"

	"github.com/cswank/kcli/internal/colors"
	"github.com/cswank/kcli/internal/kafka"
)

//feeder feeds the screen the data that it craves
type feeder interface {
	print()
	getRows() ([]string, error)
	page(page int) error
	header() string
	enter(row int) (feeder, error)
	jump(i int64) error
	search(s string, cb func(int64, int64)) (int64, error)
	row() int
}

type root struct {
	cli          *kafka.Client
	width        int
	height       int
	topics       []string
	enteredAt    int
	pg           int
	flashMessage chan<- string
}

func newRoot(cli *kafka.Client, width, height int, flashMessage chan<- string) (*root, error) {
	topics, err := cli.GetTopics()
	if len(topics) == 0 {
		return nil, fmt.Errorf("no topics found in kafka")
	}

	sort.Strings(topics)
	return &root{
		cli:          cli,
		width:        width,
		height:       height,
		topics:       topics,
		flashMessage: flashMessage,
	}, err
}

func (r *root) print() {
	fmt.Println(r.header())
	for _, t := range r.topics {
		fmt.Println(t)
	}
}

func (r *root) page(pg int) error {
	if (r.pg == 0 && pg < 0) || (r.pg+pg)*r.height > len(r.topics) {
		return nil
	}
	r.pg += pg
	return nil
}

func (r *root) getRows() ([]string, error) {
	start := r.pg * r.height
	end := r.pg*r.height + r.height
	if end >= len(r.topics) {
		end = len(r.topics)
	}
	return r.topics[start:end], nil
}

func (r *root) enter(row int) (feeder, error) {
	if row >= len(r.topics) {
		r.flashMessage <- "nothing to see here"
		return nil, errNoData
	}
	r.enteredAt = row
	return newTopic(r.cli, r.topics[row], r.width, r.height, r.flashMessage)
}

func (r *root) jump(_ int64) error                                   { return nil }
func (r *root) search(_ string, _ func(int64, int64)) (int64, error) { return -1, nil }

func (r *root) row() int { return r.enteredAt }

func (r *root) header() string {
	return "topics"
}

type topic struct {
	cli    *kafka.Client
	height int
	width  int
	offset int

	topic        string
	partitions   []kafka.Partition
	fmt          string
	enteredAt    int
	flashMessage chan<- string
}

func newTopic(cli *kafka.Client, t string, width, height int, flashMessage chan<- string) (feeder, error) {
	partitions, err := cli.GetTopic(t)
	return &topic{
		cli:          cli,
		width:        width,
		height:       height,
		topic:        t,
		partitions:   partitions,
		fmt:          c2("%-13d %-22d %-22d %-22d %d"),
		flashMessage: flashMessage,
	}, err
}

func (t *topic) search(s string, cb func(int64, int64)) (int64, error) {
	results, err := t.cli.SearchTopic(t.partitions, s, false, cb)
	if err != nil || len(results) == 0 {
		return -1, err
	}
	t.partitions = results

	return int64(len(results)), nil
}

func (t *topic) jump(i int64) error {
	if int(i) >= len(t.partitions) || int(i) < 0 {
		t.flashMessage <- "nothing to see here"
		return nil
	}
	t.offset = int(i)
	return nil
}

func (t *topic) row() int { return t.enteredAt }

func (t *topic) header() string {
	return "partition     1st offset             current offset         last offset            size"
}

func (t *topic) setOffset(n int64) error {
	for i, part := range t.partitions {
		if n > 0 {
			end := part.Offset + n
			if end >= part.End {
				end = part.End - 1
				if end < 0 {
					end = 0
				}
			}
			part.Offset = end
		} else {
			end := part.End + n
			if end <= part.Start {
				end = part.Start
			}
			part.Offset = end
		}
		t.partitions[i] = part
	}
	return nil
}

func (t *topic) page(pg int) error {
	offset := t.offset + (t.height * pg)
	if offset > len(t.partitions) {
		return nil
	}
	if offset < 0 {
		offset = 0
	}
	t.offset = offset
	return nil
}

func (t *topic) getRows() ([]string, error) {
	end := t.offset + t.height
	if end >= len(t.partitions) {
		end = len(t.partitions)
	}

	chunk := t.partitions[t.offset:end]
	out := make([]string, len(chunk))
	for i, p := range chunk {
		out[i] = fmt.Sprintf(t.fmt, p.Partition, p.Start, p.Offset, p.End, p.End-p.Start)
	}

	return out, nil
}

func (t *topic) enter(row int) (feeder, error) {
	t.enteredAt = row
	row = t.offset + row
	if row >= len(t.partitions) {
		go func() { t.flashMessage <- "nothing to see here" }()
		return nil, errNoData
	}
	p := t.partitions[row]
	if p.End-p.Start == 0 {
		go func() { t.flashMessage <- "nothing to see here" }()
		return nil, errNoData
	}
	return newPartition(t.cli, p, t.width, t.height, t.flashMessage)
}

func (t *topic) print() {
	fmt.Println(t.header())
	f := t.fmt + "\n"
	for _, p := range t.partitions {
		fmt.Printf(f, p.Partition, p.Start, p.Offset, p.End, p.End-p.Start)
	}
}

type partition struct {
	cli          *kafka.Client
	height       int
	width        int
	partition    kafka.Partition
	rows         []kafka.Message
	enteredAt    int
	fmt          string
	pg           int
	flashMessage chan<- string
}

func newPartition(cli *kafka.Client, p kafka.Partition, width, height int, flashMessage chan<- string) (feeder, error) {
	rows, err := cli.GetPartition(p, height, func(_ []byte) bool { return true })
	return &partition{
		cli:          cli,
		width:        width,
		height:       height,
		partition:    p,
		rows:         rows,
		fmt:          "%-12d %s",
		flashMessage: flashMessage,
	}, err
}

func (p *partition) search(s string, cb func(int64, int64)) (int64, error) {
	i, err := p.cli.Search(p.partition, s, cb)
	if err != nil || i == -1 {
		return i, err
	}

	return i, p.jump(i)
}

func (p *partition) jump(i int64) error {
	if i >= p.partition.End {
		return nil
	}

	p.pg = int(i) / p.height
	p.partition.Offset = i
	rows, err := p.cli.GetPartition(p.partition, p.height, func(_ []byte) bool { return true })
	if err != nil {
		return err
	}
	p.rows = rows
	return nil
}

func (p *partition) row() int { return p.enteredAt }

func (p *partition) header() string {
	return fmt.Sprintf(
		"offset       message    topic: %s partition: %d start: %d end: %d",
		p.partition.Topic,
		p.partition.Partition,
		p.partition.Start,
		p.partition.End,
	)
}

func (p *partition) getRows() ([]string, error) {
	out := make([]string, len(p.rows))
	for i, msg := range p.rows {
		end := p.width
		if len(msg.Value) < end {
			end = len(msg.Value)
		}
		out[i] = fmt.Sprintf(p.fmt, p.partition.Offset+int64(i), string(msg.Value[:end]))
	}

	return out, nil
}

func (p *partition) page(pg int) error {
	if p.pg == 0 && pg < 0 && p.partition.Offset == p.partition.Start {
		return nil
	} else if p.pg == 0 && pg < 0 && p.partition.Offset > p.partition.Start {
		pg = 0
	}

	o := int64((p.pg+pg)*p.height) + p.partition.Start
	if o >= p.partition.End {
		return nil
	}
	p.pg += pg
	p.partition.Offset = o
	rows, err := p.cli.GetPartition(p.partition, p.height, func(_ []byte) bool { return true })
	if err != nil {
		return err
	}
	p.rows = rows
	return nil
}

func (p *partition) enter(row int) (feeder, error) {
	if row >= len(p.rows) {
		go func() { p.flashMessage <- "nothing to see here" }()
		return nil, errNoData
	}
	p.enteredAt = row
	return newMessage(p.rows[row], p.width, p.height, p.flashMessage)
}

func (p *partition) print() {
	p.cli.Fetch(p.partition, p.partition.End, func(s string) {
		fmt.Println(s)
	})
}

type message struct {
	height       int
	width        int
	msg          kafka.Message
	enteredAt    int
	body         []string
	pg           int
	offset       int
	flashMessage chan<- string
}

func newMessage(msg kafka.Message, width, height int, flashMessage chan<- string) (feeder, error) {
	buf, err := prettyMessage(msg.Value)
	if err != nil {
		return nil, err
	}

	var body []string
	scanner := bufio.NewScanner(buf)
	for scanner.Scan() {
		body = append(body, scanner.Text())
	}

	return &message{
		width:        width,
		height:       height,
		msg:          msg,
		body:         body,
		flashMessage: flashMessage,
	}, nil
}

func (m *message) print() {
	for _, r := range m.body {
		fmt.Println(r)
	}
}

func (m *message) search(s string, cb func(int64, int64)) (int64, error) {
	for i, r := range m.body {
		j := strings.Index(r, s)
		if j > -1 {
			return int64(j), m.jump(int64(i))
		}
	}

	return -1, nil
}

func (m *message) jump(i int64) error {
	m.pg = int(i) / m.height
	m.offset = int(i) % m.height
	return nil
}

func (m *message) row() int { return m.enteredAt }

func (m *message) header() string {
	return fmt.Sprintf(
		"topic: %s partition: %d offset: %d",
		m.msg.Partition.Topic,
		m.msg.Partition.Partition,
		m.msg.Offset,
	)
}

func (m *message) page(pg int) error {
	if m.pg == 0 && pg < 0 {
		m.offset = 0
		return nil
	}

	if ((pg+m.pg)*m.height)+m.offset > len(m.body) {
		return nil
	}
	m.pg += pg
	if m.pg == 0 {
		m.offset = 0
	}
	return nil
}

func (m *message) enter(row int) (feeder, error) {
	m.enteredAt = row
	return nil, errNoData
}

func (m *message) getRows() ([]string, error) {
	start := (m.pg * m.height) + m.offset
	end := start + m.height
	if end >= len(m.body) {
		end = len(m.body)
	}
	return m.body[start:end], nil
}

func prettyMessage(val []byte) (io.Reader, error) {
	var i interface{}
	if err := json.Unmarshal(val, &i); err != nil {
		//not json, so return original data
		return bytes.NewBuffer(val), nil
	}

	d, err := colors.Marshal(i)
	buf := bytes.NewBuffer(d)
	return buf, err
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"go.opencensus.io/trace"
	"golang.org/x/mod/semver"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/fetch"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/postgres"
	"golang.org/x/pkgsite/internal/proxy"
	"golang.org/x/pkgsite/internal/source"
	"golang.org/x/pkgsite/internal/stdlib"
)

// ProxyRemoved is a set of module@version that have been removed from the proxy,
// even though they are still in the index.
var ProxyRemoved = map[string]bool{}

// fetchTask represents the result of a fetch task that was processed.
type fetchTask struct {
	fetch.FetchResult
	timings map[string]time.Duration
}

// A Fetcher holds state for fetching modules.
type Fetcher struct {
	ProxyClient  *proxy.Client
	SourceClient *source.Client
	DB           *postgres.DB
}

// FetchAndUpdateState fetches and processes a module version, and then updates
// the module_version_states table according to the result. It returns an HTTP
// status code representing the result of the fetch operation, and a non-nil
// error if this status code is not 200.
func (f *Fetcher) FetchAndUpdateState(ctx context.Context, modulePath, requestedVersion, appVersionLabel string, disableProxyFetch bool) (_ int, resolvedVersion string, err error) {
	defer derrors.Wrap(&err, "FetchAndUpdateState(%q, %q, %q, %t)", modulePath, requestedVersion, appVersionLabel, disableProxyFetch)
	tctx, span := trace.StartSpan(ctx, "FetchAndUpdateState")
	ctx = experiment.NewContext(tctx, experiment.FromContext(ctx).Active()...)
	ctx = log.NewContextWithLabel(ctx, "fetch", modulePath+"@"+requestedVersion)
	if !utf8.ValidString(modulePath) {
		log.Errorf(ctx, "module path %q is not valid UTF-8", modulePath)
	}
	if !utf8.ValidString(requestedVersion) {
		log.Errorf(ctx, "requested version %q is not valid UTF-8", requestedVersion)
	}
	span.AddAttributes(
		trace.StringAttribute("modulePath", modulePath),
		trace.StringAttribute("version", requestedVersion))
	defer span.End()

	ft := f.fetchAndInsertModule(ctx, modulePath, requestedVersion, disableProxyFetch)
	span.AddAttributes(trace.Int64Attribute("numPackages", int64(len(ft.PackageVersionStates))))

	// If there were any errors processing the module then we didn't insert it.
	// Delete it in case we are reprocessing an existing module.
	// However, don't delete if the error was internal, or we are shedding load.
	if ft.Status >= 400 && ft.Status < 500 {
		if err := deleteModule(ctx, f.DB, ft); err != nil {
			log.Error(ctx, err)
			ft.Error = err
			ft.Status = http.StatusInternalServerError
		}
		// Do not return an error here, because we want to insert into
		// module_version_states below.
	}
	// Regardless of what the status code is, insert the result into
	// version_map, so that a response can be returned for frontend_fetch.
	if err := updateVersionMap(ctx, f.DB, ft); err != nil {
		log.Error(ctx, err)
		if ft.Status != http.StatusInternalServerError {
			ft.Error = err
			ft.Status = http.StatusInternalServerError
		}
		// Do not return an error here, because we want to insert into
		// module_version_states below.
	}
	if !semver.IsValid(ft.ResolvedVersion) {
		// If the requestedVersion was not successfully resolved to a semantic
		// version, then at this point it will be the same as the
		// resolvedVersion. This fetch request does not need to be recorded in
		// module_version_states, since that table is only used to track
		// modules that have been published to index.golang.org.
		return ft.Status, ft.ResolvedVersion, ft.Error
	}

	// Update the module_version_states table with the new status of
	// module@version. This must happen last, because if it succeeds with a
	// code < 500 but a later action fails, we will never retry the later
	// action.
	// TODO(golang/go#39628): Split UpsertModuleVersionState into
	// InsertModuleVersionState and UpdateModuleVersionState.
	start := time.Now()
	err = f.DB.UpsertModuleVersionState(ctx, ft.ModulePath, ft.ResolvedVersion, appVersionLabel,
		time.Time{}, ft.Status, ft.GoModPath, ft.Error, ft.PackageVersionStates)
	ft.timings["db.UpsertModuleVersionState"] = time.Since(start)
	if err != nil {
		log.Error(ctx, err)
		if ft.Error != nil {
			ft.Status = http.StatusInternalServerError
			ft.Error = fmt.Errorf("db.UpsertModuleVersionState: %v, original error: %v", err, ft.Error)
		}
		logTaskResult(ctx, ft, "Failed to update module version state")
		return http.StatusInternalServerError, ft.ResolvedVersion, ft.Error
	}
	logTaskResult(ctx, ft, "Updated module version state")
	return ft.Status, ft.ResolvedVersion, ft.Error
}

// fetchAndInsertModule fetches the given module version from the module proxy
// or (in the case of the standard library) from the Go repo and writes the
// resulting data to the database.
//
// The given parentCtx is used for tracing, but fetches actually execute in a
// detached context with fixed timeout, so that fetches are allowed to complete
// even for short-lived requests.
func (f *Fetcher) fetchAndInsertModule(ctx context.Context, modulePath, requestedVersion string, disableProxyFetch bool) *fetchTask {
	ft := &fetchTask{
		FetchResult: fetch.FetchResult{
			ModulePath:       modulePath,
			RequestedVersion: requestedVersion,
		},
		timings: map[string]time.Duration{},
	}
	defer func() {
		derrors.Wrap(&ft.Error, "fetchAndInsertModule(%q, %q)", modulePath, requestedVersion)
		if ft.Error != nil {
			ft.Status = derrors.ToStatus(ft.Error)
			ft.ResolvedVersion = requestedVersion
		}
	}()

	if ProxyRemoved[modulePath+"@"+requestedVersion] {
		log.Infof(ctx, "not fetching %s@%s because it is on the ProxyRemoved list", modulePath, requestedVersion)
		ft.Error = derrors.Excluded
		return ft
	}

	exc, err := f.DB.IsExcluded(ctx, modulePath)
	if err != nil {
		ft.Error = err
		return ft
	}
	if exc {
		ft.Error = derrors.Excluded
		return ft
	}

	// Fetch the module, and the current @main and @master version of this module.
	// The @main and @master version will be used to update the version_map
	// target if applicable.
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		start := time.Now()
		fr := fetch.FetchModule(ctx, modulePath, requestedVersion, f.ProxyClient, f.SourceClient, disableProxyFetch)
		if fr == nil {
			panic("fetch.FetchModule should never return a nil FetchResult")
		}
		defer fr.Defer()
		ft.FetchResult = *fr
		ft.timings["fetch.FetchModule"] = time.Since(start)
	}()
	// Do not resolve the @main and @master version if disableProxyFetch is on.
	var main string
	wg.Add(1)
	go func() {
		defer wg.Done()
		if !disableProxyFetch {
			main = resolvedVersion(ctx, modulePath, internal.MainVersion, f.ProxyClient)
		}
	}()
	var master string
	wg.Add(1)
	go func() {
		defer wg.Done()
		if !disableProxyFetch {
			master = resolvedVersion(ctx, modulePath, internal.MasterVersion, f.ProxyClient)
		}
	}()
	wg.Wait()
	ft.MainVersion = main
	ft.MasterVersion = master

	// There was an error fetching this module.
	if ft.Error != nil {
		logf := log.Infof
		if ft.Status == http.StatusServiceUnavailable {
			logf = log.Warningf
		} else if ft.Status >= 500 && ft.Status != derrors.ToStatus(derrors.ProxyTimedOut) {
			logf = log.Errorf
		}
		logf(ctx, "Error executing fetch: %v (code %d)", ft.Error, ft.Status)
		return ft
	}

	// The module was successfully fetched.
	log.Infof(ctx, "fetch.FetchModule succeeded for %s@%s", ft.ModulePath, ft.RequestedVersion)
	start := time.Now()
	err = f.DB.InsertModule(ctx, ft.Module)
	ft.timings["db.InsertModule"] = time.Since(start)
	if err != nil {
		log.Error(ctx, err)

		ft.Status = derrors.ToStatus(err)
		ft.Error = err
		return ft
	}
	log.Infof(ctx, "db.InsertModule succeeded for %s@%s", ft.ModulePath, ft.RequestedVersion)
	return ft
}

func resolvedVersion(ctx context.Context, modulePath, requestedVersion string, proxyClient *proxy.Client) string {
	if modulePath == stdlib.ModulePath && requestedVersion == internal.MainVersion {
		return ""
	}
	info, err := fetch.GetInfo(ctx, modulePath, requestedVersion, proxyClient, false)
	if err != nil {
		if !errors.Is(err, derrors.NotFound) {
			// If an error occurs, log it and insert the module as
			// normal.
			log.Errorf(ctx, "fetch.GetInfo(ctx, %q, %q, f.ProxyClient, false): %v", modulePath, requestedVersion, err)
		}
		log.Infof(ctx, "fetch.GetInfo(ctx, %q, %q, f.ProxyClient, false): %v", modulePath, requestedVersion, err)
		return ""
	}
	return info.Version
}

func updateVersionMap(ctx context.Context, db *postgres.DB, ft *fetchTask) (err error) {
	start := time.Now()
	defer func() {
		ft.timings["worker.updatedVersionMap"] = time.Since(start)
		derrors.Wrap(&err, "updateVersionMap(%q, %q, %q, %d, %v)",
			ft.ModulePath, ft.RequestedVersion, ft.ResolvedVersion, ft.Status, ft.Error)
	}()
	ctx, span := trace.StartSpan(ctx, "worker.updateVersionMap")
	defer span.End()

	var errMsg string
	if ft.Error != nil {
		errMsg = ft.Error.Error()
	}

	// If the resolved version for the this module version is also the resolved
	// version for @main or @master, update version_map to match.
	requestedVersions := []string{ft.RequestedVersion}
	if ft.MainVersion == ft.ResolvedVersion {
		requestedVersions = append(requestedVersions, internal.MainVersion)
	}
	if ft.MasterVersion == ft.ResolvedVersion {
		requestedVersions = append(requestedVersions, internal.MasterVersion)
	}
	for _, v := range requestedVersions {
		v := v
		vm := &internal.VersionMap{
			ModulePath:       ft.ModulePath,
			RequestedVersion: v,
			ResolvedVersion:  ft.ResolvedVersion,
			Status:           ft.Status,
			GoModPath:        ft.GoModPath,
			Error:            errMsg,
		}
		if err := db.UpsertVersionMap(ctx, vm); err != nil {
			return err
		}
	}
	return nil
}

func deleteModule(ctx context.Context, db *postgres.DB, ft *fetchTask) (err error) {
	start := time.Now()
	defer func() {
		ft.timings["worker.deleteModule"] = time.Since(start)
		derrors.Wrap(&err, "deleteModule(%q, %q, %q, %d, %v)",
			ft.ModulePath, ft.RequestedVersion, ft.ResolvedVersion, ft.Status, ft.Error)
	}()
	ctx, span := trace.StartSpan(ctx, "worker.deleteModule")
	defer span.End()

	log.Infof(ctx, "%s@%s: code=%d, deleting", ft.ModulePath, ft.ResolvedVersion, ft.Status)
	if err := db.DeleteModule(ctx, ft.ModulePath, ft.ResolvedVersion); err != nil {
		return err
	}
	// If this was an alternative path (ft.Status == 491) and there is an older
	// version in search_documents, delete it. This is the case where a module's
	// canonical path was changed by the addition of a go.mod file. For example,
	// versions of logrus before it acquired a go.mod file could have the path
	// github.com/Sirupsen/logrus, but once the go.mod file specifies that the
	// path is all lower-case, the old versions should not show up in search. We
	// still leave their pages in the database so users of those old versions
	// can still view documentation.
	if ft.Status == derrors.ToStatus(derrors.AlternativeModule) {
		log.Infof(ctx, "%s@%s: code=491, deleting older version from search", ft.ModulePath, ft.ResolvedVersion)
		if err := db.DeleteOlderVersionFromSearchDocuments(ctx, ft.ModulePath, ft.ResolvedVersion); err != nil {
			return err
		}
	}
	return nil
}

func logTaskResult(ctx context.Context, ft *fetchTask, prefix string) {
	var times []string
	for k, v := range ft.timings {
		times = append(times, fmt.Sprintf("%s=%.3fs", k, v.Seconds()))
	}
	sort.Strings(times)
	msg := strings.Join(times, ", ")
	logf := log.Infof
	if ft.Status == http.StatusInternalServerError {
		logf = log.Errorf
	}
	logf(ctx, "%s for %s@%s: code=%d, num_packages=%d, err=%v; timings: %s",
		prefix, ft.ModulePath, ft.ResolvedVersion, ft.Status, len(ft.PackageVersionStates), ft.Error, msg)
}
<commit_msg>internal/worker: delete unnecessary log<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"go.opencensus.io/trace"
	"golang.org/x/mod/semver"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/fetch"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/postgres"
	"golang.org/x/pkgsite/internal/proxy"
	"golang.org/x/pkgsite/internal/source"
	"golang.org/x/pkgsite/internal/stdlib"
)

// ProxyRemoved is a set of module@version that have been removed from the proxy,
// even though they are still in the index.
var ProxyRemoved = map[string]bool{}

// fetchTask represents the result of a fetch task that was processed.
type fetchTask struct {
	fetch.FetchResult
	timings map[string]time.Duration
}

// A Fetcher holds state for fetching modules.
type Fetcher struct {
	ProxyClient  *proxy.Client
	SourceClient *source.Client
	DB           *postgres.DB
}

// FetchAndUpdateState fetches and processes a module version, and then updates
// the module_version_states table according to the result. It returns an HTTP
// status code representing the result of the fetch operation, and a non-nil
// error if this status code is not 200.
func (f *Fetcher) FetchAndUpdateState(ctx context.Context, modulePath, requestedVersion, appVersionLabel string, disableProxyFetch bool) (_ int, resolvedVersion string, err error) {
	defer derrors.Wrap(&err, "FetchAndUpdateState(%q, %q, %q, %t)", modulePath, requestedVersion, appVersionLabel, disableProxyFetch)
	tctx, span := trace.StartSpan(ctx, "FetchAndUpdateState")
	ctx = experiment.NewContext(tctx, experiment.FromContext(ctx).Active()...)
	ctx = log.NewContextWithLabel(ctx, "fetch", modulePath+"@"+requestedVersion)
	if !utf8.ValidString(modulePath) {
		log.Errorf(ctx, "module path %q is not valid UTF-8", modulePath)
	}
	if !utf8.ValidString(requestedVersion) {
		log.Errorf(ctx, "requested version %q is not valid UTF-8", requestedVersion)
	}
	span.AddAttributes(
		trace.StringAttribute("modulePath", modulePath),
		trace.StringAttribute("version", requestedVersion))
	defer span.End()

	ft := f.fetchAndInsertModule(ctx, modulePath, requestedVersion, disableProxyFetch)
	span.AddAttributes(trace.Int64Attribute("numPackages", int64(len(ft.PackageVersionStates))))

	// If there were any errors processing the module then we didn't insert it.
	// Delete it in case we are reprocessing an existing module.
	// However, don't delete if the error was internal, or we are shedding load.
	if ft.Status >= 400 && ft.Status < 500 {
		if err := deleteModule(ctx, f.DB, ft); err != nil {
			log.Error(ctx, err)
			ft.Error = err
			ft.Status = http.StatusInternalServerError
		}
		// Do not return an error here, because we want to insert into
		// module_version_states below.
	}
	// Regardless of what the status code is, insert the result into
	// version_map, so that a response can be returned for frontend_fetch.
	if err := updateVersionMap(ctx, f.DB, ft); err != nil {
		log.Error(ctx, err)
		if ft.Status != http.StatusInternalServerError {
			ft.Error = err
			ft.Status = http.StatusInternalServerError
		}
		// Do not return an error here, because we want to insert into
		// module_version_states below.
	}
	if !semver.IsValid(ft.ResolvedVersion) {
		// If the requestedVersion was not successfully resolved to a semantic
		// version, then at this point it will be the same as the
		// resolvedVersion. This fetch request does not need to be recorded in
		// module_version_states, since that table is only used to track
		// modules that have been published to index.golang.org.
		return ft.Status, ft.ResolvedVersion, ft.Error
	}

	// Update the module_version_states table with the new status of
	// module@version. This must happen last, because if it succeeds with a
	// code < 500 but a later action fails, we will never retry the later
	// action.
	// TODO(golang/go#39628): Split UpsertModuleVersionState into
	// InsertModuleVersionState and UpdateModuleVersionState.
	start := time.Now()
	err = f.DB.UpsertModuleVersionState(ctx, ft.ModulePath, ft.ResolvedVersion, appVersionLabel,
		time.Time{}, ft.Status, ft.GoModPath, ft.Error, ft.PackageVersionStates)
	ft.timings["db.UpsertModuleVersionState"] = time.Since(start)
	if err != nil {
		log.Error(ctx, err)
		if ft.Error != nil {
			ft.Status = http.StatusInternalServerError
			ft.Error = fmt.Errorf("db.UpsertModuleVersionState: %v, original error: %v", err, ft.Error)
		}
		logTaskResult(ctx, ft, "Failed to update module version state")
		return http.StatusInternalServerError, ft.ResolvedVersion, ft.Error
	}
	logTaskResult(ctx, ft, "Updated module version state")
	return ft.Status, ft.ResolvedVersion, ft.Error
}

// fetchAndInsertModule fetches the given module version from the module proxy
// or (in the case of the standard library) from the Go repo and writes the
// resulting data to the database.
//
// The given parentCtx is used for tracing, but fetches actually execute in a
// detached context with fixed timeout, so that fetches are allowed to complete
// even for short-lived requests.
func (f *Fetcher) fetchAndInsertModule(ctx context.Context, modulePath, requestedVersion string, disableProxyFetch bool) *fetchTask {
	ft := &fetchTask{
		FetchResult: fetch.FetchResult{
			ModulePath:       modulePath,
			RequestedVersion: requestedVersion,
		},
		timings: map[string]time.Duration{},
	}
	defer func() {
		derrors.Wrap(&ft.Error, "fetchAndInsertModule(%q, %q)", modulePath, requestedVersion)
		if ft.Error != nil {
			ft.Status = derrors.ToStatus(ft.Error)
			ft.ResolvedVersion = requestedVersion
		}
	}()

	if ProxyRemoved[modulePath+"@"+requestedVersion] {
		log.Infof(ctx, "not fetching %s@%s because it is on the ProxyRemoved list", modulePath, requestedVersion)
		ft.Error = derrors.Excluded
		return ft
	}

	exc, err := f.DB.IsExcluded(ctx, modulePath)
	if err != nil {
		ft.Error = err
		return ft
	}
	if exc {
		ft.Error = derrors.Excluded
		return ft
	}

	// Fetch the module, and the current @main and @master version of this module.
	// The @main and @master version will be used to update the version_map
	// target if applicable.
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		start := time.Now()
		fr := fetch.FetchModule(ctx, modulePath, requestedVersion, f.ProxyClient, f.SourceClient, disableProxyFetch)
		if fr == nil {
			panic("fetch.FetchModule should never return a nil FetchResult")
		}
		defer fr.Defer()
		ft.FetchResult = *fr
		ft.timings["fetch.FetchModule"] = time.Since(start)
	}()
	// Do not resolve the @main and @master version if disableProxyFetch is on.
	var main string
	wg.Add(1)
	go func() {
		defer wg.Done()
		if !disableProxyFetch {
			main = resolvedVersion(ctx, modulePath, internal.MainVersion, f.ProxyClient)
		}
	}()
	var master string
	wg.Add(1)
	go func() {
		defer wg.Done()
		if !disableProxyFetch {
			master = resolvedVersion(ctx, modulePath, internal.MasterVersion, f.ProxyClient)
		}
	}()
	wg.Wait()
	ft.MainVersion = main
	ft.MasterVersion = master

	// There was an error fetching this module.
	if ft.Error != nil {
		logf := log.Infof
		if ft.Status == http.StatusServiceUnavailable {
			logf = log.Warningf
		} else if ft.Status >= 500 && ft.Status != derrors.ToStatus(derrors.ProxyTimedOut) {
			logf = log.Errorf
		}
		logf(ctx, "Error executing fetch: %v (code %d)", ft.Error, ft.Status)
		return ft
	}

	// The module was successfully fetched.
	log.Infof(ctx, "fetch.FetchModule succeeded for %s@%s", ft.ModulePath, ft.RequestedVersion)
	start := time.Now()
	err = f.DB.InsertModule(ctx, ft.Module)
	ft.timings["db.InsertModule"] = time.Since(start)
	if err != nil {
		log.Error(ctx, err)

		ft.Status = derrors.ToStatus(err)
		ft.Error = err
		return ft
	}
	log.Infof(ctx, "db.InsertModule succeeded for %s@%s", ft.ModulePath, ft.RequestedVersion)
	return ft
}

func resolvedVersion(ctx context.Context, modulePath, requestedVersion string, proxyClient *proxy.Client) string {
	if modulePath == stdlib.ModulePath && requestedVersion == internal.MainVersion {
		return ""
	}
	info, err := fetch.GetInfo(ctx, modulePath, requestedVersion, proxyClient, false)
	if err != nil {
		if !errors.Is(err, derrors.NotFound) {
			// If an error occurs, log it and insert the module as normal.
			log.Errorf(ctx, "fetch.GetInfo(ctx, %q, %q, f.ProxyClient, false): %v", modulePath, requestedVersion, err)
		}
		return ""
	}
	return info.Version
}

func updateVersionMap(ctx context.Context, db *postgres.DB, ft *fetchTask) (err error) {
	start := time.Now()
	defer func() {
		ft.timings["worker.updatedVersionMap"] = time.Since(start)
		derrors.Wrap(&err, "updateVersionMap(%q, %q, %q, %d, %v)",
			ft.ModulePath, ft.RequestedVersion, ft.ResolvedVersion, ft.Status, ft.Error)
	}()
	ctx, span := trace.StartSpan(ctx, "worker.updateVersionMap")
	defer span.End()

	var errMsg string
	if ft.Error != nil {
		errMsg = ft.Error.Error()
	}

	// If the resolved version for the this module version is also the resolved
	// version for @main or @master, update version_map to match.
	requestedVersions := []string{ft.RequestedVersion}
	if ft.MainVersion == ft.ResolvedVersion {
		requestedVersions = append(requestedVersions, internal.MainVersion)
	}
	if ft.MasterVersion == ft.ResolvedVersion {
		requestedVersions = append(requestedVersions, internal.MasterVersion)
	}
	for _, v := range requestedVersions {
		v := v
		vm := &internal.VersionMap{
			ModulePath:       ft.ModulePath,
			RequestedVersion: v,
			ResolvedVersion:  ft.ResolvedVersion,
			Status:           ft.Status,
			GoModPath:        ft.GoModPath,
			Error:            errMsg,
		}
		if err := db.UpsertVersionMap(ctx, vm); err != nil {
			return err
		}
	}
	return nil
}

func deleteModule(ctx context.Context, db *postgres.DB, ft *fetchTask) (err error) {
	start := time.Now()
	defer func() {
		ft.timings["worker.deleteModule"] = time.Since(start)
		derrors.Wrap(&err, "deleteModule(%q, %q, %q, %d, %v)",
			ft.ModulePath, ft.RequestedVersion, ft.ResolvedVersion, ft.Status, ft.Error)
	}()
	ctx, span := trace.StartSpan(ctx, "worker.deleteModule")
	defer span.End()

	log.Infof(ctx, "%s@%s: code=%d, deleting", ft.ModulePath, ft.ResolvedVersion, ft.Status)
	if err := db.DeleteModule(ctx, ft.ModulePath, ft.ResolvedVersion); err != nil {
		return err
	}
	// If this was an alternative path (ft.Status == 491) and there is an older
	// version in search_documents, delete it. This is the case where a module's
	// canonical path was changed by the addition of a go.mod file. For example,
	// versions of logrus before it acquired a go.mod file could have the path
	// github.com/Sirupsen/logrus, but once the go.mod file specifies that the
	// path is all lower-case, the old versions should not show up in search. We
	// still leave their pages in the database so users of those old versions
	// can still view documentation.
	if ft.Status == derrors.ToStatus(derrors.AlternativeModule) {
		log.Infof(ctx, "%s@%s: code=491, deleting older version from search", ft.ModulePath, ft.ResolvedVersion)
		if err := db.DeleteOlderVersionFromSearchDocuments(ctx, ft.ModulePath, ft.ResolvedVersion); err != nil {
			return err
		}
	}
	return nil
}

func logTaskResult(ctx context.Context, ft *fetchTask, prefix string) {
	var times []string
	for k, v := range ft.timings {
		times = append(times, fmt.Sprintf("%s=%.3fs", k, v.Seconds()))
	}
	sort.Strings(times)
	msg := strings.Join(times, ", ")
	logf := log.Infof
	if ft.Status == http.StatusInternalServerError {
		logf = log.Errorf
	}
	logf(ctx, "%s for %s@%s: code=%d, num_packages=%d, err=%v; timings: %s",
		prefix, ft.ModulePath, ft.ResolvedVersion, ft.Status, len(ft.PackageVersionStates), ft.Error, msg)
}
<|endoftext|>


<commit_before>package chunkymonkey

import (
    "os"
    "io"
    "log"
    "net"
    "math"
    "bytes"

    "chunkymonkey/proto"
    .   "chunkymonkey/types"
)

type Player struct {
    Entity
    game        *Game
    conn        net.Conn
    name        string
    position    AbsXYZ
    look        LookDegrees
    currentItem ItemID
    txQueue     chan []byte
}

const StanceNormal = 1.62

func StartPlayer(game *Game, conn net.Conn, name string) {
    player := &Player{
        game:     game,
        conn:     conn,
        name:     name,
        position: StartPosition,
        look:     LookDegrees{0, 0},
        txQueue:  make(chan []byte, 128),
    }

    game.Enqueue(func(game *Game) {
        game.AddPlayer(player)
        // TODO pass proper map seed and dimension
        proto.ServerWriteLogin(conn, player.Entity.EntityID, 0, DimensionNormal)
        player.start()
        player.postLogin()
    })
}

func (player *Player) start() {
    go player.ReceiveLoop()
    go player.TransmitLoop()
}

func (player *Player) PacketKeepAlive() {
}

func (player *Player) PacketChatMessage(message string) {
    player.game.Enqueue(func(game *Game) { game.SendChatMessage(message) })
}

func (player *Player) PacketEntityAction(entityID EntityID, action EntityAction) {
}

func (player *Player) PacketUseEntity(user EntityID, target EntityID, leftClick bool) {
}

func (player *Player) PacketRespawn() {
}

func (player *Player) PacketPlayer(onGround bool) {
}

func (player *Player) PacketPlayerPosition(position *AbsXYZ, stance AbsCoord, onGround bool) {
    // TODO: Should keep track of when players enter/leave their mutual radius
    // of "awareness". I.e a client should receive a RemoveEntity packet when
    // the player walks out of range, and no longer receive WriteEntityTeleport
    // packets for them. The converse should happen when players come in range
    // of each other.

    player.game.Enqueue(func(game *Game) {
        var delta = AbsXYZ{position.X - player.position.X,
            position.Y - player.position.Y,
            position.Z - player.position.Z}
        distance := math.Sqrt(float64(delta.X*delta.X + delta.Y*delta.Y + delta.Z*delta.Z))
        if distance > 10 {
            log.Printf("Discarding player position that is too far removed (%.2f, %.2f, %.2f)",
                position.X, position.Y, position.Z)
            return
        }

        player.position = *position

        buf := &bytes.Buffer{}
        proto.WriteEntityTeleport(
            buf,
            player.EntityID,
            player.position.ToAbsIntXYZ(),
            player.look.ToLookBytes())
        game.MulticastPacket(buf.Bytes(), player)
    })
}

func (player *Player) PacketPlayerLook(look *LookDegrees, onGround bool) {
    player.game.Enqueue(func(game *Game) {
        // TODO input validation
        player.look = *look

        buf := &bytes.Buffer{}
        proto.WriteEntityLook(buf, player.EntityID, look.ToLookBytes())
        game.MulticastPacket(buf.Bytes(), player)
    })
}

func (player *Player) PacketPlayerDigging(status DigStatus, blockLoc *BlockXYZ, face Face) {
    // TODO validate that the player is actually somewhere near the block

    if status == DigBlockBroke {
        // TODO validate that the player has dug long enough to stop speed
        // hacking (based on block type and tool used - non-trivial).

        player.game.Enqueue(func(game *Game) {
            chunkLoc, subLoc := blockLoc.ToChunkLocal()

            chunk := game.chunkManager.Get(chunkLoc)

            if chunk == nil {
                return
            }

            chunk.DestroyBlock(subLoc)
        })
    }
}

func (player *Player) PacketPlayerBlockPlacement(itemID ItemID, blockLoc *BlockXYZ, face Face, amount ItemCount, uses ItemUses) {
}

func (player *Player) PacketHoldingChange(itemID ItemID) {
}

func (player *Player) PacketEntityAnimation(entityID EntityID, animation EntityAnimation) {
}

func (player *Player) PacketWindowClose(windowID WindowID) {
}

func (player *Player) PacketWindowClick(windowID WindowID, slot SlotID, rightClick bool, txID TxID, itemID ItemID, amount ItemCount, uses ItemUses) {
}

func (player *Player) PacketSignUpdate(position *BlockXYZ, lines [4]string) {
}

func (player *Player) PacketDisconnect(reason string) {
    log.Printf("Player %s disconnected reason=%s", player.name, reason)
    player.game.Enqueue(func(game *Game) {
        game.RemovePlayer(player)
        close(player.txQueue)
        player.conn.Close()
    })
}

func (player *Player) ReceiveLoop() {
    for {
        err := proto.ServerReadPacket(player.conn, player)
        if err != nil {
            if err != os.EOF {
                log.Print("ReceiveLoop failed: ", err.String())
            }
            return
        }
    }
}

func (player *Player) TransmitLoop() {
    for {
        bs := <-player.txQueue
        if bs == nil {
            return // txQueue closed
        }

        _, err := player.conn.Write(bs)
        if err != nil {
            if err != os.EOF {
                log.Print("TransmitLoop failed: ", err.String())
            }
            return
        }
    }
}

func (player *Player) sendChunks(writer io.Writer) {
    playerChunkLoc := player.position.ToChunkXZ()

    for chunk := range player.game.chunkManager.ChunksInRadius(playerChunkLoc) {
        proto.WritePreChunk(writer, &chunk.XZ, ChunkInit)
    }

    for chunk := range player.game.chunkManager.ChunksInRadius(playerChunkLoc) {
        chunk.SendChunkData(writer)
    }
}

func (player *Player) TransmitPacket(packet []byte) {
    if packet == nil {
        return // skip empty packets
    }
    player.txQueue <- packet
}

func (player *Player) postLogin() {
    buf := &bytes.Buffer{}
    proto.WriteSpawnPosition(buf, player.position.ToBlockXYZ())
    player.sendChunks(buf)
    proto.ServerWritePlayerPositionLook(buf, &player.position, &player.look,
        player.position.Y+StanceNormal, false)
    player.TransmitPacket(buf.Bytes())
}
<commit_msg>Added expvar for cumulative number of player connects/disconnects.<commit_after>package chunkymonkey

import (
    "bytes"
    "expvar"
    "io"
    "log"
    "math"
    "net"
    "os"

    "chunkymonkey/proto"
    .   "chunkymonkey/types"
)

var (
    expVarPlayerConnectionCount    *expvar.Int
    expVarPlayerDisconnectionCount *expvar.Int
)

func init() {
    expVarPlayerConnectionCount = expvar.NewInt("player-connection-count")
    expVarPlayerDisconnectionCount = expvar.NewInt("player-disconnection-count")
}

type Player struct {
    Entity
    game        *Game
    conn        net.Conn
    name        string
    position    AbsXYZ
    look        LookDegrees
    currentItem ItemID
    txQueue     chan []byte
}

const StanceNormal = 1.62

func StartPlayer(game *Game, conn net.Conn, name string) {
    player := &Player{
        game:     game,
        conn:     conn,
        name:     name,
        position: StartPosition,
        look:     LookDegrees{0, 0},
        txQueue:  make(chan []byte, 128),
    }

    game.Enqueue(func(game *Game) {
        game.AddPlayer(player)
        // TODO pass proper map seed and dimension
        proto.ServerWriteLogin(conn, player.Entity.EntityID, 0, DimensionNormal)
        player.start()
        player.postLogin()
    })
}

func (player *Player) start() {
    expVarPlayerConnectionCount.Add(1)
    go player.ReceiveLoop()
    go player.TransmitLoop()
}

func (player *Player) PacketKeepAlive() {
}

func (player *Player) PacketChatMessage(message string) {
    player.game.Enqueue(func(game *Game) { game.SendChatMessage(message) })
}

func (player *Player) PacketEntityAction(entityID EntityID, action EntityAction) {
}

func (player *Player) PacketUseEntity(user EntityID, target EntityID, leftClick bool) {
}

func (player *Player) PacketRespawn() {
}

func (player *Player) PacketPlayer(onGround bool) {
}

func (player *Player) PacketPlayerPosition(position *AbsXYZ, stance AbsCoord, onGround bool) {
    // TODO: Should keep track of when players enter/leave their mutual radius
    // of "awareness". I.e a client should receive a RemoveEntity packet when
    // the player walks out of range, and no longer receive WriteEntityTeleport
    // packets for them. The converse should happen when players come in range
    // of each other.

    player.game.Enqueue(func(game *Game) {
        var delta = AbsXYZ{position.X - player.position.X,
            position.Y - player.position.Y,
            position.Z - player.position.Z}
        distance := math.Sqrt(float64(delta.X*delta.X + delta.Y*delta.Y + delta.Z*delta.Z))
        if distance > 10 {
            log.Printf("Discarding player position that is too far removed (%.2f, %.2f, %.2f)",
                position.X, position.Y, position.Z)
            return
        }

        player.position = *position

        buf := &bytes.Buffer{}
        proto.WriteEntityTeleport(
            buf,
            player.EntityID,
            player.position.ToAbsIntXYZ(),
            player.look.ToLookBytes())
        game.MulticastPacket(buf.Bytes(), player)
    })
}

func (player *Player) PacketPlayerLook(look *LookDegrees, onGround bool) {
    player.game.Enqueue(func(game *Game) {
        // TODO input validation
        player.look = *look

        buf := &bytes.Buffer{}
        proto.WriteEntityLook(buf, player.EntityID, look.ToLookBytes())
        game.MulticastPacket(buf.Bytes(), player)
    })
}

func (player *Player) PacketPlayerDigging(status DigStatus, blockLoc *BlockXYZ, face Face) {
    // TODO validate that the player is actually somewhere near the block

    if status == DigBlockBroke {
        // TODO validate that the player has dug long enough to stop speed
        // hacking (based on block type and tool used - non-trivial).

        player.game.Enqueue(func(game *Game) {
            chunkLoc, subLoc := blockLoc.ToChunkLocal()

            chunk := game.chunkManager.Get(chunkLoc)

            if chunk == nil {
                return
            }

            chunk.DestroyBlock(subLoc)
        })
    }
}

func (player *Player) PacketPlayerBlockPlacement(itemID ItemID, blockLoc *BlockXYZ, face Face, amount ItemCount, uses ItemUses) {
}

func (player *Player) PacketHoldingChange(itemID ItemID) {
}

func (player *Player) PacketEntityAnimation(entityID EntityID, animation EntityAnimation) {
}

func (player *Player) PacketWindowClose(windowID WindowID) {
}

func (player *Player) PacketWindowClick(windowID WindowID, slot SlotID, rightClick bool, txID TxID, itemID ItemID, amount ItemCount, uses ItemUses) {
}

func (player *Player) PacketSignUpdate(position *BlockXYZ, lines [4]string) {
}

func (player *Player) PacketDisconnect(reason string) {
    log.Printf("Player %s disconnected reason=%s", player.name, reason)
    player.game.Enqueue(func(game *Game) {
        game.RemovePlayer(player)
        close(player.txQueue)
        player.conn.Close()
    })
}

func (player *Player) ReceiveLoop() {
    for {
        err := proto.ServerReadPacket(player.conn, player)
        if err != nil {
            if err != os.EOF {
                log.Print("ReceiveLoop failed: ", err.String())
            }
            expVarPlayerDisconnectionCount.Add(1)
            return
        }
    }
}

func (player *Player) TransmitLoop() {
    for {
        bs := <-player.txQueue
        if bs == nil {
            return // txQueue closed
        }

        _, err := player.conn.Write(bs)
        if err != nil {
            if err != os.EOF {
                log.Print("TransmitLoop failed: ", err.String())
            }
            return
        }
    }
}

func (player *Player) sendChunks(writer io.Writer) {
    playerChunkLoc := player.position.ToChunkXZ()

    for chunk := range player.game.chunkManager.ChunksInRadius(playerChunkLoc) {
        proto.WritePreChunk(writer, &chunk.XZ, ChunkInit)
    }

    for chunk := range player.game.chunkManager.ChunksInRadius(playerChunkLoc) {
        chunk.SendChunkData(writer)
    }
}

func (player *Player) TransmitPacket(packet []byte) {
    if packet == nil {
        return // skip empty packets
    }
    player.txQueue <- packet
}

func (player *Player) postLogin() {
    buf := &bytes.Buffer{}
    proto.WriteSpawnPosition(buf, player.position.ToBlockXYZ())
    player.sendChunks(buf)
    proto.ServerWritePlayerPositionLook(buf, &player.position, &player.look,
        player.position.Y+StanceNormal, false)
    player.TransmitPacket(buf.Bytes())
}
<|endoftext|>


<commit_before>package search

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"

	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
	"github.com/wasanx25/sreq/config"
	"github.com/wasanx25/sreq/history"
)

// Content is structure that scraping content from Qiita
type Content struct {
	ID    string
	Title string
	Desc  string
}

func search(argument string, pagenation int, sort string) ([]*Content, error) {
	doc, err := goquery.NewDocument(config.GetPageURL(argument, sort, strconv.Itoa(pagenation)))
	if err != nil {
		return nil, err
	}

	var contents []*Content

	doc.Find(".searchResult").Each(func(_ int, s *goquery.Selection) {
		itemID, _ := s.Attr("data-uuid")
		title := s.Find(".searchResult_itemTitle a").Text()
		desc := s.Find(".searchResult_snippet").Text()

		content := &Content{
			ID:    itemID,
			Title: title,
			Desc:  desc,
		}

		contents = append(contents, content)
	})

	return contents, nil
}

func viewList(contents []*Content) {
	for num, content := range contents {
		fmt.Print(color.YellowString(strconv.Itoa(num) + " -> "))
		fmt.Println(content.Title)
		fmt.Println(color.GreenString(content.Desc))
		fmt.Print("\n")
	}
	if len(contents) == 10 {
		fmt.Println(color.YellowString("n -> ") + "next page")
	}
	fmt.Print("SELECT > ")
}

func scan(contents []*Content, argument string, lynx bool) bool {
	var num string
	if _, err := fmt.Scanf("%s", &num); err != nil {
		fmt.Println(err)
	}

	if num == "n" {
		return false
	}

	index, _ := strconv.Atoi(num)
	target := contents[index]

	resp, err := http.Get(config.GetAPIURL(target.ID))
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
	}
	var qiita *config.Qiita
	json.Unmarshal(b, &qiita)

	writeHistory(qiita, argument)

	if lynx {
		openFile(qiita.HTML, "/tmp/sreq.html", "lynx", "-display_charset=utf-8", "-assume_charset=utf-8")
		return true
	}

	openFile(qiita.Markdown, "/tmp/sreq.txt", "less")
	return true
}

func openFile(body string, file string, cmdName ...string) {
	text := []byte(body)
	ioutil.WriteFile(file, text, os.ModePerm)
	cmdName = append(cmdName, file)
	cmd := exec.Command(cmdName[0], cmdName[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Run()
}

func writeHistory(content *config.Qiita, argument string) {
	var snippets history.Snippets
	snippets.Load()
	url := content.URL
	newSnippet := history.Snippet{
		SearchKeyword: argument,
		URL:           url,
		Title:         content.Title,
	}
	snippets.Snippets = append(snippets.Snippets, newSnippet)
	if err := snippets.Save(); err != nil {
		fmt.Printf("Failed. %v", err)
		os.Exit(2)
	}
}
<commit_msg>Reset search.go<commit_after>package search

// import (
// 	"encoding/json"
// 	"fmt"
// 	"io/ioutil"
// 	"net/http"
// 	"os"
// 	"os/exec"
// 	"strconv"
//
// 	"github.com/PuerkitoBio/goquery"
// 	"github.com/fatih/color"
// 	"github.com/wasanx25/sreq/config"
// 	"github.com/wasanx25/sreq/history"
// )
//
// // Content is structure that scraping content from Qiita
// type Content struct {
// 	ID    string
// 	Title string
// 	Desc  string
// }
//
// func search(argument string, pagenation int, sort string) ([]*Content, error) {
// 	doc, err := goquery.NewDocument(config.GetPageURL(argument, sort, strconv.Itoa(pagenation)))
// 	if err != nil {
// 		return nil, err
// 	}
//
// 	var contents []*Content
//
// 	doc.Find(".searchResult").Each(func(_ int, s *goquery.Selection) {
// 		itemID, _ := s.Attr("data-uuid")
// 		title := s.Find(".searchResult_itemTitle a").Text()
// 		desc := s.Find(".searchResult_snippet").Text()
//
// 		content := &Content{
// 			ID:    itemID,
// 			Title: title,
// 			Desc:  desc,
// 		}
//
// 		contents = append(contents, content)
// 	})
//
// 	return contents, nil
// }
//
// func viewList(contents []*Content) {
// 	for num, content := range contents {
// 		fmt.Print(color.YellowString(strconv.Itoa(num) + " -> "))
// 		fmt.Println(content.Title)
// 		fmt.Println(color.GreenString(content.Desc))
// 		fmt.Print("\n")
// 	}
// 	if len(contents) == 10 {
// 		fmt.Println(color.YellowString("n -> ") + "next page")
// 	}
// 	fmt.Print("SELECT > ")
// }
//
// func scan(contents []*Content, argument string, lynx bool) bool {
// 	var num string
// 	if _, err := fmt.Scanf("%s", &num); err != nil {
// 		fmt.Println(err)
// 	}
//
// 	if num == "n" {
// 		return false
// 	}
//
// 	index, _ := strconv.Atoi(num)
// 	target := contents[index]
//
// 	resp, err := http.Get(config.GetAPIURL(target.ID))
// 	if err != nil {
// 		fmt.Println(err)
// 	}
// 	defer resp.Body.Close()
//
// 	b, err := ioutil.ReadAll(resp.Body)
// 	if err != nil {
// 		fmt.Println(err)
// 	}
// 	var qiita *config.Qiita
// 	json.Unmarshal(b, &qiita)
//
// 	writeHistory(qiita, argument)
//
// 	if lynx {
// 		openFile(qiita.HTML, "/tmp/sreq.html", "lynx", "-display_charset=utf-8", "-assume_charset=utf-8")
// 		return true
// 	}
//
// 	openFile(qiita.Markdown, "/tmp/sreq.txt", "less")
// 	return true
// }
//
// func openFile(body string, file string, cmdName ...string) {
// 	text := []byte(body)
// 	ioutil.WriteFile(file, text, os.ModePerm)
// 	cmdName = append(cmdName, file)
// 	cmd := exec.Command(cmdName[0], cmdName[1:]...)
// 	cmd.Stdin = os.Stdin
// 	cmd.Stdout = os.Stdout
// 	cmd.Run()
// }
//
// func writeHistory(content *config.Qiita, argument string) {
// 	var snippets history.Snippets
// 	snippets.Load()
// 	url := content.URL
// 	newSnippet := history.Snippet{
// 		SearchKeyword: argument,
// 		URL:           url,
// 		Title:         content.Title,
// 	}
// 	snippets.Snippets = append(snippets.Snippets, newSnippet)
// 	if err := snippets.Save(); err != nil {
// 		fmt.Printf("Failed. %v", err)
// 		os.Exit(2)
// 	}
// }
<|endoftext|>


<commit_before>package auctioneer_runner

import (
	"os/exec"
	"strings"
	"time"

	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

type AuctioneerRunner struct {
	auctioneerBin string
	etcdCluster   []string
	natsCluster   []string
	Session       *gexec.Session
}

func New(auctioneerBin string, etcdCluster, natsCluster []string) *AuctioneerRunner {
	return &AuctioneerRunner{
		auctioneerBin: auctioneerBin,
		etcdCluster:   etcdCluster,
		natsCluster:   natsCluster,
	}
}

func (r *AuctioneerRunner) Start() {
	r.StartWithoutCheck()
	Eventually(r.Session, 5*time.Second).Should(gbytes.Say("auctioneer.started"))
}

func (r *AuctioneerRunner) StartWithoutCheck() {
	executorSession, err := gexec.Start(
		exec.Command(
			r.auctioneerBin,
			"-etcdCluster", strings.Join(r.etcdCluster, ","),
			"-natsAddresses", strings.Join(r.natsCluster, ","),
		),
		ginkgo.GinkgoWriter,
		ginkgo.GinkgoWriter,
	)
	Ω(err).ShouldNot(HaveOccurred())
	r.Session = executorSession
}

func (r *AuctioneerRunner) Stop() {
	if r.Session != nil {
		r.Session.Terminate().Wait(5 * time.Second)
	}
}

func (r *AuctioneerRunner) KillWithFire() {
	if r.Session != nil {
		r.Session.Kill().Wait(5 * time.Second)
	}
}
<commit_msg>Add coloured prefix to runner output<commit_after>package auctioneer_runner

import (
	"os/exec"
	"strings"
	"time"

	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

type AuctioneerRunner struct {
	auctioneerBin string
	etcdCluster   []string
	natsCluster   []string
	Session       *gexec.Session
}

func New(auctioneerBin string, etcdCluster, natsCluster []string) *AuctioneerRunner {
	return &AuctioneerRunner{
		auctioneerBin: auctioneerBin,
		etcdCluster:   etcdCluster,
		natsCluster:   natsCluster,
	}
}

func (r *AuctioneerRunner) Start() {
	r.StartWithoutCheck()
	Eventually(r.Session, 5*time.Second).Should(gbytes.Say("auctioneer.started"))
}

func (r *AuctioneerRunner) StartWithoutCheck() {
	executorSession, err := gexec.Start(
		exec.Command(
			r.auctioneerBin,
			"-etcdCluster", strings.Join(r.etcdCluster, ","),
			"-natsAddresses", strings.Join(r.natsCluster, ","),
		),
		gexec.NewPrefixedWriter("\x1b[32m[o]\x1b[93m[auctioneer]\x1b[0m ", ginkgo.GinkgoWriter),
		gexec.NewPrefixedWriter("\x1b[91m[e]\x1b[93m[auctioneer]\x1b[0m ", ginkgo.GinkgoWriter),
	)
	Ω(err).ShouldNot(HaveOccurred())
	r.Session = executorSession
}

func (r *AuctioneerRunner) Stop() {
	if r.Session != nil {
		r.Session.Terminate().Wait(5 * time.Second)
	}
}

func (r *AuctioneerRunner) KillWithFire() {
	if r.Session != nil {
		r.Session.Kill().Wait(5 * time.Second)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fi

import (
	"context"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"k8s.io/klog/v2"
	"k8s.io/kops/util/pkg/hashing"
)

func DownloadURL(url string, dest string, hash *hashing.Hash) (*hashing.Hash, error) {
	if hash != nil {
		match, err := fileHasHash(dest, hash)
		if err != nil {
			return nil, err
		}
		if match {
			return hash, nil
		}
	}

	dirMode := os.FileMode(0755)
	err := downloadURLAlways(url, dest, dirMode)
	if err != nil {
		return nil, err
	}

	if hash != nil {
		match, err := fileHasHash(dest, hash)
		if err != nil {
			return nil, err
		}
		if !match {
			return nil, fmt.Errorf("downloaded from %q but hash did not match expected %q", url, hash)
		}
	} else {
		hash, err = hashing.HashAlgorithmSHA256.HashFile(dest)
		if err != nil {
			return nil, err
		}
	}

	return hash, nil
}

func downloadURLAlways(url string, destPath string, dirMode os.FileMode) error {
	err := os.MkdirAll(path.Dir(destPath), dirMode)
	if err != nil {
		return fmt.Errorf("error creating directories for destination file %q: %v", destPath, err)
	}

	output, err := os.Create(destPath)
	if err != nil {
		return fmt.Errorf("error creating file for download %q: %v", destPath, err)
	}
	defer output.Close()

	klog.Infof("Downloading %q", url)

	// Create a client with custom timeouts
	// to avoid idle downloads to hang the program
	httpClient := &http.Client{
		Transport: &http.Transport{
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).DialContext,
			TLSHandshakeTimeout:   10 * time.Second,
			ResponseHeaderTimeout: 10 * time.Second,
			IdleConnTimeout:       30 * time.Second,
		},
	}

	// this will stop slow downloads after 3 minutes
	// and interrupt reading of the Response.Body
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return fmt.Errorf("Cannot create request: %v", err)
	}

	response, err := httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("error doing HTTP fetch of %q: %v", url, err)
	}
	defer response.Body.Close()

	if response.StatusCode >= 400 {
		return fmt.Errorf("error response from %q: HTTP %v", url, response.StatusCode)
	}

	start := time.Now()
	defer klog.Infof("Copying %q to %q took %q seconds", url, destPath, time.Since(start))

	_, err = io.Copy(output, response.Body)
	if err != nil {
		return fmt.Errorf("error downloading HTTP content from %q: %v", url, err)
	}
	return nil
}
<commit_msg>leverage proxy env variables<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fi

import (
	"context"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"k8s.io/klog/v2"
	"k8s.io/kops/util/pkg/hashing"
)

func DownloadURL(url string, dest string, hash *hashing.Hash) (*hashing.Hash, error) {
	if hash != nil {
		match, err := fileHasHash(dest, hash)
		if err != nil {
			return nil, err
		}
		if match {
			return hash, nil
		}
	}

	dirMode := os.FileMode(0755)
	err := downloadURLAlways(url, dest, dirMode)
	if err != nil {
		return nil, err
	}

	if hash != nil {
		match, err := fileHasHash(dest, hash)
		if err != nil {
			return nil, err
		}
		if !match {
			return nil, fmt.Errorf("downloaded from %q but hash did not match expected %q", url, hash)
		}
	} else {
		hash, err = hashing.HashAlgorithmSHA256.HashFile(dest)
		if err != nil {
			return nil, err
		}
	}

	return hash, nil
}

func downloadURLAlways(url string, destPath string, dirMode os.FileMode) error {
	err := os.MkdirAll(path.Dir(destPath), dirMode)
	if err != nil {
		return fmt.Errorf("error creating directories for destination file %q: %v", destPath, err)
	}

	output, err := os.Create(destPath)
	if err != nil {
		return fmt.Errorf("error creating file for download %q: %v", destPath, err)
	}
	defer output.Close()

	klog.Infof("Downloading %q", url)

	// Create a client with custom timeouts
	// to avoid idle downloads to hang the program
	httpClient := &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).DialContext,
			TLSHandshakeTimeout:   10 * time.Second,
			ResponseHeaderTimeout: 10 * time.Second,
			IdleConnTimeout:       30 * time.Second,
		},
	}

	// this will stop slow downloads after 3 minutes
	// and interrupt reading of the Response.Body
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return fmt.Errorf("Cannot create request: %v", err)
	}

	response, err := httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("error doing HTTP fetch of %q: %v", url, err)
	}
	defer response.Body.Close()

	if response.StatusCode >= 400 {
		return fmt.Errorf("error response from %q: HTTP %v", url, response.StatusCode)
	}

	start := time.Now()
	defer klog.Infof("Copying %q to %q took %q seconds", url, destPath, time.Since(start))

	_, err = io.Copy(output, response.Body)
	if err != nil {
		return fmt.Errorf("error downloading HTTP content from %q: %v", url, err)
	}
	return nil
}
<|endoftext|>