<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright 2016-2021 Authors of Cilium

// This module implements Cilium's network device detection.

package cmd

import (
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/cilium/cilium/pkg/datapath/linux/probes"
	"github.com/cilium/cilium/pkg/k8s"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/mac"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
	"github.com/vishvananda/netlink"
	"golang.org/x/sys/unix"
)

var (
	excludedDevicePrefixes = []string{
		"cilium_",
		"lo",
		"lxc",
		"cni",
		"docker",
	}

	// Route filter to look at all routing tables.
	routeFilter = netlink.Route{
		Table: unix.RT_TABLE_UNSPEC,
	}
	routeFilterMask = netlink.RT_FILTER_TABLE
)

type DeviceManager struct {
	lock.Mutex
	devices map[string]struct{}
}

func NewDeviceManager() *DeviceManager {
	return &DeviceManager{
		devices: make(map[string]struct{}),
	}
}

// Detect tries to detect devices to which BPF programs may be loaded.
// See areDevicesRequired() for features that require the device information.
//
// The devices are detected by looking at all the configured global unicast
// routes in the system.
func (dm *DeviceManager) Detect() error {
	dm.Lock()
	defer dm.Unlock()
	dm.devices = make(map[string]struct{})

	if err := expandDevices(); err != nil {
		return err
	}

	l3DevOK := true
	if !option.Config.EnableHostLegacyRouting {
		// Probe whether fast redirect is supported for L3 devices. This will
		// invoke bpftool and requires root privileges, so we're only probing
		// when necessary.
		l3DevOK = supportL3Dev()
	}

	if len(option.Config.Devices) == 0 && areDevicesRequired() {
		// Detect the devices from the system routing table by finding the devices
		// which have global unicast routes.
		family := netlink.FAMILY_ALL
		if option.Config.EnableIPv4 && !option.Config.EnableIPv6 {
			family = netlink.FAMILY_V4
		} else if !option.Config.EnableIPv4 && option.Config.EnableIPv6 {
			family = netlink.FAMILY_V6
		}

		routes, err := netlink.RouteListFiltered(family, &routeFilter, routeFilterMask)
		if err != nil {
			return fmt.Errorf("cannot retrieve routes for device detection: %w", err)
		}
		dm.updateDevicesFromRoutes(l3DevOK, routes)
	} else {
		for _, dev := range option.Config.Devices {
			dm.devices[dev] = struct{}{}
		}
	}

	detectDirectRoutingDev := option.Config.EnableNodePort
	if option.Config.DirectRoutingDevice != "" {
		dm.devices[option.Config.DirectRoutingDevice] = struct{}{}
		detectDirectRoutingDev = false
	}

	detectIPv6MCastDev := option.Config.EnableIPv6NDP
	if option.Config.IPv6MCastDevice != "" {
		dm.devices[option.Config.IPv6MCastDevice] = struct{}{}
		detectIPv6MCastDev = false
	}

	if detectDirectRoutingDev || detectIPv6MCastDev {
		k8sNodeDev := ""
		k8sNodeLink, err := findK8SNodeIPLink()
		if err == nil {
			k8sNodeDev = k8sNodeLink.Attrs().Name
			dm.devices[k8sNodeDev] = struct{}{}
		} else if k8s.IsEnabled() {
			return fmt.Errorf("k8s is enabled, but still failed to find node IP: %w", err)
		}

		if detectDirectRoutingDev {
			// If only one device found, use that one. Otherwise use the device with k8s node IP.
			if len(dm.devices) == 1 {
				for dev := range dm.devices {
					option.Config.DirectRoutingDevice = dev
					break
				}
			} else if k8sNodeDev != "" {
				option.Config.DirectRoutingDevice = k8sNodeDev
			} else {
				return fmt.Errorf("Unable to determine direct routing device. Use --%s to specify it",
					option.DirectRoutingDevice)
			}
			log.WithField(option.DirectRoutingDevice, option.Config.DirectRoutingDevice).
				Info("Direct routing device detected")
		}

		if detectIPv6MCastDev {
			if k8sNodeLink != nil && k8sNodeLink.Attrs().Flags&net.FlagMulticast != 0 {
				option.Config.IPv6MCastDevice = k8sNodeDev
				log.WithField(option.IPv6MCastDevice, option.Config.IPv6MCastDevice).Info("IPv6 multicast device detected")
			} else {
				return fmt.Errorf("Unable to determine Multicast device. Use --%s to specify it",
					option.IPv6MCastDevice)
			}
		}
	}

	option.Config.Devices = dm.getDevices()
	log.WithField(logfields.Devices, option.Config.Devices).Info("Detected devices")

	return nil
}

// GetDevices returns the current list of devices Cilium should attach programs to.
func (dm *DeviceManager) GetDevices() []string {
	dm.Lock()
	defer dm.Unlock()
	return dm.getDevices()
}

func (dm *DeviceManager) getDevices() []string {
	devs := make([]string, 0, len(dm.devices))
	for dev := range dm.devices {
		devs = append(devs, dev)
	}
	sort.Strings(devs)
	return devs
}

// isViableDevice returns true if the given link is usable and Cilium should attach
// programs to it.
func (dm *DeviceManager) isViableDevice(l3DevOK, hasDefaultRoute bool, link netlink.Link) bool {
	name := link.Attrs().Name

	// Do not consider any of the excluded devices.
	for _, p := range excludedDevicePrefixes {
		if strings.HasPrefix(name, p) {
			log.WithField(logfields.Device, name).
				Debugf("Skipping device as it has excluded prefix '%s'", p)
			return false
		}
	}

	// Skip lower bond devices.
	if link.Attrs().RawFlags&unix.IFF_SLAVE != 0 {
		log.WithField(logfields.Device, name).Debug("Skipping bonded device")
		return false
	}

	// Ignore L3 devices if we cannot support them.
	if !l3DevOK && !mac.LinkHasMacAddr(link) {
		log.WithField(logfields.Device, name).
			Info("Ignoring L3 device; >= 5.8 kernel is required.")
		return false
	}

	// Skip veth devices that don't have a default route.
	// This is a workaround for kubernetes-in-docker. We want to avoid
	// veth devices in general as they may be leftovers from another CNI.
	if !hasDefaultRoute {
		_, virtual := link.(*netlink.Veth)
		if virtual {
			log.WithField(logfields.Device, name).
				Debug("Ignoring veth device as it has no default route")
			return false
		}

	}
	return true
}

type linkInfo struct {
	hasDefaultRoute bool
}

// updateDevicesFromRoutes processes a batch of routes and updates the set of
// devices. Returns true if devices changed.
func (dm *DeviceManager) updateDevicesFromRoutes(l3DevOK bool, routes []netlink.Route) bool {
	linkInfos := make(map[int]linkInfo)

	// Collect all link indices mentioned in the route update batch
	for _, route := range routes {
		// Only consider devices that have global unicast routes,
		// e.g. skip loopback, multicast and link local routes.
		if route.Dst != nil && !route.Dst.IP.IsGlobalUnicast() {
			continue
		}
		if route.Table == unix.RT_TABLE_LOCAL {
			continue
		}
		linkInfo := linkInfos[route.LinkIndex]
		linkInfo.hasDefaultRoute = linkInfo.hasDefaultRoute || route.Dst == nil
		linkInfos[route.LinkIndex] = linkInfo
	}

	changed := false
	for index, info := range linkInfos {
		link, err := netlink.LinkByIndex(index)
		if err != nil {
			log.WithError(err).WithField(logfields.LinkIndex, index).
				Warn("Failed to get link by index")
			continue
		}
		name := link.Attrs().Name

		// Skip devices we already know.
		if _, exists := dm.devices[name]; exists {
			continue
		}

		viable := dm.isViableDevice(l3DevOK, info.hasDefaultRoute, link)
		if viable {
			dm.devices[name] = struct{}{}
			changed = true
		}
	}
	return changed
}

// expandDevices expands all wildcard device names to concrete devices.
// e.g. device "eth+" expands to "eth0,eth1" etc. Non-matching wildcards are ignored.
func expandDevices() error {
	allLinks, err := netlink.LinkList()
	if err != nil {
		return fmt.Errorf("Device wildcard expansion failed to fetch devices: %w", err)
	}
	expandedDevices := make(map[string]struct{})
	for _, iface := range option.Config.Devices {
		if strings.HasSuffix(iface, "+") {
			prefix := strings.TrimRight(iface, "+")
			for _, link := range allLinks {
				attrs := link.Attrs()
				if strings.HasPrefix(attrs.Name, prefix) {
					expandedDevices[attrs.Name] = struct{}{}
				}
			}
		} else {
			expandedDevices[iface] = struct{}{}
		}
	}
	if len(option.Config.Devices) > 0 && len(expandedDevices) == 0 {
		// User defined devices, but expansion yielded no devices. Fail here to not
		// surprise with auto-detection.
		return fmt.Errorf("Device wildcard expansion failed to detect devices. Please verify --%s option.",
			option.Devices)
	}

	option.Config.Devices = make([]string, 0, len(expandedDevices))
	for dev := range expandedDevices {
		option.Config.Devices = append(option.Config.Devices, dev)
	}
	sort.Strings(option.Config.Devices)
	return nil
}

func areDevicesRequired() bool {
	return option.Config.EnableNodePort ||
		option.Config.EnableHostFirewall ||
		option.Config.EnableBandwidthManager ||
		option.Config.EnableIPSec
}

func findK8SNodeIPLink() (netlink.Link, error) {
	nodeIP := node.GetK8sNodeIP()

	if nodeIP == nil {
		return nil, fmt.Errorf("Failed to find K8s node device as node IP is not known")
	}

	var family int
	if nodeIP.To4() != nil {
		family = netlink.FAMILY_V4
	} else {
		family = netlink.FAMILY_V6
	}

	if addrs, err := netlink.AddrList(nil, family); err == nil {
		for _, a := range addrs {
			if a.IP.Equal(nodeIP) {
				link, err := netlink.LinkByIndex(a.LinkIndex)
				if err != nil {
					return nil, err
				}
				return link, nil
			}
		}
	}
	return nil, fmt.Errorf("K8s node device not found")
}

// supportL3Dev returns true if the kernel is new enough to support fast redirection of
// packets coming from L3 devices using bpf_skb_redirect_peer.
func supportL3Dev() bool {
	probesManager := probes.NewProbeManager()
	if h := probesManager.GetHelpers("sched_cls"); h != nil {
		_, found := h["bpf_skb_change_head"]
		return found
	}
	return false
}
<commit_msg>daemon: Do not detect devices for IPsec<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright 2016-2021 Authors of Cilium

// This module implements Cilium's network device detection.

package cmd

import (
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/cilium/cilium/pkg/datapath/linux/probes"
	"github.com/cilium/cilium/pkg/k8s"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/mac"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
	"github.com/vishvananda/netlink"
	"golang.org/x/sys/unix"
)

var (
	excludedDevicePrefixes = []string{
		"cilium_",
		"lo",
		"lxc",
		"cni",
		"docker",
	}

	// Route filter to look at all routing tables.
	routeFilter = netlink.Route{
		Table: unix.RT_TABLE_UNSPEC,
	}
	routeFilterMask = netlink.RT_FILTER_TABLE
)

type DeviceManager struct {
	lock.Mutex
	devices map[string]struct{}
}

func NewDeviceManager() *DeviceManager {
	return &DeviceManager{
		devices: make(map[string]struct{}),
	}
}

// Detect tries to detect devices to which BPF programs may be loaded.
// See areDevicesRequired() for features that require the device information.
//
// The devices are detected by looking at all the configured global unicast
// routes in the system.
func (dm *DeviceManager) Detect() error {
	dm.Lock()
	defer dm.Unlock()
	dm.devices = make(map[string]struct{})

	if err := expandDevices(); err != nil {
		return err
	}

	l3DevOK := true
	if !option.Config.EnableHostLegacyRouting {
		// Probe whether fast redirect is supported for L3 devices. This will
		// invoke bpftool and requires root privileges, so we're only probing
		// when necessary.
		l3DevOK = supportL3Dev()
	}

	if len(option.Config.Devices) == 0 && areDevicesRequired() {
		// Detect the devices from the system routing table by finding the devices
		// which have global unicast routes.
		family := netlink.FAMILY_ALL
		if option.Config.EnableIPv4 && !option.Config.EnableIPv6 {
			family = netlink.FAMILY_V4
		} else if !option.Config.EnableIPv4 && option.Config.EnableIPv6 {
			family = netlink.FAMILY_V6
		}

		routes, err := netlink.RouteListFiltered(family, &routeFilter, routeFilterMask)
		if err != nil {
			return fmt.Errorf("cannot retrieve routes for device detection: %w", err)
		}
		dm.updateDevicesFromRoutes(l3DevOK, routes)
	} else {
		for _, dev := range option.Config.Devices {
			dm.devices[dev] = struct{}{}
		}
	}

	detectDirectRoutingDev := option.Config.EnableNodePort
	if option.Config.DirectRoutingDevice != "" {
		dm.devices[option.Config.DirectRoutingDevice] = struct{}{}
		detectDirectRoutingDev = false
	}

	detectIPv6MCastDev := option.Config.EnableIPv6NDP
	if option.Config.IPv6MCastDevice != "" {
		dm.devices[option.Config.IPv6MCastDevice] = struct{}{}
		detectIPv6MCastDev = false
	}

	if detectDirectRoutingDev || detectIPv6MCastDev {
		k8sNodeDev := ""
		k8sNodeLink, err := findK8SNodeIPLink()
		if err == nil {
			k8sNodeDev = k8sNodeLink.Attrs().Name
			dm.devices[k8sNodeDev] = struct{}{}
		} else if k8s.IsEnabled() {
			return fmt.Errorf("k8s is enabled, but still failed to find node IP: %w", err)
		}

		if detectDirectRoutingDev {
			// If only one device found, use that one. Otherwise use the device with k8s node IP.
			if len(dm.devices) == 1 {
				for dev := range dm.devices {
					option.Config.DirectRoutingDevice = dev
					break
				}
			} else if k8sNodeDev != "" {
				option.Config.DirectRoutingDevice = k8sNodeDev
			} else {
				return fmt.Errorf("Unable to determine direct routing device. Use --%s to specify it",
					option.DirectRoutingDevice)
			}
			log.WithField(option.DirectRoutingDevice, option.Config.DirectRoutingDevice).
				Info("Direct routing device detected")
		}

		if detectIPv6MCastDev {
			if k8sNodeLink != nil && k8sNodeLink.Attrs().Flags&net.FlagMulticast != 0 {
				option.Config.IPv6MCastDevice = k8sNodeDev
				log.WithField(option.IPv6MCastDevice, option.Config.IPv6MCastDevice).Info("IPv6 multicast device detected")
			} else {
				return fmt.Errorf("Unable to determine Multicast device. Use --%s to specify it",
					option.IPv6MCastDevice)
			}
		}
	}

	option.Config.Devices = dm.getDevices()
	log.WithField(logfields.Devices, option.Config.Devices).Info("Detected devices")

	return nil
}

// GetDevices returns the current list of devices Cilium should attach programs to.
func (dm *DeviceManager) GetDevices() []string {
	dm.Lock()
	defer dm.Unlock()
	return dm.getDevices()
}

func (dm *DeviceManager) getDevices() []string {
	devs := make([]string, 0, len(dm.devices))
	for dev := range dm.devices {
		devs = append(devs, dev)
	}
	sort.Strings(devs)
	return devs
}

// isViableDevice returns true if the given link is usable and Cilium should attach
// programs to it.
func (dm *DeviceManager) isViableDevice(l3DevOK, hasDefaultRoute bool, link netlink.Link) bool {
	name := link.Attrs().Name

	// Do not consider any of the excluded devices.
	for _, p := range excludedDevicePrefixes {
		if strings.HasPrefix(name, p) {
			log.WithField(logfields.Device, name).
				Debugf("Skipping device as it has excluded prefix '%s'", p)
			return false
		}
	}

	// Skip lower bond devices.
	if link.Attrs().RawFlags&unix.IFF_SLAVE != 0 {
		log.WithField(logfields.Device, name).Debug("Skipping bonded device")
		return false
	}

	// Ignore L3 devices if we cannot support them.
	if !l3DevOK && !mac.LinkHasMacAddr(link) {
		log.WithField(logfields.Device, name).
			Info("Ignoring L3 device; >= 5.8 kernel is required.")
		return false
	}

	// Skip veth devices that don't have a default route.
	// This is a workaround for kubernetes-in-docker. We want to avoid
	// veth devices in general as they may be leftovers from another CNI.
	if !hasDefaultRoute {
		_, virtual := link.(*netlink.Veth)
		if virtual {
			log.WithField(logfields.Device, name).
				Debug("Ignoring veth device as it has no default route")
			return false
		}

	}
	return true
}

type linkInfo struct {
	hasDefaultRoute bool
}

// updateDevicesFromRoutes processes a batch of routes and updates the set of
// devices. Returns true if devices changed.
func (dm *DeviceManager) updateDevicesFromRoutes(l3DevOK bool, routes []netlink.Route) bool {
	linkInfos := make(map[int]linkInfo)

	// Collect all link indices mentioned in the route update batch
	for _, route := range routes {
		// Only consider devices that have global unicast routes,
		// e.g. skip loopback, multicast and link local routes.
		if route.Dst != nil && !route.Dst.IP.IsGlobalUnicast() {
			continue
		}
		if route.Table == unix.RT_TABLE_LOCAL {
			continue
		}
		linkInfo := linkInfos[route.LinkIndex]
		linkInfo.hasDefaultRoute = linkInfo.hasDefaultRoute || route.Dst == nil
		linkInfos[route.LinkIndex] = linkInfo
	}

	changed := false
	for index, info := range linkInfos {
		link, err := netlink.LinkByIndex(index)
		if err != nil {
			log.WithError(err).WithField(logfields.LinkIndex, index).
				Warn("Failed to get link by index")
			continue
		}
		name := link.Attrs().Name

		// Skip devices we already know.
		if _, exists := dm.devices[name]; exists {
			continue
		}

		viable := dm.isViableDevice(l3DevOK, info.hasDefaultRoute, link)
		if viable {
			dm.devices[name] = struct{}{}
			changed = true
		}
	}
	return changed
}

// expandDevices expands all wildcard device names to concrete devices.
// e.g. device "eth+" expands to "eth0,eth1" etc. Non-matching wildcards are ignored.
func expandDevices() error {
	allLinks, err := netlink.LinkList()
	if err != nil {
		return fmt.Errorf("Device wildcard expansion failed to fetch devices: %w", err)
	}
	expandedDevices := make(map[string]struct{})
	for _, iface := range option.Config.Devices {
		if strings.HasSuffix(iface, "+") {
			prefix := strings.TrimRight(iface, "+")
			for _, link := range allLinks {
				attrs := link.Attrs()
				if strings.HasPrefix(attrs.Name, prefix) {
					expandedDevices[attrs.Name] = struct{}{}
				}
			}
		} else {
			expandedDevices[iface] = struct{}{}
		}
	}
	if len(option.Config.Devices) > 0 && len(expandedDevices) == 0 {
		// User defined devices, but expansion yielded no devices. Fail here to not
		// surprise with auto-detection.
		return fmt.Errorf("Device wildcard expansion failed to detect devices. Please verify --%s option.",
			option.Devices)
	}

	option.Config.Devices = make([]string, 0, len(expandedDevices))
	for dev := range expandedDevices {
		option.Config.Devices = append(option.Config.Devices, dev)
	}
	sort.Strings(option.Config.Devices)
	return nil
}

func areDevicesRequired() bool {
	return option.Config.EnableNodePort ||
		option.Config.EnableHostFirewall ||
		option.Config.EnableBandwidthManager
}

func findK8SNodeIPLink() (netlink.Link, error) {
	nodeIP := node.GetK8sNodeIP()

	if nodeIP == nil {
		return nil, fmt.Errorf("Failed to find K8s node device as node IP is not known")
	}

	var family int
	if nodeIP.To4() != nil {
		family = netlink.FAMILY_V4
	} else {
		family = netlink.FAMILY_V6
	}

	if addrs, err := netlink.AddrList(nil, family); err == nil {
		for _, a := range addrs {
			if a.IP.Equal(nodeIP) {
				link, err := netlink.LinkByIndex(a.LinkIndex)
				if err != nil {
					return nil, err
				}
				return link, nil
			}
		}
	}
	return nil, fmt.Errorf("K8s node device not found")
}

// supportL3Dev returns true if the kernel is new enough to support fast redirection of
// packets coming from L3 devices using bpf_skb_redirect_peer.
func supportL3Dev() bool {
	probesManager := probes.NewProbeManager()
	if h := probesManager.GetHelpers("sched_cls"); h != nil {
		_, found := h["bpf_skb_change_head"]
		return found
	}
	return false
}
<|endoftext|>


<commit_before>// Package os runs processes locally
package os

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"

	"github.com/micro/go-micro/v3/runtime/local/process"
)

func (p *Process) Exec(exe *process.Executable) error {
	cmd := exec.Command(exe.Package.Path)
	return cmd.Run()
}

func (p *Process) Fork(exe *process.Executable) (*process.PID, error) {
	// create command
	cmd := exec.Command(exe.Package.Path, exe.Args...)
	// set env vars
	cmd.Env = append(cmd.Env, exe.Env...)

	in, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	out, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	er, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	// start the process
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &process.PID{
		ID:     fmt.Sprintf("%d", cmd.Process.Pid),
		Input:  in,
		Output: out,
		Error:  er,
	}, nil
}

func (p *Process) Kill(pid *process.PID) error {
	id, err := strconv.Atoi(pid.ID)
	if err != nil {
		return err
	}

	pr, err := os.FindProcess(id)
	if err != nil {
		return err
	}

	// now kill it
	err = pr.Kill()

	// return the kill error
	return err
}

func (p *Process) Wait(pid *process.PID) error {
	id, err := strconv.Atoi(pid.ID)
	if err != nil {
		return err
	}

	pr, err := os.FindProcess(id)
	if err != nil {
		return err
	}

	ps, err := pr.Wait()
	if err != nil {
		return err
	}

	if ps.Success() {
		return nil
	}

	return fmt.Errorf(ps.String())
}
<commit_msg>executable is now os<commit_after>// Package os runs processes locally
package os

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"

	"github.com/micro/go-micro/v3/runtime/local/process"
)

func (p *Process) Exec(exe *process.Exec) error {
	cmd := exec.Command(exe.Package.Path)
	return cmd.Run()
}

func (p *Process) Fork(exe *process.Exec) (*process.PID, error) {
	// create command
	cmd := exec.Command(exe.Package.Path, exe.Args...)
	// set env vars
	cmd.Env = append(cmd.Env, exe.Env...)

	in, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	out, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	er, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	// start the process
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &process.PID{
		ID:     fmt.Sprintf("%d", cmd.Process.Pid),
		Input:  in,
		Output: out,
		Error:  er,
	}, nil
}

func (p *Process) Kill(pid *process.PID) error {
	id, err := strconv.Atoi(pid.ID)
	if err != nil {
		return err
	}

	pr, err := os.FindProcess(id)
	if err != nil {
		return err
	}

	// now kill it
	err = pr.Kill()

	// return the kill error
	return err
}

func (p *Process) Wait(pid *process.PID) error {
	id, err := strconv.Atoi(pid.ID)
	if err != nil {
		return err
	}

	pr, err := os.FindProcess(id)
	if err != nil {
		return err
	}

	ps, err := pr.Wait()
	if err != nil {
		return err
	}

	if ps.Success() {
		return nil
	}

	return fmt.Errorf(ps.String())
}
<|endoftext|>


<commit_before>package planner

import "strings"

var (
	RuntimeK3S  = "k3s"
	RuntimeRKE2 = "rke2"
)

func GetRuntimeCommand(kubernetesVersion string) string {
	return strings.ToLower(GetRuntime(kubernetesVersion))
}

func GetRuntimeServerUnit(kubernetesVersion string) string {
	if GetRuntime(kubernetesVersion) == RuntimeK3S {
		return RuntimeK3S
	}
	return RuntimeRKE2 + "-server"
}

func GetRuntimeEnv(kubernetesVersion string) string {
	return strings.ToUpper(GetRuntimeEnv(kubernetesVersion))
}

func GetRuntime(kubernetesVersion string) string {
	if strings.Contains(kubernetesVersion, RuntimeK3S) {
		return RuntimeK3S
	}
	return RuntimeRKE2
}
<commit_msg>Fix stack overflow<commit_after>package planner

import "strings"

var (
	RuntimeK3S  = "k3s"
	RuntimeRKE2 = "rke2"
)

func GetRuntimeCommand(kubernetesVersion string) string {
	return strings.ToLower(GetRuntime(kubernetesVersion))
}

func GetRuntimeServerUnit(kubernetesVersion string) string {
	if GetRuntime(kubernetesVersion) == RuntimeK3S {
		return RuntimeK3S
	}
	return RuntimeRKE2 + "-server"
}

func GetRuntimeEnv(kubernetesVersion string) string {
	return strings.ToUpper(GetRuntime(kubernetesVersion))
}

func GetRuntime(kubernetesVersion string) string {
	if strings.Contains(kubernetesVersion, RuntimeK3S) {
		return RuntimeK3S
	}
	return RuntimeRKE2
}
<|endoftext|>


<commit_before>// package tcpreaderwrapper wraps a gopacket tcpassembly.tcpreader.ReaderStream
package tcpreaderwrapper

import (
	"code.google.com/p/gopacket/tcpassembly"
	"code.google.com/p/gopacket/tcpassembly/tcpreader"
)

type ReaderStreamWrapper struct {
	tcpreader.ReaderStream
	Reassemblies []ReassemblyInfo
}

// NewReaderStream returns a new ReaderStreamWrapper object.
func NewReaderStreamWrapper() ReaderStreamWrapper {
	r := ReaderStreamWrapper{
		ReaderStream: tcpreader.NewReaderStream(),
		Reassemblies: make([]ReassemblyInfo, 0),
	}
	r.ReaderStream.ReaderStreamOptions.LossErrors = true
	return r
}

// Reassembled implements tcpassembly.Stream's Reassembled function.
func (r *ReaderStreamWrapper) Reassembled(reassembly []tcpassembly.Reassembly) {
	// keep track of sizes and times to reconstruct
	for _, re := range reassembly {
		r.Reassemblies = append(r.Reassemblies, newReassemblyInfo(re))
	}
	r.ReaderStream.Reassembled(reassembly)
}
<commit_msg>Disable LossErrors<commit_after>// package tcpreaderwrapper wraps a gopacket tcpassembly.tcpreader.ReaderStream
// and holds recent resassemblies to fetch timing information
package tcpreaderwrapper

import (
	"code.google.com/p/gopacket/tcpassembly"
	"code.google.com/p/gopacket/tcpassembly/tcpreader"
)

type ReaderStreamWrapper struct {
	tcpreader.ReaderStream
	Reassemblies []ReassemblyInfo
}

// NewReaderStream returns a new ReaderStreamWrapper object.
func NewReaderStreamWrapper() ReaderStreamWrapper {
	r := ReaderStreamWrapper{
		ReaderStream: tcpreader.NewReaderStream(),
		Reassemblies: make([]ReassemblyInfo, 0),
	}
	//	r.ReaderStream.ReaderStreamOptions.LossErrors = true
	return r
}

// Reassembled implements tcpassembly.Stream's Reassembled function.
func (r *ReaderStreamWrapper) Reassembled(reassembly []tcpassembly.Reassembly) {
	// keep track of sizes and times to reconstruct
	for _, re := range reassembly {
		r.Reassemblies = append(r.Reassemblies, newReassemblyInfo(re))
	}
	r.ReaderStream.Reassembled(reassembly)
}
<|endoftext|>


<commit_before>// Package cloudflare is a dns Provider for cloudflare
package cloudflare

import (
	"context"
	"strings"
	"time"

	"github.com/cloudflare/cloudflare-go"
	miekdns "github.com/miekg/dns"
	"github.com/pkg/errors"

	"github.com/micro/go-micro/util/log"
	dns "github.com/micro/micro/network/dns/proto/dns"
	"github.com/micro/micro/network/dns/provider"
)

type cfProvider struct {
	api    *cloudflare.API
	zoneID string
}

// New returns a configured cloudflare DNS provider
func New(apiToken, zoneID string) (provider.Provider, error) {
	api, err := cloudflare.NewWithAPIToken(apiToken)
	if err != nil {
		return nil, err
	}

	return &cfProvider{
		api:    api,
		zoneID: zoneID,
	}, nil
}

func (cf *cfProvider) Advertise(records ...*dns.Record) error {
	for _, r := range records {
		_, err := cf.api.CreateDNSRecord(cf.zoneID, cloudflare.DNSRecord{
			Name:     r.GetName(),
			Content:  r.GetValue(),
			Type:     r.GetType(),
			Priority: int(r.GetPriority()),
			TTL:      1,
		})
		if err != nil {
			return err
		}

	}
	return nil
}

func (cf *cfProvider) Remove(records ...*dns.Record) error {
	existing := make(map[string]map[string]cloudflare.DNSRecord)
	existingRecords, err := cf.api.DNSRecords(cf.zoneID, cloudflare.DNSRecord{})
	if err != nil {
		return err
	}
	for _, e := range existingRecords {
		if _, found := existing[e.Name]; !found {
			existing[e.Name] = make(map[string]cloudflare.DNSRecord)
		}
		existing[e.Name][e.Content] = e
	}
	for _, r := range records {
		if _, found := existing[r.Name]; !found {
			return errors.New("Record " + r.Name + " could not be deleted as it doesn't exist")
		}
		toDelete, found := existing[r.Name][r.Value]
		if !found {
			return errors.New("Record " + r.Name + " with address " + r.Value + " could not be deleted as it doesn't exist")
		}
		err := cf.api.DeleteDNSRecord(cf.zoneID, toDelete.ID)
		if err != nil {
			return err
		}
	}
	return nil
}

func (cf *cfProvider) Resolve(name, recordType string) ([]*dns.Record, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	dnstype, found := miekdns.StringToType[recordType]
	if !found {
		return nil, errors.New(recordType + " is not a valid record type")
	}
	m := new(miekdns.Msg)
	m.SetQuestion(miekdns.Fqdn(name), dnstype)
	r, err := miekdns.ExchangeContext(ctx, m, "8.8.8.8:53")
	if err != nil {
		return nil, err
	}
	var response []*dns.Record
	for _, answer := range r.Answer {
		h := answer.Header()
		rec := &dns.Record{
			Name: h.Name,
			Type: miekdns.TypeToString[h.Rrtype],
			Ttl:  answer.Header().Ttl,
		}
		if rec.Type != recordType {
			log.Trace("Tried to look up a " + recordType + " record but got a " + rec.Type)
			continue
		}
		switch rec.Type {
		case "A":
			arecord, _ := answer.(*miekdns.A)
			rec.Value = arecord.A.String()
		case "AAAA":
			aaaarecord := answer.(*miekdns.AAAA)
			rec.Value = aaaarecord.AAAA.String()
		case "TXT":
			txtrecord := answer.(*miekdns.TXT)
			rec.Value = strings.Join(txtrecord.Txt, "")
		case "MX":
			mxrecord := answer.(*miekdns.MX)
			rec.Value = mxrecord.Mx
			rec.Priority = uint32(mxrecord.Preference)
		default:
			return nil, errors.New("Can't handle record type " + rec.Type)
		}
		response = append(response, rec)
	}
	return response, nil
}
<commit_msg>Cloudflare resolver should use Cloudflare DNS<commit_after>// Package cloudflare is a dns Provider for cloudflare
package cloudflare

import (
	"context"
	"strings"
	"time"

	"github.com/cloudflare/cloudflare-go"
	miekdns "github.com/miekg/dns"
	"github.com/pkg/errors"

	"github.com/micro/go-micro/util/log"
	dns "github.com/micro/micro/network/dns/proto/dns"
	"github.com/micro/micro/network/dns/provider"
)

type cfProvider struct {
	api    *cloudflare.API
	zoneID string
}

// New returns a configured cloudflare DNS provider
func New(apiToken, zoneID string) (provider.Provider, error) {
	api, err := cloudflare.NewWithAPIToken(apiToken)
	if err != nil {
		return nil, err
	}

	return &cfProvider{
		api:    api,
		zoneID: zoneID,
	}, nil
}

func (cf *cfProvider) Advertise(records ...*dns.Record) error {
	for _, r := range records {
		_, err := cf.api.CreateDNSRecord(cf.zoneID, cloudflare.DNSRecord{
			Name:     r.GetName(),
			Content:  r.GetValue(),
			Type:     r.GetType(),
			Priority: int(r.GetPriority()),
			TTL:      1,
		})
		if err != nil {
			return err
		}

	}
	return nil
}

func (cf *cfProvider) Remove(records ...*dns.Record) error {
	existing := make(map[string]map[string]cloudflare.DNSRecord)
	existingRecords, err := cf.api.DNSRecords(cf.zoneID, cloudflare.DNSRecord{})
	if err != nil {
		return err
	}
	for _, e := range existingRecords {
		if _, found := existing[e.Name]; !found {
			existing[e.Name] = make(map[string]cloudflare.DNSRecord)
		}
		existing[e.Name][e.Content] = e
	}
	for _, r := range records {
		if _, found := existing[r.Name]; !found {
			return errors.New("Record " + r.Name + " could not be deleted as it doesn't exist")
		}
		toDelete, found := existing[r.Name][r.Value]
		if !found {
			return errors.New("Record " + r.Name + " with address " + r.Value + " could not be deleted as it doesn't exist")
		}
		err := cf.api.DeleteDNSRecord(cf.zoneID, toDelete.ID)
		if err != nil {
			return err
		}
	}
	return nil
}

func (cf *cfProvider) Resolve(name, recordType string) ([]*dns.Record, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	dnstype, found := miekdns.StringToType[recordType]
	if !found {
		return nil, errors.New(recordType + " is not a valid record type")
	}
	m := new(miekdns.Msg)
	m.SetQuestion(miekdns.Fqdn(name), dnstype)
	r, err := miekdns.ExchangeContext(ctx, m, "1.0.0.1:53")
	if err != nil {
		return nil, err
	}
	var response []*dns.Record
	for _, answer := range r.Answer {
		h := answer.Header()
		rec := &dns.Record{
			Name: h.Name,
			Type: miekdns.TypeToString[h.Rrtype],
			Ttl:  answer.Header().Ttl,
		}
		if rec.Type != recordType {
			log.Trace("Tried to look up a " + recordType + " record but got a " + rec.Type)
			continue
		}
		switch rec.Type {
		case "A":
			arecord, _ := answer.(*miekdns.A)
			rec.Value = arecord.A.String()
		case "AAAA":
			aaaarecord := answer.(*miekdns.AAAA)
			rec.Value = aaaarecord.AAAA.String()
		case "TXT":
			txtrecord := answer.(*miekdns.TXT)
			rec.Value = strings.Join(txtrecord.Txt, "")
		case "MX":
			mxrecord := answer.(*miekdns.MX)
			rec.Value = mxrecord.Mx
			rec.Priority = uint32(mxrecord.Preference)
		default:
			return nil, errors.New("Can't handle record type " + rec.Type)
		}
		response = append(response, rec)
	}
	return response, nil
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"

	buildv1 "github.com/openshift/api/build/v1"
)

// PredicateFunc is testing an argument and decides does it meet some criteria or not.
type PredicateFunc func(interface{}) bool

// FilterBuilds returns array of builds that satisfies predicate function.
func FilterBuilds(builds []buildv1.Build, predicate PredicateFunc) []buildv1.Build {
	if len(builds) == 0 {
		return builds
	}

	result := make([]buildv1.Build, 0)
	for _, build := range builds {
		if predicate(build) {
			result = append(result, build)
		}
	}

	return result
}

// ByBuildConfigPredicate matches all builds that have build config annotation or label with specified value.
func ByBuildConfigPredicate(labelValue string) PredicateFunc {
	return func(arg interface{}) bool {
		return hasBuildConfigAnnotation(arg.(buildv1.Build), BuildConfigAnnotation, labelValue) ||
			hasBuildConfigLabel(arg.(buildv1.Build), BuildConfigLabel, labelValue)
	}
}

func hasBuildConfigLabel(build buildv1.Build, labelName, labelValue string) bool {
	value, ok := build.Labels[labelName]
	return ok && value == labelValue
}

func hasBuildConfigAnnotation(build buildv1.Build, annotationName, annotationValue string) bool {
	if build.Annotations == nil {
		return false
	}
	value, ok := build.Annotations[annotationName]
	return ok && value == annotationValue
}

// BuildNameForConfigVersion returns the name of the version-th build
// for the config that has the provided name.
func BuildNameForConfigVersion(name string, version int) string {
	return fmt.Sprintf("%s-%d", name, version)
}
<commit_msg>build: move StrategyType to staging<commit_after>package build

import (
	"fmt"

	buildv1 "github.com/openshift/api/build/v1"
)

// PredicateFunc is testing an argument and decides does it meet some criteria or not.
type PredicateFunc func(interface{}) bool

// FilterBuilds returns array of builds that satisfies predicate function.
func FilterBuilds(builds []buildv1.Build, predicate PredicateFunc) []buildv1.Build {
	if len(builds) == 0 {
		return builds
	}

	result := make([]buildv1.Build, 0)
	for _, build := range builds {
		if predicate(build) {
			result = append(result, build)
		}
	}

	return result
}

// ByBuildConfigPredicate matches all builds that have build config annotation or label with specified value.
func ByBuildConfigPredicate(labelValue string) PredicateFunc {
	return func(arg interface{}) bool {
		return hasBuildConfigAnnotation(arg.(buildv1.Build), BuildConfigAnnotation, labelValue) ||
			hasBuildConfigLabel(arg.(buildv1.Build), BuildConfigLabel, labelValue)
	}
}

func hasBuildConfigLabel(build buildv1.Build, labelName, labelValue string) bool {
	value, ok := build.Labels[labelName]
	return ok && value == labelValue
}

func hasBuildConfigAnnotation(build buildv1.Build, annotationName, annotationValue string) bool {
	if build.Annotations == nil {
		return false
	}
	value, ok := build.Annotations[annotationName]
	return ok && value == annotationValue
}

// BuildNameForConfigVersion returns the name of the version-th build
// for the config that has the provided name.
func BuildNameForConfigVersion(name string, version int) string {
	return fmt.Sprintf("%s-%d", name, version)
}

func StrategyType(strategy buildv1.BuildStrategy) string {
	switch {
	case strategy.DockerStrategy != nil:
		return "Docker"
	case strategy.CustomStrategy != nil:
		return "Custom"
	case strategy.SourceStrategy != nil:
		return "Source"
	case strategy.JenkinsPipelineStrategy != nil:
		return "JenkinsPipeline"
	}
	return ""
}<|endoftext|>


<commit_before>package annotations

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"regexp"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
	Write(contentUUID string, thing interface{}) (err error)
	Read(contentUUID string) (thing interface{}, found bool, err error)
	Delete(contentUUID string) (found bool, err error)
	Check() (err error)
	DecodeJSON(*json.Decoder) (thing interface{}, err error)
	Count() (int, error)
	Initialise() error
}

//holds the Neo4j-specific information
type service struct {
	cypherRunner    neoutils.CypherRunner
	indexManager    neoutils.IndexManager
	platformVersion string
}

//NewAnnotationsService instantiate driver
func NewAnnotationsService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager, platformVersion string) service {
	if platformVersion == "" {
		log.Fatalf("PlatformVersion was not specified!")
	}
	return service{cypherRunner, indexManager, platformVersion}
}

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
	a := annotations{}
	err := dec.Decode(&a)
	return a, err
}

func (s service) Read(contentUUID string) (thing interface{}, found bool, err error) {
	results := []annotation{}

	//TODO shouldn't return Provenances if none of the scores, agentRole or atTime are set
	statementTemplate := `
					MATCH (c:Thing{uuid:{contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
					WITH c, cc, rel, {id:cc.uuid,prefLabel:cc.prefLabel,types:labels(cc),predicate:type(rel)} as thing,
					collect(
						{scores:[
							{scoringSystem:'%s', value:rel.relevanceScore},
							{scoringSystem:'%s', value:rel.confidenceScore}],
						agentRole:rel.annotatedBy,
						atTime:rel.annotatedDate}) as provenances
					RETURN thing, provenances ORDER BY thing.id
									`
	statement := fmt.Sprintf(statementTemplate, relevanceScoringSystem, confidenceScoringSystem)

	query := &neoism.CypherQuery{
		Statement:  statement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion},
		Result:     &results,
	}
	err = s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v", contentUUID, query.Statement, err)
		return annotations{}, false, fmt.Errorf("Error accessing Annotations datastore for uuid: %s", contentUUID)
	}
	log.Debugf("CypherResult Read Annotations for uuid: %s was: %+v", contentUUID, results)
	if (len(results)) == 0 {
		return annotations{}, false, nil
	}

	for idx := range results {
		mapToResponseFormat(&results[idx])
	}

	return results, true, nil
}

//Delete removes all the annotations for this content. Ignore the nodes on either end -
//may leave nodes that are only 'things' inserted by this writer: clean up
//as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string) (bool, error) {

	var deleteStatement string

	if s.platformVersion == "v2" {
		deleteStatement = `MATCH (c:Thing{uuid: {contentUUID}})-[rel:MENTIONS{platformVersion:{platformVersion}}]->(cc:Thing) DELETE rel`
	} else {
		deleteStatement = `MATCH (c:Thing{uuid: {contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing) DELETE rel`
	}

	query := &neoism.CypherQuery{
		Statement:    deleteStatement,
		Parameters:   neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion},
		IncludeStats: true,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	stats, err := query.Stats()
	if err != nil {
		return false, err
	}

	var found bool
	if stats.ContainsUpdates {
		found = true
	}

	return found, err
}

//Write a set of annotations associated with a piece of content. Any annotations
//already there will be removed
func (s service) Write(contentUUID string, thing interface{}) (err error) {
	annotationsToWrite := thing.(annotations)

	if contentUUID == "" {
		return errors.New("Content uuid is required")
	}
	if err := validateAnnotations(&annotationsToWrite); err != nil {
		log.Warnf("Validation of supplied annotations failed")
		return err
	}

	if len(annotationsToWrite) == 0 {
		log.Warnf("No new annotations supplied for content uuid: %s", contentUUID)
	}

	queries := append([]*neoism.CypherQuery{}, dropAllAnnotationsQuery(contentUUID, s.platformVersion))

	var statements = []string{}
	for _, annotationToWrite := range annotationsToWrite {
		query, err := createAnnotationQuery(contentUUID, annotationToWrite, s.platformVersion)
		if err != nil {
			return err
		}
		statements = append(statements, query.Statement)
		queries = append(queries, query)
	}
	log.Infof("Updated Annotations for content uuid: %s", contentUUID)
	log.Debugf("For update, ran statements: %+v", statements)

	return s.cypherRunner.CypherBatch(queries)
}

// Check tests neo4j by running a simple cypher query
func (s service) Check() error {
	return neoutils.Check(s.cypherRunner)
}

func (s service) Count() (int, error) {
	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement:  `MATCH ()-[r{platformVersion:{platformVersion}}]->() RETURN count(r) as c`,
		Parameters: neoism.Props{"platformVersion": s.platformVersion},
		Result:     &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (s service) Initialise() error {
	return nil // No constraints need to be set up
}

func createAnnotationRelationship(relation string) (statement string) {
	stmt := `
                MERGE (content:Thing{uuid:{contentID}})
                MERGE (upp:Identifier:UPPIdentifier{value:{conceptID}})
                MERGE (upp)-[:IDENTIFIES]->(concept:Thing) ON CREATE SET concept.uuid = {conceptID}
                MERGE (content)-[pred:%s{platformVersion:{platformVersion}}]->(concept)
                SET pred={annProps}
                `
	statement = fmt.Sprintf(stmt, relation)
	return statement
}

func getRelationshipFromPredicate(predicate string) (relation string) {
	if predicate != "" {
		relation = relations[predicate]
	} else {
		relation = relations["mentions"]
	}
	return relation
}

func createAnnotationQuery(contentUUID string, ann annotation, platformVersion string) (*neoism.CypherQuery, error) {
	query := neoism.CypherQuery{}
	thingID, err := extractUUIDFromURI(ann.Thing.ID)
	if err != nil {
		return nil, err
	}

	//todo temporary change to deal with multiple provenances
	/*if len(ann.Provenances) > 1 {
		return nil, errors.New("Cannot insert a MENTIONS annotation with multiple provenances")
	}*/

	var prov provenance
	params := map[string]interface{}{}
	params["platformVersion"] = platformVersion

	if len(ann.Provenances) >= 1 {
		prov = ann.Provenances[0]
		annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, supplied, err := extractDataFromProvenance(&prov)

		if err != nil {
			log.Infof("ERROR=%s", err)
			return nil, err
		}

		if supplied == true {
			if annotatedBy != "" {
				params["annotatedBy"] = annotatedBy
			}
			if prov.AtTime != "" {
				params["annotatedDateEpoch"] = annotatedDateEpoch
				params["annotatedDate"] = prov.AtTime
			}
			params["relevanceScore"] = relevanceScore
			params["confidenceScore"] = confidenceScore
		}
	}

	relation := getRelationshipFromPredicate(ann.Thing.Predicate)
	query.Statement = createAnnotationRelationship(relation)
	query.Parameters = map[string]interface{}{
		"contentID":       contentUUID,
		"conceptID":       thingID,
		"platformVersion": platformVersion,
		"annProps":        params,
	}
	return &query, nil
}

func extractDataFromProvenance(prov *provenance) (string, int64, float64, float64, bool, error) {
	if len(prov.Scores) == 0 {
		return "", -1, -1, -1, false, nil
	}
	var annotatedBy string
	var annotatedDateEpoch int64
	var confidenceScore, relevanceScore float64
	var err error
	if prov.AgentRole != "" {
		annotatedBy, err = extractUUIDFromURI(prov.AgentRole)
	}
	if prov.AtTime != "" {
		annotatedDateEpoch, err = convertAnnotatedDateToEpoch(prov.AtTime)
	}
	relevanceScore, confidenceScore, err = extractScores(prov.Scores)

	if err != nil {
		return "", -1, -1, -1, true, err
	}
	return annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, true, nil
}

func extractUUIDFromURI(uri string) (string, error) {
	result := uuidExtractRegex.FindStringSubmatch(uri)
	if len(result) == 2 {
		return result[1], nil
	}
	return "", fmt.Errorf("Couldn't extract uuid from uri %s", uri)
}

func convertAnnotatedDateToEpoch(annotatedDateString string) (int64, error) {
	datetimeEpoch, err := time.Parse(time.RFC3339, annotatedDateString)

	if err != nil {
		return 0, err
	}

	return datetimeEpoch.Unix(), nil
}

func extractScores(scores []score) (float64, float64, error) {
	var relevanceScore, confidenceScore float64
	for _, score := range scores {
		scoringSystem := score.ScoringSystem
		value := score.Value
		switch scoringSystem {
		case relevanceScoringSystem:
			relevanceScore = value
		case confidenceScoringSystem:
			confidenceScore = value
		}
	}
	return relevanceScore, confidenceScore, nil
}

func dropAllAnnotationsQuery(contentUUID string, platformVersion string) *neoism.CypherQuery {

	var matchStmtTemplate string

	//TODO hard-coded verification:
	// -> necessary for brands - which got written by content-api with isClassifiedBy relationship, and should not be deleted by annotations-rw
	// -> so far brands are the only v2 concepts which have isClassifiedBy relationship; as soon as this changes: implementation needs to be updated
	if platformVersion == "v2" {
		matchStmtTemplate = `OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r:MENTIONS{platformVersion:{platformVersion}}]->(t:Thing)
                        DELETE r`
	} else {
		matchStmtTemplate = `OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r]->(t:Thing)
			WHERE r.platformVersion={platformVersion}
                        DELETE r`
	}

	query := neoism.CypherQuery{}
	query.Statement = matchStmtTemplate
	query.Parameters = neoism.Props{"contentID": contentUUID, "platformVersion": platformVersion}
	return &query
}

func validateAnnotations(annotations *annotations) error {
	//TODO - for consistency, we should probably just not create the annotation?
	for _, annotation := range *annotations {
		if annotation.Thing.ID == "" {
			return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
		}
	}
	return nil
}

//ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
	Msg string
}

func (v ValidationError) Error() string {
	return v.Msg
}

func mapToResponseFormat(ann *annotation) {
	ann.Thing.ID = mapper.IDURL(ann.Thing.ID)
	// We expect only ONE provenance - provenance value is considered valid even if the AgentRole is not specified. See: v1 - isClassifiedBy
	for idx := range ann.Provenances {
		if ann.Provenances[idx].AgentRole != "" {
			ann.Provenances[idx].AgentRole = mapper.IDURL(ann.Provenances[idx].AgentRole)
		}
	}
}
<commit_msg>Changing cypher to include lifecycle, TODO: Tests<commit_after>package annotations

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"regexp"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
	Write(contentUUID string, thing interface{}) (err error)
	Read(contentUUID string) (thing interface{}, found bool, err error)
	Delete(contentUUID string) (found bool, err error)
	Check() (err error)
	DecodeJSON(*json.Decoder) (thing interface{}, err error)
	Count() (int, error)
	Initialise() error
}

//holds the Neo4j-specific information
type service struct {
	cypherRunner    neoutils.CypherRunner
	indexManager    neoutils.IndexManager
	platformVersion string
}

//NewAnnotationsService instantiate driver
func NewAnnotationsService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager, platformVersion string) service {
	if platformVersion == "" {
		log.Fatalf("PlatformVersion was not specified!")
	}
	return service{cypherRunner, indexManager, platformVersion}
}

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
	a := annotations{}
	err := dec.Decode(&a)
	return a, err
}

func (s service) Read(contentUUID string) (thing interface{}, found bool, err error) {
	results := []annotation{}

	//TODO shouldn't return Provenances if none of the scores, agentRole or atTime are set
	statementTemplate := `
					MATCH (c:Thing{uuid:{contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
					WHERE rel.lifecycle = {lifecycle}
					OR rel.lifecycle IS NULL
					WITH c, cc, rel, {id:cc.uuid,prefLabel:cc.prefLabel,types:labels(cc),predicate:type(rel)} as thing,
					collect(
						{scores:[
							{scoringSystem:'%s', value:rel.relevanceScore},
							{scoringSystem:'%s', value:rel.confidenceScore}],
						agentRole:rel.annotatedBy,
						atTime:rel.annotatedDate}) as provenances
					RETURN thing, provenances ORDER BY thing.id
									`
	statement := fmt.Sprintf(statementTemplate, relevanceScoringSystem, confidenceScoringSystem)

	query := &neoism.CypherQuery{
		Statement:  statement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		Result:     &results,
	}
	err = s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v", contentUUID, query.Statement, err)
		return annotations{}, false, fmt.Errorf("Error accessing Annotations datastore for uuid: %s", contentUUID)
	}
	log.Debugf("CypherResult Read Annotations for uuid: %s was: %+v", contentUUID, results)
	if (len(results)) == 0 {
		return annotations{}, false, nil
	}

	for idx := range results {
		mapToResponseFormat(&results[idx])
	}

	return results, true, nil
}

//Delete removes all the annotations for this content. Ignore the nodes on either end -
//may leave nodes that are only 'things' inserted by this writer: clean up
//as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string) (bool, error) {
	deleteStatement := `MATCH (c:Thing{uuid: {contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
											WHERE rel.lifecycle = {lifecycle}
											OR rel.lifecycle IS NULL
	 										DELETE rel`
	query := &neoism.CypherQuery{
		Statement:    deleteStatement,
		Parameters:   neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		IncludeStats: true,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	stats, err := query.Stats()
	if err != nil {
		return false, err
	}

	var found bool
	if stats.ContainsUpdates {
		found = true
	}

	return found, err
}

//Write a set of annotations associated with a piece of content. Any annotations
//already there will be removed
func (s service) Write(contentUUID string, thing interface{}) (err error) {
	annotationsToWrite := thing.(annotations)

	if contentUUID == "" {
		return errors.New("Content uuid is required")
	}
	if err := validateAnnotations(&annotationsToWrite); err != nil {
		log.Warnf("Validation of supplied annotations failed")
		return err
	}

	if len(annotationsToWrite) == 0 {
		log.Warnf("No new annotations supplied for content uuid: %s", contentUUID)
	}

	queries := append([]*neoism.CypherQuery{}, dropAllAnnotationsQuery(contentUUID, s.platformVersion))

	var statements = []string{}
	for _, annotationToWrite := range annotationsToWrite {
		query, err := createAnnotationQuery(contentUUID, annotationToWrite, s.platformVersion)
		if err != nil {
			return err
		}
		statements = append(statements, query.Statement)
		queries = append(queries, query)
	}
	log.Infof("Updated Annotations for content uuid: %s", contentUUID)
	log.Debugf("For update, ran statements: %+v", statements)

	return s.cypherRunner.CypherBatch(queries)
}

// Check tests neo4j by running a simple cypher query
func (s service) Check() error {
	return neoutils.Check(s.cypherRunner)
}

func (s service) Count() (int, error) {
	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH ()-[r{platformVersion:{platformVersion}}]->()
								WHERE rel.lifecycle = {lifecycle}
								OR rel.lifecycle IS NULL
								RETURN count(r) as c`,
		Parameters: neoism.Props{"platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		Result:     &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (s service) Initialise() error {
	return nil // No constraints need to be set up
}

func createAnnotationRelationship(relation string) (statement string) {
	stmt := `
                MERGE (content:Thing{uuid:{contentID}})
                MERGE (upp:Identifier:UPPIdentifier{value:{conceptID}})
                MERGE (upp)-[:IDENTIFIES]->(concept:Thing) ON CREATE SET concept.uuid = {conceptID}
                MERGE (content)-[pred:%s {platformVersion:{platformVersion}, lifecycle: {lifecycle}}]->(concept)
                SET pred={annProps}
          `
	statement = fmt.Sprintf(stmt, relation)
	return statement
}

func getRelationshipFromPredicate(predicate string) (relation string) {
	if predicate != "" {
		relation = relations[predicate]
	} else {
		relation = relations["mentions"]
	}
	return relation
}

func createAnnotationQuery(contentUUID string, ann annotation, platformVersion string) (*neoism.CypherQuery, error) {
	query := neoism.CypherQuery{}
	thingID, err := extractUUIDFromURI(ann.Thing.ID)
	if err != nil {
		return nil, err
	}

	//todo temporary change to deal with multiple provenances
	/*if len(ann.Provenances) > 1 {
		return nil, errors.New("Cannot insert a MENTIONS annotation with multiple provenances")
	}*/

	var prov provenance
	params := map[string]interface{}{}
	params["platformVersion"] = platformVersion

	if len(ann.Provenances) >= 1 {
		prov = ann.Provenances[0]
		annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, supplied, err := extractDataFromProvenance(&prov)

		if err != nil {
			log.Infof("ERROR=%s", err)
			return nil, err
		}

		if supplied == true {
			if annotatedBy != "" {
				params["annotatedBy"] = annotatedBy
			}
			if prov.AtTime != "" {
				params["annotatedDateEpoch"] = annotatedDateEpoch
				params["annotatedDate"] = prov.AtTime
			}
			params["relevanceScore"] = relevanceScore
			params["confidenceScore"] = confidenceScore
		}
	}

	relation := getRelationshipFromPredicate(ann.Thing.Predicate)
	query.Statement = createAnnotationRelationship(relation)
	query.Parameters = map[string]interface{}{
		"contentID":       contentUUID,
		"conceptID":       thingID,
		"platformVersion": platformVersion,
		"lifecycle":       "annotations-" + platformVersion,
		"annProps":        params,
	}
	return &query, nil
}

func extractDataFromProvenance(prov *provenance) (string, int64, float64, float64, bool, error) {
	if len(prov.Scores) == 0 {
		return "", -1, -1, -1, false, nil
	}
	var annotatedBy string
	var annotatedDateEpoch int64
	var confidenceScore, relevanceScore float64
	var err error
	if prov.AgentRole != "" {
		annotatedBy, err = extractUUIDFromURI(prov.AgentRole)
	}
	if prov.AtTime != "" {
		annotatedDateEpoch, err = convertAnnotatedDateToEpoch(prov.AtTime)
	}
	relevanceScore, confidenceScore, err = extractScores(prov.Scores)

	if err != nil {
		return "", -1, -1, -1, true, err
	}
	return annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, true, nil
}

func extractUUIDFromURI(uri string) (string, error) {
	result := uuidExtractRegex.FindStringSubmatch(uri)
	if len(result) == 2 {
		return result[1], nil
	}
	return "", fmt.Errorf("Couldn't extract uuid from uri %s", uri)
}

func convertAnnotatedDateToEpoch(annotatedDateString string) (int64, error) {
	datetimeEpoch, err := time.Parse(time.RFC3339, annotatedDateString)

	if err != nil {
		return 0, err
	}

	return datetimeEpoch.Unix(), nil
}

func extractScores(scores []score) (float64, float64, error) {
	var relevanceScore, confidenceScore float64
	for _, score := range scores {
		scoringSystem := score.ScoringSystem
		value := score.Value
		switch scoringSystem {
		case relevanceScoringSystem:
			relevanceScore = value
		case confidenceScoringSystem:
			confidenceScore = value
		}
	}
	return relevanceScore, confidenceScore, nil
}

func dropAllAnnotationsQuery(contentUUID string, platformVersion string) *neoism.CypherQuery {
	matchStmtTemplate := `OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r {platformVersion:{platformVersion}, lifecycle: {lifecycle}}]->(t:Thing)
											DELETE r`

	query := neoism.CypherQuery{}
	query.Statement = matchStmtTemplate
	query.Parameters = neoism.Props{"contentID": contentUUID, "platformVersion": platformVersion, "lifecycle": "annotations-" + platformVersion}
	return &query
}

func validateAnnotations(annotations *annotations) error {
	//TODO - for consistency, we should probably just not create the annotation?
	for _, annotation := range *annotations {
		if annotation.Thing.ID == "" {
			return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
		}
	}
	return nil
}

//ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
	Msg string
}

func (v ValidationError) Error() string {
	return v.Msg
}

func mapToResponseFormat(ann *annotation) {
	ann.Thing.ID = mapper.IDURL(ann.Thing.ID)
	// We expect only ONE provenance - provenance value is considered valid even if the AgentRole is not specified. See: v1 - isClassifiedBy
	for idx := range ann.Provenances {
		if ann.Provenances[idx].AgentRole != "" {
			ann.Provenances[idx].AgentRole = mapper.IDURL(ann.Provenances[idx].AgentRole)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017-2020 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package grub implements a grub config file parser.
//
// See the grub manual https://www.gnu.org/software/grub/manual/grub/ for
// a reference of the configuration format
// In particular the following pages:
// - https://www.gnu.org/software/grub/manual/grub/html_node/Shell_002dlike-scripting.html
// - https://www.gnu.org/software/grub/manual/grub/html_node/Commands.html
//
// Currently, only the linux[16|efi], initrd[16|efi], menuentry and set
// directives are partially supported.
package grub

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/u-root/u-root/pkg/boot"
	"github.com/u-root/u-root/pkg/boot/multiboot"
	"github.com/u-root/u-root/pkg/curl"
	"github.com/u-root/u-root/pkg/shlex"
	"github.com/u-root/u-root/pkg/uio"
)

var (
	// ErrInitrdUsedWithoutLinux is returned when an initrd directive is
	// not following a linux directive in the same menu entry
	ErrInitrdUsedWithoutLinux = errors.New("missing linux directive before initrd")
	// ErrModuleUsedWithoutMultiboot is returned when a module directive is
	// not following a multiboot directive in the same menu entry
	ErrModuleUsedWithoutMultiboot = errors.New("missing multiboot directive before module")
)

var probeGrubFiles = []string{
	"boot/grub/grub.cfg",
	"grub/grub.cfg",
	"grub2/grub.cfg",
}

// ParseLocalConfig looks for a GRUB config in the disk partition mounted at
// diskDir and parses out OSes to boot.
//
// This... is at best crude, at worst totally wrong, since we fundamentally
// assume that the kernels we boot are only on this one partition. But so is
// this whole parser.
func ParseLocalConfig(ctx context.Context, diskDir string) ([]boot.OSImage, error) {
	wd := &url.URL{
		Scheme: "file",
		Path:   diskDir,
	}

	for _, relname := range probeGrubFiles {
		c, err := ParseConfigFile(ctx, curl.DefaultSchemes, relname, wd)
		if curl.IsURLError(err) {
			continue
		}
		return c, err
	}
	return nil, fmt.Errorf("no valid grub config found")
}

// ParseConfigFile parses a grub configuration as specified in
// https://www.gnu.org/software/grub/manual/grub/
//
// Currently, only the linux[16|efi], initrd[16|efi], menuentry and set
// directives are partially supported.
//
// `wd` is the default scheme, host, and path for any files named as a
// relative path - e.g. kernel, include, and initramfs paths are requested
// relative to the wd.
func ParseConfigFile(ctx context.Context, s curl.Schemes, configFile string, wd *url.URL) ([]boot.OSImage, error) {
	p := newParser(wd, s)
	if err := p.appendFile(ctx, configFile); err != nil {
		return nil, err
	}

	// Don't add entries twice.
	//
	// Multiple labels can refer to the same image, so we have to dedup by pointer.
	seenLinux := make(map[*boot.LinuxImage]struct{})
	seenMB := make(map[*boot.MultibootImage]struct{})

	if len(p.defaultEntry) > 0 {
		p.labelOrder = append([]string{p.defaultEntry}, p.labelOrder...)
	}

	var images []boot.OSImage
	for _, label := range p.labelOrder {
		if img, ok := p.linuxEntries[label]; ok {
			if _, ok := seenLinux[img]; !ok {
				images = append(images, img)
				seenLinux[img] = struct{}{}
			}
		}

		if img, ok := p.mbEntries[label]; ok {
			if _, ok := seenMB[img]; !ok {
				images = append(images, img)
				seenMB[img] = struct{}{}
			}
		}
	}
	return images, nil
}

type parser struct {
	linuxEntries map[string]*boot.LinuxImage
	mbEntries    map[string]*boot.MultibootImage

	labelOrder   []string
	defaultEntry string

	W io.Writer

	// parser internals.
	numEntry int

	// curEntry is the current entry number as a string.
	curEntry string

	// curLabel is the last parsed label from a "menuentry".
	curLabel string

	wd      *url.URL
	schemes curl.Schemes
}

// newParser returns a new grub parser using working directory `wd`
// and schemes `s`.
//
// If a path encountered in a configuration file is relative instead of a full
// URL, `wd` is used as the "working directory" of that relative path; the
// resulting URL is roughly `wd.String()/path`.
//
// `s` is used to get files referred to by URLs.
func newParser(wd *url.URL, s curl.Schemes) *parser {
	return &parser{
		linuxEntries: make(map[string]*boot.LinuxImage),
		mbEntries:    make(map[string]*boot.MultibootImage),
		wd:           wd,
		schemes:      s,
	}
}

func parseURL(surl string, wd *url.URL) (*url.URL, error) {
	u, err := url.Parse(surl)
	if err != nil {
		return nil, fmt.Errorf("could not parse URL %q: %v", surl, err)
	}

	if len(u.Scheme) == 0 {
		u.Scheme = wd.Scheme

		if len(u.Host) == 0 {
			// If this is not there, it was likely just a path.
			u.Host = wd.Host
			u.Path = filepath.Join(wd.Path, filepath.Clean(u.Path))
		}
	}
	return u, nil
}

// getFile parses `url` relative to the config's working directory and returns
// an io.Reader for the requested url.
//
// If url is just a relative path and not a full URL, c.wd is used as the
// "working directory" of that relative path; the resulting URL is roughly
// path.Join(wd.String(), url).
func (c *parser) getFile(url string) (io.ReaderAt, error) {
	u, err := parseURL(url, c.wd)
	if err != nil {
		return nil, err
	}

	return c.schemes.LazyFetch(u)
}

// appendFile parses the config file downloaded from `url` and adds it to `c`.
func (c *parser) appendFile(ctx context.Context, url string) error {
	u, err := parseURL(url, c.wd)
	if err != nil {
		return err
	}

	log.Printf("Fetching %s", u)

	r, err := c.schemes.Fetch(ctx, u)
	if err != nil {
		return err
	}

	config, err := uio.ReadAll(r)
	if err != nil {
		return err
	}
	if len(config) > 500 {
		// Avoid flooding the console on real systems
		// TODO: do we want to pass a verbose flag or a logger?
		log.Printf("Got config file %s", r)
	} else {
		log.Printf("Got config file %s:\n%s\n", r, string(config))
	}
	return c.append(ctx, string(config))
}

// CmdlineQuote quotes the command line as grub-core/lib/cmdline.c does
func cmdlineQuote(args []string) string {
	q := make([]string, len(args))
	for i, s := range args {
		s = strings.Replace(s, `\`, `\\`, -1)
		s = strings.Replace(s, `'`, `\'`, -1)
		s = strings.Replace(s, `"`, `\"`, -1)
		if strings.ContainsRune(s, ' ') {
			s = `"` + s + `"`
		}
		q[i] = s
	}
	return strings.Join(q, " ")
}

// append parses `config` and adds the respective configuration to `c`.
func (c *parser) append(ctx context.Context, config string) error {
	// Here's a shitty parser.
	for _, line := range strings.Split(config, "\n") {
		kv := shlex.Argv(line)
		if len(kv) < 1 {
			continue
		}
		directive := strings.ToLower(kv[0])
		// Used by tests (allow no parameters here)
		if c.W != nil && directive == "echo" {
			fmt.Fprintf(c.W, "echo:%#v\n", kv[1:])
		}

		if len(kv) <= 1 {
			continue
		}
		arg := kv[1]

		switch directive {
		case "set":
			vals := strings.SplitN(arg, "=", 2)
			if len(vals) == 2 {
				//TODO handle vars? bootVars[vals[0]] = vals[1]
				//log.Printf("grubvar: %s=%s", vals[0], vals[1])
				if vals[0] == "default" {
					c.defaultEntry = vals[1]
				}
			}

		case "configfile":
			// TODO test that
			if err := c.appendFile(ctx, arg); err != nil {
				return err
			}

		case "menuentry":
			c.curEntry = strconv.Itoa(c.numEntry)
			c.curLabel = arg
			c.numEntry++
			c.labelOrder = append(c.labelOrder, c.curEntry, c.curLabel)

		case "linux", "linux16", "linuxefi":
			k, err := c.getFile(arg)
			if err != nil {
				return err
			}
			// from grub manual: "Any initrd must be reloaded after using this command" so we can replace the entry
			entry := &boot.LinuxImage{
				Name:    c.curLabel,
				Kernel:  k,
				Cmdline: cmdlineQuote(kv[2:]),
			}
			c.linuxEntries[c.curEntry] = entry
			c.linuxEntries[c.curLabel] = entry

		case "initrd", "initrd16", "initrdefi":
			i, err := c.getFile(arg)
			if err != nil {
				return err
			}
			entry, ok := c.linuxEntries[c.curEntry]
			if !ok {
				return ErrInitrdUsedWithoutLinux
			}
			entry.Initrd = i

		case "multiboot":
			// TODO handle --quirk-* arguments ? (change parsing)
			k, err := c.getFile(arg)
			if err != nil {
				return err
			}
			// from grub manual: "Any initrd must be reloaded after using this command" so we can replace the entry
			entry := &boot.MultibootImage{
				Name:    c.curLabel,
				Kernel:  k,
				Cmdline: cmdlineQuote(kv[2:]),
			}
			c.mbEntries[c.curEntry] = entry
			c.mbEntries[c.curLabel] = entry

		case "module":
			// TODO handle --nounzip arguments ? (change parsing)
			m, err := c.getFile(arg)
			if err != nil {
				return err
			}
			entry, ok := c.mbEntries[c.curEntry]
			if !ok {
				return ErrModuleUsedWithoutMultiboot
			}
			// TODO: Lasy tryGzipFilter(m)
			mod := multiboot.Module{
				Module:  m,
				Name:    arg,
				CmdLine: cmdlineQuote(kv[2:]),
			}
			entry.Modules = append(entry.Modules, mod)

		}
	}
	return nil

}
<commit_msg>grub: make entry access safe and don't fatal out<commit_after>// Copyright 2017-2020 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package grub implements a grub config file parser.
//
// See the grub manual https://www.gnu.org/software/grub/manual/grub/ for
// a reference of the configuration format
// In particular the following pages:
// - https://www.gnu.org/software/grub/manual/grub/html_node/Shell_002dlike-scripting.html
// - https://www.gnu.org/software/grub/manual/grub/html_node/Commands.html
//
// Currently, only the linux[16|efi], initrd[16|efi], menuentry and set
// directives are partially supported.
package grub

import (
	"context"
	"fmt"
	"io"
	"log"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/u-root/u-root/pkg/boot"
	"github.com/u-root/u-root/pkg/boot/multiboot"
	"github.com/u-root/u-root/pkg/curl"
	"github.com/u-root/u-root/pkg/shlex"
	"github.com/u-root/u-root/pkg/uio"
)

var probeGrubFiles = []string{
	"boot/grub/grub.cfg",
	"grub/grub.cfg",
	"grub2/grub.cfg",
}

// ParseLocalConfig looks for a GRUB config in the disk partition mounted at
// diskDir and parses out OSes to boot.
//
// This... is at best crude, at worst totally wrong, since we fundamentally
// assume that the kernels we boot are only on this one partition. But so is
// this whole parser.
func ParseLocalConfig(ctx context.Context, diskDir string) ([]boot.OSImage, error) {
	wd := &url.URL{
		Scheme: "file",
		Path:   diskDir,
	}

	for _, relname := range probeGrubFiles {
		c, err := ParseConfigFile(ctx, curl.DefaultSchemes, relname, wd)
		if curl.IsURLError(err) {
			continue
		}
		return c, err
	}
	return nil, fmt.Errorf("no valid grub config found")
}

// ParseConfigFile parses a grub configuration as specified in
// https://www.gnu.org/software/grub/manual/grub/
//
// Currently, only the linux[16|efi], initrd[16|efi], menuentry and set
// directives are partially supported.
//
// `wd` is the default scheme, host, and path for any files named as a
// relative path - e.g. kernel, include, and initramfs paths are requested
// relative to the wd.
func ParseConfigFile(ctx context.Context, s curl.Schemes, configFile string, wd *url.URL) ([]boot.OSImage, error) {
	p := newParser(wd, s)
	if err := p.appendFile(ctx, configFile); err != nil {
		return nil, err
	}

	// Don't add entries twice.
	//
	// Multiple labels can refer to the same image, so we have to dedup by pointer.
	seenLinux := make(map[*boot.LinuxImage]struct{})
	seenMB := make(map[*boot.MultibootImage]struct{})

	if len(p.defaultEntry) > 0 {
		p.labelOrder = append([]string{p.defaultEntry}, p.labelOrder...)
	}

	var images []boot.OSImage
	for _, label := range p.labelOrder {
		if img, ok := p.linuxEntries[label]; ok {
			if _, ok := seenLinux[img]; !ok {
				images = append(images, img)
				seenLinux[img] = struct{}{}
			}
		}

		if img, ok := p.mbEntries[label]; ok {
			if _, ok := seenMB[img]; !ok {
				images = append(images, img)
				seenMB[img] = struct{}{}
			}
		}
	}
	return images, nil
}

type parser struct {
	linuxEntries map[string]*boot.LinuxImage
	mbEntries    map[string]*boot.MultibootImage

	labelOrder   []string
	defaultEntry string

	W io.Writer

	// parser internals.
	numEntry int

	// curEntry is the current entry number as a string.
	curEntry string

	// curLabel is the last parsed label from a "menuentry".
	curLabel string

	wd      *url.URL
	schemes curl.Schemes
}

// newParser returns a new grub parser using working directory `wd`
// and schemes `s`.
//
// If a path encountered in a configuration file is relative instead of a full
// URL, `wd` is used as the "working directory" of that relative path; the
// resulting URL is roughly `wd.String()/path`.
//
// `s` is used to get files referred to by URLs.
func newParser(wd *url.URL, s curl.Schemes) *parser {
	return &parser{
		linuxEntries: make(map[string]*boot.LinuxImage),
		mbEntries:    make(map[string]*boot.MultibootImage),
		wd:           wd,
		schemes:      s,
	}
}

func parseURL(surl string, wd *url.URL) (*url.URL, error) {
	u, err := url.Parse(surl)
	if err != nil {
		return nil, fmt.Errorf("could not parse URL %q: %v", surl, err)
	}

	if len(u.Scheme) == 0 {
		u.Scheme = wd.Scheme

		if len(u.Host) == 0 {
			// If this is not there, it was likely just a path.
			u.Host = wd.Host
			u.Path = filepath.Join(wd.Path, filepath.Clean(u.Path))
		}
	}
	return u, nil
}

// getFile parses `url` relative to the config's working directory and returns
// an io.Reader for the requested url.
//
// If url is just a relative path and not a full URL, c.wd is used as the
// "working directory" of that relative path; the resulting URL is roughly
// path.Join(wd.String(), url).
func (c *parser) getFile(url string) (io.ReaderAt, error) {
	u, err := parseURL(url, c.wd)
	if err != nil {
		return nil, err
	}

	return c.schemes.LazyFetch(u)
}

// appendFile parses the config file downloaded from `url` and adds it to `c`.
func (c *parser) appendFile(ctx context.Context, url string) error {
	u, err := parseURL(url, c.wd)
	if err != nil {
		return err
	}

	log.Printf("Fetching %s", u)

	r, err := c.schemes.Fetch(ctx, u)
	if err != nil {
		return err
	}

	config, err := uio.ReadAll(r)
	if err != nil {
		return err
	}
	if len(config) > 500 {
		// Avoid flooding the console on real systems
		// TODO: do we want to pass a verbose flag or a logger?
		log.Printf("Got config file %s", r)
	} else {
		log.Printf("Got config file %s:\n%s\n", r, string(config))
	}
	return c.append(ctx, string(config))
}

// CmdlineQuote quotes the command line as grub-core/lib/cmdline.c does
func cmdlineQuote(args []string) string {
	q := make([]string, len(args))
	for i, s := range args {
		s = strings.Replace(s, `\`, `\\`, -1)
		s = strings.Replace(s, `'`, `\'`, -1)
		s = strings.Replace(s, `"`, `\"`, -1)
		if strings.ContainsRune(s, ' ') {
			s = `"` + s + `"`
		}
		q[i] = s
	}
	return strings.Join(q, " ")
}

// append parses `config` and adds the respective configuration to `c`.
//
// NOTE: This parser has outlived its usefulness already, given that it doesn't
// even understand the {} scoping in GRUB. But let's get the tests to pass, and
// then we can do a rewrite.
func (c *parser) append(ctx context.Context, config string) error {
	// Here's a shitty parser.
	for _, line := range strings.Split(config, "\n") {
		kv := shlex.Argv(line)
		if len(kv) < 1 {
			continue
		}
		directive := strings.ToLower(kv[0])
		// Used by tests (allow no parameters here)
		if c.W != nil && directive == "echo" {
			fmt.Fprintf(c.W, "echo:%#v\n", kv[1:])
		}

		if len(kv) <= 1 {
			continue
		}
		arg := kv[1]

		switch directive {
		case "set":
			vals := strings.SplitN(arg, "=", 2)
			if len(vals) == 2 {
				//TODO handle vars? bootVars[vals[0]] = vals[1]
				//log.Printf("grubvar: %s=%s", vals[0], vals[1])
				if vals[0] == "default" {
					c.defaultEntry = vals[1]
				}
			}

		case "configfile":
			// TODO test that
			if err := c.appendFile(ctx, arg); err != nil {
				return err
			}

		case "menuentry":
			c.curEntry = strconv.Itoa(c.numEntry)
			c.curLabel = arg
			c.numEntry++
			c.labelOrder = append(c.labelOrder, c.curEntry, c.curLabel)

		case "linux", "linux16", "linuxefi":
			k, err := c.getFile(arg)
			if err != nil {
				return err
			}
			// from grub manual: "Any initrd must be reloaded after using this command" so we can replace the entry
			entry := &boot.LinuxImage{
				Name:    c.curLabel,
				Kernel:  k,
				Cmdline: cmdlineQuote(kv[2:]),
			}
			c.linuxEntries[c.curEntry] = entry
			c.linuxEntries[c.curLabel] = entry

		case "initrd", "initrd16", "initrdefi":
			if e, ok := c.linuxEntries[c.curEntry]; ok {
				i, err := c.getFile(arg)
				if err != nil {
					return err
				}
				e.Initrd = i
			}

		case "multiboot":
			// TODO handle --quirk-* arguments ? (change parsing)
			k, err := c.getFile(arg)
			if err != nil {
				return err
			}
			// from grub manual: "Any initrd must be reloaded after using this command" so we can replace the entry
			entry := &boot.MultibootImage{
				Name:    c.curLabel,
				Kernel:  k,
				Cmdline: cmdlineQuote(kv[2:]),
			}
			c.mbEntries[c.curEntry] = entry
			c.mbEntries[c.curLabel] = entry

		case "module":
			// TODO handle --nounzip arguments ? (change parsing)
			if e, ok := c.mbEntries[c.curEntry]; ok {
				m, err := c.getFile(arg)
				if err != nil {
					return err
				}
				// TODO: Lasy tryGzipFilter(m)
				mod := multiboot.Module{
					Module:  m,
					Name:    arg,
					CmdLine: cmdlineQuote(kv[2:]),
				}
				e.Modules = append(e.Modules, mod)
			}
		}
	}
	return nil

}
<|endoftext|>


<commit_before>package kit

// InstrumentingTemplate
var InstrumentingTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/metrics"
	"golang.org/x/net/context"
)

func DefaultMiddlewares(method string, requestCount metrics.Counter, requestLatency metrics.TimeHistogram, logger log.Logger) endpoint.Middleware {
	return endpoint.Chain(
		RequestLatencyMiddleware(method, requestLatency),
		RequestCountMiddleware(method, requestCount),
		RequestLoggingMiddleware(method, logger),
	)
}

func RequestCountMiddleware(method string, requestCount metrics.Counter) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func() {
				methodField := metrics.Field{Key: "method", Value: method}
				errorField := metrics.Field{Key: "error", Value: fmt.Sprintf("%v", err)}
				requestCount.With(methodField).With(errorField).Add(1)
			}()

			response, err = next(ctx, request)
			return
		}
	}
}

func RequestLatencyMiddleware(method string, requestLatency metrics.TimeHistogram) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func(begin time.Time) {
				methodField := metrics.Field{Key: "method", Value: method}
				errorField := metrics.Field{Key: "error", Value: fmt.Sprintf("%v", err)}
				requestLatency.With(methodField).With(errorField).Observe(time.Since(begin))
			}(time.Now())

			response, err = next(ctx, request)
			return
		}
	}
}

func RequestLoggingMiddleware(method string, logger log.Logger) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func(begin time.Time) {
				input, _ := json.Marshal(request)
				output, _ := json.Marshal(response)
				_ = logger.Log(
					"method", method,
					"input", string(input),
					"output", string(output),
					"err", err,
					"took", time.Since(begin),
				)
			}(time.Now())
			response, err = next(ctx, request)
			return
		}
	}
}
`

// InterfaceTemplate
var InterfaceTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

type {{$title}}Service interface {
{{range $funcKey, $funcValue := $schema.Functions}}
{{$funcKey}}(ctx context.Context, req *{{Argumentize $funcValue.Properties.incoming}}) (res *{{Argumentize $funcValue.Properties.outgoing}}, err error){{end}}
}
`

// TransportHTTPServerTemplate
var TransportHTTPServerTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
)

// Handler functions

{{range $funcKey, $funcValue := $schema.Functions}}
func New{{$funcKey}}Handler(ctx context.Context, svc {{$title}}Service, middleware endpoint.Middleware, options ...httptransport.ServerOption) *httptransport.Server {
	return httptransport.NewServer(
		ctx,
		middleware(make{{$funcKey}}Endpoint(svc)),
		decode{{$funcKey}}Request,
		encodeResponse,
		options...,
	)
}
{{end}}

// Endpoint functions

{{range $funcKey, $funcValue := $schema.Functions}}
func make{{$funcKey}}Endpoint(svc {{$title}}Service) endpoint.Endpoint {
	return func(ctx context.Context, request interface{}) (interface{}, error) {
		req := request.(*{{Argumentize $funcValue.Properties.incoming}})
		return svc.{{$funcKey}}(ctx, req)
	}
}
{{end}}
`

// TransportHTTPClientTemplate
var TransportHTTPClientTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	jujuratelimit "github.com/juju/ratelimit"
	"github.com/sony/gobreaker"
	"golang.org/x/net/context"

	"github.com/go-kit/kit/circuitbreaker"
	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/loadbalancer"
	"github.com/go-kit/kit/loadbalancer/static"
	"github.com/go-kit/kit/log"
	kitratelimit "github.com/go-kit/kit/ratelimit"
	httptransport "github.com/go-kit/kit/transport/http"
)

// client
type {{$title}}Client struct {
{{range $funcKey, $funcValue := $schema.Functions}}
	{{$funcKey}}Endpoint endpoint.Endpoint
{{end}}}

// constructor
func  New{{$title}}Client(proxies []string, ctx context.Context, maxAttempt int, maxTime time.Duration, qps int, logger log.Logger) *{{$title}}Client {
return &{{$title}}Client{
{{range $funcKey, $funcValue := $schema.Functions}}
{{$funcKey}}Endpoint : new{{$funcKey}}ClientEndpoint(proxies, ctx, maxAttempt, maxTime, qps, logger),{{end}}
}
}

{{range $funcKey, $funcValue := $schema.Functions}}
func ({{Pointerize $title}} *{{$title}}Client) {{$funcKey}}(ctx context.Context, req *{{Argumentize $funcValue.Properties.incoming}}) (*{{Argumentize $funcValue.Properties.outgoing}}, error) {
	res, err := {{Pointerize $title}}.{{$funcKey}}Endpoint(ctx, req)
	if err != nil {
		return nil, err
	}

	return res.(*{{Argumentize $funcValue.Properties.outgoing}}), nil
}
{{end}}

// Client Endpoint functions
{{range $funcKey, $funcValue := $schema.Functions}}
func new{{$funcKey}}ClientEndpoint(proxies []string, ctx context.Context, maxAttempt int, maxTime time.Duration, qps int, logger log.Logger) endpoint.Endpoint {
	factory := createFactory(ctx, qps, make{{$funcKey}}Proxy)
	return defaultClientEndpointCreator(proxies, maxAttempt, maxTime, logger, factory)
}
{{end}}

{{range $funcKey, $funcValue := $schema.Functions}}
func make{{$funcKey}}Proxy(ctx context.Context, instance string) endpoint.Endpoint {
	return httptransport.NewClient(
		"POST",
		createProxyURL(instance, "{{ToLower $funcKey}}"),
		encodeRequest,
		decode{{$funcKey}}Response,
	).Endpoint()
}
{{end}}

// Proxy functions

func createProxyURL(instance, endpoint string) *url.URL {
	if !strings.HasPrefix(instance, "http") {
		instance = "http://" + instance
	}
	u, err := url.Parse(instance)
	if err != nil {
		panic(err)
	}
	if u.Path == "" {
		u.Path = endpoint
	}

	return u
}

type proxyFunc func(context.Context, string) endpoint.Endpoint

func createFactory(ctx context.Context, qps int, pf proxyFunc) loadbalancer.Factory {
	return func(instance string) (endpoint.Endpoint, io.Closer, error) {
		var e endpoint.Endpoint
		e = pf(ctx, instance)
		e = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{}))(e)
		e = kitratelimit.NewTokenBucketLimiter(jujuratelimit.NewBucketWithRate(float64(qps), int64(qps)))(e)
		return e, nil, nil
	}
}

func defaultClientEndpointCreator(
	proxies []string,
	maxAttempts int,
	maxTime time.Duration,
	logger log.Logger,
	factory loadbalancer.Factory,
) endpoint.Endpoint {

	publisher := static.NewPublisher(
		proxies,
		factory,
		logger,
	)

	lb := loadbalancer.NewRoundRobin(publisher)
	return loadbalancer.Retry(maxAttempts, maxTime, lb)
}

`

// TransportHTTPSemioticsTemplate
var TransportHTTPSemioticsTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
)

// Decode Request functions

{{range $funcKey, $funcValue := $schema.Functions}}
func decode{{$funcKey}}Request(r *http.Request) (interface{}, error) {
	var req {{Argumentize $funcValue.Properties.incoming}}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return nil, err
	}
	return &req, nil
}
{{end}}

// Decode Response functions

{{range $funcKey, $funcValue := $schema.Functions}}
func decode{{$funcKey}}Response(r *http.Response) (interface{}, error) {
	var res {{Argumentize $funcValue.Properties.incoming}}
	if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
		return nil, err
	}
	return &res, nil
}
{{end}}

// Encode request function

func encodeRequest(r *http.Request, request interface{}) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(request); err != nil {
		return err
	}
	r.Body = ioutil.NopCloser(&buf)
	return nil
}

// Encode response function

func encodeResponse(rw http.ResponseWriter, response interface{}) error {
	return json.NewEncoder(rw).Encode(response)
}
`
<commit_msg>Kit: better go formatting<commit_after>package kit

// InstrumentingTemplate
var InstrumentingTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/metrics"
	"golang.org/x/net/context"
)

func DefaultMiddlewares(method string, requestCount metrics.Counter, requestLatency metrics.TimeHistogram, logger log.Logger) endpoint.Middleware {
	return endpoint.Chain(
		RequestLatencyMiddleware(method, requestLatency),
		RequestCountMiddleware(method, requestCount),
		RequestLoggingMiddleware(method, logger),
	)
}

func RequestCountMiddleware(method string, requestCount metrics.Counter) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func() {
				methodField := metrics.Field{Key: "method", Value: method}
				errorField := metrics.Field{Key: "error", Value: fmt.Sprintf("%v", err)}
				requestCount.With(methodField).With(errorField).Add(1)
			}()

			response, err = next(ctx, request)
			return
		}
	}
}

func RequestLatencyMiddleware(method string, requestLatency metrics.TimeHistogram) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func(begin time.Time) {
				methodField := metrics.Field{Key: "method", Value: method}
				errorField := metrics.Field{Key: "error", Value: fmt.Sprintf("%v", err)}
				requestLatency.With(methodField).With(errorField).Observe(time.Since(begin))
			}(time.Now())

			response, err = next(ctx, request)
			return
		}
	}
}

func RequestLoggingMiddleware(method string, logger log.Logger) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func(begin time.Time) {
				input, _ := json.Marshal(request)
				output, _ := json.Marshal(response)
				_ = logger.Log(
					"method", method,
					"input", string(input),
					"output", string(output),
					"err", err,
					"took", time.Since(begin),
				)
			}(time.Now())
			response, err = next(ctx, request)
			return
		}
	}
}
`

// InterfaceTemplate
var InterfaceTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

type {{$title}}Service interface {
{{range $funcKey, $funcValue := $schema.Functions}}
{{$funcKey}}(ctx context.Context, req *{{Argumentize $funcValue.Properties.incoming}}) (res *{{Argumentize $funcValue.Properties.outgoing}}, err error){{end}}
}
`

// TransportHTTPServerTemplate
var TransportHTTPServerTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
)

// Handler functions

{{range $funcKey, $funcValue := $schema.Functions}}
func New{{$funcKey}}Handler(ctx context.Context, svc {{$title}}Service, middleware endpoint.Middleware, options ...httptransport.ServerOption) *httptransport.Server {
	return httptransport.NewServer(
		ctx,
		middleware(make{{$funcKey}}Endpoint(svc)),
		decode{{$funcKey}}Request,
		encodeResponse,
		options...,
	)
}
{{end}}

// Endpoint functions

{{range $funcKey, $funcValue := $schema.Functions}}
func make{{$funcKey}}Endpoint(svc {{$title}}Service) endpoint.Endpoint {
	return func(ctx context.Context, request interface{}) (interface{}, error) {
		req := request.(*{{Argumentize $funcValue.Properties.incoming}})
		return svc.{{$funcKey}}(ctx, req)
	}
}
{{end}}
`

// TransportHTTPClientTemplate
var TransportHTTPClientTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	jujuratelimit "github.com/juju/ratelimit"
	"github.com/sony/gobreaker"
	"golang.org/x/net/context"

	"github.com/go-kit/kit/circuitbreaker"
	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/loadbalancer"
	"github.com/go-kit/kit/loadbalancer/static"
	"github.com/go-kit/kit/log"
	kitratelimit "github.com/go-kit/kit/ratelimit"
	httptransport "github.com/go-kit/kit/transport/http"
)

// {{$title}}Client holds remote endpoint functions
// Satisfies {{$title}}Service interface
type {{$title}}Client struct {
{{range $funcKey, $funcValue := $schema.Functions}}// {{$funcKey}}Endpoint provides remote call to {{ToLower $funcKey}} endpoint
	{{$funcKey}}Endpoint endpoint.Endpoint

{{end}}}

// New{{$title}}Client creates a new client for {{$title}}Service
func  New{{$title}}Client(proxies []string, ctx context.Context, maxAttempt int, maxTime time.Duration, qps int, logger log.Logger) *{{$title}}Client {
return &{{$title}}Client{
{{range $funcKey, $funcValue := $schema.Functions}}
{{$funcKey}}Endpoint : new{{$funcKey}}ClientEndpoint(proxies, ctx, maxAttempt, maxTime, qps, logger),{{end}}
}
}

{{range $funcKey, $funcValue := $schema.Functions}}
{{AsComment $funcValue.Description}}func ({{Pointerize $title}} *{{$title}}Client) {{$funcKey}}(ctx context.Context, req *{{Argumentize $funcValue.Properties.incoming}}) (*{{Argumentize $funcValue.Properties.outgoing}}, error) {
	res, err := {{Pointerize $title}}.{{$funcKey}}Endpoint(ctx, req)
	if err != nil {
		return nil, err
	}

	return res.(*{{Argumentize $funcValue.Properties.outgoing}}), nil
}
{{end}}

// Client Endpoint functions
{{range $funcKey, $funcValue := $schema.Functions}}
func new{{$funcKey}}ClientEndpoint(proxies []string, ctx context.Context, maxAttempt int, maxTime time.Duration, qps int, logger log.Logger) endpoint.Endpoint {
	factory := createFactory(ctx, qps, make{{$funcKey}}Proxy)
	return defaultClientEndpointCreator(proxies, maxAttempt, maxTime, logger, factory)
}
{{end}}

{{range $funcKey, $funcValue := $schema.Functions}}
func make{{$funcKey}}Proxy(ctx context.Context, instance string) endpoint.Endpoint {
	return httptransport.NewClient(
		"POST",
		createProxyURL(instance, "{{ToLower $funcKey}}"),
		encodeRequest,
		decode{{$funcKey}}Response,
	).Endpoint()
}
{{end}}

// Proxy functions

func createProxyURL(instance, endpoint string) *url.URL {
	if !strings.HasPrefix(instance, "http") {
		instance = "http://" + instance
	}
	u, err := url.Parse(instance)
	if err != nil {
		panic(err)
	}
	if u.Path == "" {
		u.Path = endpoint
	}

	return u
}

type proxyFunc func(context.Context, string) endpoint.Endpoint

func createFactory(ctx context.Context, qps int, pf proxyFunc) loadbalancer.Factory {
	return func(instance string) (endpoint.Endpoint, io.Closer, error) {
		var e endpoint.Endpoint
		e = pf(ctx, instance)
		e = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{}))(e)
		e = kitratelimit.NewTokenBucketLimiter(jujuratelimit.NewBucketWithRate(float64(qps), int64(qps)))(e)
		return e, nil, nil
	}
}

func defaultClientEndpointCreator(
	proxies []string,
	maxAttempts int,
	maxTime time.Duration,
	logger log.Logger,
	factory loadbalancer.Factory,
) endpoint.Endpoint {

	publisher := static.NewPublisher(
		proxies,
		factory,
		logger,
	)

	lb := loadbalancer.NewRoundRobin(publisher)
	return loadbalancer.Retry(maxAttempts, maxTime, lb)
}

`

// TransportHTTPSemioticsTemplate
var TransportHTTPSemioticsTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
)

// Decode Request functions

{{range $funcKey, $funcValue := $schema.Functions}}
func decode{{$funcKey}}Request(r *http.Request) (interface{}, error) {
	var req {{Argumentize $funcValue.Properties.incoming}}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return nil, err
	}
	return &req, nil
}
{{end}}

// Decode Response functions

{{range $funcKey, $funcValue := $schema.Functions}}
func decode{{$funcKey}}Response(r *http.Response) (interface{}, error) {
	var res {{Argumentize $funcValue.Properties.incoming}}
	if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
		return nil, err
	}
	return &res, nil
}
{{end}}

// Encode request function

func encodeRequest(r *http.Request, request interface{}) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(request); err != nil {
		return err
	}
	r.Body = ioutil.NopCloser(&buf)
	return nil
}

// Encode response function

func encodeResponse(rw http.ResponseWriter, response interface{}) error {
	return json.NewEncoder(rw).Encode(response)
}
`
<|endoftext|>


<commit_before>package config

type RawGit struct {
	*RawGitExport        `yaml:",inline"`
	As                   string                `yaml:"as,omitempty"`
	Url                  string                `yaml:"url,omitempty"`
	Branch               string                `yaml:"branch,omitempty"`
	Commit               string                `yaml:"commit,omitempty"`
	RawStageDependencies *RawStageDependencies `yaml:"stageDependencies,omitempty"`

	RawDimg *RawDimg `yaml:"-"` // parent

	UnsupportedAttributes map[string]interface{} `yaml:",inline"`
}

func (c *RawGit) Type() string {
	if c.Url != "" {
		return "remote"
	}
	return "local"
}

func (c *RawGit) UnmarshalYAML(unmarshal func(interface{}) error) error {
	if parent, ok := ParentStack.Peek().(*RawDimg); ok {
		c.RawDimg = parent
	}

	ParentStack.Push(c)
	type plain RawGit
	err := unmarshal((*plain)(c))
	ParentStack.Pop()
	if err != nil {
		return err
	}

	if err := CheckOverflow(c.UnsupportedAttributes, c); err != nil {
		return err
	}

	return nil
}

func (c *RawGit) ToGitLocalDirective() (gitLocal *GitLocal, err error) {
	gitLocal = &GitLocal{}

	if gitExport, err := c.RawGitExport.ToDirective(); err != nil {
		return nil, err
	} else {
		gitLocal.GitExport = gitExport
	}

	if c.RawStageDependencies != nil {
		if stageDependencies, err := c.RawStageDependencies.ToDirective(); err != nil {
			return nil, err
		} else {
			gitLocal.StageDependencies = stageDependencies
		}
	}

	gitLocal.As = c.As

	gitLocal.Raw = c

	if err := c.ValidateGitLocalDirective(gitLocal); err != nil {
		return nil, err
	}

	return gitLocal, nil
}

func (c *RawGit) ValidateGitLocalDirective(gitLocal *GitLocal) (err error) {
	if err := gitLocal.Validate(); err != nil {
		return err
	}

	return nil
}

func (c *RawGit) ToGitRemoteDirective() (gitRemote *GitRemote, err error) {
	gitRemote = &GitRemote{}

	if gitLocal, err := c.ToGitLocalDirective(); err != nil {
		return nil, err
	} else {
		gitRemote.GitLocal = gitLocal
	}

	gitRemote.Branch = c.Branch
	gitRemote.Commit = c.Commit
	gitRemote.Url = c.Url
	// TODO: gitRemote.Name = вычленить имя из c.Url

	gitRemote.Raw = c

	if err := c.ValidateGitRemoteDirective(gitRemote); err != nil {
		return nil, err
	}

	return gitRemote, nil
}

func (c *RawGit) ValidateGitRemoteDirective(gitRemote *GitRemote) (err error) {
	if err := gitRemote.Validate(); err != nil {
		return err
	}

	return nil
}
<commit_msg>fix panic "Option ,inline needs a struct value field"<commit_after>package config

type RawGit struct {
	RawGitExport         `yaml:",inline"`
	As                   string                `yaml:"as,omitempty"`
	Url                  string                `yaml:"url,omitempty"`
	Branch               string                `yaml:"branch,omitempty"`
	Commit               string                `yaml:"commit,omitempty"`
	RawStageDependencies *RawStageDependencies `yaml:"stageDependencies,omitempty"`

	RawDimg *RawDimg `yaml:"-"` // parent

	UnsupportedAttributes map[string]interface{} `yaml:",inline"`
}

func (c *RawGit) Type() string {
	if c.Url != "" {
		return "remote"
	}
	return "local"
}

func (c *RawGit) UnmarshalYAML(unmarshal func(interface{}) error) error {
	if parent, ok := ParentStack.Peek().(*RawDimg); ok {
		c.RawDimg = parent
	}

	ParentStack.Push(c)
	type plain RawGit
	err := unmarshal((*plain)(c))
	ParentStack.Pop()
	if err != nil {
		return err
	}

	if err := CheckOverflow(c.UnsupportedAttributes, c); err != nil {
		return err
	}

	return nil
}

func (c *RawGit) ToGitLocalDirective() (gitLocal *GitLocal, err error) {
	gitLocal = &GitLocal{}

	if gitExport, err := c.RawGitExport.ToDirective(); err != nil {
		return nil, err
	} else {
		gitLocal.GitExport = gitExport
	}

	if c.RawStageDependencies != nil {
		if stageDependencies, err := c.RawStageDependencies.ToDirective(); err != nil {
			return nil, err
		} else {
			gitLocal.StageDependencies = stageDependencies
		}
	}

	gitLocal.As = c.As

	gitLocal.Raw = c

	if err := c.ValidateGitLocalDirective(gitLocal); err != nil {
		return nil, err
	}

	return gitLocal, nil
}

func (c *RawGit) ValidateGitLocalDirective(gitLocal *GitLocal) (err error) {
	if err := gitLocal.Validate(); err != nil {
		return err
	}

	return nil
}

func (c *RawGit) ToGitRemoteDirective() (gitRemote *GitRemote, err error) {
	gitRemote = &GitRemote{}

	if gitLocal, err := c.ToGitLocalDirective(); err != nil {
		return nil, err
	} else {
		gitRemote.GitLocal = gitLocal
	}

	gitRemote.Branch = c.Branch
	gitRemote.Commit = c.Commit
	gitRemote.Url = c.Url
	// TODO: gitRemote.Name = вычленить имя из c.Url

	gitRemote.Raw = c

	if err := c.ValidateGitRemoteDirective(gitRemote); err != nil {
		return nil, err
	}

	return gitRemote, nil
}

func (c *RawGit) ValidateGitRemoteDirective(gitRemote *GitRemote) (err error) {
	if err := gitRemote.Validate(); err != nil {
		return err
	}

	return nil
}
<|endoftext|>