<commit_before>package plugin

import (
	"fmt"
	"net"
	osexec "os/exec"
	"strings"
	"time"

	log "github.com/golang/glog"

	"github.com/openshift/origin/pkg/sdn/plugin/cniserver"

	osclient "github.com/openshift/origin/pkg/client"
	osapi "github.com/openshift/origin/pkg/sdn/api"
	"github.com/openshift/origin/pkg/util/ipcmd"
	"github.com/openshift/origin/pkg/util/netutils"
	"github.com/openshift/origin/pkg/util/ovs"

	docker "github.com/fsouza/go-dockerclient"

	kapi "k8s.io/kubernetes/pkg/api"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	knetwork "k8s.io/kubernetes/pkg/kubelet/network"
	"k8s.io/kubernetes/pkg/labels"
	kexec "k8s.io/kubernetes/pkg/util/exec"
	kubeutilnet "k8s.io/kubernetes/pkg/util/net"
	kwait "k8s.io/kubernetes/pkg/util/wait"
)

type OsdnNode struct {
	multitenant        bool
	kClient            *kclient.Client
	osClient           *osclient.Client
	ovs                *ovs.Interface
	networkInfo        *NetworkInfo
	podManager         *podManager
	localSubnetCIDR    string
	localIP            string
	hostName           string
	podNetworkReady    chan struct{}
	kubeletInitReady   chan struct{}
	vnids              *nodeVNIDMap
	iptablesSyncPeriod time.Duration
	mtu                uint32
	egressPolicies     map[uint32][]*osapi.EgressNetworkPolicy

	host             knetwork.Host
	kubeletCniPlugin knetwork.NetworkPlugin

	clearLbr0IptablesRule bool
}

// Called by higher layers to create the plugin SDN node instance
func NewNodePlugin(pluginName string, osClient *osclient.Client, kClient *kclient.Client, hostname string, selfIP string, iptablesSyncPeriod time.Duration, mtu uint32) (*OsdnNode, error) {
	if !osapi.IsOpenShiftNetworkPlugin(pluginName) {
		return nil, nil
	}

	log.Infof("Initializing SDN node of type %q with configured hostname %q (IP %q), iptables sync period %q", pluginName, hostname, selfIP, iptablesSyncPeriod.String())
	if hostname == "" {
		output, err := kexec.New().Command("uname", "-n").CombinedOutput()
		if err != nil {
			return nil, err
		}
		hostname = strings.TrimSpace(string(output))
		log.Infof("Resolved hostname to %q", hostname)
	}
	if selfIP == "" {
		var err error
		selfIP, err = netutils.GetNodeIP(hostname)
		if err != nil {
			log.V(5).Infof("Failed to determine node address from hostname %s; using default interface (%v)", hostname, err)
			var defaultIP net.IP
			defaultIP, err = kubeutilnet.ChooseHostInterface()
			if err != nil {
				return nil, err
			}
			selfIP = defaultIP.String()
			log.Infof("Resolved IP address to %q", selfIP)
		}
	}

	ovsif, err := ovs.New(kexec.New(), BR)
	if err != nil {
		return nil, err
	}

	plugin := &OsdnNode{
		multitenant:        osapi.IsOpenShiftMultitenantNetworkPlugin(pluginName),
		kClient:            kClient,
		osClient:           osClient,
		ovs:                ovsif,
		localIP:            selfIP,
		hostName:           hostname,
		vnids:              newNodeVNIDMap(),
		podNetworkReady:    make(chan struct{}),
		kubeletInitReady:   make(chan struct{}),
		iptablesSyncPeriod: iptablesSyncPeriod,
		mtu:                mtu,
		egressPolicies:     make(map[uint32][]*osapi.EgressNetworkPolicy),
	}

	if err := plugin.dockerPreCNICleanup(); err != nil {
		return nil, err
	}

	return plugin, nil
}

// Detect whether we are upgrading from a pre-CNI openshift and clean up
// interfaces and iptables rules that are no longer required
func (node *OsdnNode) dockerPreCNICleanup() error {
	exec := kexec.New()
	itx := ipcmd.NewTransaction(exec, "lbr0")
	itx.SetLink("down")
	if err := itx.EndTransaction(); err != nil {
		// no cleanup required
		return nil
	}

	node.clearLbr0IptablesRule = true

	// Restart docker to kill old pods and make it use docker0 again.
	// "systemctl restart" will bail out (unnecessarily) in the
	// OpenShift-in-a-container case, so we work around that by sending
	// the messages by hand.
	if err := osexec.Command("dbus-send", "--system", "--print-reply", "--reply-timeout=2000", "--type=method_call", "--dest=org.freedesktop.systemd1", "/org/freedesktop/systemd1", "org.freedesktop.systemd1.Manager.Reload"); err != nil {
		log.Error(err)
	}
	if err := osexec.Command("dbus-send", "--system", "--print-reply", "--reply-timeout=2000", "--type=method_call", "--dest=org.freedesktop.systemd1", "/org/freedesktop/systemd1", "org.freedesktop.systemd1.Manager.RestartUnit", "string:'docker.service' string:'replace'"); err != nil {
		log.Error(err)
	}

	// Delete pre-CNI interfaces
	for _, intf := range []string{"lbr0", "vovsbr", "vlinuxbr"} {
		itx := ipcmd.NewTransaction(exec, intf)
		itx.DeleteLink()
		itx.IgnoreError()
		itx.EndTransaction()
	}

	// Wait until docker has restarted since kubelet will exit it docker isn't running
	dockerClient, err := docker.NewClientFromEnv()
	if err != nil {
		return fmt.Errorf("failed to get docker client: %v", err)
	}
	err = kwait.ExponentialBackoff(
		kwait.Backoff{
			Duration: 100 * time.Millisecond,
			Factor:   1.2,
			Steps:    6,
		},
		func() (bool, error) {
			if err := dockerClient.Ping(); err != nil {
				// wait longer
				return false, nil
			}
			return true, nil
		})
	if err != nil {
		return fmt.Errorf("failed to connect to docker after SDN cleanup restart: %v", err)
	}

	log.Infof("Cleaned up left-over openshift-sdn docker bridge and interfaces")

	return nil
}

func (node *OsdnNode) Start() error {
	var err error
	node.networkInfo, err = getNetworkInfo(node.osClient)
	if err != nil {
		return fmt.Errorf("Failed to get network information: %v", err)
	}

	nodeIPTables := newNodeIPTables(node.networkInfo.ClusterNetwork.String(), node.iptablesSyncPeriod)
	if err = nodeIPTables.Setup(); err != nil {
		return fmt.Errorf("Failed to set up iptables: %v", err)
	}

	node.localSubnetCIDR, err = node.getLocalSubnet()
	if err != nil {
		return err
	}

	networkChanged, err := node.SetupSDN()
	if err != nil {
		return err
	}

	err = node.SubnetStartNode()
	if err != nil {
		return err
	}

	if node.multitenant {
		if err = node.VnidStartNode(); err != nil {
			return err
		}
		if err = node.SetupEgressNetworkPolicy(); err != nil {
			return err
		}
	}

	log.V(5).Infof("Creating and initializing openshift-sdn pod manager")
	node.podManager, err = newPodManager(node.host, node.multitenant, node.localSubnetCIDR, node.networkInfo, node.kClient, node.vnids, node.mtu)
	if err != nil {
		return err
	}
	if err := node.podManager.Start(cniserver.CNIServerSocketPath); err != nil {
		return err
	}

	// Wait for kubelet to init the plugin so we get a knetwork.Host
	log.V(5).Infof("Waiting for kubelet network plugin initialization")
	<-node.kubeletInitReady

	if networkChanged {
		var pods []kapi.Pod
		pods, err = node.GetLocalPods(kapi.NamespaceAll)
		if err != nil {
			return err
		}
		for _, p := range pods {
			err = node.UpdatePod(p)
			if err != nil {
				log.Warningf("Could not update pod %q: %s", p.Name, err)
			}
		}
	}

	log.V(5).Infof("openshift-sdn network plugin ready")
	node.markPodNetworkReady()

	return nil
}

// FIXME: this should eventually go into kubelet via a CNI UPDATE/CHANGE action
// See https://github.com/containernetworking/cni/issues/89
func (node *OsdnNode) UpdatePod(pod kapi.Pod) error {
	req := &cniserver.PodRequest{
		Command:      cniserver.CNI_UPDATE,
		PodNamespace: pod.Namespace,
		PodName:      pod.Name,
		ContainerId:  getPodContainerID(&pod),
		// netns is read from docker if needed, since we don't get it from kubelet
		Result: make(chan *cniserver.PodResult),
	}

	// Send request and wait for the result
	_, err := node.podManager.handleCNIRequest(req)
	return err
}

func (node *OsdnNode) GetLocalPods(namespace string) ([]kapi.Pod, error) {
	fieldSelector := fields.Set{"spec.nodeName": node.hostName}.AsSelector()
	opts := kapi.ListOptions{
		LabelSelector: labels.Everything(),
		FieldSelector: fieldSelector,
	}
	podList, err := node.kClient.Pods(namespace).List(opts)
	if err != nil {
		return nil, err
	}

	// Filter running pods
	pods := make([]kapi.Pod, 0, len(podList.Items))
	for _, pod := range podList.Items {
		if pod.Status.Phase == kapi.PodRunning {
			pods = append(pods, pod)
		}
	}
	return pods, nil
}

func (node *OsdnNode) markPodNetworkReady() {
	close(node.podNetworkReady)
}

func (node *OsdnNode) IsPodNetworkReady() error {
	select {
	case <-node.podNetworkReady:
		return nil
	default:
		return fmt.Errorf("SDN pod network is not ready")
	}
}
<commit_msg>Fix ordering of SDN startup threads<commit_after>package plugin

import (
	"fmt"
	"net"
	osexec "os/exec"
	"strings"
	"time"

	log "github.com/golang/glog"

	"github.com/openshift/origin/pkg/sdn/plugin/cniserver"

	osclient "github.com/openshift/origin/pkg/client"
	osapi "github.com/openshift/origin/pkg/sdn/api"
	"github.com/openshift/origin/pkg/util/ipcmd"
	"github.com/openshift/origin/pkg/util/netutils"
	"github.com/openshift/origin/pkg/util/ovs"

	docker "github.com/fsouza/go-dockerclient"

	kapi "k8s.io/kubernetes/pkg/api"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	knetwork "k8s.io/kubernetes/pkg/kubelet/network"
	"k8s.io/kubernetes/pkg/labels"
	kexec "k8s.io/kubernetes/pkg/util/exec"
	kubeutilnet "k8s.io/kubernetes/pkg/util/net"
	kwait "k8s.io/kubernetes/pkg/util/wait"
)

type OsdnNode struct {
	multitenant        bool
	kClient            *kclient.Client
	osClient           *osclient.Client
	ovs                *ovs.Interface
	networkInfo        *NetworkInfo
	podManager         *podManager
	localSubnetCIDR    string
	localIP            string
	hostName           string
	podNetworkReady    chan struct{}
	kubeletInitReady   chan struct{}
	vnids              *nodeVNIDMap
	iptablesSyncPeriod time.Duration
	mtu                uint32
	egressPolicies     map[uint32][]*osapi.EgressNetworkPolicy

	host             knetwork.Host
	kubeletCniPlugin knetwork.NetworkPlugin

	clearLbr0IptablesRule bool
}

// Called by higher layers to create the plugin SDN node instance
func NewNodePlugin(pluginName string, osClient *osclient.Client, kClient *kclient.Client, hostname string, selfIP string, iptablesSyncPeriod time.Duration, mtu uint32) (*OsdnNode, error) {
	if !osapi.IsOpenShiftNetworkPlugin(pluginName) {
		return nil, nil
	}

	log.Infof("Initializing SDN node of type %q with configured hostname %q (IP %q), iptables sync period %q", pluginName, hostname, selfIP, iptablesSyncPeriod.String())
	if hostname == "" {
		output, err := kexec.New().Command("uname", "-n").CombinedOutput()
		if err != nil {
			return nil, err
		}
		hostname = strings.TrimSpace(string(output))
		log.Infof("Resolved hostname to %q", hostname)
	}
	if selfIP == "" {
		var err error
		selfIP, err = netutils.GetNodeIP(hostname)
		if err != nil {
			log.V(5).Infof("Failed to determine node address from hostname %s; using default interface (%v)", hostname, err)
			var defaultIP net.IP
			defaultIP, err = kubeutilnet.ChooseHostInterface()
			if err != nil {
				return nil, err
			}
			selfIP = defaultIP.String()
			log.Infof("Resolved IP address to %q", selfIP)
		}
	}

	ovsif, err := ovs.New(kexec.New(), BR)
	if err != nil {
		return nil, err
	}

	plugin := &OsdnNode{
		multitenant:        osapi.IsOpenShiftMultitenantNetworkPlugin(pluginName),
		kClient:            kClient,
		osClient:           osClient,
		ovs:                ovsif,
		localIP:            selfIP,
		hostName:           hostname,
		vnids:              newNodeVNIDMap(),
		podNetworkReady:    make(chan struct{}),
		kubeletInitReady:   make(chan struct{}),
		iptablesSyncPeriod: iptablesSyncPeriod,
		mtu:                mtu,
		egressPolicies:     make(map[uint32][]*osapi.EgressNetworkPolicy),
	}

	if err := plugin.dockerPreCNICleanup(); err != nil {
		return nil, err
	}

	return plugin, nil
}

// Detect whether we are upgrading from a pre-CNI openshift and clean up
// interfaces and iptables rules that are no longer required
func (node *OsdnNode) dockerPreCNICleanup() error {
	exec := kexec.New()
	itx := ipcmd.NewTransaction(exec, "lbr0")
	itx.SetLink("down")
	if err := itx.EndTransaction(); err != nil {
		// no cleanup required
		return nil
	}

	node.clearLbr0IptablesRule = true

	// Restart docker to kill old pods and make it use docker0 again.
	// "systemctl restart" will bail out (unnecessarily) in the
	// OpenShift-in-a-container case, so we work around that by sending
	// the messages by hand.
	if err := osexec.Command("dbus-send", "--system", "--print-reply", "--reply-timeout=2000", "--type=method_call", "--dest=org.freedesktop.systemd1", "/org/freedesktop/systemd1", "org.freedesktop.systemd1.Manager.Reload"); err != nil {
		log.Error(err)
	}
	if err := osexec.Command("dbus-send", "--system", "--print-reply", "--reply-timeout=2000", "--type=method_call", "--dest=org.freedesktop.systemd1", "/org/freedesktop/systemd1", "org.freedesktop.systemd1.Manager.RestartUnit", "string:'docker.service' string:'replace'"); err != nil {
		log.Error(err)
	}

	// Delete pre-CNI interfaces
	for _, intf := range []string{"lbr0", "vovsbr", "vlinuxbr"} {
		itx := ipcmd.NewTransaction(exec, intf)
		itx.DeleteLink()
		itx.IgnoreError()
		itx.EndTransaction()
	}

	// Wait until docker has restarted since kubelet will exit it docker isn't running
	dockerClient, err := docker.NewClientFromEnv()
	if err != nil {
		return fmt.Errorf("failed to get docker client: %v", err)
	}
	err = kwait.ExponentialBackoff(
		kwait.Backoff{
			Duration: 100 * time.Millisecond,
			Factor:   1.2,
			Steps:    6,
		},
		func() (bool, error) {
			if err := dockerClient.Ping(); err != nil {
				// wait longer
				return false, nil
			}
			return true, nil
		})
	if err != nil {
		return fmt.Errorf("failed to connect to docker after SDN cleanup restart: %v", err)
	}

	log.Infof("Cleaned up left-over openshift-sdn docker bridge and interfaces")

	return nil
}

func (node *OsdnNode) Start() error {
	var err error
	node.networkInfo, err = getNetworkInfo(node.osClient)
	if err != nil {
		return fmt.Errorf("Failed to get network information: %v", err)
	}

	nodeIPTables := newNodeIPTables(node.networkInfo.ClusterNetwork.String(), node.iptablesSyncPeriod)
	if err = nodeIPTables.Setup(); err != nil {
		return fmt.Errorf("Failed to set up iptables: %v", err)
	}

	node.localSubnetCIDR, err = node.getLocalSubnet()
	if err != nil {
		return err
	}

	networkChanged, err := node.SetupSDN()
	if err != nil {
		return err
	}

	err = node.SubnetStartNode()
	if err != nil {
		return err
	}

	if node.multitenant {
		if err = node.VnidStartNode(); err != nil {
			return err
		}
		if err = node.SetupEgressNetworkPolicy(); err != nil {
			return err
		}
	}

	// Wait for kubelet to init the plugin so we get a knetwork.Host
	log.V(5).Infof("Waiting for kubelet network plugin initialization")
	<-node.kubeletInitReady

	log.V(5).Infof("Creating and initializing openshift-sdn pod manager")
	node.podManager, err = newPodManager(node.host, node.multitenant, node.localSubnetCIDR, node.networkInfo, node.kClient, node.vnids, node.mtu)
	if err != nil {
		return err
	}
	if err := node.podManager.Start(cniserver.CNIServerSocketPath); err != nil {
		return err
	}

	if networkChanged {
		var pods []kapi.Pod
		pods, err = node.GetLocalPods(kapi.NamespaceAll)
		if err != nil {
			return err
		}
		for _, p := range pods {
			err = node.UpdatePod(p)
			if err != nil {
				log.Warningf("Could not update pod %q: %s", p.Name, err)
			}
		}
	}

	log.V(5).Infof("openshift-sdn network plugin ready")
	node.markPodNetworkReady()

	return nil
}

// FIXME: this should eventually go into kubelet via a CNI UPDATE/CHANGE action
// See https://github.com/containernetworking/cni/issues/89
func (node *OsdnNode) UpdatePod(pod kapi.Pod) error {
	req := &cniserver.PodRequest{
		Command:      cniserver.CNI_UPDATE,
		PodNamespace: pod.Namespace,
		PodName:      pod.Name,
		ContainerId:  getPodContainerID(&pod),
		// netns is read from docker if needed, since we don't get it from kubelet
		Result: make(chan *cniserver.PodResult),
	}

	// Send request and wait for the result
	_, err := node.podManager.handleCNIRequest(req)
	return err
}

func (node *OsdnNode) GetLocalPods(namespace string) ([]kapi.Pod, error) {
	fieldSelector := fields.Set{"spec.nodeName": node.hostName}.AsSelector()
	opts := kapi.ListOptions{
		LabelSelector: labels.Everything(),
		FieldSelector: fieldSelector,
	}
	podList, err := node.kClient.Pods(namespace).List(opts)
	if err != nil {
		return nil, err
	}

	// Filter running pods
	pods := make([]kapi.Pod, 0, len(podList.Items))
	for _, pod := range podList.Items {
		if pod.Status.Phase == kapi.PodRunning {
			pods = append(pods, pod)
		}
	}
	return pods, nil
}

func (node *OsdnNode) markPodNetworkReady() {
	close(node.podNetworkReady)
}

func (node *OsdnNode) IsPodNetworkReady() error {
	select {
	case <-node.podNetworkReady:
		return nil
	default:
		return fmt.Errorf("SDN pod network is not ready")
	}
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/engine"
	"github.com/Aptomi/aptomi/pkg/engine/apply"
	"github.com/Aptomi/aptomi/pkg/engine/diff"
	"github.com/Aptomi/aptomi/pkg/engine/resolve"
	"github.com/Aptomi/aptomi/pkg/event"
	"github.com/Aptomi/aptomi/pkg/runtime"
	log "github.com/Sirupsen/logrus"
	"time"
)

func logError(err interface{}) {
	log.Errorf("Error while enforcing policy: %s", err)
}

func (server *Server) enforceLoop() error {
	for {
		err := server.enforce()
		if err != nil {
			logError(err)
		}

		// sleep for a specified time or wait until policy has changed, whichever comes first
		timer := time.NewTimer(server.cfg.Enforcer.Interval)
		select {
		case <-server.policyChanged:
			break
		case <-timer.C:
			break
		}
		timer.Stop()
	}
}

func (server *Server) enforce() error {
	server.enforcementIdx++

	defer func() {
		if err := recover(); err != nil {
			logError(err)
		}
	}()

	// todo think about initial state when there is no revision at all
	currRevision, err := server.store.GetRevision(runtime.LastGen)
	if err != nil {
		return fmt.Errorf("unable to get curr revision: %s", err)
	}

	// Mark last Revision as failed if it wasn't completed
	if currRevision != nil && currRevision.Status == engine.RevisionStatusInProgress {
		currRevision.Status = engine.RevisionStatusError
		currRevision.AppliedAt = time.Now()
		revErr := server.store.UpdateRevision(currRevision)
		if revErr != nil {
			log.Warnf("(enforce-%d) Error while setting current revision that is in progress to error state: %s", server.enforcementIdx, revErr)
		}
		log.Infof("(enforce-%d) Current revision that is in progress was reset to error state", server.enforcementIdx)
	}

	desiredPolicy, desiredPolicyGen, err := server.store.GetPolicy(runtime.LastGen)
	if err != nil {
		return fmt.Errorf("error while getting desiredPolicy: %s", err)
	}

	// if policy is not found, it means it somehow was not initialized correctly. let's return error
	if desiredPolicy == nil {
		return fmt.Errorf("desiredPolicy is nil, does not exist in the store")
	}

	actualState, err := server.store.GetActualState()
	if err != nil {
		return fmt.Errorf("error while getting actual state: %s", err)
	}

	resolveLog := event.NewLog(fmt.Sprintf("enforce-%d-resolve", server.enforcementIdx), true)
	resolver := resolve.NewPolicyResolver(desiredPolicy, server.externalData, resolveLog)
	desiredState, err := resolver.ResolveAllDependencies()
	if err != nil {
		server.saveErrRevision(currRevision, desiredPolicyGen, resolveLog)

		return fmt.Errorf("cannot resolve desiredPolicy: %s", err)
	}

	stateDiff := diff.NewPolicyResolutionDiff(desiredState, actualState)

	nextRevision, err := server.store.NewRevision(desiredPolicyGen)
	if err != nil {
		return fmt.Errorf("unable to get next revision: %s", err)
	}
	nextRevision.ResolveLog = resolveLog.AsAPIEvents()

	// policy changed while no actions needed to achieve desired state
	if len(stateDiff.Actions) <= 0 && currRevision != nil && currRevision.Policy == nextRevision.Policy {
		log.Infof("(enforce-%d) No changes, policy gen %d", server.enforcementIdx, desiredPolicyGen)
		return nil
	}
	log.Infof("(enforce-%d) New revision %d, policy gen %d, %d actions need to be applied", server.enforcementIdx, nextRevision.GetGeneration(), desiredPolicyGen, len(stateDiff.Actions))

	// Save revision
	err = server.store.SaveRevision(nextRevision)
	if err != nil {
		return fmt.Errorf("error while saving new revision: %s", err)
	}

	if server.cfg.Enforcer.Noop {
		log.Infof("(enforce-%d) Applying changes in noop mode (sleep per action = %s)", server.enforcementIdx, server.cfg.Enforcer.NoopSleep)
	} else {
		log.Infof("(enforce-%d) Applying changes", server.enforcementIdx)
	}

	pluginRegistry := server.pluginRegistryFactory()
	applyLog := event.NewLog(fmt.Sprintf("enforce-%d-apply", server.enforcementIdx), true)
	applier := apply.NewEngineApply(desiredPolicy, desiredState, actualState, server.store.GetActualStateUpdater(), server.externalData, pluginRegistry, stateDiff.Actions, applyLog, server.store.GetRevisionProgressUpdater(nextRevision))
	_, err = applier.Apply()

	// reload revision to have progress data saved into it
	nextRevision, saveErr := server.store.GetRevision(runtime.LastGen)
	if saveErr != nil {
		return fmt.Errorf("error while reloading last revision to have progress loaded: %s", saveErr)
	}
	nextRevision.ApplyLog = applyLog.AsAPIEvents()

	// save apply log
	saveErr = server.store.UpdateRevision(nextRevision)
	if saveErr != nil {
		return fmt.Errorf("error while saving new revision with apply log: %s", saveErr)
	}

	if err != nil {
		return fmt.Errorf("error while applying new revision: %s", err)
	}
	log.Infof("(enforce-%d) New revision %d successfully applied, %d component instances", server.enforcementIdx, nextRevision.GetGeneration(), len(desiredState.GetComponentProcessingOrder()))

	return nil
}

func (server *Server) saveErrRevision(currRevision *engine.Revision, desiredPolicyGen runtime.Generation, resolveLog *event.Log) {
	if currRevision == nil || currRevision.Policy != desiredPolicyGen || currRevision.Status != engine.RevisionStatusError {
		rev, err := server.store.NewRevision(desiredPolicyGen)
		if err != nil {
			log.Warnf("(enforce-%d) Error while creating revision to record resolution error: %s", server.enforcementIdx, err)
		}

		rev.Status = engine.RevisionStatusError
		rev.ResolveLog = resolveLog.AsAPIEvents()

		err = server.store.SaveRevision(rev)
		if err != nil {
			log.Warnf("(enforce-%d) Error while saving revision to record resolution error: %s", server.enforcementIdx, err)
		}
	}
}
<commit_msg>linter fix (ineffective break statement)<commit_after>package server

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/engine"
	"github.com/Aptomi/aptomi/pkg/engine/apply"
	"github.com/Aptomi/aptomi/pkg/engine/diff"
	"github.com/Aptomi/aptomi/pkg/engine/resolve"
	"github.com/Aptomi/aptomi/pkg/event"
	"github.com/Aptomi/aptomi/pkg/runtime"
	log "github.com/Sirupsen/logrus"
	"time"
)

func logError(err interface{}) {
	log.Errorf("Error while enforcing policy: %s", err)
}

func (server *Server) enforceLoop() error {
	for {
		err := server.enforce()
		if err != nil {
			logError(err)
		}

		// sleep for a specified time or wait until policy has changed, whichever comes first
		timer := time.NewTimer(server.cfg.Enforcer.Interval)
		select {
		case <-server.policyChanged:
			break // nolint: megacheck
		case <-timer.C:
			break // nolint: megacheck
		}
		timer.Stop()
	}
}

func (server *Server) enforce() error {
	server.enforcementIdx++

	defer func() {
		if err := recover(); err != nil {
			logError(err)
		}
	}()

	// todo think about initial state when there is no revision at all
	currRevision, err := server.store.GetRevision(runtime.LastGen)
	if err != nil {
		return fmt.Errorf("unable to get curr revision: %s", err)
	}

	// Mark last Revision as failed if it wasn't completed
	if currRevision != nil && currRevision.Status == engine.RevisionStatusInProgress {
		currRevision.Status = engine.RevisionStatusError
		currRevision.AppliedAt = time.Now()
		revErr := server.store.UpdateRevision(currRevision)
		if revErr != nil {
			log.Warnf("(enforce-%d) Error while setting current revision that is in progress to error state: %s", server.enforcementIdx, revErr)
		}
		log.Infof("(enforce-%d) Current revision that is in progress was reset to error state", server.enforcementIdx)
	}

	desiredPolicy, desiredPolicyGen, err := server.store.GetPolicy(runtime.LastGen)
	if err != nil {
		return fmt.Errorf("error while getting desiredPolicy: %s", err)
	}

	// if policy is not found, it means it somehow was not initialized correctly. let's return error
	if desiredPolicy == nil {
		return fmt.Errorf("desiredPolicy is nil, does not exist in the store")
	}

	actualState, err := server.store.GetActualState()
	if err != nil {
		return fmt.Errorf("error while getting actual state: %s", err)
	}

	resolveLog := event.NewLog(fmt.Sprintf("enforce-%d-resolve", server.enforcementIdx), true)
	resolver := resolve.NewPolicyResolver(desiredPolicy, server.externalData, resolveLog)
	desiredState, err := resolver.ResolveAllDependencies()
	if err != nil {
		server.saveErrRevision(currRevision, desiredPolicyGen, resolveLog)

		return fmt.Errorf("cannot resolve desiredPolicy: %s", err)
	}

	stateDiff := diff.NewPolicyResolutionDiff(desiredState, actualState)

	nextRevision, err := server.store.NewRevision(desiredPolicyGen)
	if err != nil {
		return fmt.Errorf("unable to get next revision: %s", err)
	}
	nextRevision.ResolveLog = resolveLog.AsAPIEvents()

	// policy changed while no actions needed to achieve desired state
	if len(stateDiff.Actions) <= 0 && currRevision != nil && currRevision.Policy == nextRevision.Policy {
		log.Infof("(enforce-%d) No changes, policy gen %d", server.enforcementIdx, desiredPolicyGen)
		return nil
	}
	log.Infof("(enforce-%d) New revision %d, policy gen %d, %d actions need to be applied", server.enforcementIdx, nextRevision.GetGeneration(), desiredPolicyGen, len(stateDiff.Actions))

	// Save revision
	err = server.store.SaveRevision(nextRevision)
	if err != nil {
		return fmt.Errorf("error while saving new revision: %s", err)
	}

	if server.cfg.Enforcer.Noop {
		log.Infof("(enforce-%d) Applying changes in noop mode (sleep per action = %s)", server.enforcementIdx, server.cfg.Enforcer.NoopSleep)
	} else {
		log.Infof("(enforce-%d) Applying changes", server.enforcementIdx)
	}

	pluginRegistry := server.pluginRegistryFactory()
	applyLog := event.NewLog(fmt.Sprintf("enforce-%d-apply", server.enforcementIdx), true)
	applier := apply.NewEngineApply(desiredPolicy, desiredState, actualState, server.store.GetActualStateUpdater(), server.externalData, pluginRegistry, stateDiff.Actions, applyLog, server.store.GetRevisionProgressUpdater(nextRevision))
	_, err = applier.Apply()

	// reload revision to have progress data saved into it
	nextRevision, saveErr := server.store.GetRevision(runtime.LastGen)
	if saveErr != nil {
		return fmt.Errorf("error while reloading last revision to have progress loaded: %s", saveErr)
	}
	nextRevision.ApplyLog = applyLog.AsAPIEvents()

	// save apply log
	saveErr = server.store.UpdateRevision(nextRevision)
	if saveErr != nil {
		return fmt.Errorf("error while saving new revision with apply log: %s", saveErr)
	}

	if err != nil {
		return fmt.Errorf("error while applying new revision: %s", err)
	}
	log.Infof("(enforce-%d) New revision %d successfully applied, %d component instances", server.enforcementIdx, nextRevision.GetGeneration(), len(desiredState.GetComponentProcessingOrder()))

	return nil
}

func (server *Server) saveErrRevision(currRevision *engine.Revision, desiredPolicyGen runtime.Generation, resolveLog *event.Log) {
	if currRevision == nil || currRevision.Policy != desiredPolicyGen || currRevision.Status != engine.RevisionStatusError {
		rev, err := server.store.NewRevision(desiredPolicyGen)
		if err != nil {
			log.Warnf("(enforce-%d) Error while creating revision to record resolution error: %s", server.enforcementIdx, err)
		}

		rev.Status = engine.RevisionStatusError
		rev.ResolveLog = resolveLog.AsAPIEvents()

		err = server.store.SaveRevision(rev)
		if err != nil {
			log.Warnf("(enforce-%d) Error while saving revision to record resolution error: %s", server.enforcementIdx, err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/html"
	"golang.org/x/net/context"
	"google.golang.org/appengine/v2"
	"google.golang.org/appengine/v2/log"
	"google.golang.org/appengine/v2/user"
)

// This file contains common middleware for UI handlers (auth, html templates, etc).

type contextHandler func(c context.Context, w http.ResponseWriter, r *http.Request) error

func handlerWrapper(fn contextHandler) http.Handler {
	return handleContext(handleAuth(fn))
}

func handleContext(fn contextHandler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)
		if err := fn(c, w, r); err != nil {
			hdr := commonHeaderRaw(c, r)
			data := &struct {
				Header *uiHeader
				Error  string
			}{
				Header: hdr,
				Error:  err.Error(),
			}
			if err == ErrAccess {
				if hdr.LoginLink != "" {
					http.Redirect(w, r, hdr.LoginLink, http.StatusTemporaryRedirect)
					return
				}
				http.Error(w, "403 Forbidden", http.StatusForbidden)
				return
			}
			if redir, ok := err.(ErrRedirect); ok {
				http.Redirect(w, r, redir.Error(), http.StatusFound)
				return
			}
			logf := log.Errorf
			if _, dontlog := err.(ErrDontLog); dontlog {
				// We don't log these as errors because they can be provoked
				// by invalid user requests, so we don't wan't to pollute error log.
				logf = log.Warningf
			}
			logf(c, "%v", err)
			w.WriteHeader(http.StatusInternalServerError)
			if err1 := templates.ExecuteTemplate(w, "error.html", data); err1 != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
		}
	})
}

type (
	ErrDontLog  struct{ error }
	ErrRedirect struct{ error }
)

func handleAuth(fn contextHandler) contextHandler {
	return func(c context.Context, w http.ResponseWriter, r *http.Request) error {
		if err := checkAccessLevel(c, r, config.AccessLevel); err != nil {
			return err
		}
		return fn(c, w, r)
	}
}

func serveTemplate(w http.ResponseWriter, name string, data interface{}) error {
	buf := new(bytes.Buffer)
	if err := templates.ExecuteTemplate(buf, name, data); err != nil {
		return err
	}
	w.Write(buf.Bytes())
	return nil
}

type uiHeader struct {
	Admin               bool
	URLPath             string
	LoginLink           string
	AnalyticsTrackingID string
	Subpage             string
	Namespace           string
	Cached              *Cached
	Namespaces          []uiNamespace
}

type uiNamespace struct {
	Name    string
	Caption string
}

type cookieData struct {
	Namespace string `json:"namespace"`
}

func commonHeaderRaw(c context.Context, r *http.Request) *uiHeader {
	h := &uiHeader{
		Admin:               accessLevel(c, r) == AccessAdmin,
		URLPath:             r.URL.Path,
		AnalyticsTrackingID: config.AnalyticsTrackingID,
	}
	if user.Current(c) == nil {
		h.LoginLink, _ = user.LoginURL(c, r.URL.String())
	}
	return h
}

func commonHeader(c context.Context, r *http.Request, w http.ResponseWriter, ns string) (*uiHeader, error) {
	accessLevel := accessLevel(c, r)
	if ns == "" {
		ns = strings.ToLower(r.URL.Path)
		if ns != "" && ns[0] == '/' {
			ns = ns[1:]
		}
		if pos := strings.IndexByte(ns, '/'); pos != -1 {
			ns = ns[:pos]
		}
	}
	h := commonHeaderRaw(c, r)
	const adminPage = "admin"
	isAdminPage := r.URL.Path == "/"+adminPage
	found := false
	for ns1, cfg := range config.Namespaces {
		if accessLevel < cfg.AccessLevel {
			if ns1 == ns {
				return nil, ErrAccess
			}
			continue
		}
		if cfg.Decommissioned {
			continue
		}
		if ns1 == ns {
			found = true
		}
		h.Namespaces = append(h.Namespaces, uiNamespace{
			Name:    ns1,
			Caption: cfg.DisplayTitle,
		})
	}
	sort.Slice(h.Namespaces, func(i, j int) bool {
		return h.Namespaces[i].Caption < h.Namespaces[j].Caption
	})
	cookie := decodeCookie(r)
	if !found {
		ns = config.DefaultNamespace
		if cfg := config.Namespaces[cookie.Namespace]; cfg != nil && cfg.AccessLevel <= accessLevel {
			ns = cookie.Namespace
		}
		if accessLevel == AccessAdmin {
			ns = adminPage
		}
		if ns != adminPage || !isAdminPage {
			return nil, ErrRedirect{fmt.Errorf("/%v", ns)}
		}
	}
	if ns != adminPage {
		h.Namespace = ns
		cookie.Namespace = ns
		encodeCookie(w, cookie)
		cached, err := CacheGet(c, r, ns)
		if err != nil {
			return nil, err
		}
		h.Cached = cached
	}
	return h, nil
}

const cookieName = "syzkaller"

func decodeCookie(r *http.Request) *cookieData {
	cd := new(cookieData)
	cookie, err := r.Cookie(cookieName)
	if err != nil {
		return cd
	}
	decoded, err := base64.StdEncoding.DecodeString(cookie.Value)
	if err != nil {
		return cd
	}
	json.Unmarshal(decoded, cd)
	return cd
}

func encodeCookie(w http.ResponseWriter, cd *cookieData) {
	data, err := json.Marshal(cd)
	if err != nil {
		return
	}
	cookie := &http.Cookie{
		Name:    cookieName,
		Value:   base64.StdEncoding.EncodeToString(data),
		Expires: time.Now().Add(time.Hour * 24 * 365),
	}
	http.SetCookie(w, cookie)
}

var templates = html.CreateGlob("*.html")
<commit_msg>dashboard/app: fix reported error depth (#3138)<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/html"
	"golang.org/x/net/context"
	"google.golang.org/appengine/v2"
	"google.golang.org/appengine/v2/log"
	"google.golang.org/appengine/v2/user"
)

// This file contains common middleware for UI handlers (auth, html templates, etc).

type contextHandler func(c context.Context, w http.ResponseWriter, r *http.Request) error

func handlerWrapper(fn contextHandler) http.Handler {
	return handleContext(handleAuth(fn))
}

func handleContext(fn contextHandler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)
		if err := fn(c, w, r); err != nil {
			hdr := commonHeaderRaw(c, r)
			data := &struct {
				Header *uiHeader
				Error  string
			}{
				Header: hdr,
				Error:  err.Error(),
			}
			if err == ErrAccess {
				if hdr.LoginLink != "" {
					http.Redirect(w, r, hdr.LoginLink, http.StatusTemporaryRedirect)
					return
				}
				http.Error(w, "403 Forbidden", http.StatusForbidden)
				return
			}
			if redir, ok := err.(ErrRedirect); ok {
				http.Redirect(w, r, redir.Error(), http.StatusFound)
				return
			}
			logf := log.Errorf
			if _, dontlog := err.(ErrDontLog); dontlog {
				// We don't log these as errors because they can be provoked
				// by invalid user requests, so we don't wan't to pollute error log.
				logf = log.Warningf
			}
			logf(c, "%v", err)
			w.WriteHeader(http.StatusInternalServerError)
			if err1 := templates.ExecuteTemplate(w, "error.html", data); err1 != nil {
				combinedError := fmt.Sprintf("got err \"%v\" processing ExecuteTemplate() for err \"%v\"", err1, err)
				http.Error(w, combinedError, http.StatusInternalServerError)
			}
		}
	})
}

type (
	ErrDontLog  struct{ error }
	ErrRedirect struct{ error }
)

func handleAuth(fn contextHandler) contextHandler {
	return func(c context.Context, w http.ResponseWriter, r *http.Request) error {
		if err := checkAccessLevel(c, r, config.AccessLevel); err != nil {
			return err
		}
		return fn(c, w, r)
	}
}

func serveTemplate(w http.ResponseWriter, name string, data interface{}) error {
	buf := new(bytes.Buffer)
	if err := templates.ExecuteTemplate(buf, name, data); err != nil {
		return err
	}
	w.Write(buf.Bytes())
	return nil
}

type uiHeader struct {
	Admin               bool
	URLPath             string
	LoginLink           string
	AnalyticsTrackingID string
	Subpage             string
	Namespace           string
	Cached              *Cached
	Namespaces          []uiNamespace
}

type uiNamespace struct {
	Name    string
	Caption string
}

type cookieData struct {
	Namespace string `json:"namespace"`
}

func commonHeaderRaw(c context.Context, r *http.Request) *uiHeader {
	h := &uiHeader{
		Admin:               accessLevel(c, r) == AccessAdmin,
		URLPath:             r.URL.Path,
		AnalyticsTrackingID: config.AnalyticsTrackingID,
	}
	if user.Current(c) == nil {
		h.LoginLink, _ = user.LoginURL(c, r.URL.String())
	}
	return h
}

func commonHeader(c context.Context, r *http.Request, w http.ResponseWriter, ns string) (*uiHeader, error) {
	accessLevel := accessLevel(c, r)
	if ns == "" {
		ns = strings.ToLower(r.URL.Path)
		if ns != "" && ns[0] == '/' {
			ns = ns[1:]
		}
		if pos := strings.IndexByte(ns, '/'); pos != -1 {
			ns = ns[:pos]
		}
	}
	h := commonHeaderRaw(c, r)
	const adminPage = "admin"
	isAdminPage := r.URL.Path == "/"+adminPage
	found := false
	for ns1, cfg := range config.Namespaces {
		if accessLevel < cfg.AccessLevel {
			if ns1 == ns {
				return nil, ErrAccess
			}
			continue
		}
		if cfg.Decommissioned {
			continue
		}
		if ns1 == ns {
			found = true
		}
		h.Namespaces = append(h.Namespaces, uiNamespace{
			Name:    ns1,
			Caption: cfg.DisplayTitle,
		})
	}
	sort.Slice(h.Namespaces, func(i, j int) bool {
		return h.Namespaces[i].Caption < h.Namespaces[j].Caption
	})
	cookie := decodeCookie(r)
	if !found {
		ns = config.DefaultNamespace
		if cfg := config.Namespaces[cookie.Namespace]; cfg != nil && cfg.AccessLevel <= accessLevel {
			ns = cookie.Namespace
		}
		if accessLevel == AccessAdmin {
			ns = adminPage
		}
		if ns != adminPage || !isAdminPage {
			return nil, ErrRedirect{fmt.Errorf("/%v", ns)}
		}
	}
	if ns != adminPage {
		h.Namespace = ns
		cookie.Namespace = ns
		encodeCookie(w, cookie)
		cached, err := CacheGet(c, r, ns)
		if err != nil {
			return nil, err
		}
		h.Cached = cached
	}
	return h, nil
}

const cookieName = "syzkaller"

func decodeCookie(r *http.Request) *cookieData {
	cd := new(cookieData)
	cookie, err := r.Cookie(cookieName)
	if err != nil {
		return cd
	}
	decoded, err := base64.StdEncoding.DecodeString(cookie.Value)
	if err != nil {
		return cd
	}
	json.Unmarshal(decoded, cd)
	return cd
}

func encodeCookie(w http.ResponseWriter, cd *cookieData) {
	data, err := json.Marshal(cd)
	if err != nil {
		return
	}
	cookie := &http.Cookie{
		Name:    cookieName,
		Value:   base64.StdEncoding.EncodeToString(data),
		Expires: time.Now().Add(time.Hour * 24 * 365),
	}
	http.SetCookie(w, cookie)
}

var templates = html.CreateGlob("*.html")
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scorecard

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/forseti-security/config-validator/pkg/api/validator"
	"github.com/forseti-security/config-validator/pkg/gcv"
	"github.com/pkg/errors"
)

// ScoringConfig holds settings for generating a score
type ScoringConfig struct {
	PolicyPath  string                           // the directory path of a policy library to use
	categories  map[string]*constraintCategory   // available constraint categories
	constraints map[string]*constraintViolations // a map of constraints violated and their violations
	validator   *gcv.Validator                   // the validator instance used for scoring
}

// NewScoringConfig creates a scoring engine for the given policy library
func NewScoringConfig(policyPath string) (*ScoringConfig, error) {
	config := &ScoringConfig{}

	config.PolicyPath = policyPath

	v, err := gcv.NewValidator(
		gcv.PolicyPath(filepath.Join(config.PolicyPath, "policies")),
		gcv.PolicyLibraryDir(filepath.Join(config.PolicyPath, "lib")),
	)
	if err != nil {
		return nil, errors.Wrap(err, "initializing gcv validator")
	}
	config.validator = v

	return config, nil
}

const otherCategoryKey = "other"

// constraintCategory holds constraints by category
type constraintCategory struct {
	Name        string
	constraints []*constraintViolations
}

func (c constraintCategory) Count() int {
	sum := 0
	for _, cv := range c.constraints {
		sum += cv.Count()
	}
	return sum
}

// constraintViolations holds violations for a particular constraint
type constraintViolations struct {
	constraint *validator.Constraint
	Violations []*validator.Violation `protobuf:"bytes,1,rep,name=violations,proto3" json:"violations,omitempty"`
}

func (cv constraintViolations) Count() int {
	return len(cv.Violations)
}

func (cv constraintViolations) GetName() string {
	return cv.constraint.GetMetadata().GetStructValue().GetFields()["name"].GetStringValue()
}

var availableCategories = map[string]string{
	"operational-efficiency": "Operational Efficiency",
	"security":               "Security",
	"reliability":            "Reliability",
	otherCategoryKey:         "Other",
}

func (config *ScoringConfig) getConstraintForViolation(violation *validator.Violation) (*constraintViolations, error) {
	key := violation.GetConstraint()
	cv, found := config.constraints[key]
	if !found {
		constraint := violation.GetConstraintConfig()
		cv = &constraintViolations{
			constraint: constraint,
		}
		config.constraints[key] = cv

		metadata := constraint.GetMetadata()
		annotations := metadata.GetStructValue().GetFields()["annotations"].GetStructValue().GetFields()

		categoryKey := otherCategoryKey
		categoryValue, found := annotations["bundles.validator.forsetisecurity.org/scorecard-v1"]
		if found {
			categoryKey = categoryValue.GetStringValue()
		}

		category, found := config.categories[categoryKey]
		if !found {
			return nil, fmt.Errorf("Unknown constraint category %v for constraint %v", categoryKey, key)
		}
		category.constraints = append(category.constraints, cv)
	}
	return cv, nil
}

// attachViolations puts violations into their appropriate categories
func (config *ScoringConfig) attachViolations(audit *validator.AuditResponse) error {
	// Build map of categories
	config.categories = make(map[string]*constraintCategory)
	for k, name := range availableCategories {
		config.categories[k] = &constraintCategory{
			Name: name,
		}
	}

	// Categorize violations
	config.constraints = make(map[string]*constraintViolations)
	for _, v := range audit.Violations {
		cv, err := config.getConstraintForViolation(v)
		if err != nil {
			return errors.Wrap(err, "Categorizing violation")
		}

		cv.Violations = append(cv.Violations, v)
	}

	return nil
}

// Score creates a Scorecard for an inventory
func (inventory *InventoryConfig) Score(config *ScoringConfig, outputPath string, outputFormat string) error {
	auditResult, err := getViolations(inventory, config)
	if err != nil {
		return err
	}

	err = config.attachViolations(auditResult)
	var dest io.Writer

	if len(auditResult.Violations) > 0 {
		if outputPath == "" {
			dest = os.Stdout
		} else {
			outputFile := "scorecard." + outputFormat
			dest, err = os.Create(filepath.Join(outputPath, outputFile))
			if err != nil {
				return err
			}
		}
		switch outputFormat {
		case "json":
			type violationOutput struct {
				Category   string
				Constraint string
				Resource   string
				Message    string
			}
			var vOutput violationOutput
			for _, category := range config.categories {
				for _, cv := range category.constraints {
					for _, v := range cv.Violations {
						vOutput.Category = category.Name
						vOutput.Constraint = v.Constraint
						vOutput.Resource = v.Resource
						vOutput.Message = v.Message
						byteContent, err := json.MarshalIndent(vOutput, "", "  ")
						if err != nil {
							return err
						}
						io.WriteString(dest, string(byteContent)+"\n")
						Log.Debug("Violation metadata", "metadata", v.GetMetadata())
					}
				}
			}
		case "csv":
			io.WriteString(dest, "Category,Constraint,Resource,Message\n")
			for _, category := range config.categories {
				for _, cv := range category.constraints {
					for _, v := range cv.Violations {
						io.WriteString(dest, fmt.Sprintf("%v,%v,%v,%v\n", category.Name, v.Constraint, v.Resource, v.Message))
						Log.Debug("Violation metadata", "metadata", v.GetMetadata())
					}
				}
			}
		case "txt":
			io.WriteString(dest, fmt.Sprintf("\n\n%v total issues found\n", len(auditResult.Violations)))
			for _, category := range config.categories {
				io.WriteString(dest, fmt.Sprintf("\n\n%v: %v issues found\n", category.Name, category.Count()))
				io.WriteString(dest, fmt.Sprintf("----------\n"))
				for _, cv := range category.constraints {
					io.WriteString(dest, fmt.Sprintf("%v: %v issues\n", cv.GetName(), cv.Count()))
					for _, v := range cv.Violations {
						io.WriteString(dest, fmt.Sprintf("- %v\n\n",
							v.Message,
						))
						Log.Debug("Violation metadata", "metadata", v.GetMetadata())
					}
				}
			}
		default:
			return fmt.Errorf("Unsupported output format %v", outputFormat)
		}
	} else {
		fmt.Println("No issues found found! You have a perfect score.")
	}

	return nil
}
<commit_msg>RichViolation struct<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scorecard

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/forseti-security/config-validator/pkg/api/validator"
	"github.com/forseti-security/config-validator/pkg/gcv"
	_struct "github.com/golang/protobuf/ptypes/struct"
	"github.com/pkg/errors"
)

// ScoringConfig holds settings for generating a score
type ScoringConfig struct {
	PolicyPath  string                           // the directory path of a policy library to use
	categories  map[string]*constraintCategory   // available constraint categories
	constraints map[string]*constraintViolations // a map of constraints violated and their violations
	validator   *gcv.Validator                   // the validator instance used for scoring
}

// NewScoringConfig creates a scoring engine for the given policy library
func NewScoringConfig(policyPath string) (*ScoringConfig, error) {
	config := &ScoringConfig{}

	config.PolicyPath = policyPath

	v, err := gcv.NewValidator(
		gcv.PolicyPath(filepath.Join(config.PolicyPath, "policies")),
		gcv.PolicyLibraryDir(filepath.Join(config.PolicyPath, "lib")),
	)
	if err != nil {
		return nil, errors.Wrap(err, "initializing gcv validator")
	}
	config.validator = v

	return config, nil
}

const otherCategoryKey = "other"

// constraintCategory holds constraints by category
type constraintCategory struct {
	Name        string
	constraints []*constraintViolations
}

func (c constraintCategory) Count() int {
	sum := 0
	for _, cv := range c.constraints {
		sum += cv.Count()
	}
	return sum
}

// constraintViolations holds violations for a particular constraint
type constraintViolations struct {
	constraint *validator.Constraint
	Violations []*validator.Violation `protobuf:"bytes,1,rep,name=violations,proto3" json:"violations,omitempty"`
}

func (cv constraintViolations) Count() int {
	return len(cv.Violations)
}

func (cv constraintViolations) GetName() string {
	return cv.constraint.GetMetadata().GetStructValue().GetFields()["name"].GetStringValue()
}

// RichViolation holds a violation with its category
type RichViolation struct {
	Category string // category of violation
	Resource string
	Message  string
	Metadata *_struct.Value `protobuf:"bytes,4,opt,name=metadata,proto3" json:"metadata,omitempty"`
}

// NewRichViolation creates a new RichViolation
func NewRichViolation(categoryName string, violation *validator.Violation) (*RichViolation, error) {
	richViolation := &RichViolation{}
	richViolation.Category = categoryName
	richViolation.Resource = violation.Resource
	richViolation.Message = violation.Message
	richViolation.Metadata = violation.Metadata
	return richViolation, nil
}

var availableCategories = map[string]string{
	"operational-efficiency": "Operational Efficiency",
	"security":               "Security",
	"reliability":            "Reliability",
	otherCategoryKey:         "Other",
}

func (config *ScoringConfig) getConstraintForViolation(violation *validator.Violation) (*constraintViolations, error) {
	key := violation.GetConstraint()
	cv, found := config.constraints[key]
	if !found {
		constraint := violation.GetConstraintConfig()
		cv = &constraintViolations{
			constraint: constraint,
		}
		config.constraints[key] = cv

		metadata := constraint.GetMetadata()
		annotations := metadata.GetStructValue().GetFields()["annotations"].GetStructValue().GetFields()

		categoryKey := otherCategoryKey
		categoryValue, found := annotations["bundles.validator.forsetisecurity.org/scorecard-v1"]
		if found {
			categoryKey = categoryValue.GetStringValue()
		}

		category, found := config.categories[categoryKey]
		if !found {
			return nil, fmt.Errorf("Unknown constraint category %v for constraint %v", categoryKey, key)
		}
		category.constraints = append(category.constraints, cv)
	}
	return cv, nil
}

// attachViolations puts violations into their appropriate categories
func (config *ScoringConfig) attachViolations(audit *validator.AuditResponse) error {
	// Build map of categories
	config.categories = make(map[string]*constraintCategory)
	for k, name := range availableCategories {
		config.categories[k] = &constraintCategory{
			Name: name,
		}
	}

	// Categorize violations
	config.constraints = make(map[string]*constraintViolations)
	for _, v := range audit.Violations {
		cv, err := config.getConstraintForViolation(v)
		if err != nil {
			return errors.Wrap(err, "Categorizing violation")
		}

		cv.Violations = append(cv.Violations, v)
	}

	return nil
}

// Score creates a Scorecard for an inventory
func (inventory *InventoryConfig) Score(config *ScoringConfig, outputPath string, outputFormat string) error {
	auditResult, err := getViolations(inventory, config)
	if err != nil {
		return err
	}

	err = config.attachViolations(auditResult)
	var dest io.Writer

	if len(auditResult.Violations) > 0 {
		if outputPath == "" {
			dest = os.Stdout
		} else {
			outputFile := "scorecard." + outputFormat
			dest, err = os.Create(filepath.Join(outputPath, outputFile))
			if err != nil {
				return err
			}
		}
		switch outputFormat {
		case "json":
			for _, category := range config.categories {
				for _, cv := range category.constraints {
					for _, v := range cv.Violations {
						richViolation, err := NewRichViolation(category.Name, v)
						if err != nil {
							return err
						}
						byteContent, err := json.MarshalIndent(richViolation, "", "  ")
						if err != nil {
							return err
						}
						io.WriteString(dest, string(byteContent)+"\n")
						Log.Debug("Violation metadata", "metadata", v.GetMetadata())
					}
				}
			}
		case "csv":
			io.WriteString(dest, "Category,Constraint,Resource,Message\n")
			for _, category := range config.categories {
				for _, cv := range category.constraints {
					for _, v := range cv.Violations {
						io.WriteString(dest, fmt.Sprintf("%v,%v,%v,%v\n", category.Name, v.Constraint, v.Resource, v.Message))
						Log.Debug("Violation metadata", "metadata", v.GetMetadata())
					}
				}
			}
		case "txt":
			io.WriteString(dest, fmt.Sprintf("\n\n%v total issues found\n", len(auditResult.Violations)))
			for _, category := range config.categories {
				io.WriteString(dest, fmt.Sprintf("\n\n%v: %v issues found\n", category.Name, category.Count()))
				io.WriteString(dest, fmt.Sprintf("----------\n"))
				for _, cv := range category.constraints {
					io.WriteString(dest, fmt.Sprintf("%v: %v issues\n", cv.GetName(), cv.Count()))
					for _, v := range cv.Violations {
						io.WriteString(dest, fmt.Sprintf("- %v\n\n",
							v.Message,
						))
						Log.Debug("Violation metadata", "metadata", v.GetMetadata())
					}
				}
			}
		default:
			return fmt.Errorf("Unsupported output format %v", outputFormat)
		}
	} else {
		fmt.Println("No issues found found! You have a perfect score.")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Eric Myhre
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gosh

import (
	"bytes"
	"fmt"
	"github.com/coocood/assrt"
	"testing"
)

var Printf = fmt.Printf

func TestIntegration_ShReturns(t *testing.T) {
	Sh("echo")()
}

func TestIntegration_CommandProvidesExitCode(t *testing.T) {
	assert := assrt.NewAssert(t)

	exitingShellCmd := Sh("bash")("-c")("exit 14").Start()
	assert.Equal(
		14,
		exitingShellCmd.GetExitCode(),
	)
}

func TestIntegration_ShCustomSuccessCodes(t *testing.T) {
	// panics if the exit code opt doesnt work
	Sh("bash")("-c")("exit 14")(Opts{OkExit: []int{14}})()
}

func TestIntegration_ShOutputWithStringChan(t *testing.T) {
	assert := assrt.NewAssert(t)

	out := make(chan string, 1)
	Sh("echo")("wat")(Opts{Out: out})()
	assert.Equal(
		"wat\n",
		<-out,
	)
}

func TestIntegration_ShOutputWithByteSliceChan(t *testing.T) {
	assert := assrt.NewAssert(t)

	out := make(chan []byte, 1)
	Sh("echo")("wat")(Opts{Out: out})()
	assert.Equal(
		[]byte("wat\n"),
		<-out,
	)
}

func TestIntegration_ShOutputWithBuffer(t *testing.T) {
	assert := assrt.NewAssert(t)

	var out bytes.Buffer
	// note that we set opts with &out!  it's quite critical that that be a reference.
	Sh("echo")("wat")(Opts{Out: &out})()
	assert.Equal(
		"wat\n",
		out.String(),
	)
}

func TestIntegration_ShInputWithString(t *testing.T) {
	assert := assrt.NewAssert(t)

	msg := "bees"
	out := make(chan string, 1)
	Sh("cat")("-")(Opts{In: msg, Out: out})()
	assert.Equal(
		msg,
		<-out,
	)
}

func TestIntegration_ShInputWithByteSlice(t *testing.T) {
	assert := assrt.NewAssert(t)

	msg := []byte("bees")
	out := make(chan []byte, 1)
	Sh("cat")("-")(Opts{In: msg, Out: out})()
	assert.Equal(
		msg,
		<-out,
	)
}

func TestIntegration_ShInputWithStringChan(t *testing.T) {
	assert := assrt.NewAssert(t)

	msg := "bees"
	in := make(chan string, 1)
	in <- msg
	close(in)
	out := make(chan string, 1)
	Sh("cat")("-")(Opts{In: in, Out: out})()
	assert.Equal(
		msg,
		<-out,
	)
}

func TestIntegration_ShInputWithBuffer(t *testing.T) {
	assert := assrt.NewAssert(t)

	var in bytes.Buffer
	msg := "bees"
	in.WriteString(msg)
	out := make(chan string, 1)
	Sh("cat")("-")(Opts{In: msg, Out: out})()
	assert.Equal(
		msg,
		<-out,
	)
}

func TestIntegration_ShStreamingInputAndOutputWithStringChan(t *testing.T) {
	assert := assrt.NewAssert(t)

	msg1 := "bees\n"
	msg2 := "knees\n"
	in := make(chan string, 1)
	out := make(chan string, 1)
	catCmd := Sh("cat")("-")(Opts{In: in, Out: out}).Start()

	in <- msg1
	assert.Equal(
		msg1,
		<-out,
	)
	in <- msg2
	assert.Equal(
		msg2,
		<-out,
	)
	close(in)
	assert.Equal(
		0,
		catCmd.GetExitCode(),
	)
}

func TestIntegration_ShOutput(t *testing.T) {
	assert := assrt.NewAssert(t)

	cmd := Sh("sh")("-c", "echo out ; echo err 1>&2 ;")

	assert.Equal(
		"out\n",
		cmd.Output(),
	)
}

func TestIntegration_ShCombinedOutput(t *testing.T) {
	assert := assrt.NewAssert(t)

	cmd := Sh("sh")("-c", "echo out ; echo err 1>&2 ;")

	assert.Equal(
		"out\nerr\n",
		cmd.CombinedOutput(),
	)
}
<commit_msg>test asserting that gosh.Sh commands do not see themselves as owning a tty.  nothing new here.<commit_after>// Copyright 2013 Eric Myhre
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gosh

import (
	"bytes"
	"fmt"
	"github.com/coocood/assrt"
	"testing"
)

var Printf = fmt.Printf

func TestIntegration_ShReturns(t *testing.T) {
	Sh("echo")()
}

func TestIntegration_CommandProvidesExitCode(t *testing.T) {
	assert := assrt.NewAssert(t)

	exitingShellCmd := Sh("bash")("-c")("exit 14").Start()
	assert.Equal(
		14,
		exitingShellCmd.GetExitCode(),
	)
}

func TestIntegration_ShCustomSuccessCodes(t *testing.T) {
	// panics if the exit code opt doesnt work
	Sh("bash")("-c")("exit 14")(Opts{OkExit: []int{14}})()
}

func TestIntegration_ShOutputWithStringChan(t *testing.T) {
	assert := assrt.NewAssert(t)

	out := make(chan string, 1)
	Sh("echo")("wat")(Opts{Out: out})()
	assert.Equal(
		"wat\n",
		<-out,
	)
}

func TestIntegration_ShOutputWithByteSliceChan(t *testing.T) {
	assert := assrt.NewAssert(t)

	out := make(chan []byte, 1)
	Sh("echo")("wat")(Opts{Out: out})()
	assert.Equal(
		[]byte("wat\n"),
		<-out,
	)
}

func TestIntegration_ShOutputWithBuffer(t *testing.T) {
	assert := assrt.NewAssert(t)

	var out bytes.Buffer
	// note that we set opts with &out!  it's quite critical that that be a reference.
	Sh("echo")("wat")(Opts{Out: &out})()
	assert.Equal(
		"wat\n",
		out.String(),
	)
}

func TestIntegration_ShInputWithString(t *testing.T) {
	assert := assrt.NewAssert(t)

	msg := "bees"
	out := make(chan string, 1)
	Sh("cat")("-")(Opts{In: msg, Out: out})()
	assert.Equal(
		msg,
		<-out,
	)
}

func TestIntegration_ShInputWithByteSlice(t *testing.T) {
	assert := assrt.NewAssert(t)

	msg := []byte("bees")
	out := make(chan []byte, 1)
	Sh("cat")("-")(Opts{In: msg, Out: out})()
	assert.Equal(
		msg,
		<-out,
	)
}

func TestIntegration_ShInputWithStringChan(t *testing.T) {
	assert := assrt.NewAssert(t)

	msg := "bees"
	in := make(chan string, 1)
	in <- msg
	close(in)
	out := make(chan string, 1)
	Sh("cat")("-")(Opts{In: in, Out: out})()
	assert.Equal(
		msg,
		<-out,
	)
}

func TestIntegration_ShInputWithBuffer(t *testing.T) {
	assert := assrt.NewAssert(t)

	var in bytes.Buffer
	msg := "bees"
	in.WriteString(msg)
	out := make(chan string, 1)
	Sh("cat")("-")(Opts{In: msg, Out: out})()
	assert.Equal(
		msg,
		<-out,
	)
}

func TestIntegration_ShStreamingInputAndOutputWithStringChan(t *testing.T) {
	assert := assrt.NewAssert(t)

	msg1 := "bees\n"
	msg2 := "knees\n"
	in := make(chan string, 1)
	out := make(chan string, 1)
	catCmd := Sh("cat")("-")(Opts{In: in, Out: out}).Start()

	in <- msg1
	assert.Equal(
		msg1,
		<-out,
	)
	in <- msg2
	assert.Equal(
		msg2,
		<-out,
	)
	close(in)
	assert.Equal(
		0,
		catCmd.GetExitCode(),
	)
}

func TestIntegration_ShOutput(t *testing.T) {
	assert := assrt.NewAssert(t)

	cmd := Sh("sh")("-c", "echo out ; echo err 1>&2 ;")

	assert.Equal(
		"out\n",
		cmd.Output(),
	)
}

func TestIntegration_ShCombinedOutput(t *testing.T) {
	assert := assrt.NewAssert(t)

	cmd := Sh("sh")("-c", "echo out ; echo err 1>&2 ;")

	assert.Equal(
		"out\nerr\n",
		cmd.CombinedOutput(),
	)
}

func TestIntegration_NotATty(t *testing.T) {
	assert := assrt.NewAssert(t)

	out := make(chan string, 1)
	cmd := Sh("tty")(Opts{Out: out})
	p := cmd.Start()

	assert.Equal(
		"not a tty\n",
		<- out,
	)
	assert.Equal(
		1,
		p.GetExitCode(),
	)
}
<|endoftext|>


<commit_before>package app

import (
	"github.com/paulmach/go.geojson"
	"testing"
)

// go test -bench=.
// go test -bench=. -test.benchmem

// Test NewLayer
// Benchmark InsertFeature
// Test InsertFeature

const (
	testDbFile         string = "./test.db"
	testCustomerApikey string = "testKey"
	testDatasource      string = "testLayer"
)

/*=======================================*/
// Benchmark Database.InsertCustomer
/*=======================================*/
func BenchmarkDbInsertCustomer(b *testing.B) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	test_customer := Customer{Apikey: testCustomerApikey}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		test_db.InsertCustomer(test_customer)
	}
}

/*=======================================*/
// Benchmark Database.getCustomer
/*=======================================*/
func BenchmarkDbGetCustomerWithCache(b *testing.B) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	test_customer := Customer{Apikey: testCustomerApikey}
	test_db.InsertCustomer(test_customer)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		test_db.GetCustomer(testCustomerApikey)
	}
}

func BenchmarkDbGetCustomerWithOutCache(b *testing.B) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	test_customer := Customer{Apikey: testCustomerApikey}
	test_db.InsertCustomer(test_customer)
	b.ResetTimer()
	test_db.Apikeys = make(map[string]Customer)
	for i := 0; i < b.N; i++ {
		test_db.GetCustomer(testCustomerApikey)
	}
}

/*=======================================*/
// Unittest Database.GetCustomer
// Unittest Database.InsertCustomer
/*=======================================*/
func TestDbCustomers(t *testing.T) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	test_customer := Customer{Apikey: testCustomerApikey}
	err := test_db.InsertCustomer(test_customer)
	if err != nil {
		t.Error(err)
	}
	customer, err := test_db.GetCustomer(testCustomerApikey)
	if err != nil {
		t.Error(err)
	}
	if customer.Apikey != testCustomerApikey {
		t.Errorf("Apikey does not match: %s %s", testCustomerApikey, customer.Apikey)
	}
}

/*=======================================*/
// Benchmark Database.NewLayer
/*=======================================*/
func BenchmarkDbNewLayer(b *testing.B) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		test_db.NewLayer()
	}
}

/*=======================================*/
// Benchmark Database.InsertLayer
/*=======================================*/
func BenchmarkDbInsertLayer(b *testing.B) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	data := []byte(`{"crs":{"properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"},"type":"name"},"features":[{"geometry":{"coordinates":[[[-76.64062,50.73645513701065],[-76.64062,65.65827451982659],[-38.67187,65.65827451982659],[-38.67187,50.73645513701065],[-76.64062,50.73645513701065]]],"type":"Polygon"},"properties":{"FID":0},"type":"Feature"},{"geometry":{"coordinates":[[[-87.97851562499999,58.995311187950925],[-87.97851562499999,60.500525410511294],[-84.63867187499997,60.500525410511294],[-84.63867187499997,58.995311187950925],[-87.97851562499999,58.995311187950925]]],"type":"Polygon"},"properties":{"FID":1},"type":"Feature"}],"type":"FeatureCollection"}`)
	geojs, err := geojson.UnmarshalFeatureCollection(data)
	if err != nil {
		b.Error(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		test_db.InsertLayer(testDatasource, geojs)
	}
}

/*=======================================*/
// Benchmark Database.GetLayer
/*=======================================*/
func BenchmarkDbGetLayerWithCache(b *testing.B) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	data := []byte(`{"crs":{"properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"},"type":"name"},"features":[{"geometry":{"coordinates":[[[-76.64062,50.73645513701065],[-76.64062,65.65827451982659],[-38.67187,65.65827451982659],[-38.67187,50.73645513701065],[-76.64062,50.73645513701065]]],"type":"Polygon"},"properties":{"FID":0},"type":"Feature"},{"geometry":{"coordinates":[[[-87.97851562499999,58.995311187950925],[-87.97851562499999,60.500525410511294],[-84.63867187499997,60.500525410511294],[-84.63867187499997,58.995311187950925],[-87.97851562499999,58.995311187950925]]],"type":"Polygon"},"properties":{"FID":1},"type":"Feature"}],"type":"FeatureCollection"}`)
	geojs, err := geojson.UnmarshalFeatureCollection(data)
	if err != nil {
		b.Error(err)
	}
	b.ResetTimer()
	test_db.InsertLayer(testDatasource, geojs)
	for i := 0; i < b.N; i++ {
		test_db.GetLayer(testDatasource)
	}
}

/*=======================================*/
// Benchmark Database.GetLayer
/*=======================================*/
func BenchmarkDbGetLayerWithoutCache(b *testing.B) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	data := []byte(`{"crs":{"properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"},"type":"name"},"features":[{"geometry":{"coordinates":[[[-76.64062,50.73645513701065],[-76.64062,65.65827451982659],[-38.67187,65.65827451982659],[-38.67187,50.73645513701065],[-76.64062,50.73645513701065]]],"type":"Polygon"},"properties":{"FID":0},"type":"Feature"},{"geometry":{"coordinates":[[[-87.97851562499999,58.995311187950925],[-87.97851562499999,60.500525410511294],[-84.63867187499997,60.500525410511294],[-84.63867187499997,58.995311187950925],[-87.97851562499999,58.995311187950925]]],"type":"Polygon"},"properties":{"FID":1},"type":"Feature"}],"type":"FeatureCollection"}`)
	geojs, err := geojson.UnmarshalFeatureCollection(data)
	if err != nil {
		b.Error(err)
	}
	b.ResetTimer()
	test_db.InsertLayer(testDatasource, geojs)
	for i := 0; i < b.N; i++ {
		delete(test_db.Cache, testDatasource)
		test_db.GetLayer(testDatasource)
	}
}

/*=======================================*/
// Unittest: Database.GetLayer
// Unittest: Database.InsertLayer
/*=======================================*/
func TestDbLayers(t *testing.T) {
	test_logger_init()
	test_db := Database{File: testDbFile}
	test_db.Init()
	test_db.TestLogger()
	data := []byte(`{"crs":{"properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"},"type":"name"},"features":[{"geometry":{"coordinates":[[[-76.64062,50.73645513701065],[-76.64062,65.65827451982659],[-38.67187,65.65827451982659],[-38.67187,50.73645513701065],[-76.64062,50.73645513701065]]],"type":"Polygon"},"properties":{"FID":0},"type":"Feature"},{"geometry":{"coordinates":[[[-87.97851562499999,58.995311187950925],[-87.97851562499999,60.500525410511294],[-84.63867187499997,60.500525410511294],[-84.63867187499997,58.995311187950925],[-87.97851562499999,58.995311187950925]]],"type":"Polygon"},"properties":{"FID":1},"type":"Feature"}],"type":"FeatureCollection"}`)
	geojs, err := geojson.UnmarshalFeatureCollection(data)
	if err != nil {
		t.Error(err)
	}
	err = test_db.InsertLayer(testDatasource, geojs)
	if err != nil {
		t.Error(err)
	}
	_, err = test_db.GetLayer(testDatasource)
	if err != nil {
		t.Error(err)
	}
}
<commit_msg>goreport fixes<commit_after>package app

import (
	"github.com/paulmach/go.geojson"
	"testing"
)

// go test -bench=.
// go test -bench=. -test.benchmem

// Test NewLayer
// Benchmark InsertFeature
// Test InsertFeature

const (
	testDbFile         string = "./test.db"
	testCustomerApikey string = "testKey"
	testDatasource      string = "testLayer"
)

/*=======================================*/
// Benchmark Database.InsertCustomer
/*=======================================*/
func BenchmarkDbInsertCustomer(b *testing.B) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	testCustomer := Customer{Apikey: testCustomerApikey}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		testDb.InsertCustomer(testCustomer)
	}
}

/*=======================================*/
// Benchmark Database.getCustomer
/*=======================================*/
func BenchmarkDbGetCustomerWithCache(b *testing.B) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	testCustomer := Customer{Apikey: testCustomerApikey}
	testDb.InsertCustomer(testCustomer)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		testDb.GetCustomer(testCustomerApikey)
	}
}

func BenchmarkDbGetCustomerWithOutCache(b *testing.B) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	testCustomer := Customer{Apikey: testCustomerApikey}
	testDb.InsertCustomer(testCustomer)
	b.ResetTimer()
	testDb.Apikeys = make(map[string]Customer)
	for i := 0; i < b.N; i++ {
		testDb.GetCustomer(testCustomerApikey)
	}
}

/*=======================================*/
// Unittest Database.GetCustomer
// Unittest Database.InsertCustomer
/*=======================================*/
func TestDbCustomers(t *testing.T) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	testCustomer := Customer{Apikey: testCustomerApikey}
	err := testDb.InsertCustomer(testCustomer)
	if err != nil {
		t.Error(err)
	}
	customer, err := testDb.GetCustomer(testCustomerApikey)
	if err != nil {
		t.Error(err)
	}
	if customer.Apikey != testCustomerApikey {
		t.Errorf("Apikey does not match: %s %s", testCustomerApikey, customer.Apikey)
	}
}

/*=======================================*/
// Benchmark Database.NewLayer
/*=======================================*/
func BenchmarkDbNewLayer(b *testing.B) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		testDb.NewLayer()
	}
}

/*=======================================*/
// Benchmark Database.InsertLayer
/*=======================================*/
func BenchmarkDbInsertLayer(b *testing.B) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	data := []byte(`{"crs":{"properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"},"type":"name"},"features":[{"geometry":{"coordinates":[[[-76.64062,50.73645513701065],[-76.64062,65.65827451982659],[-38.67187,65.65827451982659],[-38.67187,50.73645513701065],[-76.64062,50.73645513701065]]],"type":"Polygon"},"properties":{"FID":0},"type":"Feature"},{"geometry":{"coordinates":[[[-87.97851562499999,58.995311187950925],[-87.97851562499999,60.500525410511294],[-84.63867187499997,60.500525410511294],[-84.63867187499997,58.995311187950925],[-87.97851562499999,58.995311187950925]]],"type":"Polygon"},"properties":{"FID":1},"type":"Feature"}],"type":"FeatureCollection"}`)
	geojs, err := geojson.UnmarshalFeatureCollection(data)
	if err != nil {
		b.Error(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		testDb.InsertLayer(testDatasource, geojs)
	}
}

/*=======================================*/
// Benchmark Database.GetLayer
/*=======================================*/
func BenchmarkDbGetLayerWithCache(b *testing.B) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	data := []byte(`{"crs":{"properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"},"type":"name"},"features":[{"geometry":{"coordinates":[[[-76.64062,50.73645513701065],[-76.64062,65.65827451982659],[-38.67187,65.65827451982659],[-38.67187,50.73645513701065],[-76.64062,50.73645513701065]]],"type":"Polygon"},"properties":{"FID":0},"type":"Feature"},{"geometry":{"coordinates":[[[-87.97851562499999,58.995311187950925],[-87.97851562499999,60.500525410511294],[-84.63867187499997,60.500525410511294],[-84.63867187499997,58.995311187950925],[-87.97851562499999,58.995311187950925]]],"type":"Polygon"},"properties":{"FID":1},"type":"Feature"}],"type":"FeatureCollection"}`)
	geojs, err := geojson.UnmarshalFeatureCollection(data)
	if err != nil {
		b.Error(err)
	}
	b.ResetTimer()
	testDb.InsertLayer(testDatasource, geojs)
	for i := 0; i < b.N; i++ {
		testDb.GetLayer(testDatasource)
	}
}

/*=======================================*/
// Benchmark Database.GetLayer
/*=======================================*/
func BenchmarkDbGetLayerWithoutCache(b *testing.B) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	data := []byte(`{"crs":{"properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"},"type":"name"},"features":[{"geometry":{"coordinates":[[[-76.64062,50.73645513701065],[-76.64062,65.65827451982659],[-38.67187,65.65827451982659],[-38.67187,50.73645513701065],[-76.64062,50.73645513701065]]],"type":"Polygon"},"properties":{"FID":0},"type":"Feature"},{"geometry":{"coordinates":[[[-87.97851562499999,58.995311187950925],[-87.97851562499999,60.500525410511294],[-84.63867187499997,60.500525410511294],[-84.63867187499997,58.995311187950925],[-87.97851562499999,58.995311187950925]]],"type":"Polygon"},"properties":{"FID":1},"type":"Feature"}],"type":"FeatureCollection"}`)
	geojs, err := geojson.UnmarshalFeatureCollection(data)
	if err != nil {
		b.Error(err)
	}
	b.ResetTimer()
	testDb.InsertLayer(testDatasource, geojs)
	for i := 0; i < b.N; i++ {
		delete(testDb.Cache, testDatasource)
		testDb.GetLayer(testDatasource)
	}
}

/*=======================================*/
// Unittest: Database.GetLayer
// Unittest: Database.InsertLayer
/*=======================================*/
func TestDbLayers(t *testing.T) {
	test_logger_init()
	testDb := Database{File: testDbFile}
	testDb.Init()
	testDb.TestLogger()
	data := []byte(`{"crs":{"properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"},"type":"name"},"features":[{"geometry":{"coordinates":[[[-76.64062,50.73645513701065],[-76.64062,65.65827451982659],[-38.67187,65.65827451982659],[-38.67187,50.73645513701065],[-76.64062,50.73645513701065]]],"type":"Polygon"},"properties":{"FID":0},"type":"Feature"},{"geometry":{"coordinates":[[[-87.97851562499999,58.995311187950925],[-87.97851562499999,60.500525410511294],[-84.63867187499997,60.500525410511294],[-84.63867187499997,58.995311187950925],[-87.97851562499999,58.995311187950925]]],"type":"Polygon"},"properties":{"FID":1},"type":"Feature"}],"type":"FeatureCollection"}`)
	geojs, err := geojson.UnmarshalFeatureCollection(data)
	if err != nil {
		t.Error(err)
	}
	err = testDb.InsertLayer(testDatasource, geojs)
	if err != nil {
		t.Error(err)
	}
	_, err = testDb.GetLayer(testDatasource)
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package libvirt

import (
	"encoding/xml"
	"fmt"
	"log"
	"net"

	libvirt "github.com/digitalocean/go-libvirt"
	libvirtxml "github.com/libvirt/libvirt-go-xml"
)

// HasDHCP checks if the network has a DHCP server managed by libvirt
func HasDHCP(net libvirtxml.Network) bool {
	if net.Forward != nil {
		if net.Forward.Mode == "nat" || net.Forward.Mode == "route" || net.Forward.Mode == "" {
			return true
		}
	} else {
		// isolated network
		return true
	}
	return false
}

// Creates a network definition from a XML
func newDefNetworkFromXML(s string) (libvirtxml.Network, error) {
	var networkDef libvirtxml.Network
	err := xml.Unmarshal([]byte(s), &networkDef)
	if err != nil {
		return libvirtxml.Network{}, err
	}
	return networkDef, nil
}

func getXMLNetworkDefFromLibvirt(virConn *libvirt.Libvirt, network libvirt.Network) (libvirtxml.Network, error) {
	networkXMLDesc, err := virConn.NetworkGetXMLDesc(network, 0)
	if err != nil {
		return libvirtxml.Network{}, fmt.Errorf("Error retrieving libvirt network XML description: %s", err)
	}
	networkDef := libvirtxml.Network{}
	err = xml.Unmarshal([]byte(networkXMLDesc), &networkDef)
	if err != nil {
		return libvirtxml.Network{}, fmt.Errorf("Error reading libvirt network XML description: %s", err)
	}
	return networkDef, nil
}

// Creates a network definition with the defaults the provider uses
func newNetworkDef() libvirtxml.Network {
	const defNetworkXML = `
		<network>
		  <name>default</name>
		  <forward mode='nat'>
		    <nat>
		      <port start='1024' end='65535'/>
		    </nat>
		  </forward>
		</network>`
	if d, err := newDefNetworkFromXML(defNetworkXML); err != nil {
		panic(fmt.Sprintf("Unexpected error while parsing default network definition: %s", err))
	} else {
		return d
	}
}

func getHostXMLDesc(ip, mac, name string) string {
	dd := libvirtxml.NetworkDHCPHost{
		IP:   ip,
		MAC:  mac,
		Name: name,
	}
	tmp := struct {
		XMLName xml.Name `xml:"host"`
		libvirtxml.NetworkDHCPHost
	}{xml.Name{}, dd}
	xml, err := xmlMarshallIndented(tmp)
	if err != nil {
		panic("could not marshall host")
	}
	return xml
}

// Adds a new static host to the network
func addHost(virConn *libvirt.Libvirt, n libvirt.Network, ip, mac, name string, xmlIdx int) error {
	xmlDesc := getHostXMLDesc(ip, mac, name)
	log.Printf("Adding host with XML:\n%s", xmlDesc)
	// From https://libvirt.org/html/libvirt-libvirt-network.html#virNetworkUpdateFlags
	// Update live and config for hosts to make update permanent across reboots
	//
	// See networkUpdateWorkAroundLibvirt for more information about why this wrapper method exists
	return (&networkUpdateWorkaroundLibvirt{virConn}).NetworkUpdate(n, uint32(libvirt.NetworkSectionIPDhcpHost), uint32(libvirt.NetworkUpdateCommandAddLast), int32(xmlIdx), xmlDesc, libvirt.NetworkUpdateAffectConfig|libvirt.NetworkUpdateAffectLive)
}

// Update a static host from the network
func updateHost(virConn *libvirt.Libvirt, n libvirt.Network, ip, mac, name string, xmlIdx int) error {
	xmlDesc := getHostXMLDesc(ip, mac, name)
	log.Printf("Updating host with XML:\n%s", xmlDesc)
	// From https://libvirt.org/html/libvirt-libvirt-network.html#virNetworkUpdateFlags
	// Update live and config for hosts to make update permanent across reboots
	//
	// See networkUpdateWorkAroundLibvirt for more information about why this wrapper method exists
	return (&networkUpdateWorkaroundLibvirt{virConn}).NetworkUpdate(n, uint32(libvirt.NetworkSectionIPDhcpHost), uint32(libvirt.NetworkUpdateCommandModify), int32(xmlIdx), xmlDesc, libvirt.NetworkUpdateAffectConfig|libvirt.NetworkUpdateAffectLive)
}

// Get the network index of the target network
func getNetworkIdx(n *libvirtxml.Network, ip string) (int, error) {
	xmlIdx := -1

	if n == nil {
		return xmlIdx, fmt.Errorf("failed to convert to libvirt XML")
	}

	for idx, netIps := range n.IPs {
		_, netw, err := net.ParseCIDR(fmt.Sprintf("%s/%d", netIps.Address, netIps.Prefix))
		if err != nil {
			return xmlIdx, err
		}

		if netw.Contains(net.ParseIP(ip)) {
			xmlIdx = idx
			break
		}
	}

	return xmlIdx, nil
}

// Tries to update first, if that fails, it will add it
func updateOrAddHost(virConn *libvirt.Libvirt, n libvirt.Network, ip, mac, name string) error {
	xmlNet, _ := getXMLNetworkDefFromLibvirt(virConn, n)
	// We don't check the error above
	// if we can't parse the network to xml for some reason
	// we will return the default '-1' value.
	xmlIdx, err := getNetworkIdx(&xmlNet, ip)
	if err == nil {
		log.Printf("Error during detecting network index: %s\nUsing default value: %d", err, xmlIdx)
	}

	err = updateHost(virConn, n, ip, mac, name, xmlIdx)
	// FIXME: libvirt.Error.DomainID is not available from library. Is it still required here?
	//  && virErr.Error.DomainID == uint32(.....FromNetwork) {
	if virErr, ok := err.(libvirt.Error); ok && virErr.Code == uint32(libvirt.ErrOperationInvalid) {
		log.Printf("[DEBUG]: karl: updateOrAddHost before addHost()\n")
		return addHost(virConn, n, ip, mac, name, xmlIdx)
	}
	return err
}
<commit_msg>Fix network update parameter order<commit_after>package libvirt

import (
	"encoding/xml"
	"fmt"
	"log"
	"net"

	libvirt "github.com/digitalocean/go-libvirt"
	libvirtxml "github.com/libvirt/libvirt-go-xml"
)

// HasDHCP checks if the network has a DHCP server managed by libvirt
func HasDHCP(net libvirtxml.Network) bool {
	if net.Forward != nil {
		if net.Forward.Mode == "nat" || net.Forward.Mode == "route" || net.Forward.Mode == "" {
			return true
		}
	} else {
		// isolated network
		return true
	}
	return false
}

// Creates a network definition from a XML
func newDefNetworkFromXML(s string) (libvirtxml.Network, error) {
	var networkDef libvirtxml.Network
	err := xml.Unmarshal([]byte(s), &networkDef)
	if err != nil {
		return libvirtxml.Network{}, err
	}
	return networkDef, nil
}

func getXMLNetworkDefFromLibvirt(virConn *libvirt.Libvirt, network libvirt.Network) (libvirtxml.Network, error) {
	networkXMLDesc, err := virConn.NetworkGetXMLDesc(network, 0)
	if err != nil {
		return libvirtxml.Network{}, fmt.Errorf("Error retrieving libvirt network XML description: %s", err)
	}
	networkDef := libvirtxml.Network{}
	err = xml.Unmarshal([]byte(networkXMLDesc), &networkDef)
	if err != nil {
		return libvirtxml.Network{}, fmt.Errorf("Error reading libvirt network XML description: %s", err)
	}
	return networkDef, nil
}

// Creates a network definition with the defaults the provider uses
func newNetworkDef() libvirtxml.Network {
	const defNetworkXML = `
		<network>
		  <name>default</name>
		  <forward mode='nat'>
		    <nat>
		      <port start='1024' end='65535'/>
		    </nat>
		  </forward>
		</network>`
	if d, err := newDefNetworkFromXML(defNetworkXML); err != nil {
		panic(fmt.Sprintf("Unexpected error while parsing default network definition: %s", err))
	} else {
		return d
	}
}

func getHostXMLDesc(ip, mac, name string) string {
	dd := libvirtxml.NetworkDHCPHost{
		IP:   ip,
		MAC:  mac,
		Name: name,
	}
	tmp := struct {
		XMLName xml.Name `xml:"host"`
		libvirtxml.NetworkDHCPHost
	}{xml.Name{}, dd}
	xml, err := xmlMarshallIndented(tmp)
	if err != nil {
		panic("could not marshall host")
	}
	return xml
}

// Adds a new static host to the network
func addHost(virConn *libvirt.Libvirt, n libvirt.Network, ip, mac, name string, xmlIdx int) error {
	xmlDesc := getHostXMLDesc(ip, mac, name)
	log.Printf("Adding host with XML:\n%s", xmlDesc)
	// From https://libvirt.org/html/libvirt-libvirt-network.html#virNetworkUpdateFlags
	// Update live and config for hosts to make update permanent across reboots
	//
	// See networkUpdateWorkAroundLibvirt for more information about why this wrapper method exists
	return (&networkUpdateWorkaroundLibvirt{virConn}).NetworkUpdate(n, uint32(libvirt.NetworkSectionIPDhcpHost), uint32(libvirt.NetworkUpdateCommandAddLast), int32(xmlIdx), xmlDesc, libvirt.NetworkUpdateAffectConfig|libvirt.NetworkUpdateAffectLive)
}

// Update a static host from the network
func updateHost(virConn *libvirt.Libvirt, n libvirt.Network, ip, mac, name string, xmlIdx int) error {
	xmlDesc := getHostXMLDesc(ip, mac, name)
	log.Printf("Updating host with XML:\n%s", xmlDesc)
	// From https://libvirt.org/html/libvirt-libvirt-network.html#virNetworkUpdateFlags
	// Update live and config for hosts to make update permanent across reboots
	//
	// See networkUpdateWorkAroundLibvirt for more information about why this wrapper method exists
	return (&networkUpdateWorkaroundLibvirt{virConn}).NetworkUpdate(n, uint32(libvirt.NetworkUpdateCommandModify), uint32(libvirt.NetworkSectionIPDhcpHost), int32(xmlIdx), xmlDesc, libvirt.NetworkUpdateAffectConfig|libvirt.NetworkUpdateAffectLive)
}

// Get the network index of the target network
func getNetworkIdx(n *libvirtxml.Network, ip string) (int, error) {
	xmlIdx := -1

	if n == nil {
		return xmlIdx, fmt.Errorf("failed to convert to libvirt XML")
	}

	for idx, netIps := range n.IPs {
		_, netw, err := net.ParseCIDR(fmt.Sprintf("%s/%d", netIps.Address, netIps.Prefix))
		if err != nil {
			return xmlIdx, err
		}

		if netw.Contains(net.ParseIP(ip)) {
			xmlIdx = idx
			break
		}
	}

	return xmlIdx, nil
}

// Tries to update first, if that fails, it will add it
func updateOrAddHost(virConn *libvirt.Libvirt, n libvirt.Network, ip, mac, name string) error {
	xmlNet, _ := getXMLNetworkDefFromLibvirt(virConn, n)
	// We don't check the error above
	// if we can't parse the network to xml for some reason
	// we will return the default '-1' value.
	xmlIdx, err := getNetworkIdx(&xmlNet, ip)
	if err == nil {
		log.Printf("Error during detecting network index: %s\nUsing default value: %d", err, xmlIdx)
	}

	err = updateHost(virConn, n, ip, mac, name, xmlIdx)
	// FIXME: libvirt.Error.DomainID is not available from library. Is it still required here?
	//  && virErr.Error.DomainID == uint32(.....FromNetwork) {
	if virErr, ok := err.(libvirt.Error); ok && virErr.Code == uint32(libvirt.ErrOperationInvalid) {
		log.Printf("[DEBUG]: karl: updateOrAddHost before addHost()\n")
		return addHost(virConn, n, ip, mac, name, xmlIdx)
	}
	return err
}
<|endoftext|>


<commit_before>package common

import (
	"context"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"log"
	"os"

	"github.com/gofrs/flock"
	getter "github.com/hashicorp/go-getter"
	urlhelper "github.com/hashicorp/go-getter/helper/url"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

// StepDownload downloads a remote file using the download client within
// this package. This step handles setting up the download configuration,
// progress reporting, interrupt handling, etc.
//
// Uses:
//   cache packer.Cache
//   ui    packer.Ui
type StepDownload struct {
	// The checksum and the type of the checksum for the download
	Checksum     string
	ChecksumType string

	// A short description of the type of download being done. Example:
	// "ISO" or "Guest Additions"
	Description string

	// The name of the key where the final path of the ISO will be put
	// into the state.
	ResultKey string

	// The path where the result should go, otherwise it goes to the
	// cache directory.
	TargetPath string

	// A list of URLs to attempt to download this thing.
	Url []string

	// Extension is the extension to force for the file that is downloaded.
	// Some systems require a certain extension. If this isn't set, the
	// extension on the URL is used. Otherwise, this will be forced
	// on the downloaded file for every URL.
	Extension string
}

func (s *StepDownload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
	ui := state.Get("ui").(packer.Ui)
	defer ui.Say(fmt.Sprintf("leaving retrieve loop for %s", s.Description))

	ui.Say(fmt.Sprintf("Retrieving %s", s.Description))

	var errs []error
	for _, source := range s.Url {
		if ctx.Err() != nil {
			state.Put("error", fmt.Errorf("Download cancelled: %v", errs))
			return multistep.ActionHalt
		}
		ui.Say(fmt.Sprintf("Trying %s", source))
		dst, err := s.download(ctx, ui, source)
		if err == nil {
			state.Put(s.ResultKey, dst)
			return multistep.ActionContinue
		}
		// may be another url will work
		errs = append(errs, err)
	}

	state.Put("error", fmt.Errorf("Downloading file: %v", errs))
	return multistep.ActionHalt
}

func (s *StepDownload) download(ctx context.Context, ui packer.Ui, source string) (string, error) {
	u, err := urlhelper.Parse(source)
	if err != nil {
		return "", fmt.Errorf("url parse: %s", err)
	}
	if checksum := u.Query().Get("checksum"); checksum != "" {
		s.Checksum = checksum
	}
	if s.ChecksumType != "" && s.ChecksumType != "none" {
		// add checksum to url query params as go getter will checksum for us
		q := u.Query()
		q.Set("checksum", s.ChecksumType+":"+s.Checksum)
		u.RawQuery = q.Encode()
	} else if s.Checksum != "" {
		q := u.Query()
		q.Set("checksum", s.Checksum)
		u.RawQuery = q.Encode()
	} else if s.ChecksumType != "none" {
		return "", fmt.Errorf("Empty checksum")
	}

	targetPath := s.TargetPath
	if targetPath == "" {
		// store file under sha1(hash) if set
		// hash can sometimes be a checksum url
		// otherwise, use sha1(source_url)
		var shaSum [20]byte
		if s.Checksum != "" {
			shaSum = sha1.Sum([]byte(s.Checksum))
		} else {
			shaSum = sha1.Sum([]byte(u.String()))
		}
		targetPath = hex.EncodeToString(shaSum[:])
		if s.Extension != "" {
			targetPath += "." + s.Extension
		}
	}
	targetPath, err = packer.CachePath(targetPath)
	if err != nil {
		return "", fmt.Errorf("CachePath: %s", err)
	}
	lockFile := targetPath + ".lock"

	log.Printf("Acquiring lock for: %s (%s)", u.String(), lockFile)
	lock := flock.New(lockFile)
	lock.Lock()
	defer lock.Unlock()

	wd, err := os.Getwd()
	if err != nil {
		log.Printf("get working directory: %v", err)
		// here we ignore the error in case the
		// working directory is not needed.
		// It would be better if the go-getter
		// could guess it only in cases it is
		// necessary.
	}

	ui.Say(fmt.Sprintf("Trying %s", u.String()))
	gc := getter.Client{
		Ctx:              ctx,
		Dst:              targetPath,
		Src:              u.String(),
		ProgressListener: ui,
		Pwd:              wd,
		Dir:              false,
	}

	switch err := gc.Get(); err.(type) {
	case nil: // success !
		ui.Say(fmt.Sprintf("%s => %s", u.String(), targetPath))
		return targetPath, nil
	case *getter.ChecksumError:
		ui.Say(fmt.Sprintf("Checksum did not match, removing %s", targetPath))
		if err := os.Remove(targetPath); err != nil {
			ui.Error(fmt.Sprintf("Failed to remove cache file. Please remove manually: %s", targetPath))
		}
		return "", err
	default:
		ui.Say(fmt.Sprintf("Download failed %s", err))
		return "", err
	}
}

func (s *StepDownload) Cleanup(multistep.StateBag) {}
<commit_msg>step download: ovf files usually point to a file in the same directory, using them in place is the only way<commit_after>package common

import (
	"context"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/gofrs/flock"
	getter "github.com/hashicorp/go-getter"
	urlhelper "github.com/hashicorp/go-getter/helper/url"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

// StepDownload downloads a remote file using the download client within
// this package. This step handles setting up the download configuration,
// progress reporting, interrupt handling, etc.
//
// Uses:
//   cache packer.Cache
//   ui    packer.Ui
type StepDownload struct {
	// The checksum and the type of the checksum for the download
	Checksum     string
	ChecksumType string

	// A short description of the type of download being done. Example:
	// "ISO" or "Guest Additions"
	Description string

	// The name of the key where the final path of the ISO will be put
	// into the state.
	ResultKey string

	// The path where the result should go, otherwise it goes to the
	// cache directory.
	TargetPath string

	// A list of URLs to attempt to download this thing.
	Url []string

	// Extension is the extension to force for the file that is downloaded.
	// Some systems require a certain extension. If this isn't set, the
	// extension on the URL is used. Otherwise, this will be forced
	// on the downloaded file for every URL.
	Extension string
}

func (s *StepDownload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
	ui := state.Get("ui").(packer.Ui)
	defer ui.Say(fmt.Sprintf("leaving retrieve loop for %s", s.Description))

	ui.Say(fmt.Sprintf("Retrieving %s", s.Description))

	var errs []error
	for _, source := range s.Url {
		if ctx.Err() != nil {
			state.Put("error", fmt.Errorf("Download cancelled: %v", errs))
			return multistep.ActionHalt
		}
		ui.Say(fmt.Sprintf("Trying %s", source))
		var err error
		var dst string
		if s.Description == "OVF/OVA" && strings.HasSuffix(source, ".ovf") {
			// TODO(adrien): make go-getter allow using files in place.
			// ovf files usually point to a file in the same directory, so
			// using them in place is the only way.
			ui.Say(fmt.Sprintf("Using ovf inplace"))
			dst = source
		} else {
			dst, err = s.download(ctx, ui, source)
		}
		if err == nil {
			state.Put(s.ResultKey, dst)
			return multistep.ActionContinue
		}
		// may be another url will work
		errs = append(errs, err)
	}

	state.Put("error", fmt.Errorf("Downloading file: %v", errs))
	return multistep.ActionHalt
}

func (s *StepDownload) download(ctx context.Context, ui packer.Ui, source string) (string, error) {
	u, err := urlhelper.Parse(source)
	if err != nil {
		return "", fmt.Errorf("url parse: %s", err)
	}
	if checksum := u.Query().Get("checksum"); checksum != "" {
		s.Checksum = checksum
	}
	if s.ChecksumType != "" && s.ChecksumType != "none" {
		// add checksum to url query params as go getter will checksum for us
		q := u.Query()
		q.Set("checksum", s.ChecksumType+":"+s.Checksum)
		u.RawQuery = q.Encode()
	} else if s.Checksum != "" {
		q := u.Query()
		q.Set("checksum", s.Checksum)
		u.RawQuery = q.Encode()
	} else if s.ChecksumType != "none" {
		return "", fmt.Errorf("Empty checksum")
	}

	targetPath := s.TargetPath
	if targetPath == "" {
		// store file under sha1(hash) if set
		// hash can sometimes be a checksum url
		// otherwise, use sha1(source_url)
		var shaSum [20]byte
		if s.Checksum != "" {
			shaSum = sha1.Sum([]byte(s.Checksum))
		} else {
			shaSum = sha1.Sum([]byte(u.String()))
		}
		targetPath = hex.EncodeToString(shaSum[:])
		if s.Extension != "" {
			targetPath += "." + s.Extension
		}
	}
	targetPath, err = packer.CachePath(targetPath)
	if err != nil {
		return "", fmt.Errorf("CachePath: %s", err)
	}
	lockFile := targetPath + ".lock"

	log.Printf("Acquiring lock for: %s (%s)", u.String(), lockFile)
	lock := flock.New(lockFile)
	lock.Lock()
	defer lock.Unlock()

	wd, err := os.Getwd()
	if err != nil {
		log.Printf("get working directory: %v", err)
		// here we ignore the error in case the
		// working directory is not needed.
		// It would be better if the go-getter
		// could guess it only in cases it is
		// necessary.
	}

	ui.Say(fmt.Sprintf("Trying %s", u.String()))
	gc := getter.Client{
		Ctx:              ctx,
		Dst:              targetPath,
		Src:              u.String(),
		ProgressListener: ui,
		Pwd:              wd,
		Dir:              false,
	}

	switch err := gc.Get(); err.(type) {
	case nil: // success !
		ui.Say(fmt.Sprintf("%s => %s", u.String(), targetPath))
		return targetPath, nil
	case *getter.ChecksumError:
		ui.Say(fmt.Sprintf("Checksum did not match, removing %s", targetPath))
		if err := os.Remove(targetPath); err != nil {
			ui.Error(fmt.Sprintf("Failed to remove cache file. Please remove manually: %s", targetPath))
		}
		return "", err
	default:
		ui.Say(fmt.Sprintf("Download failed %s", err))
		return "", err
	}
}

func (s *StepDownload) Cleanup(multistep.StateBag) {}
<|endoftext|>