<commit_before>package netpol

import (
	"crypto/sha256"
	"encoding/base32"
	"strings"

	api "k8s.io/api/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog/v2"
)

func (npc *NetworkPolicyController) newPodEventHandler() cache.ResourceEventHandler {
	return cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			if podObj, ok := obj.(*api.Pod); ok {
				// If the pod isn't yet actionable there is no action to take here anyway, so skip it. When it becomes
				// actionable, we'll get an update below.
				if isNetPolActionable(podObj) {
					npc.OnPodUpdate(obj)
				}
			}
		},
		UpdateFunc: func(oldObj, newObj interface{}) {
			var newPodObj, oldPodObj *api.Pod
			var ok bool

			// If either of these objects are not pods, quit now
			if newPodObj, ok = newObj.(*api.Pod); !ok {
				return
			}
			if oldPodObj, ok = oldObj.(*api.Pod); !ok {
				return
			}

			// We don't check isNetPolActionable here, because if it is transitioning in or out of the actionable state
			// we want to run the full sync so that it can be added or removed from the existing network policy of the host
			// For the network policies, we are only interested in some changes, most pod changes aren't relevant to network policy
			if isPodUpdateNetPolRelevant(oldPodObj, newPodObj) {
				npc.OnPodUpdate(newObj)
			}
		},
		DeleteFunc: func(obj interface{}) {
			npc.handlePodDelete(obj)
		},
	}
}

// OnPodUpdate handles updates to pods from the Kubernetes api server
func (npc *NetworkPolicyController) OnPodUpdate(obj interface{}) {
	pod := obj.(*api.Pod)
	klog.V(2).Infof("Received update to pod: %s/%s", pod.Namespace, pod.Name)

	npc.RequestFullSync()
}

func (npc *NetworkPolicyController) handlePodDelete(obj interface{}) {
	pod, ok := obj.(*api.Pod)
	if !ok {
		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
		if !ok {
			klog.Errorf("unexpected object type: %v", obj)
			return
		}
		if pod, ok = tombstone.Obj.(*api.Pod); !ok {
			klog.Errorf("unexpected object type: %v", obj)
			return
		}
	}
	klog.V(2).Infof("Received pod: %s/%s delete event", pod.Namespace, pod.Name)

	npc.RequestFullSync()
}

func (npc *NetworkPolicyController) syncPodFirewallChains(networkPoliciesInfo []networkPolicyInfo, version string) (map[string]bool, error) {

	activePodFwChains := make(map[string]bool)

	dropUnmarkedTrafficRules := func(podName, podNamespace, podFwChainName string) error {
		// add rule to log the packets that will be dropped due to network policy enforcement
		comment := "\"rule to log dropped traffic POD name:" + podName + " namespace: " + podNamespace + "\""
		args := []string{"-A", podFwChainName, "-m", "comment", "--comment", comment, "-m", "mark", "!", "--mark", "0x10000/0x10000", "-j", "NFLOG", "--nflog-group", "100", "-m", "limit", "--limit", "10/minute", "--limit-burst", "10", "\n"}
		// This used to be AppendUnique when we were using iptables directly, this checks to make sure we didn't drop unmarked for this chain already
		if strings.Contains(npc.filterTableRules.String(), strings.Join(args, " ")) {
			return nil
		}
		npc.filterTableRules.WriteString(strings.Join(args, " "))

		// add rule to DROP if no applicable network policy permits the traffic
		comment = "\"rule to REJECT traffic destined for POD name:" + podName + " namespace: " + podNamespace + "\""
		args = []string{"-A", podFwChainName, "-m", "comment", "--comment", comment, "-m", "mark", "!", "--mark", "0x10000/0x10000", "-j", "REJECT", "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))

		// reset mark to let traffic pass through rest of the chains
		args = []string{"-A", podFwChainName, "-j", "MARK", "--set-mark", "0/0x10000", "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))

		return nil
	}

	// loop through the pods running on the node
	allLocalPods, err := npc.getLocalPods(npc.nodeIP.String())
	if err != nil {
		return nil, err
	}
	for _, pod := range *allLocalPods {

		// ensure pod specific firewall chain exist for all the pods that need ingress firewall
		podFwChainName := podFirewallChainName(pod.namespace, pod.name, version)
		npc.filterTableRules.WriteString(":" + podFwChainName + "\n")

		activePodFwChains[podFwChainName] = true

		// setup rules to run pod inbound traffic through applicable ingress network policies
		err = npc.setupPodIngressRules(&pod, podFwChainName, networkPoliciesInfo, version)
		if err != nil {
			return nil, err
		}

		// setup rules to intercept inbound traffic to the pods
		err = npc.interceptPodInboundTraffic(&pod, podFwChainName)
		if err != nil {
			return nil, err
		}

		// setup rules to run pod inbound traffic through applicable ingress network policies
		err = npc.setupPodEgressRules(&pod, podFwChainName, networkPoliciesInfo, version)
		if err != nil {
			return nil, err
		}

		// setup rules to intercept inbound traffic to the pods
		err = npc.interceptPodOutboundTraffic(&pod, podFwChainName)
		if err != nil {
			return nil, err
		}

		err = dropUnmarkedTrafficRules(pod.name, pod.namespace, podFwChainName)
		if err != nil {
			return nil, err
		}

		// set mark to indicate traffic from/to the pod passed network policies.
		// Mark will be checked to explictly ACCEPT the traffic
		comment := "\"set mark to ACCEPT traffic that comply to network policies\""
		args := []string{"-A", podFwChainName, "-m", "comment", "--comment", comment, "-j", "MARK", "--set-mark", "0x20000/0x20000", "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	return activePodFwChains, nil
}

// setup rules to jump to applicable network policy chaings for the pod inbound traffic
func (npc *NetworkPolicyController) setupPodIngressRules(pod *podInfo, podFwChainName string, networkPoliciesInfo []networkPolicyInfo, version string) error {

	// ensure statefull firewall, that permits return traffic for the traffic originated by the pod
	comment := "\"rule for stateful firewall for pod\""
	args := []string{"-I", podFwChainName, "1", "-m", "comment", "--comment", comment, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT", "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

	// add entries in pod firewall to run through required network policies
	for _, policy := range networkPoliciesInfo {
		if _, ok := policy.targetPods[pod.ip]; ok {
			comment := "\"run through nw policy " + policy.name + "\""
			policyChainName := networkPolicyChainName(policy.namespace, policy.name, version)
			args := []string{"-I", podFwChainName, "1", "-m", "comment", "--comment", comment, "-j", policyChainName, "\n"}
			npc.filterTableRules.WriteString(strings.Join(args, " "))
		}
	}

	// if pod does not have any network policy which applies rules for pod's ingress traffic
	// then apply default network policy
	if !npc.isIngressNetworkPolicyEnabledPod(networkPoliciesInfo, pod) {
		comment := "\"run through default ingress policy  chain\""
		args := []string{"-I", podFwChainName, "1", "-d", pod.ip, "-m", "comment", "--comment", comment, "-j", kubeDefaultNetpolChain, "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	comment = "\"rule to permit the traffic traffic to pods when source is the pod's local node\""
	args = []string{"-I", podFwChainName, "1", "-m", "comment", "--comment", comment, "-m", "addrtype", "--src-type", "LOCAL", "-d", pod.ip, "-j", "ACCEPT", "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

	return nil
}

func (npc *NetworkPolicyController) interceptPodInboundTraffic(pod *podInfo, podFwChainName string) error {
	// ensure there is rule in filter table and FORWARD chain to jump to pod specific firewall chain
	// this rule applies to the traffic getting routed (coming for other node pods)
	comment := "\"rule to jump traffic destined to POD name:" + pod.name + " namespace: " + pod.namespace +
		" to chain " + podFwChainName + "\""
	args := []string{"-I", kubeForwardChainName, "1", "-m", "comment", "--comment", comment, "-d", pod.ip, "-j", podFwChainName + "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

	// ensure there is rule in filter table and OUTPUT chain to jump to pod specific firewall chain
	// this rule applies to the traffic from a pod getting routed back to another pod on same node by service proxy
	args = []string{"-I", kubeOutputChainName, "1", "-m", "comment", "--comment", comment, "-d", pod.ip, "-j", podFwChainName + "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

	// ensure there is rule in filter table and forward chain to jump to pod specific firewall chain
	// this rule applies to the traffic getting switched (coming for same node pods)
	comment = "\"rule to jump traffic destined to POD name:" + pod.name + " namespace: " + pod.namespace +
		" to chain " + podFwChainName + "\""
	args = []string{"-I", kubeForwardChainName, "1", "-m", "physdev", "--physdev-is-bridged",
		"-m", "comment", "--comment", comment,
		"-d", pod.ip,
		"-j", podFwChainName, "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))
	return nil
}

// setup rules to jump to applicable network policy chains for the pod outbound traffic
func (npc *NetworkPolicyController) setupPodEgressRules(pod *podInfo, podFwChainName string, networkPoliciesInfo []networkPolicyInfo, version string) error {

	// ensure statefull firewall, that permits return traffic for the traffic originated by the pod
	comment := "\"rule for stateful firewall for pod\""
	args := []string{"-I", podFwChainName, "1", "-m", "comment", "--comment", comment, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT", "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

	// add entries in pod firewall to run through required network policies
	for _, policy := range networkPoliciesInfo {
		if _, ok := policy.targetPods[pod.ip]; ok {
			comment := "\"run through nw policy " + policy.name + "\""
			policyChainName := networkPolicyChainName(policy.namespace, policy.name, version)
			args := []string{"-I", podFwChainName, "1", "-m", "comment", "--comment", comment, "-j", policyChainName, "\n"}
			npc.filterTableRules.WriteString(strings.Join(args, " "))
		}
	}

	// if pod does not have any network policy which applies rules for pod's egress traffic
	// then apply default network policy
	if !npc.isEgressNetworkPolicyEnabledPod(networkPoliciesInfo, pod) {
		comment := "\"run through default network policy  chain\""
		args := []string{"-I", podFwChainName, "1", "-s", pod.ip, "-m", "comment", "--comment", comment, "-j", kubeDefaultNetpolChain, "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	return nil
}

// setup iptable rules to intercept outbound traffic from pods and run it across the
// firewall chain corresponding to the pod so that egress network policies are enforced
func (npc *NetworkPolicyController) interceptPodOutboundTraffic(pod *podInfo, podFwChainName string) error {
	egressFilterChains := []string{kubeInputChainName, kubeForwardChainName, kubeOutputChainName}
	for _, chain := range egressFilterChains {
		// ensure there is rule in filter table and FORWARD chain to jump to pod specific firewall chain
		// this rule applies to the traffic getting forwarded/routed (traffic from the pod destinted
		// to pod on a different node)
		comment := "\"rule to jump traffic from POD name:" + pod.name + " namespace: " + pod.namespace +
			" to chain " + podFwChainName + "\""
		args := []string{"-I", chain, "1", "-m", "comment", "--comment", comment, "-s", pod.ip, "-j", podFwChainName, "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	// ensure there is rule in filter table and forward chain to jump to pod specific firewall chain
	// this rule applies to the traffic getting switched (coming for same node pods)
	comment := "\"rule to jump traffic from POD name:" + pod.name + " namespace: " + pod.namespace +
		" to chain " + podFwChainName + "\""
	args := []string{"-I", kubeForwardChainName, "1", "-m", "physdev", "--physdev-is-bridged",
		"-m", "comment", "--comment", comment,
		"-s", pod.ip,
		"-j", podFwChainName, "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))
	return nil
}

func (npc *NetworkPolicyController) getLocalPods(nodeIP string) (*map[string]podInfo, error) {
	localPods := make(map[string]podInfo)
	for _, obj := range npc.podLister.List() {
		pod := obj.(*api.Pod)
		// ignore the pods running on the different node and pods that are not actionable
		if strings.Compare(pod.Status.HostIP, nodeIP) != 0 || !isNetPolActionable(pod) {
			continue
		}
		localPods[pod.Status.PodIP] = podInfo{ip: pod.Status.PodIP,
			name:      pod.ObjectMeta.Name,
			namespace: pod.ObjectMeta.Namespace,
			labels:    pod.ObjectMeta.Labels}
	}
	return &localPods, nil
}

func (npc *NetworkPolicyController) isIngressNetworkPolicyEnabledPod(networkPoliciesInfo []networkPolicyInfo, pod *podInfo) bool {
	for _, policy := range networkPoliciesInfo {
		if policy.namespace != pod.namespace {
			continue
		}
		_, ok := policy.targetPods[pod.ip]
		if ok && (policy.policyType == "both" || policy.policyType == "ingress") {
			return true
		}
	}
	return false
}

func (npc *NetworkPolicyController) isEgressNetworkPolicyEnabledPod(networkPoliciesInfo []networkPolicyInfo, pod *podInfo) bool {
	for _, policy := range networkPoliciesInfo {
		if policy.namespace != pod.namespace {
			continue
		}
		_, ok := policy.targetPods[pod.ip]
		if ok && (policy.policyType == "both" || policy.policyType == "egress") {
			return true
		}
	}
	return false
}

func podFirewallChainName(namespace, podName string, version string) string {
	hash := sha256.Sum256([]byte(namespace + podName + version))
	encoded := base32.StdEncoding.EncodeToString(hash[:])
	return kubePodFirewallChainPrefix + encoded[:16]
}
<commit_msg>addressing review comments<commit_after>package netpol

import (
	"crypto/sha256"
	"encoding/base32"
	"strings"

	api "k8s.io/api/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog/v2"
)

func (npc *NetworkPolicyController) newPodEventHandler() cache.ResourceEventHandler {
	return cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			if podObj, ok := obj.(*api.Pod); ok {
				// If the pod isn't yet actionable there is no action to take here anyway, so skip it. When it becomes
				// actionable, we'll get an update below.
				if isNetPolActionable(podObj) {
					npc.OnPodUpdate(obj)
				}
			}
		},
		UpdateFunc: func(oldObj, newObj interface{}) {
			var newPodObj, oldPodObj *api.Pod
			var ok bool

			// If either of these objects are not pods, quit now
			if newPodObj, ok = newObj.(*api.Pod); !ok {
				return
			}
			if oldPodObj, ok = oldObj.(*api.Pod); !ok {
				return
			}

			// We don't check isNetPolActionable here, because if it is transitioning in or out of the actionable state
			// we want to run the full sync so that it can be added or removed from the existing network policy of the host
			// For the network policies, we are only interested in some changes, most pod changes aren't relevant to network policy
			if isPodUpdateNetPolRelevant(oldPodObj, newPodObj) {
				npc.OnPodUpdate(newObj)
			}
		},
		DeleteFunc: func(obj interface{}) {
			npc.handlePodDelete(obj)
		},
	}
}

// OnPodUpdate handles updates to pods from the Kubernetes api server
func (npc *NetworkPolicyController) OnPodUpdate(obj interface{}) {
	pod := obj.(*api.Pod)
	klog.V(2).Infof("Received update to pod: %s/%s", pod.Namespace, pod.Name)

	npc.RequestFullSync()
}

func (npc *NetworkPolicyController) handlePodDelete(obj interface{}) {
	pod, ok := obj.(*api.Pod)
	if !ok {
		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
		if !ok {
			klog.Errorf("unexpected object type: %v", obj)
			return
		}
		if pod, ok = tombstone.Obj.(*api.Pod); !ok {
			klog.Errorf("unexpected object type: %v", obj)
			return
		}
	}
	klog.V(2).Infof("Received pod: %s/%s delete event", pod.Namespace, pod.Name)

	npc.RequestFullSync()
}

func (npc *NetworkPolicyController) syncPodFirewallChains(networkPoliciesInfo []networkPolicyInfo, version string) (map[string]bool, error) {

	activePodFwChains := make(map[string]bool)

	dropUnmarkedTrafficRules := func(podName, podNamespace, podFwChainName string) error {
		// add rule to log the packets that will be dropped due to network policy enforcement
		comment := "\"rule to log dropped traffic POD name:" + podName + " namespace: " + podNamespace + "\""
		args := []string{"-A", podFwChainName, "-m", "comment", "--comment", comment, "-m", "mark", "!", "--mark", "0x10000/0x10000", "-j", "NFLOG", "--nflog-group", "100", "-m", "limit", "--limit", "10/minute", "--limit-burst", "10", "\n"}
		// This used to be AppendUnique when we were using iptables directly, this checks to make sure we didn't drop unmarked for this chain already
		if strings.Contains(npc.filterTableRules.String(), strings.Join(args, " ")) {
			return nil
		}
		npc.filterTableRules.WriteString(strings.Join(args, " "))

		// add rule to DROP if no applicable network policy permits the traffic
		comment = "\"rule to REJECT traffic destined for POD name:" + podName + " namespace: " + podNamespace + "\""
		args = []string{"-A", podFwChainName, "-m", "comment", "--comment", comment, "-m", "mark", "!", "--mark", "0x10000/0x10000", "-j", "REJECT", "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))

		// reset mark to let traffic pass through rest of the chains
		args = []string{"-A", podFwChainName, "-j", "MARK", "--set-mark", "0/0x10000", "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))

		return nil
	}

	// loop through the pods running on the node
	allLocalPods, err := npc.getLocalPods(npc.nodeIP.String())
	if err != nil {
		return nil, err
	}
	for _, pod := range *allLocalPods {

		// ensure pod specific firewall chain exist for all the pods that need ingress firewall
		podFwChainName := podFirewallChainName(pod.namespace, pod.name, version)
		npc.filterTableRules.WriteString(":" + podFwChainName + "\n")

		activePodFwChains[podFwChainName] = true

		// setup rules to run through applicable ingress/egress network policies for the pod
		npc.setupPodNetpolRules(&pod, podFwChainName, networkPoliciesInfo, version)

		// setup rules to intercept inbound traffic to the pods
		npc.interceptPodInboundTraffic(&pod, podFwChainName)

		// setup rules to intercept inbound traffic to the pods
		npc.interceptPodOutboundTraffic(&pod, podFwChainName)

		err = dropUnmarkedTrafficRules(pod.name, pod.namespace, podFwChainName)
		if err != nil {
			return nil, err
		}

		// set mark to indicate traffic from/to the pod passed network policies.
		// Mark will be checked to explictly ACCEPT the traffic
		comment := "\"set mark to ACCEPT traffic that comply to network policies\""
		args := []string{"-A", podFwChainName, "-m", "comment", "--comment", comment, "-j", "MARK", "--set-mark", "0x20000/0x20000", "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	return activePodFwChains, nil
}

// setup rules to jump to applicable network policy chains for the traffic from/to the pod
func (npc *NetworkPolicyController) setupPodNetpolRules(pod *podInfo, podFwChainName string, networkPoliciesInfo []networkPolicyInfo, version string) {

	hasIngressPolicy := false
	hasEgressPolicy := false

	// add entries in pod firewall to run through applicable network policies
	for _, policy := range networkPoliciesInfo {
		if _, ok := policy.targetPods[pod.ip]; !ok {
			continue
		}
		comment := "\"run through nw policy " + policy.name + "\""
		policyChainName := networkPolicyChainName(policy.namespace, policy.name, version)
		var args []string
		switch policy.policyType {
		case "both":
			hasIngressPolicy = true
			hasEgressPolicy = true
			args = []string{"-I", podFwChainName, "1", "-m", "comment", "--comment", comment, "-j", policyChainName, "\n"}
		case "ingress":
			hasIngressPolicy = true
			args = []string{"-I", podFwChainName, "1", "-d", pod.ip, "-m", "comment", "--comment", comment, "-j", policyChainName, "\n"}
		case "egress":
			hasEgressPolicy = true
			args = []string{"-I", podFwChainName, "1", "-s", pod.ip, "-m", "comment", "--comment", comment, "-j", policyChainName, "\n"}
		}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	// if pod does not have any network policy which applies rules for pod's ingress traffic
	// then apply default network policy
	if !hasIngressPolicy {
		comment := "\"run through default ingress network policy  chain\""
		args := []string{"-I", podFwChainName, "1", "-d", pod.ip, "-m", "comment", "--comment", comment, "-j", kubeDefaultNetpolChain, "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	// if pod does not have any network policy which applies rules for pod's egress traffic
	// then apply default network policy
	if !hasEgressPolicy {
		comment := "\"run through default egress network policy  chain\""
		args := []string{"-I", podFwChainName, "1", "-s", pod.ip, "-m", "comment", "--comment", comment, "-j", kubeDefaultNetpolChain, "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	comment := "\"rule to permit the traffic traffic to pods when source is the pod's local node\""
	args := []string{"-I", podFwChainName, "1", "-m", "comment", "--comment", comment, "-m", "addrtype", "--src-type", "LOCAL", "-d", pod.ip, "-j", "ACCEPT", "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

	// ensure statefull firewall that permits RELATED,ESTABLISHED traffic from/to the pod
	comment = "\"rule for stateful firewall for pod\""
	args = []string{"-I", podFwChainName, "1", "-m", "comment", "--comment", comment, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT", "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

}

func (npc *NetworkPolicyController) interceptPodInboundTraffic(pod *podInfo, podFwChainName string) {
	// ensure there is rule in filter table and FORWARD chain to jump to pod specific firewall chain
	// this rule applies to the traffic getting routed (coming for other node pods)
	comment := "\"rule to jump traffic destined to POD name:" + pod.name + " namespace: " + pod.namespace +
		" to chain " + podFwChainName + "\""
	args := []string{"-I", kubeForwardChainName, "1", "-m", "comment", "--comment", comment, "-d", pod.ip, "-j", podFwChainName + "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

	// ensure there is rule in filter table and OUTPUT chain to jump to pod specific firewall chain
	// this rule applies to the traffic from a pod getting routed back to another pod on same node by service proxy
	args = []string{"-I", kubeOutputChainName, "1", "-m", "comment", "--comment", comment, "-d", pod.ip, "-j", podFwChainName + "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))

	// ensure there is rule in filter table and forward chain to jump to pod specific firewall chain
	// this rule applies to the traffic getting switched (coming for same node pods)
	comment = "\"rule to jump traffic destined to POD name:" + pod.name + " namespace: " + pod.namespace +
		" to chain " + podFwChainName + "\""
	args = []string{"-I", kubeForwardChainName, "1", "-m", "physdev", "--physdev-is-bridged",
		"-m", "comment", "--comment", comment,
		"-d", pod.ip,
		"-j", podFwChainName, "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))
}

// setup iptable rules to intercept outbound traffic from pods and run it across the
// firewall chain corresponding to the pod so that egress network policies are enforced
func (npc *NetworkPolicyController) interceptPodOutboundTraffic(pod *podInfo, podFwChainName string) {
	egressFilterChains := []string{kubeInputChainName, kubeForwardChainName, kubeOutputChainName}
	for _, chain := range egressFilterChains {
		// ensure there is rule in filter table and FORWARD chain to jump to pod specific firewall chain
		// this rule applies to the traffic getting forwarded/routed (traffic from the pod destinted
		// to pod on a different node)
		comment := "\"rule to jump traffic from POD name:" + pod.name + " namespace: " + pod.namespace +
			" to chain " + podFwChainName + "\""
		args := []string{"-I", chain, "1", "-m", "comment", "--comment", comment, "-s", pod.ip, "-j", podFwChainName, "\n"}
		npc.filterTableRules.WriteString(strings.Join(args, " "))
	}

	// ensure there is rule in filter table and forward chain to jump to pod specific firewall chain
	// this rule applies to the traffic getting switched (coming for same node pods)
	comment := "\"rule to jump traffic from POD name:" + pod.name + " namespace: " + pod.namespace +
		" to chain " + podFwChainName + "\""
	args := []string{"-I", kubeForwardChainName, "1", "-m", "physdev", "--physdev-is-bridged",
		"-m", "comment", "--comment", comment,
		"-s", pod.ip,
		"-j", podFwChainName, "\n"}
	npc.filterTableRules.WriteString(strings.Join(args, " "))
}

func (npc *NetworkPolicyController) getLocalPods(nodeIP string) (*map[string]podInfo, error) {
	localPods := make(map[string]podInfo)
	for _, obj := range npc.podLister.List() {
		pod := obj.(*api.Pod)
		// ignore the pods running on the different node and pods that are not actionable
		if strings.Compare(pod.Status.HostIP, nodeIP) != 0 || !isNetPolActionable(pod) {
			continue
		}
		localPods[pod.Status.PodIP] = podInfo{ip: pod.Status.PodIP,
			name:      pod.ObjectMeta.Name,
			namespace: pod.ObjectMeta.Namespace,
			labels:    pod.ObjectMeta.Labels}
	}
	return &localPods, nil
}

func podFirewallChainName(namespace, podName string, version string) string {
	hash := sha256.Sum256([]byte(namespace + podName + version))
	encoded := base32.StdEncoding.EncodeToString(hash[:])
	return kubePodFirewallChainPrefix + encoded[:16]
}
<|endoftext|>


<commit_before>//go:build linux || solaris || darwin || freebsd
// +build linux solaris darwin freebsd

package lockfile

import (
	"bytes"
	cryptorand "crypto/rand"
	"encoding/binary"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"github.com/containers/storage/pkg/system"
	"golang.org/x/sys/unix"
)

type lockfile struct {
	// rwMutex serializes concurrent reader-writer acquisitions in the same process space
	rwMutex *sync.RWMutex
	// stateMutex is used to synchronize concurrent accesses to the state below
	stateMutex *sync.Mutex
	counter    int64
	file       string
	fd         uintptr
	lw         []byte // "last writer"-unique value valid as of the last .Touch() or .Modified(), generated by newLastWriterID()
	locktype   int16
	locked     bool
	ro         bool
}

const lastWriterIDSize = 64    // This must be the same as len(stringid.GenerateRandomID)
var lastWriterIDCounter uint64 // Private state for newLastWriterID

// newLastWriterID returns a new "last writer" ID.
// The value must be different on every call, and also differ from values
// generated by other processes.
func newLastWriterID() []byte {
	// The ID is (PID, time, per-process counter, random)
	// PID + time represents both a unique process across reboots,
	// and a specific time within the process; the per-process counter
	// is an extra safeguard for in-process concurrency.
	// The random part disambiguates across process namespaces
	// (where PID values might collide), serves as a general-purpose
	// extra safety, _and_ is used to pad the output to lastWriterIDSize,
	// because other versions of this code exist and they don't work
	// efficiently if the size of the value changes.
	pid := os.Getpid()
	tm := time.Now().UnixNano()
	counter := atomic.AddUint64(&lastWriterIDCounter, 1)

	res := make([]byte, lastWriterIDSize)
	binary.LittleEndian.PutUint64(res[0:8], uint64(tm))
	binary.LittleEndian.PutUint64(res[8:16], counter)
	binary.LittleEndian.PutUint32(res[16:20], uint32(pid))
	if n, err := cryptorand.Read(res[20:lastWriterIDSize]); err != nil || n != lastWriterIDSize-20 {
		panic(err) // This shouldn't happen
	}

	return res
}

// openLock opens the file at path and returns the corresponding file
// descriptor. The path is opened either read-only or read-write,
// depending on the value of ro argument.
//
// openLock will create the file and its parent directories,
// if necessary.
func openLock(path string, ro bool) (fd int, err error) {
	flags := unix.O_CLOEXEC | os.O_CREATE
	if ro {
		flags |= os.O_RDONLY
	} else {
		flags |= os.O_RDWR
	}
	fd, err = unix.Open(path, flags, 0o644)
	if err == nil {
		return fd, nil
	}

	// the directory of the lockfile seems to be removed, try to create it
	if os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
			return fd, fmt.Errorf("creating locker directory: %w", err)
		}

		return openLock(path, ro)
	}

	return fd, &os.PathError{Op: "open", Path: path, Err: err}
}

// createLockerForPath returns a Locker object, possibly (depending on the platform)
// working inter-process and associated with the specified path.
//
// This function will be called at most once for each path value within a single process.
//
// If ro, the lock is a read-write lock and the returned Locker should correspond to the
// “lock for reading” (shared) operation; otherwise, the lock is either an exclusive lock,
// or a read-write lock and Locker should correspond to the “lock for writing” (exclusive) operation.
//
// WARNING:
// - The lock may or MAY NOT be inter-process.
// - There may or MAY NOT be an actual object on the filesystem created for the specified path.
// - Even if ro, the lock MAY be exclusive.
func createLockerForPath(path string, ro bool) (Locker, error) {
	// Check if we can open the lock.
	fd, err := openLock(path, ro)
	if err != nil {
		return nil, err
	}
	unix.Close(fd)

	locktype := unix.F_WRLCK
	if ro {
		locktype = unix.F_RDLCK
	}
	return &lockfile{
		stateMutex: &sync.Mutex{},
		rwMutex:    &sync.RWMutex{},
		file:       path,
		lw:         newLastWriterID(),
		locktype:   int16(locktype),
		locked:     false,
		ro:         ro}, nil
}

// lock locks the lockfile via FCTNL(2) based on the specified type and
// command.
func (l *lockfile) lock(lType int16) {
	lk := unix.Flock_t{
		Type:   lType,
		Whence: int16(unix.SEEK_SET),
		Start:  0,
		Len:    0,
	}
	switch lType {
	case unix.F_RDLCK:
		l.rwMutex.RLock()
	case unix.F_WRLCK:
		l.rwMutex.Lock()
	default:
		panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", lType))
	}
	l.stateMutex.Lock()
	defer l.stateMutex.Unlock()
	if l.counter == 0 {
		// If we're the first reference on the lock, we need to open the file again.
		fd, err := openLock(l.file, l.ro)
		if err != nil {
			panic(err)
		}
		l.fd = uintptr(fd)

		// Optimization: only use the (expensive) fcntl syscall when
		// the counter is 0.  In this case, we're either the first
		// reader lock or a writer lock.
		for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil {
			time.Sleep(10 * time.Millisecond)
		}
	}
	l.locktype = lType
	l.locked = true
	l.counter++
}

// Lock locks the lockfile as a writer.  Panic if the lock is a read-only one.
func (l *lockfile) Lock() {
	if l.ro {
		panic("can't take write lock on read-only lock file")
	} else {
		l.lock(unix.F_WRLCK)
	}
}

// LockRead locks the lockfile as a reader.
func (l *lockfile) RLock() {
	l.lock(unix.F_RDLCK)
}

// Unlock unlocks the lockfile.
func (l *lockfile) Unlock() {
	l.stateMutex.Lock()
	if !l.locked {
		// Panic when unlocking an unlocked lock.  That's a violation
		// of the lock semantics and will reveal such.
		panic("calling Unlock on unlocked lock")
	}
	l.counter--
	if l.counter < 0 {
		// Panic when the counter is negative.  There is no way we can
		// recover from a corrupted lock and we need to protect the
		// storage from corruption.
		panic(fmt.Sprintf("lock %q has been unlocked too often", l.file))
	}
	if l.counter == 0 {
		// We should only release the lock when the counter is 0 to
		// avoid releasing read-locks too early; a given process may
		// acquire a read lock multiple times.
		l.locked = false
		// Close the file descriptor on the last unlock, releasing the
		// file lock.
		unix.Close(int(l.fd))
	}
	if l.locktype == unix.F_RDLCK {
		l.rwMutex.RUnlock()
	} else {
		l.rwMutex.Unlock()
	}
	l.stateMutex.Unlock()
}

// Locked checks if lockfile is locked for writing by a thread in this process.
func (l *lockfile) Locked() bool {
	l.stateMutex.Lock()
	defer l.stateMutex.Unlock()
	return l.locked && (l.locktype == unix.F_WRLCK)
}

// Touch updates the lock file with the UID of the user.
func (l *lockfile) Touch() error {
	l.stateMutex.Lock()
	if !l.locked || (l.locktype != unix.F_WRLCK) {
		panic("attempted to update last-writer in lockfile without the write lock")
	}
	defer l.stateMutex.Unlock()
	l.lw = newLastWriterID()
	n, err := unix.Pwrite(int(l.fd), l.lw, 0)
	if err != nil {
		return err
	}
	if n != len(l.lw) {
		return unix.ENOSPC
	}
	return nil
}

// Modified indicates if the lockfile has been updated since the last time it
// was loaded.
func (l *lockfile) Modified() (bool, error) {
	l.stateMutex.Lock()
	if !l.locked {
		panic("attempted to check last-writer in lockfile without locking it first")
	}
	defer l.stateMutex.Unlock()
	currentLW := make([]byte, len(l.lw))
	n, err := unix.Pread(int(l.fd), currentLW, 0)
	if err != nil {
		return true, err
	}
	// It is important to handle the partial read case, because
	// the initial size of the lock file is zero, which is a valid
	// state (no writes yet)
	currentLW = currentLW[:n]
	oldLW := l.lw
	l.lw = currentLW
	return !bytes.Equal(currentLW, oldLW), nil
}

// IsReadWriteLock indicates if the lock file is a read-write lock.
func (l *lockfile) IsReadWrite() bool {
	return !l.ro
}

// TouchedSince indicates if the lock file has been touched since the specified time
func (l *lockfile) TouchedSince(when time.Time) bool {
	st, err := system.Fstat(int(l.fd))
	if err != nil {
		return true
	}
	mtim := st.Mtim()
	touched := time.Unix(mtim.Unix())
	return when.Before(touched)
}
<commit_msg>Fix lockfiles never detecting changes<commit_after>//go:build linux || solaris || darwin || freebsd
// +build linux solaris darwin freebsd

package lockfile

import (
	"bytes"
	cryptorand "crypto/rand"
	"encoding/binary"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"github.com/containers/storage/pkg/system"
	"golang.org/x/sys/unix"
)

type lockfile struct {
	// rwMutex serializes concurrent reader-writer acquisitions in the same process space
	rwMutex *sync.RWMutex
	// stateMutex is used to synchronize concurrent accesses to the state below
	stateMutex *sync.Mutex
	counter    int64
	file       string
	fd         uintptr
	lw         []byte // "last writer"-unique value valid as of the last .Touch() or .Modified(), generated by newLastWriterID()
	locktype   int16
	locked     bool
	ro         bool
}

const lastWriterIDSize = 64    // This must be the same as len(stringid.GenerateRandomID)
var lastWriterIDCounter uint64 // Private state for newLastWriterID

// newLastWriterID returns a new "last writer" ID.
// The value must be different on every call, and also differ from values
// generated by other processes.
func newLastWriterID() []byte {
	// The ID is (PID, time, per-process counter, random)
	// PID + time represents both a unique process across reboots,
	// and a specific time within the process; the per-process counter
	// is an extra safeguard for in-process concurrency.
	// The random part disambiguates across process namespaces
	// (where PID values might collide), serves as a general-purpose
	// extra safety, _and_ is used to pad the output to lastWriterIDSize,
	// because other versions of this code exist and they don't work
	// efficiently if the size of the value changes.
	pid := os.Getpid()
	tm := time.Now().UnixNano()
	counter := atomic.AddUint64(&lastWriterIDCounter, 1)

	res := make([]byte, lastWriterIDSize)
	binary.LittleEndian.PutUint64(res[0:8], uint64(tm))
	binary.LittleEndian.PutUint64(res[8:16], counter)
	binary.LittleEndian.PutUint32(res[16:20], uint32(pid))
	if n, err := cryptorand.Read(res[20:lastWriterIDSize]); err != nil || n != lastWriterIDSize-20 {
		panic(err) // This shouldn't happen
	}

	return res
}

// openLock opens the file at path and returns the corresponding file
// descriptor. The path is opened either read-only or read-write,
// depending on the value of ro argument.
//
// openLock will create the file and its parent directories,
// if necessary.
func openLock(path string, ro bool) (fd int, err error) {
	flags := unix.O_CLOEXEC | os.O_CREATE
	if ro {
		flags |= os.O_RDONLY
	} else {
		flags |= os.O_RDWR
	}
	fd, err = unix.Open(path, flags, 0o644)
	if err == nil {
		return fd, nil
	}

	// the directory of the lockfile seems to be removed, try to create it
	if os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
			return fd, fmt.Errorf("creating locker directory: %w", err)
		}

		return openLock(path, ro)
	}

	return fd, &os.PathError{Op: "open", Path: path, Err: err}
}

// createLockerForPath returns a Locker object, possibly (depending on the platform)
// working inter-process and associated with the specified path.
//
// This function will be called at most once for each path value within a single process.
//
// If ro, the lock is a read-write lock and the returned Locker should correspond to the
// “lock for reading” (shared) operation; otherwise, the lock is either an exclusive lock,
// or a read-write lock and Locker should correspond to the “lock for writing” (exclusive) operation.
//
// WARNING:
// - The lock may or MAY NOT be inter-process.
// - There may or MAY NOT be an actual object on the filesystem created for the specified path.
// - Even if ro, the lock MAY be exclusive.
func createLockerForPath(path string, ro bool) (Locker, error) {
	// Check if we can open the lock.
	fd, err := openLock(path, ro)
	if err != nil {
		return nil, err
	}
	unix.Close(fd)

	locktype := unix.F_WRLCK
	if ro {
		locktype = unix.F_RDLCK
	}
	return &lockfile{
		stateMutex: &sync.Mutex{},
		rwMutex:    &sync.RWMutex{},
		file:       path,
		lw:         newLastWriterID(),
		locktype:   int16(locktype),
		locked:     false,
		ro:         ro}, nil
}

// lock locks the lockfile via FCTNL(2) based on the specified type and
// command.
func (l *lockfile) lock(lType int16) {
	lk := unix.Flock_t{
		Type:   lType,
		Whence: int16(unix.SEEK_SET),
		Start:  0,
		Len:    0,
	}
	switch lType {
	case unix.F_RDLCK:
		l.rwMutex.RLock()
	case unix.F_WRLCK:
		l.rwMutex.Lock()
	default:
		panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", lType))
	}
	l.stateMutex.Lock()
	defer l.stateMutex.Unlock()
	if l.counter == 0 {
		// If we're the first reference on the lock, we need to open the file again.
		fd, err := openLock(l.file, l.ro)
		if err != nil {
			panic(err)
		}
		l.fd = uintptr(fd)

		// Optimization: only use the (expensive) fcntl syscall when
		// the counter is 0.  In this case, we're either the first
		// reader lock or a writer lock.
		for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil {
			time.Sleep(10 * time.Millisecond)
		}
	}
	l.locktype = lType
	l.locked = true
	l.counter++
}

// Lock locks the lockfile as a writer.  Panic if the lock is a read-only one.
func (l *lockfile) Lock() {
	if l.ro {
		panic("can't take write lock on read-only lock file")
	} else {
		l.lock(unix.F_WRLCK)
	}
}

// LockRead locks the lockfile as a reader.
func (l *lockfile) RLock() {
	l.lock(unix.F_RDLCK)
}

// Unlock unlocks the lockfile.
func (l *lockfile) Unlock() {
	l.stateMutex.Lock()
	if !l.locked {
		// Panic when unlocking an unlocked lock.  That's a violation
		// of the lock semantics and will reveal such.
		panic("calling Unlock on unlocked lock")
	}
	l.counter--
	if l.counter < 0 {
		// Panic when the counter is negative.  There is no way we can
		// recover from a corrupted lock and we need to protect the
		// storage from corruption.
		panic(fmt.Sprintf("lock %q has been unlocked too often", l.file))
	}
	if l.counter == 0 {
		// We should only release the lock when the counter is 0 to
		// avoid releasing read-locks too early; a given process may
		// acquire a read lock multiple times.
		l.locked = false
		// Close the file descriptor on the last unlock, releasing the
		// file lock.
		unix.Close(int(l.fd))
	}
	if l.locktype == unix.F_RDLCK {
		l.rwMutex.RUnlock()
	} else {
		l.rwMutex.Unlock()
	}
	l.stateMutex.Unlock()
}

// Locked checks if lockfile is locked for writing by a thread in this process.
func (l *lockfile) Locked() bool {
	l.stateMutex.Lock()
	defer l.stateMutex.Unlock()
	return l.locked && (l.locktype == unix.F_WRLCK)
}

// Touch updates the lock file with the UID of the user.
func (l *lockfile) Touch() error {
	l.stateMutex.Lock()
	if !l.locked || (l.locktype != unix.F_WRLCK) {
		panic("attempted to update last-writer in lockfile without the write lock")
	}
	defer l.stateMutex.Unlock()
	l.lw = newLastWriterID()
	n, err := unix.Pwrite(int(l.fd), l.lw, 0)
	if err != nil {
		return err
	}
	if n != len(l.lw) {
		return unix.ENOSPC
	}
	return nil
}

// Modified indicates if the lockfile has been updated since the last time it
// was loaded.
func (l *lockfile) Modified() (bool, error) {
	l.stateMutex.Lock()
	if !l.locked {
		panic("attempted to check last-writer in lockfile without locking it first")
	}
	defer l.stateMutex.Unlock()
	currentLW := make([]byte, lastWriterIDSize)
	n, err := unix.Pread(int(l.fd), currentLW, 0)
	if err != nil {
		return true, err
	}
	// It is important to handle the partial read case, because
	// the initial size of the lock file is zero, which is a valid
	// state (no writes yet)
	currentLW = currentLW[:n]
	oldLW := l.lw
	l.lw = currentLW
	return !bytes.Equal(currentLW, oldLW), nil
}

// IsReadWriteLock indicates if the lock file is a read-write lock.
func (l *lockfile) IsReadWrite() bool {
	return !l.ro
}

// TouchedSince indicates if the lock file has been touched since the specified time
func (l *lockfile) TouchedSince(when time.Time) bool {
	st, err := system.Fstat(int(l.fd))
	if err != nil {
		return true
	}
	mtim := st.Mtim()
	touched := time.Unix(mtim.Unix())
	return when.Before(touched)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package csi

import (
	"fmt"

	"github.com/rook/rook/pkg/operator/k8sutil"

	apps "k8s.io/api/apps/v1"
	"k8s.io/client-go/kubernetes"
)

type Param struct {
	Namespace string

	RBDPluginImage    string
	CephFSPluginImage string
	RegistrarImage    string
	ProvisionerImage  string
	AttacherImage     string
	SnapshotterImage  string
}

var (
	CSIParam Param

	EnableRBD    = true
	EnableCephFS = true

	// template paths
	RBDPluginTemplatePath      string
	RBDProvisionerTemplatePath string
	RBDAttacherTemplatePath    string

	CephFSPluginTemplatePath      string
	CephFSProvisionerTemplatePath string
)

const (
	KubeMinMajor = "1"
	KubeMinMinor = "13"

	// image names
	DefaultRBDPluginImage    = "quay.io/cephcsi/rbdplugin:v1.0.0"
	DefaultCephFSPluginImage = "quay.io/cephcsi/cephfsplugin:v1.0.0"
	DefaultRegistrarImage    = "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
	DefaultProvisionerImage  = "quay.io/k8scsi/csi-provisioner:v1.0.1"
	DefaultAttacherImage     = "quay.io/k8scsi/csi-attacher:v1.0.1"
	DefaultSnapshotterImage  = "quay.io/k8scsi/csi-snapshotter:v1.0.1"

	// template
	DefaultRBDPluginTemplatePath         = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml"
	DefaultRBDProvisionerTemplatePath    = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml"
	DefaultRBDAttacherTemplatePath       = "/etc/ceph-csi/rbd/csi-rbdplugin-attacher.yaml"
	DefaultCephFSPluginTemplatePath      = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml"
	DefaultCephFSProvisionerTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml"

	ExitOnError = false // don't exit if CSI fails to deploy. Switch to true when flexdriver is disabled
)

func CSIEnabled() bool {
	if EnableRBD || EnableCephFS {
		return true
	}
	return false
}

func SetCSINamespace(namespace string) {
	CSIParam.Namespace = namespace
}

func ValidateCSIParam() error {
	if EnableRBD {
		if len(CSIParam.RBDPluginImage) == 0 {
			return fmt.Errorf("missing csi rbd plugin image")
		}
		if len(CSIParam.RegistrarImage) == 0 {
			return fmt.Errorf("missing csi registrar image")
		}
		if len(CSIParam.ProvisionerImage) == 0 {
			return fmt.Errorf("missing csi provisioner image")
		}
		if len(CSIParam.AttacherImage) == 0 {
			return fmt.Errorf("missing csi attacher image")
		}
		if len(RBDPluginTemplatePath) == 0 {
			return fmt.Errorf("missing rbd plugin template path")
		}
		if len(RBDProvisionerTemplatePath) == 0 {
			return fmt.Errorf("missing rbd provisioner template path")
		}
		if len(RBDAttacherTemplatePath) == 0 {
			return fmt.Errorf("missing rbd attacher template path")
		}
	}

	if EnableCephFS {
		if len(CSIParam.CephFSPluginImage) == 0 {
			return fmt.Errorf("missing csi cephfs plugin image")
		}
		if len(CSIParam.RegistrarImage) == 0 {
			return fmt.Errorf("missing csi registrar image")
		}
		if len(CSIParam.ProvisionerImage) == 0 {
			return fmt.Errorf("missing csi provisioner image")
		}
		if len(CephFSPluginTemplatePath) == 0 {
			return fmt.Errorf("missing cephfs plugin template path")
		}
		if len(CephFSProvisionerTemplatePath) == 0 {
			return fmt.Errorf("missing ceph provisioner template path")
		}
	}
	return nil
}

func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
	var (
		err                                            error
		rbdPlugin, cephfsPlugin                        *apps.DaemonSet
		rbdProvisioner, rbdAttacher, cephfsProvisioner *apps.StatefulSet
	)

	if EnableRBD {
		rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load rbd plugin template: %v", err)
		}
		rbdProvisioner, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load rbd provisioner template: %v", err)
		}
		rbdAttacher, err = templateToStatefulSet("rbd-attacher", RBDAttacherTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load rbd attacher template: %v", err)
		}
	}
	if EnableCephFS {
		cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load CephFS plugin template: %v", err)
		}
		cephfsProvisioner, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load CephFS provisioner template: %v", err)
		}
	}

	if rbdPlugin != nil {
		err = k8sutil.CreateDaemonSet("csi rbd plugin", namespace, clientset, rbdPlugin)
		if err != nil {
			return fmt.Errorf("failed to start rbdplugin daemonset: %v\n%v", err, rbdPlugin)
		}
	}
	if rbdProvisioner != nil {
		_, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisioner)
		if err != nil {
			return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisioner)
		}

	}
	if rbdAttacher != nil {
		_, err = k8sutil.CreateStatefulSet("csi rbd attacher", namespace, "csi-rbdplugin-attacher", clientset, rbdAttacher)
		if err != nil {
			return fmt.Errorf("failed to start rbd attacher statefulset: %v\n%v", err, rbdAttacher)
		}
	}

	if cephfsPlugin != nil {
		err = k8sutil.CreateDaemonSet("csi cephfs plugin", namespace, clientset, cephfsPlugin)
		if err != nil {
			return fmt.Errorf("failed to start cephfs plugin daemonset: %v\n%v", err, cephfsPlugin)
		}
	}
	if cephfsProvisioner != nil {
		_, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisioner)
		if err != nil {
			return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisioner)
		}

	}
	return nil
}
<commit_msg>replace fmt.Errorf with errors.New<commit_after>/*
Copyright 2019 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package csi

import (
	"errors"
	"fmt"

	"github.com/rook/rook/pkg/operator/k8sutil"

	apps "k8s.io/api/apps/v1"
	"k8s.io/client-go/kubernetes"
)

type Param struct {
	Namespace string

	RBDPluginImage    string
	CephFSPluginImage string
	RegistrarImage    string
	ProvisionerImage  string
	AttacherImage     string
	SnapshotterImage  string
}

var (
	CSIParam Param

	EnableRBD    = true
	EnableCephFS = true

	// template paths
	RBDPluginTemplatePath      string
	RBDProvisionerTemplatePath string
	RBDAttacherTemplatePath    string

	CephFSPluginTemplatePath      string
	CephFSProvisionerTemplatePath string
)

const (
	KubeMinMajor = "1"
	KubeMinMinor = "13"

	// image names
	DefaultRBDPluginImage    = "quay.io/cephcsi/rbdplugin:v1.0.0"
	DefaultCephFSPluginImage = "quay.io/cephcsi/cephfsplugin:v1.0.0"
	DefaultRegistrarImage    = "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
	DefaultProvisionerImage  = "quay.io/k8scsi/csi-provisioner:v1.0.1"
	DefaultAttacherImage     = "quay.io/k8scsi/csi-attacher:v1.0.1"
	DefaultSnapshotterImage  = "quay.io/k8scsi/csi-snapshotter:v1.0.1"

	// template
	DefaultRBDPluginTemplatePath         = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml"
	DefaultRBDProvisionerTemplatePath    = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml"
	DefaultRBDAttacherTemplatePath       = "/etc/ceph-csi/rbd/csi-rbdplugin-attacher.yaml"
	DefaultCephFSPluginTemplatePath      = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml"
	DefaultCephFSProvisionerTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml"

	ExitOnError = false // don't exit if CSI fails to deploy. Switch to true when flexdriver is disabled
)

func CSIEnabled() bool {
	if EnableRBD || EnableCephFS {
		return true
	}
	return false
}

func SetCSINamespace(namespace string) {
	CSIParam.Namespace = namespace
}

func ValidateCSIParam() error {
	if EnableRBD {
		if len(CSIParam.RBDPluginImage) == 0 {
			return errors.New("missing csi rbd plugin image")
		}
		if len(CSIParam.RegistrarImage) == 0 {
			return errors.New("missing csi registrar image")
		}
		if len(CSIParam.ProvisionerImage) == 0 {
			return errors.New("missing csi provisioner image")
		}
		if len(CSIParam.AttacherImage) == 0 {
			return errors.New("missing csi attacher image")
		}
		if len(RBDPluginTemplatePath) == 0 {
			return errors.New("missing rbd plugin template path")
		}
		if len(RBDProvisionerTemplatePath) == 0 {
			return errors.New("missing rbd provisioner template path")
		}
		if len(RBDAttacherTemplatePath) == 0 {
			return errors.New("missing rbd attacher template path")
		}
	}

	if EnableCephFS {
		if len(CSIParam.CephFSPluginImage) == 0 {
			return errors.New("missing csi cephfs plugin image")
		}
		if len(CSIParam.RegistrarImage) == 0 {
			return errors.New("missing csi registrar image")
		}
		if len(CSIParam.ProvisionerImage) == 0 {
			return errors.New("missing csi provisioner image")
		}
		if len(CephFSPluginTemplatePath) == 0 {
			return fmt.Errorf("missing cephfs plugin template path")
		}
		if len(CephFSProvisionerTemplatePath) == 0 {
			return errors.New("missing ceph provisioner template path")
		}
	}
	return nil
}

func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
	var (
		err                                            error
		rbdPlugin, cephfsPlugin                        *apps.DaemonSet
		rbdProvisioner, rbdAttacher, cephfsProvisioner *apps.StatefulSet
	)

	if EnableRBD {
		rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load rbd plugin template: %v", err)
		}
		rbdProvisioner, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load rbd provisioner template: %v", err)
		}
		rbdAttacher, err = templateToStatefulSet("rbd-attacher", RBDAttacherTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load rbd attacher template: %v", err)
		}
	}
	if EnableCephFS {
		cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load CephFS plugin template: %v", err)
		}
		cephfsProvisioner, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerTemplatePath)
		if err != nil {
			return fmt.Errorf("failed to load CephFS provisioner template: %v", err)
		}
	}

	if rbdPlugin != nil {
		err = k8sutil.CreateDaemonSet("csi rbd plugin", namespace, clientset, rbdPlugin)
		if err != nil {
			return fmt.Errorf("failed to start rbdplugin daemonset: %v\n%v", err, rbdPlugin)
		}
	}
	if rbdProvisioner != nil {
		_, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisioner)
		if err != nil {
			return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisioner)
		}

	}
	if rbdAttacher != nil {
		_, err = k8sutil.CreateStatefulSet("csi rbd attacher", namespace, "csi-rbdplugin-attacher", clientset, rbdAttacher)
		if err != nil {
			return fmt.Errorf("failed to start rbd attacher statefulset: %v\n%v", err, rbdAttacher)
		}
	}

	if cephfsPlugin != nil {
		err = k8sutil.CreateDaemonSet("csi cephfs plugin", namespace, clientset, cephfsPlugin)
		if err != nil {
			return fmt.Errorf("failed to start cephfs plugin daemonset: %v\n%v", err, cephfsPlugin)
		}
	}
	if cephfsProvisioner != nil {
		_, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisioner)
		if err != nil {
			return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisioner)
		}

	}
	return nil
}
<|endoftext|>


<commit_before>package alerting

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"time"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/models"
)

var (
	// ErrFrequencyCannotBeZeroOrLess frequency cannot be below zero
	ErrFrequencyCannotBeZeroOrLess = errors.New(`"evaluate every" cannot be zero or below`)

	// ErrFrequencyCouldNotBeParsed frequency cannot be parsed
	ErrFrequencyCouldNotBeParsed = errors.New(`"evaluate every" field could not be parsed`)
)

// Rule is the in-memory version of an alert rule.
type Rule struct {
	ID                  int64
	OrgID               int64
	DashboardID         int64
	PanelID             int64
	Frequency           int64
	Name                string
	Message             string
	LastStateChange     time.Time
	For                 time.Duration
	NoDataState         models.NoDataOption
	ExecutionErrorState models.ExecutionErrorOption
	State               models.AlertStateType
	Conditions          []Condition
	Notifications       []string
	AlertRuleTags       []*models.Tag

	StateChanges int64
}

// ValidationError is a typed error with meta data
// about the validation error.
type ValidationError struct {
	Reason      string
	Err         error
	AlertID     int64
	DashboardID int64
	PanelID     int64
}

func (e ValidationError) Error() string {
	extraInfo := e.Reason
	if e.AlertID != 0 {
		extraInfo = fmt.Sprintf("%s AlertId: %v", extraInfo, e.AlertID)
	}

	if e.PanelID != 0 {
		extraInfo = fmt.Sprintf("%s PanelId: %v", extraInfo, e.PanelID)
	}

	if e.DashboardID != 0 {
		extraInfo = fmt.Sprintf("%s DashboardId: %v", extraInfo, e.DashboardID)
	}

	if e.Err != nil {
		return fmt.Sprintf("alert validation error: %s%s", e.Err.Error(), extraInfo)
	}

	return fmt.Sprintf("alert validation error: %s", extraInfo)
}

var (
	valueFormatRegex = regexp.MustCompile(`^\d+`)
	unitFormatRegex  = regexp.MustCompile(`\w{1}$`)
)

var unitMultiplier = map[string]int{
	"s": 1,
	"m": 60,
	"h": 3600,
	"d": 86400,
}

func getTimeDurationStringToSeconds(str string) (int64, error) {
	multiplier := 1

	matches := valueFormatRegex.FindAllString(str, 1)

	if len(matches) == 0 {
		return 0, ErrFrequencyCouldNotBeParsed
	}

	value, err := strconv.Atoi(matches[0])
	if err != nil {
		return 0, err
	}

	if value == 0 {
		return 0, ErrFrequencyCannotBeZeroOrLess
	}

	unit := unitFormatRegex.FindAllString(str, 1)[0]

	if val, ok := unitMultiplier[unit]; ok {
		multiplier = val
	}

	return int64(value * multiplier), nil
}

// NewRuleFromDBAlert maps a db version of
// alert to an in-memory version.
func NewRuleFromDBAlert(ruleDef *models.Alert, logTranslationFailures bool) (*Rule, error) {
	model := &Rule{}
	model.ID = ruleDef.Id
	model.OrgID = ruleDef.OrgId
	model.DashboardID = ruleDef.DashboardId
	model.PanelID = ruleDef.PanelId
	model.Name = ruleDef.Name
	model.Message = ruleDef.Message
	model.State = ruleDef.State
	model.LastStateChange = ruleDef.NewStateDate
	model.For = ruleDef.For
	model.NoDataState = models.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
	model.ExecutionErrorState = models.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
	model.StateChanges = ruleDef.StateChanges

	model.Frequency = ruleDef.Frequency
	// frequency cannot be zero since that would not execute the alert rule.
	// so we fallback to 60 seconds if `Frequency` is missing
	if model.Frequency == 0 {
		model.Frequency = 60
	}

	for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
		jsonModel := simplejson.NewFromAny(v)
		if id, err := jsonModel.Get("id").Int64(); err == nil {
			uid, err := translateNotificationIDToUID(id, ruleDef.OrgId)
			if err != nil {
				if !errors.Is(err, models.ErrAlertNotificationFailedTranslateUniqueID) {
					logger.Error("Failed to tranlate notification id to uid", "error", err.Error(), "dashboardId", model.DashboardID, "alert", model.Name, "panelId", model.PanelID, "notificationId", id)
				}

				if logTranslationFailures {
					logger.Warn("Unable to translate notification id to uid", "dashboardId", model.DashboardID, "alert", model.Name, "panelId", model.PanelID, "notificationId", id)
				}
			} else {
				model.Notifications = append(model.Notifications, uid)
			}
		} else if uid, err := jsonModel.Get("uid").String(); err == nil {
			model.Notifications = append(model.Notifications, uid)
		} else {
			return nil, ValidationError{Reason: "Neither id nor uid is specified in 'notifications' block, " + err.Error(), DashboardID: model.DashboardID, AlertID: model.ID, PanelID: model.PanelID}
		}
	}
	model.AlertRuleTags = ruleDef.GetTagsFromSettings()

	for index, condition := range ruleDef.Settings.Get("conditions").MustArray() {
		conditionModel := simplejson.NewFromAny(condition)
		conditionType := conditionModel.Get("type").MustString()
		factory, exist := conditionFactories[conditionType]
		if !exist {
			return nil, ValidationError{Reason: "Unknown alert condition: " + conditionType, DashboardID: model.DashboardID, AlertID: model.ID, PanelID: model.PanelID}
		}
		queryCondition, err := factory(conditionModel, index)
		if err != nil {
			return nil, ValidationError{Err: err, DashboardID: model.DashboardID, AlertID: model.ID, PanelID: model.PanelID}
		}
		model.Conditions = append(model.Conditions, queryCondition)
	}

	if len(model.Conditions) == 0 {
		return nil, ValidationError{Reason: "Alert is missing conditions"}
	}

	return model, nil
}

func translateNotificationIDToUID(id int64, orgID int64) (string, error) {
	notificationUID, err := getAlertNotificationUIDByIDAndOrgID(id, orgID)
	if err != nil {
		return "", err
	}

	return notificationUID, nil
}

func getAlertNotificationUIDByIDAndOrgID(notificationID int64, orgID int64) (string, error) {
	query := &models.GetAlertNotificationUidQuery{
		OrgId: orgID,
		Id:    notificationID,
	}

	if err := bus.Dispatch(query); err != nil {
		return "", err
	}

	return query.Result, nil
}

// ConditionFactory is the function signature for creating `Conditions`.
type ConditionFactory func(model *simplejson.Json, index int) (Condition, error)

var conditionFactories = make(map[string]ConditionFactory)

// RegisterCondition adds support for alerting conditions.
func RegisterCondition(typeName string, factory ConditionFactory) {
	conditionFactories[typeName] = factory
}
<commit_msg>Chore: fix spelling mistake (#30473)<commit_after>package alerting

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"time"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/models"
)

var (
	// ErrFrequencyCannotBeZeroOrLess frequency cannot be below zero
	ErrFrequencyCannotBeZeroOrLess = errors.New(`"evaluate every" cannot be zero or below`)

	// ErrFrequencyCouldNotBeParsed frequency cannot be parsed
	ErrFrequencyCouldNotBeParsed = errors.New(`"evaluate every" field could not be parsed`)
)

// Rule is the in-memory version of an alert rule.
type Rule struct {
	ID                  int64
	OrgID               int64
	DashboardID         int64
	PanelID             int64
	Frequency           int64
	Name                string
	Message             string
	LastStateChange     time.Time
	For                 time.Duration
	NoDataState         models.NoDataOption
	ExecutionErrorState models.ExecutionErrorOption
	State               models.AlertStateType
	Conditions          []Condition
	Notifications       []string
	AlertRuleTags       []*models.Tag

	StateChanges int64
}

// ValidationError is a typed error with meta data
// about the validation error.
type ValidationError struct {
	Reason      string
	Err         error
	AlertID     int64
	DashboardID int64
	PanelID     int64
}

func (e ValidationError) Error() string {
	extraInfo := e.Reason
	if e.AlertID != 0 {
		extraInfo = fmt.Sprintf("%s AlertId: %v", extraInfo, e.AlertID)
	}

	if e.PanelID != 0 {
		extraInfo = fmt.Sprintf("%s PanelId: %v", extraInfo, e.PanelID)
	}

	if e.DashboardID != 0 {
		extraInfo = fmt.Sprintf("%s DashboardId: %v", extraInfo, e.DashboardID)
	}

	if e.Err != nil {
		return fmt.Sprintf("alert validation error: %s%s", e.Err.Error(), extraInfo)
	}

	return fmt.Sprintf("alert validation error: %s", extraInfo)
}

var (
	valueFormatRegex = regexp.MustCompile(`^\d+`)
	unitFormatRegex  = regexp.MustCompile(`\w{1}$`)
)

var unitMultiplier = map[string]int{
	"s": 1,
	"m": 60,
	"h": 3600,
	"d": 86400,
}

func getTimeDurationStringToSeconds(str string) (int64, error) {
	multiplier := 1

	matches := valueFormatRegex.FindAllString(str, 1)

	if len(matches) == 0 {
		return 0, ErrFrequencyCouldNotBeParsed
	}

	value, err := strconv.Atoi(matches[0])
	if err != nil {
		return 0, err
	}

	if value == 0 {
		return 0, ErrFrequencyCannotBeZeroOrLess
	}

	unit := unitFormatRegex.FindAllString(str, 1)[0]

	if val, ok := unitMultiplier[unit]; ok {
		multiplier = val
	}

	return int64(value * multiplier), nil
}

// NewRuleFromDBAlert maps a db version of
// alert to an in-memory version.
func NewRuleFromDBAlert(ruleDef *models.Alert, logTranslationFailures bool) (*Rule, error) {
	model := &Rule{}
	model.ID = ruleDef.Id
	model.OrgID = ruleDef.OrgId
	model.DashboardID = ruleDef.DashboardId
	model.PanelID = ruleDef.PanelId
	model.Name = ruleDef.Name
	model.Message = ruleDef.Message
	model.State = ruleDef.State
	model.LastStateChange = ruleDef.NewStateDate
	model.For = ruleDef.For
	model.NoDataState = models.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
	model.ExecutionErrorState = models.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
	model.StateChanges = ruleDef.StateChanges

	model.Frequency = ruleDef.Frequency
	// frequency cannot be zero since that would not execute the alert rule.
	// so we fallback to 60 seconds if `Frequency` is missing
	if model.Frequency == 0 {
		model.Frequency = 60
	}

	for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
		jsonModel := simplejson.NewFromAny(v)
		if id, err := jsonModel.Get("id").Int64(); err == nil {
			uid, err := translateNotificationIDToUID(id, ruleDef.OrgId)
			if err != nil {
				if !errors.Is(err, models.ErrAlertNotificationFailedTranslateUniqueID) {
					logger.Error("Failed to translate notification id to uid", "error", err.Error(), "dashboardId", model.DashboardID, "alert", model.Name, "panelId", model.PanelID, "notificationId", id)
				}

				if logTranslationFailures {
					logger.Warn("Unable to translate notification id to uid", "dashboardId", model.DashboardID, "alert", model.Name, "panelId", model.PanelID, "notificationId", id)
				}
			} else {
				model.Notifications = append(model.Notifications, uid)
			}
		} else if uid, err := jsonModel.Get("uid").String(); err == nil {
			model.Notifications = append(model.Notifications, uid)
		} else {
			return nil, ValidationError{Reason: "Neither id nor uid is specified in 'notifications' block, " + err.Error(), DashboardID: model.DashboardID, AlertID: model.ID, PanelID: model.PanelID}
		}
	}
	model.AlertRuleTags = ruleDef.GetTagsFromSettings()

	for index, condition := range ruleDef.Settings.Get("conditions").MustArray() {
		conditionModel := simplejson.NewFromAny(condition)
		conditionType := conditionModel.Get("type").MustString()
		factory, exist := conditionFactories[conditionType]
		if !exist {
			return nil, ValidationError{Reason: "Unknown alert condition: " + conditionType, DashboardID: model.DashboardID, AlertID: model.ID, PanelID: model.PanelID}
		}
		queryCondition, err := factory(conditionModel, index)
		if err != nil {
			return nil, ValidationError{Err: err, DashboardID: model.DashboardID, AlertID: model.ID, PanelID: model.PanelID}
		}
		model.Conditions = append(model.Conditions, queryCondition)
	}

	if len(model.Conditions) == 0 {
		return nil, ValidationError{Reason: "Alert is missing conditions"}
	}

	return model, nil
}

func translateNotificationIDToUID(id int64, orgID int64) (string, error) {
	notificationUID, err := getAlertNotificationUIDByIDAndOrgID(id, orgID)
	if err != nil {
		return "", err
	}

	return notificationUID, nil
}

func getAlertNotificationUIDByIDAndOrgID(notificationID int64, orgID int64) (string, error) {
	query := &models.GetAlertNotificationUidQuery{
		OrgId: orgID,
		Id:    notificationID,
	}

	if err := bus.Dispatch(query); err != nil {
		return "", err
	}

	return query.Result, nil
}

// ConditionFactory is the function signature for creating `Conditions`.
type ConditionFactory func(model *simplejson.Json, index int) (Condition, error)

var conditionFactories = make(map[string]ConditionFactory)

// RegisterCondition adds support for alerting conditions.
func RegisterCondition(typeName string, factory ConditionFactory) {
	conditionFactories[typeName] = factory
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transport

import (
	"crypto/tls"
	"fmt"
	"net"
	"sync"
)

// tlsListener overrides a TLS listener so it will reject client
// certificates with insufficient SAN credentials.
type tlsListener struct {
	net.Listener
	connc            chan net.Conn
	donec            chan struct{}
	err              error
	handshakeFailure func(*tls.Conn, error)
}

func newTLSListener(l net.Listener, tlsinfo *TLSInfo) (net.Listener, error) {
	if tlsinfo == nil || tlsinfo.Empty() {
		l.Close()
		return nil, fmt.Errorf("cannot listen on TLS for %s: KeyFile and CertFile are not presented", l.Addr().String())
	}
	tlscfg, err := tlsinfo.ServerConfig()
	if err != nil {
		return nil, err
	}
	tlsl := &tlsListener{
		Listener:         tls.NewListener(l, tlscfg),
		connc:            make(chan net.Conn),
		donec:            make(chan struct{}),
		handshakeFailure: tlsinfo.HandshakeFailure,
	}
	go tlsl.acceptLoop()
	return tlsl, nil
}

func (l *tlsListener) Accept() (net.Conn, error) {
	select {
	case conn := <-l.connc:
		return conn, nil
	case <-l.donec:
		return nil, l.err
	}
}

// acceptLoop launches each TLS handshake in a separate goroutine
// to prevent a hanging TLS connection from blocking other connections.
func (l *tlsListener) acceptLoop() {
	var wg sync.WaitGroup
	var pendingMu sync.Mutex

	pending := make(map[net.Conn]struct{})
	stopc := make(chan struct{})
	defer func() {
		close(stopc)
		pendingMu.Lock()
		for c := range pending {
			c.Close()
		}
		pendingMu.Unlock()
		wg.Wait()
		close(l.donec)
	}()

	for {
		conn, err := l.Listener.Accept()
		if err != nil {
			l.err = err
			return
		}

		pendingMu.Lock()
		pending[conn] = struct{}{}
		pendingMu.Unlock()

		wg.Add(1)
		go func() {
			defer func() {
				if conn != nil {
					conn.Close()
				}
				wg.Done()
			}()

			tlsConn := conn.(*tls.Conn)
			herr := tlsConn.Handshake()
			pendingMu.Lock()
			delete(pending, conn)
			pendingMu.Unlock()
			if herr != nil {
				if l.handshakeFailure != nil {
					l.handshakeFailure(tlsConn, herr)
				}
				return
			}

			st := tlsConn.ConnectionState()
			if len(st.PeerCertificates) > 0 {
				cert := st.PeerCertificates[0]
				if len(cert.IPAddresses) > 0 || len(cert.DNSNames) > 0 {
					addr := tlsConn.RemoteAddr().String()
					h, _, herr := net.SplitHostPort(addr)
					if herr != nil || cert.VerifyHostname(h) != nil {
						return
					}
				}
			}
			select {
			case l.connc <- tlsConn:
				conn = nil
			case <-stopc:
			}
		}()
	}
}

func (l *tlsListener) Close() error {
	err := l.Listener.Close()
	<-l.donec
	return err
}
<commit_msg>transport: resolve DNSNames when SAN checking<commit_after>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transport

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net"
	"sync"
)

// tlsListener overrides a TLS listener so it will reject client
// certificates with insufficient SAN credentials.
type tlsListener struct {
	net.Listener
	connc            chan net.Conn
	donec            chan struct{}
	err              error
	handshakeFailure func(*tls.Conn, error)
}

func newTLSListener(l net.Listener, tlsinfo *TLSInfo) (net.Listener, error) {
	if tlsinfo == nil || tlsinfo.Empty() {
		l.Close()
		return nil, fmt.Errorf("cannot listen on TLS for %s: KeyFile and CertFile are not presented", l.Addr().String())
	}
	tlscfg, err := tlsinfo.ServerConfig()
	if err != nil {
		return nil, err
	}

	hf := tlsinfo.HandshakeFailure
	if hf == nil {
		hf = func(*tls.Conn, error) {}
	}
	tlsl := &tlsListener{
		Listener:         tls.NewListener(l, tlscfg),
		connc:            make(chan net.Conn),
		donec:            make(chan struct{}),
		handshakeFailure: hf,
	}
	go tlsl.acceptLoop()
	return tlsl, nil
}

func (l *tlsListener) Accept() (net.Conn, error) {
	select {
	case conn := <-l.connc:
		return conn, nil
	case <-l.donec:
		return nil, l.err
	}
}

// acceptLoop launches each TLS handshake in a separate goroutine
// to prevent a hanging TLS connection from blocking other connections.
func (l *tlsListener) acceptLoop() {
	var wg sync.WaitGroup
	var pendingMu sync.Mutex

	pending := make(map[net.Conn]struct{})
	ctx, cancel := context.WithCancel(context.Background())
	defer func() {
		cancel()
		pendingMu.Lock()
		for c := range pending {
			c.Close()
		}
		pendingMu.Unlock()
		wg.Wait()
		close(l.donec)
	}()

	for {
		conn, err := l.Listener.Accept()
		if err != nil {
			l.err = err
			return
		}

		pendingMu.Lock()
		pending[conn] = struct{}{}
		pendingMu.Unlock()

		wg.Add(1)
		go func() {
			defer func() {
				if conn != nil {
					conn.Close()
				}
				wg.Done()
			}()

			tlsConn := conn.(*tls.Conn)
			herr := tlsConn.Handshake()
			pendingMu.Lock()
			delete(pending, conn)
			pendingMu.Unlock()
			if herr != nil {
				l.handshakeFailure(tlsConn, herr)
				return
			}

			st := tlsConn.ConnectionState()
			if len(st.PeerCertificates) > 0 {
				cert := st.PeerCertificates[0]
				addr := tlsConn.RemoteAddr().String()
				if cerr := checkCert(ctx, cert, addr); cerr != nil {
					l.handshakeFailure(tlsConn, cerr)
					return
				}
			}
			select {
			case l.connc <- tlsConn:
				conn = nil
			case <-ctx.Done():
			}
		}()
	}
}

func checkCert(ctx context.Context, cert *x509.Certificate, remoteAddr string) error {
	h, _, herr := net.SplitHostPort(remoteAddr)
	if len(cert.IPAddresses) == 0 && len(cert.DNSNames) == 0 {
		return nil
	}
	if herr != nil {
		return herr
	}
	if len(cert.IPAddresses) > 0 {
		if cerr := cert.VerifyHostname(h); cerr != nil && len(cert.DNSNames) == 0 {
			return cerr
		}
	}
	if len(cert.DNSNames) > 0 {
		for _, dns := range cert.DNSNames {
			addrs, lerr := net.DefaultResolver.LookupHost(ctx, dns)
			if lerr != nil {
				continue
			}
			for _, addr := range addrs {
				if addr == h {
					return nil
				}
			}
		}
		return fmt.Errorf("tls: %q does not match any of DNSNames %q", h, cert.DNSNames)
	}
	return nil
}

func (l *tlsListener) Close() error {
	err := l.Listener.Close()
	<-l.donec
	return err
}
<|endoftext|>


<commit_before>// +build !providerless

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awsebs

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"k8s.io/klog/v2"
	"k8s.io/mount-utils"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	cloudprovider "k8s.io/cloud-provider"
	volumehelpers "k8s.io/cloud-provider/volume/helpers"
	"k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
	"k8s.io/legacy-cloud-providers/aws"
)

const (
	diskPartitionSuffix = ""
	checkSleepDuration  = time.Second
)

// AWSDiskUtil provides operations for EBS volume.
type AWSDiskUtil struct{}

// DeleteVolume deletes an AWS EBS volume.
func (util *AWSDiskUtil) DeleteVolume(d *awsElasticBlockStoreDeleter) error {
	cloud, err := getCloudProvider(d.awsElasticBlockStore.plugin.host.GetCloudProvider())
	if err != nil {
		return err
	}

	deleted, err := cloud.DeleteDisk(d.volumeID)
	if err != nil {
		// AWS cloud provider returns volume.deletedVolumeInUseError when
		// necessary, no handling needed here.
		klog.V(2).Infof("Error deleting EBS Disk volume %s: %v", d.volumeID, err)
		return err
	}
	if deleted {
		klog.V(2).Infof("Successfully deleted EBS Disk volume %s", d.volumeID)
	} else {
		klog.V(2).Infof("Successfully deleted EBS Disk volume %s (actually already deleted)", d.volumeID)
	}
	return nil
}

// CreateVolume creates an AWS EBS volume.
// Returns: volumeID, volumeSizeGB, labels, fstype, error
func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (aws.KubernetesVolumeID, int, map[string]string, string, error) {
	cloud, err := getCloudProvider(c.awsElasticBlockStore.plugin.host.GetCloudProvider())
	if err != nil {
		return "", 0, nil, "", err
	}

	// AWS volumes don't have Name field, store the name in Name tag
	var tags map[string]string
	if c.options.CloudTags == nil {
		tags = make(map[string]string)
	} else {
		tags = *c.options.CloudTags
	}
	tags["Name"] = volumeutil.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 255) // AWS tags can have 255 characters

	capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]

	zonesWithNodes, err := getCandidateZones(cloud, node)
	if err != nil {
		return "", 0, nil, "", fmt.Errorf("error finding candidate zone for pvc: %v", err)
	}

	volumeOptions, err := populateVolumeOptions(c.plugin.GetPluginName(), c.options.PVC.Name, capacity, tags, c.options.Parameters, node, allowedTopologies, zonesWithNodes)
	if err != nil {
		klog.V(2).Infof("Error populating EBS options: %v", err)
		return "", 0, nil, "", err
	}

	// TODO: implement PVC.Selector parsing
	if c.options.PVC.Spec.Selector != nil {
		return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on AWS")
	}

	name, err := cloud.CreateDisk(volumeOptions)
	if err != nil {
		klog.V(2).Infof("Error creating EBS Disk volume: %v", err)
		return "", 0, nil, "", err
	}
	klog.V(2).Infof("Successfully created EBS Disk volume %s", name)

	labels, err := cloud.GetVolumeLabels(name)
	if err != nil {
		// We don't really want to leak the volume here...
		klog.Errorf("error building labels for new EBS volume %q: %v", name, err)
	}

	fstype := ""
	for k, v := range c.options.Parameters {
		if strings.ToLower(k) == volume.VolumeParameterFSType {
			fstype = v
		}
	}

	return name, volumeOptions.CapacityGB, labels, fstype, nil
}

// getCandidateZones finds possible zones that a volume can be created in
func getCandidateZones(cloud *aws.Cloud, selectedNode *v1.Node) (sets.String, error) {
	if selectedNode != nil {
		// For topology aware volume provisioning, node is already selected so we use the zone from
		// selected node directly instead of candidate zones.
		// We can assume the information is always available as node controller shall maintain it.
		return sets.NewString(), nil
	}

	// For non-topology-aware volumes (those that binds immediately), we fall back to original logic to query
	// cloud provider for possible zones
	return cloud.GetCandidateZonesForDynamicVolume()
}

// returns volumeOptions for EBS based on storageclass parameters and node configuration
func populateVolumeOptions(pluginName, pvcName string, capacityGB resource.Quantity, tags map[string]string, storageParams map[string]string, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, zonesWithNodes sets.String) (*aws.VolumeOptions, error) {
	requestGiB, err := volumehelpers.RoundUpToGiBInt(capacityGB)
	if err != nil {
		return nil, err
	}

	volumeOptions := &aws.VolumeOptions{
		CapacityGB: requestGiB,
		Tags:       tags,
	}

	// Apply Parameters (case-insensitive). We leave validation of
	// the values to the cloud provider.
	zonePresent := false
	zonesPresent := false
	var zone string
	var zones sets.String
	for k, v := range storageParams {
		switch strings.ToLower(k) {
		case "type":
			volumeOptions.VolumeType = v
		case "zone":
			zonePresent = true
			zone = v
		case "zones":
			zonesPresent = true
			zones, err = volumehelpers.ZonesToSet(v)
			if err != nil {
				return nil, fmt.Errorf("error parsing zones %s, must be strings separated by commas: %v", zones, err)
			}
		case "iopspergb":
			volumeOptions.IOPSPerGB, err = strconv.Atoi(v)
			if err != nil {
				return nil, fmt.Errorf("invalid iopsPerGB value %q: %v", v, err)
			}
		case "encrypted":
			volumeOptions.Encrypted, err = strconv.ParseBool(v)
			if err != nil {
				return nil, fmt.Errorf("invalid encrypted boolean value %q, must be true or false: %v", v, err)
			}
		case "kmskeyid":
			volumeOptions.KmsKeyID = v
		case volume.VolumeParameterFSType:
			// Do nothing but don't make this fail
		default:
			return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, pluginName)
		}
	}

	volumeOptions.AvailabilityZone, err = volumehelpers.SelectZoneForVolume(zonePresent, zonesPresent, zone, zones, zonesWithNodes, node, allowedTopologies, pvcName)
	if err != nil {
		return nil, err
	}
	return volumeOptions, nil
}

// Returns the first path that exists, or empty string if none exist.
func verifyDevicePath(devicePaths []string) (string, error) {
	for _, path := range devicePaths {
		if pathExists, err := mount.PathExists(path); err != nil {
			return "", fmt.Errorf("Error checking if path exists: %v", err)
		} else if pathExists {
			return path, nil
		}
	}

	return "", nil
}

// Returns list of all paths for given EBS mount
// This is more interesting on GCE (where we are able to identify volumes under /dev/disk-by-id)
// Here it is mostly about applying the partition path
func getDiskByIDPaths(volumeID aws.KubernetesVolumeID, partition string, devicePath string) []string {
	devicePaths := []string{}
	if devicePath != "" {
		devicePaths = append(devicePaths, devicePath)
	}

	if partition != "" {
		for i, path := range devicePaths {
			devicePaths[i] = path + diskPartitionSuffix + partition
		}
	}

	// We need to find NVME volumes, which are mounted on a "random" nvme path ("/dev/nvme0n1"),
	// and we have to get the volume id from the nvme interface
	awsVolumeID, err := volumeID.MapToAWSVolumeID()
	if err != nil {
		klog.Warningf("error mapping volume %q to AWS volume: %v", volumeID, err)
	} else {
		// This is the magic name on which AWS presents NVME devices under /dev/disk/by-id/
		// For example, vol-0fab1d5e3f72a5e23 creates a symlink at /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0fab1d5e3f72a5e23
		nvmeName := "nvme-Amazon_Elastic_Block_Store_" + strings.Replace(string(awsVolumeID), "-", "", -1)
		nvmePath, err := findNvmeVolume(nvmeName)
		if err != nil {
			klog.Warningf("error looking for nvme volume %q: %v", volumeID, err)
		} else if nvmePath != "" {
			devicePaths = append(devicePaths, nvmePath)
		}
	}

	return devicePaths
}

// Return cloud provider
func getCloudProvider(cloudProvider cloudprovider.Interface) (*aws.Cloud, error) {
	awsCloudProvider, ok := cloudProvider.(*aws.Cloud)
	if !ok || awsCloudProvider == nil {
		return nil, fmt.Errorf("Failed to get AWS Cloud Provider. GetCloudProvider returned %v instead", cloudProvider)
	}

	return awsCloudProvider, nil
}

// findNvmeVolume looks for the nvme volume with the specified name
// It follows the symlink (if it exists) and returns the absolute path to the device
func findNvmeVolume(findName string) (device string, err error) {
	p := filepath.Join("/dev/disk/by-id/", findName)
	stat, err := os.Lstat(p)
	if err != nil {
		if os.IsNotExist(err) {
			klog.V(6).Infof("nvme path not found %q", p)
			return "", nil
		}
		return "", fmt.Errorf("error getting stat of %q: %v", p, err)
	}

	if stat.Mode()&os.ModeSymlink != os.ModeSymlink {
		klog.Warningf("nvme file %q found, but was not a symlink", p)
		return "", nil
	}

	// Find the target, resolving to an absolute path
	// For example, /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0fab1d5e3f72a5e23 -> ../../nvme2n1
	resolved, err := filepath.EvalSymlinks(p)
	if err != nil {
		return "", fmt.Errorf("error reading target of symlink %q: %v", p, err)
	}

	if !strings.HasPrefix(resolved, "/dev") {
		return "", fmt.Errorf("resolved symlink for %q was unexpected: %q", p, resolved)
	}

	return resolved, nil
}

func formatVolumeID(volumeID string) (string, error) {
	// This is a workaround to fix the issue in converting aws volume id from globalPDPath and globalMapPath
	// There are three formats for AWSEBSVolumeSource.VolumeID and they are stored on disk in paths like so:
	// VolumeID									mountPath								mapPath
	// aws:///vol-1234					aws/vol-1234						aws:/vol-1234
	// aws://us-east-1/vol-1234 aws/us-east-1/vol-1234  aws:/us-east-1/vol-1234
	// vol-1234									vol-1234								vol-1234
	// This code is for converting volume ids from paths back to AWS style VolumeIDs
	sourceName := volumeID
	if strings.HasPrefix(volumeID, "aws/") || strings.HasPrefix(volumeID, "aws:/") {
		names := strings.Split(volumeID, "/")
		length := len(names)
		if length < 2 || length > 3 {
			return "", fmt.Errorf("invalid volume name format %q", volumeID)
		}
		volName := names[length-1]
		if !strings.HasPrefix(volName, "vol-") {
			return "", fmt.Errorf("Invalid volume name format for AWS volume (%q)", volName)
		}
		if length == 2 {
			sourceName = awsURLNamePrefix + "" + "/" + volName // empty zone label
		}
		if length == 3 {
			sourceName = awsURLNamePrefix + names[1] + "/" + volName // names[1] is the zone label
		}
		klog.V(4).Infof("Convert aws volume name from %q to %q ", volumeID, sourceName)
	}
	return sourceName, nil
}

func newAWSVolumeSpec(volumeName, volumeID string, mode v1.PersistentVolumeMode) *volume.Spec {
	awsVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
					VolumeID: volumeID,
				},
			},
			VolumeMode: &mode,
		},
	}
	return volume.NewSpecFromPersistentVolume(awsVolume, false)
}
<commit_msg>UPSTREAM: 100500: Fix mounting partitions on NVMe devices<commit_after>// +build !providerless

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awsebs

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"k8s.io/klog/v2"
	"k8s.io/mount-utils"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	cloudprovider "k8s.io/cloud-provider"
	volumehelpers "k8s.io/cloud-provider/volume/helpers"
	"k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
	"k8s.io/legacy-cloud-providers/aws"
)

const (
	diskPartitionSuffix     = ""
	nvmeDiskPartitionSuffix = "p"
	checkSleepDuration      = time.Second
)

// AWSDiskUtil provides operations for EBS volume.
type AWSDiskUtil struct{}

// DeleteVolume deletes an AWS EBS volume.
func (util *AWSDiskUtil) DeleteVolume(d *awsElasticBlockStoreDeleter) error {
	cloud, err := getCloudProvider(d.awsElasticBlockStore.plugin.host.GetCloudProvider())
	if err != nil {
		return err
	}

	deleted, err := cloud.DeleteDisk(d.volumeID)
	if err != nil {
		// AWS cloud provider returns volume.deletedVolumeInUseError when
		// necessary, no handling needed here.
		klog.V(2).Infof("Error deleting EBS Disk volume %s: %v", d.volumeID, err)
		return err
	}
	if deleted {
		klog.V(2).Infof("Successfully deleted EBS Disk volume %s", d.volumeID)
	} else {
		klog.V(2).Infof("Successfully deleted EBS Disk volume %s (actually already deleted)", d.volumeID)
	}
	return nil
}

// CreateVolume creates an AWS EBS volume.
// Returns: volumeID, volumeSizeGB, labels, fstype, error
func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (aws.KubernetesVolumeID, int, map[string]string, string, error) {
	cloud, err := getCloudProvider(c.awsElasticBlockStore.plugin.host.GetCloudProvider())
	if err != nil {
		return "", 0, nil, "", err
	}

	// AWS volumes don't have Name field, store the name in Name tag
	var tags map[string]string
	if c.options.CloudTags == nil {
		tags = make(map[string]string)
	} else {
		tags = *c.options.CloudTags
	}
	tags["Name"] = volumeutil.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 255) // AWS tags can have 255 characters

	capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]

	zonesWithNodes, err := getCandidateZones(cloud, node)
	if err != nil {
		return "", 0, nil, "", fmt.Errorf("error finding candidate zone for pvc: %v", err)
	}

	volumeOptions, err := populateVolumeOptions(c.plugin.GetPluginName(), c.options.PVC.Name, capacity, tags, c.options.Parameters, node, allowedTopologies, zonesWithNodes)
	if err != nil {
		klog.V(2).Infof("Error populating EBS options: %v", err)
		return "", 0, nil, "", err
	}

	// TODO: implement PVC.Selector parsing
	if c.options.PVC.Spec.Selector != nil {
		return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on AWS")
	}

	name, err := cloud.CreateDisk(volumeOptions)
	if err != nil {
		klog.V(2).Infof("Error creating EBS Disk volume: %v", err)
		return "", 0, nil, "", err
	}
	klog.V(2).Infof("Successfully created EBS Disk volume %s", name)

	labels, err := cloud.GetVolumeLabels(name)
	if err != nil {
		// We don't really want to leak the volume here...
		klog.Errorf("error building labels for new EBS volume %q: %v", name, err)
	}

	fstype := ""
	for k, v := range c.options.Parameters {
		if strings.ToLower(k) == volume.VolumeParameterFSType {
			fstype = v
		}
	}

	return name, volumeOptions.CapacityGB, labels, fstype, nil
}

// getCandidateZones finds possible zones that a volume can be created in
func getCandidateZones(cloud *aws.Cloud, selectedNode *v1.Node) (sets.String, error) {
	if selectedNode != nil {
		// For topology aware volume provisioning, node is already selected so we use the zone from
		// selected node directly instead of candidate zones.
		// We can assume the information is always available as node controller shall maintain it.
		return sets.NewString(), nil
	}

	// For non-topology-aware volumes (those that binds immediately), we fall back to original logic to query
	// cloud provider for possible zones
	return cloud.GetCandidateZonesForDynamicVolume()
}

// returns volumeOptions for EBS based on storageclass parameters and node configuration
func populateVolumeOptions(pluginName, pvcName string, capacityGB resource.Quantity, tags map[string]string, storageParams map[string]string, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, zonesWithNodes sets.String) (*aws.VolumeOptions, error) {
	requestGiB, err := volumehelpers.RoundUpToGiBInt(capacityGB)
	if err != nil {
		return nil, err
	}

	volumeOptions := &aws.VolumeOptions{
		CapacityGB: requestGiB,
		Tags:       tags,
	}

	// Apply Parameters (case-insensitive). We leave validation of
	// the values to the cloud provider.
	zonePresent := false
	zonesPresent := false
	var zone string
	var zones sets.String
	for k, v := range storageParams {
		switch strings.ToLower(k) {
		case "type":
			volumeOptions.VolumeType = v
		case "zone":
			zonePresent = true
			zone = v
		case "zones":
			zonesPresent = true
			zones, err = volumehelpers.ZonesToSet(v)
			if err != nil {
				return nil, fmt.Errorf("error parsing zones %s, must be strings separated by commas: %v", zones, err)
			}
		case "iopspergb":
			volumeOptions.IOPSPerGB, err = strconv.Atoi(v)
			if err != nil {
				return nil, fmt.Errorf("invalid iopsPerGB value %q: %v", v, err)
			}
		case "encrypted":
			volumeOptions.Encrypted, err = strconv.ParseBool(v)
			if err != nil {
				return nil, fmt.Errorf("invalid encrypted boolean value %q, must be true or false: %v", v, err)
			}
		case "kmskeyid":
			volumeOptions.KmsKeyID = v
		case volume.VolumeParameterFSType:
			// Do nothing but don't make this fail
		default:
			return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, pluginName)
		}
	}

	volumeOptions.AvailabilityZone, err = volumehelpers.SelectZoneForVolume(zonePresent, zonesPresent, zone, zones, zonesWithNodes, node, allowedTopologies, pvcName)
	if err != nil {
		return nil, err
	}
	return volumeOptions, nil
}

// Returns the first path that exists, or empty string if none exist.
func verifyDevicePath(devicePaths []string) (string, error) {
	for _, path := range devicePaths {
		if pathExists, err := mount.PathExists(path); err != nil {
			return "", fmt.Errorf("Error checking if path exists: %v", err)
		} else if pathExists {
			return path, nil
		}
	}

	return "", nil
}

// Returns list of all paths for given EBS mount
// This is more interesting on GCE (where we are able to identify volumes under /dev/disk-by-id)
// Here it is mostly about applying the partition path
func getDiskByIDPaths(volumeID aws.KubernetesVolumeID, partition string, devicePath string) []string {
	devicePaths := []string{}
	if devicePath != "" {
		devicePaths = append(devicePaths, devicePath)
	}

	if partition != "" {
		for i, path := range devicePaths {
			devicePaths[i] = path + diskPartitionSuffix + partition
		}
	}

	// We need to find NVME volumes, which are mounted on a "random" nvme path ("/dev/nvme0n1"),
	// and we have to get the volume id from the nvme interface
	awsVolumeID, err := volumeID.MapToAWSVolumeID()
	if err != nil {
		klog.Warningf("error mapping volume %q to AWS volume: %v", volumeID, err)
	} else {
		// This is the magic name on which AWS presents NVME devices under /dev/disk/by-id/
		// For example, vol-0fab1d5e3f72a5e23 creates a symlink at /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0fab1d5e3f72a5e23
		nvmeName := "nvme-Amazon_Elastic_Block_Store_" + strings.Replace(string(awsVolumeID), "-", "", -1)
		nvmePath, err := findNvmeVolume(nvmeName)
		if err != nil {
			klog.Warningf("error looking for nvme volume %q: %v", volumeID, err)
		} else if nvmePath != "" {
			if partition != "" {
				nvmePath = nvmePath + nvmeDiskPartitionSuffix + partition
			}
			devicePaths = append(devicePaths, nvmePath)
		}
	}

	return devicePaths
}

// Return cloud provider
func getCloudProvider(cloudProvider cloudprovider.Interface) (*aws.Cloud, error) {
	awsCloudProvider, ok := cloudProvider.(*aws.Cloud)
	if !ok || awsCloudProvider == nil {
		return nil, fmt.Errorf("Failed to get AWS Cloud Provider. GetCloudProvider returned %v instead", cloudProvider)
	}

	return awsCloudProvider, nil
}

// findNvmeVolume looks for the nvme volume with the specified name
// It follows the symlink (if it exists) and returns the absolute path to the device
func findNvmeVolume(findName string) (device string, err error) {
	p := filepath.Join("/dev/disk/by-id/", findName)
	stat, err := os.Lstat(p)
	if err != nil {
		if os.IsNotExist(err) {
			klog.V(6).Infof("nvme path not found %q", p)
			return "", nil
		}
		return "", fmt.Errorf("error getting stat of %q: %v", p, err)
	}

	if stat.Mode()&os.ModeSymlink != os.ModeSymlink {
		klog.Warningf("nvme file %q found, but was not a symlink", p)
		return "", nil
	}

	// Find the target, resolving to an absolute path
	// For example, /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0fab1d5e3f72a5e23 -> ../../nvme2n1
	resolved, err := filepath.EvalSymlinks(p)
	if err != nil {
		return "", fmt.Errorf("error reading target of symlink %q: %v", p, err)
	}

	if !strings.HasPrefix(resolved, "/dev") {
		return "", fmt.Errorf("resolved symlink for %q was unexpected: %q", p, resolved)
	}

	return resolved, nil
}

func formatVolumeID(volumeID string) (string, error) {
	// This is a workaround to fix the issue in converting aws volume id from globalPDPath and globalMapPath
	// There are three formats for AWSEBSVolumeSource.VolumeID and they are stored on disk in paths like so:
	// VolumeID									mountPath								mapPath
	// aws:///vol-1234					aws/vol-1234						aws:/vol-1234
	// aws://us-east-1/vol-1234 aws/us-east-1/vol-1234  aws:/us-east-1/vol-1234
	// vol-1234									vol-1234								vol-1234
	// This code is for converting volume ids from paths back to AWS style VolumeIDs
	sourceName := volumeID
	if strings.HasPrefix(volumeID, "aws/") || strings.HasPrefix(volumeID, "aws:/") {
		names := strings.Split(volumeID, "/")
		length := len(names)
		if length < 2 || length > 3 {
			return "", fmt.Errorf("invalid volume name format %q", volumeID)
		}
		volName := names[length-1]
		if !strings.HasPrefix(volName, "vol-") {
			return "", fmt.Errorf("Invalid volume name format for AWS volume (%q)", volName)
		}
		if length == 2 {
			sourceName = awsURLNamePrefix + "" + "/" + volName // empty zone label
		}
		if length == 3 {
			sourceName = awsURLNamePrefix + names[1] + "/" + volName // names[1] is the zone label
		}
		klog.V(4).Infof("Convert aws volume name from %q to %q ", volumeID, sourceName)
	}
	return sourceName, nil
}

func newAWSVolumeSpec(volumeName, volumeID string, mode v1.PersistentVolumeMode) *volume.Spec {
	awsVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
					VolumeID: volumeID,
				},
			},
			VolumeMode: &mode,
		},
	}
	return volume.NewSpecFromPersistentVolume(awsVolume, false)
}
<|endoftext|>