<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package interpodaffinity

import (
	"context"
	"fmt"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/klog"
	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
	"k8s.io/kubernetes/pkg/scheduler/nodeinfo"
	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
)

// preScoreStateKey is the key in CycleState to InterPodAffinity pre-computed data for Scoring.
const preScoreStateKey = "PreScore" + Name

// preScoreState computed at PreScore and used at Score.
type preScoreState struct {
	topologyScore     map[string]map[string]int64
	affinityTerms     []*weightedAffinityTerm
	antiAffinityTerms []*weightedAffinityTerm
}

// Clone implements the mandatory Clone interface. We don't really copy the data since
// there is no need for that.
func (s *preScoreState) Clone() framework.StateData {
	return s
}

// A "processed" representation of v1.WeightedAffinityTerm.
type weightedAffinityTerm struct {
	affinityTerm
	weight int32
}

func newWeightedAffinityTerm(pod *v1.Pod, term *v1.PodAffinityTerm, weight int32) (*weightedAffinityTerm, error) {
	namespaces := schedutil.GetNamespacesFromPodAffinityTerm(pod, term)
	selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector)
	if err != nil {
		return nil, err
	}
	return &weightedAffinityTerm{affinityTerm: affinityTerm{namespaces: namespaces, selector: selector, topologyKey: term.TopologyKey}, weight: weight}, nil
}

func getWeightedAffinityTerms(pod *v1.Pod, v1Terms []v1.WeightedPodAffinityTerm) ([]*weightedAffinityTerm, error) {
	if v1Terms == nil {
		return nil, nil
	}

	var terms []*weightedAffinityTerm
	for i := range v1Terms {
		p, err := newWeightedAffinityTerm(pod, &v1Terms[i].PodAffinityTerm, v1Terms[i].Weight)
		if err != nil {
			return nil, err
		}
		terms = append(terms, p)
	}
	return terms, nil
}

func (pl *InterPodAffinity) processTerm(
	state *preScoreState,
	term *weightedAffinityTerm,
	podToCheck *v1.Pod,
	fixedNode *v1.Node,
	multiplier int,
) {
	if len(fixedNode.Labels) == 0 {
		return
	}

	match := schedutil.PodMatchesTermsNamespaceAndSelector(podToCheck, term.namespaces, term.selector)
	tpValue, tpValueExist := fixedNode.Labels[term.topologyKey]
	if match && tpValueExist {
		pl.Lock()
		if state.topologyScore[term.topologyKey] == nil {
			state.topologyScore[term.topologyKey] = make(map[string]int64)
		}
		state.topologyScore[term.topologyKey][tpValue] += int64(term.weight * int32(multiplier))
		pl.Unlock()
	}
	return
}

func (pl *InterPodAffinity) processTerms(state *preScoreState, terms []*weightedAffinityTerm, podToCheck *v1.Pod, fixedNode *v1.Node, multiplier int) error {
	for _, term := range terms {
		pl.processTerm(state, term, podToCheck, fixedNode, multiplier)
	}
	return nil
}

func (pl *InterPodAffinity) processExistingPod(state *preScoreState, existingPod *v1.Pod, existingPodNodeInfo *nodeinfo.NodeInfo, incomingPod *v1.Pod) error {
	existingPodAffinity := existingPod.Spec.Affinity
	existingHasAffinityConstraints := existingPodAffinity != nil && existingPodAffinity.PodAffinity != nil
	existingHasAntiAffinityConstraints := existingPodAffinity != nil && existingPodAffinity.PodAntiAffinity != nil
	existingPodNode := existingPodNodeInfo.Node()

	// For every soft pod affinity term of <pod>, if <existingPod> matches the term,
	// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPods>`s node by the term`s weight.
	pl.processTerms(state, state.affinityTerms, existingPod, existingPodNode, 1)

	// For every soft pod anti-affinity term of <pod>, if <existingPod> matches the term,
	// decrement <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>`s node by the term`s weight.
	pl.processTerms(state, state.antiAffinityTerms, existingPod, existingPodNode, -1)

	if existingHasAffinityConstraints {
		// For every hard pod affinity term of <existingPod>, if <pod> matches the term,
		// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
		// value as that of <existingPod>'s node by the constant <ipa.hardPodAffinityWeight>
		if *pl.HardPodAffinityWeight > 0 {
			terms := existingPodAffinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution
			// TODO: Uncomment this block when implement RequiredDuringSchedulingRequiredDuringExecution.
			//if len(existingPodAffinity.PodAffinity.RequiredDuringSchedulingRequiredDuringExecution) != 0 {
			//	terms = append(terms, existingPodAffinity.PodAffinity.RequiredDuringSchedulingRequiredDuringExecution...)
			//}
			for i := range terms {
				term := &terms[i]
				processedTerm, err := newWeightedAffinityTerm(existingPod, term, *pl.HardPodAffinityWeight)
				if err != nil {
					return err
				}
				pl.processTerm(state, processedTerm, incomingPod, existingPodNode, 1)
			}
		}
		// For every soft pod affinity term of <existingPod>, if <pod> matches the term,
		// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
		// value as that of <existingPod>'s node by the term's weight.
		terms, err := getWeightedAffinityTerms(existingPod, existingPodAffinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
		if err != nil {
			klog.Error(err)
			return nil
		}

		pl.processTerms(state, terms, incomingPod, existingPodNode, 1)
	}
	if existingHasAntiAffinityConstraints {
		// For every soft pod anti-affinity term of <existingPod>, if <pod> matches the term,
		// decrement <pm.counts> for every node in the cluster with the same <term.TopologyKey>
		// value as that of <existingPod>'s node by the term's weight.
		terms, err := getWeightedAffinityTerms(existingPod, existingPodAffinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
		if err != nil {
			return err
		}
		pl.processTerms(state, terms, incomingPod, existingPodNode, -1)
	}
	return nil
}

// PreScore builds and writes cycle state used by Score and NormalizeScore.
func (pl *InterPodAffinity) PreScore(
	pCtx context.Context,
	cycleState *framework.CycleState,
	pod *v1.Pod,
	nodes []*v1.Node,
) *framework.Status {
	if len(nodes) == 0 {
		// No nodes to score.
		return nil
	}

	if pl.sharedLister == nil {
		return framework.NewStatus(framework.Error, fmt.Sprintf("BuildTopologyPairToScore with empty shared lister"))
	}

	affinity := pod.Spec.Affinity
	hasAffinityConstraints := affinity != nil && affinity.PodAffinity != nil
	hasAntiAffinityConstraints := affinity != nil && affinity.PodAntiAffinity != nil

	// Unless the pod being scheduled has affinity terms, we only
	// need to process nodes hosting pods with affinity.
	allNodes, err := pl.sharedLister.NodeInfos().HavePodsWithAffinityList()
	if err != nil {
		framework.NewStatus(framework.Error, fmt.Sprintf("get pods with affinity list error, err: %v", err))
	}
	if hasAffinityConstraints || hasAntiAffinityConstraints {
		allNodes, err = pl.sharedLister.NodeInfos().List()
		if err != nil {
			framework.NewStatus(framework.Error, fmt.Sprintf("get all nodes from shared lister error, err: %v", err))
		}
	}

	var affinityTerms []*weightedAffinityTerm
	var antiAffinityTerms []*weightedAffinityTerm
	if hasAffinityConstraints {
		if affinityTerms, err = getWeightedAffinityTerms(pod, affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution); err != nil {
			klog.Error(err)
			return nil
		}
	}
	if hasAntiAffinityConstraints {
		if antiAffinityTerms, err = getWeightedAffinityTerms(pod, affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution); err != nil {
			klog.Error(err)
			return nil
		}
	}

	state := &preScoreState{
		topologyScore:     make(map[string]map[string]int64),
		affinityTerms:     affinityTerms,
		antiAffinityTerms: antiAffinityTerms,
	}

	errCh := schedutil.NewErrorChannel()
	ctx, cancel := context.WithCancel(pCtx)
	processNode := func(i int) {
		nodeInfo := allNodes[i]
		if nodeInfo.Node() == nil {
			return
		}
		// Unless the pod being scheduled has affinity terms, we only
		// need to process pods with affinity in the node.
		podsToProcess := nodeInfo.PodsWithAffinity()
		if hasAffinityConstraints || hasAntiAffinityConstraints {
			// We need to process all the pods.
			podsToProcess = nodeInfo.Pods()
		}

		for _, existingPod := range podsToProcess {
			if err := pl.processExistingPod(state, existingPod, nodeInfo, pod); err != nil {
				errCh.SendErrorWithCancel(err, cancel)
				return
			}
		}
	}
	workqueue.ParallelizeUntil(ctx, 16, len(allNodes), processNode)
	if err := errCh.ReceiveError(); err != nil {
		return framework.NewStatus(framework.Error, err.Error())
	}

	cycleState.Write(preScoreStateKey, state)
	return nil
}

func getPreScoreState(cycleState *framework.CycleState) (*preScoreState, error) {
	c, err := cycleState.Read(preScoreStateKey)
	if err != nil {
		return nil, fmt.Errorf("Error reading %q from cycleState: %v", preScoreStateKey, err)
	}

	s, ok := c.(*preScoreState)
	if !ok {
		return nil, fmt.Errorf("%+v  convert to interpodaffinity.preScoreState error", c)
	}
	return s, nil
}

// Score invoked at the Score extension point.
// The "score" returned in this function is the matching number of pods on the `nodeName`,
// it is normalized later.
func (pl *InterPodAffinity) Score(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
	nodeInfo, err := pl.sharedLister.NodeInfos().Get(nodeName)
	if err != nil || nodeInfo.Node() == nil {
		return 0, framework.NewStatus(framework.Error, fmt.Sprintf("getting node %q from Snapshot: %v, node is nil: %v", nodeName, err, nodeInfo.Node() == nil))
	}
	node := nodeInfo.Node()

	s, err := getPreScoreState(cycleState)
	if err != nil {
		return 0, framework.NewStatus(framework.Error, err.Error())
	}
	var score int64
	for tpKey, tpValues := range s.topologyScore {
		if v, exist := node.Labels[tpKey]; exist {
			score += tpValues[v]
		}
	}

	return score, nil
}

// NormalizeScore normalizes the score for each filteredNode.
// The basic rule is: the bigger the score(matching number of pods) is, the smaller the
// final normalized score will be.
func (pl *InterPodAffinity) NormalizeScore(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
	s, err := getPreScoreState(cycleState)
	if err != nil {
		return framework.NewStatus(framework.Error, err.Error())
	}
	if len(s.topologyScore) == 0 {
		return nil
	}

	var maxCount, minCount int64
	for i := range scores {
		score := scores[i].Score
		if score > maxCount {
			maxCount = score
		}
		if score < minCount {
			minCount = score
		}
	}

	maxMinDiff := maxCount - minCount
	for i := range scores {
		fScore := float64(0)
		if maxMinDiff > 0 {
			fScore = float64(framework.MaxNodeScore) * (float64(scores[i].Score-minCount) / float64(maxMinDiff))
		}

		scores[i].Score = int64(fScore)
	}

	return nil
}

// ScoreExtensions of the Score plugin.
func (pl *InterPodAffinity) ScoreExtensions() framework.ScoreExtensions {
	return pl
}
<commit_msg>Reduce locking when calculating affinity scores<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package interpodaffinity

import (
	"context"
	"fmt"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/klog"
	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
	"k8s.io/kubernetes/pkg/scheduler/nodeinfo"
	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
)

// preScoreStateKey is the key in CycleState to InterPodAffinity pre-computed data for Scoring.
const preScoreStateKey = "PreScore" + Name

type scoreMap map[string]map[string]int64

// preScoreState computed at PreScore and used at Score.
type preScoreState struct {
	topologyScore     scoreMap
	affinityTerms     []*weightedAffinityTerm
	antiAffinityTerms []*weightedAffinityTerm
}

// Clone implements the mandatory Clone interface. We don't really copy the data since
// there is no need for that.
func (s *preScoreState) Clone() framework.StateData {
	return s
}

// A "processed" representation of v1.WeightedAffinityTerm.
type weightedAffinityTerm struct {
	affinityTerm
	weight int32
}

func newWeightedAffinityTerm(pod *v1.Pod, term *v1.PodAffinityTerm, weight int32) (*weightedAffinityTerm, error) {
	namespaces := schedutil.GetNamespacesFromPodAffinityTerm(pod, term)
	selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector)
	if err != nil {
		return nil, err
	}
	return &weightedAffinityTerm{affinityTerm: affinityTerm{namespaces: namespaces, selector: selector, topologyKey: term.TopologyKey}, weight: weight}, nil
}

func getWeightedAffinityTerms(pod *v1.Pod, v1Terms []v1.WeightedPodAffinityTerm) ([]*weightedAffinityTerm, error) {
	if v1Terms == nil {
		return nil, nil
	}

	var terms []*weightedAffinityTerm
	for i := range v1Terms {
		p, err := newWeightedAffinityTerm(pod, &v1Terms[i].PodAffinityTerm, v1Terms[i].Weight)
		if err != nil {
			return nil, err
		}
		terms = append(terms, p)
	}
	return terms, nil
}

func (m scoreMap) processTerm(
	term *weightedAffinityTerm,
	podToCheck *v1.Pod,
	fixedNode *v1.Node,
	multiplier int,
) {
	if len(fixedNode.Labels) == 0 {
		return
	}

	match := schedutil.PodMatchesTermsNamespaceAndSelector(podToCheck, term.namespaces, term.selector)
	tpValue, tpValueExist := fixedNode.Labels[term.topologyKey]
	if match && tpValueExist {
		if m[term.topologyKey] == nil {
			m[term.topologyKey] = make(map[string]int64)
		}
		m[term.topologyKey][tpValue] += int64(term.weight * int32(multiplier))
	}
	return
}

func (m scoreMap) processTerms(terms []*weightedAffinityTerm, podToCheck *v1.Pod, fixedNode *v1.Node, multiplier int) {
	for _, term := range terms {
		m.processTerm(term, podToCheck, fixedNode, multiplier)
	}
}

func (m scoreMap) append(other scoreMap) {
	for topology, oScores := range other {
		scores := m[topology]
		if scores == nil {
			m[topology] = oScores
			continue
		}
		for k, v := range oScores {
			scores[k] += v
		}
	}
}

func (pl *InterPodAffinity) processExistingPod(state *preScoreState, existingPod *v1.Pod, existingPodNodeInfo *nodeinfo.NodeInfo, incomingPod *v1.Pod, topoScore scoreMap) error {
	existingPodAffinity := existingPod.Spec.Affinity
	existingHasAffinityConstraints := existingPodAffinity != nil && existingPodAffinity.PodAffinity != nil
	existingHasAntiAffinityConstraints := existingPodAffinity != nil && existingPodAffinity.PodAntiAffinity != nil
	existingPodNode := existingPodNodeInfo.Node()

	// For every soft pod affinity term of <pod>, if <existingPod> matches the term,
	// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPods>`s node by the term`s weight.
	topoScore.processTerms(state.affinityTerms, existingPod, existingPodNode, 1)

	// For every soft pod anti-affinity term of <pod>, if <existingPod> matches the term,
	// decrement <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>`s node by the term`s weight.
	topoScore.processTerms(state.antiAffinityTerms, existingPod, existingPodNode, -1)

	if existingHasAffinityConstraints {
		// For every hard pod affinity term of <existingPod>, if <pod> matches the term,
		// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
		// value as that of <existingPod>'s node by the constant <ipa.hardPodAffinityWeight>
		if *pl.HardPodAffinityWeight > 0 {
			terms := existingPodAffinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution
			// TODO: Uncomment this block when implement RequiredDuringSchedulingRequiredDuringExecution.
			//if len(existingPodAffinity.PodAffinity.RequiredDuringSchedulingRequiredDuringExecution) != 0 {
			//	terms = append(terms, existingPodAffinity.PodAffinity.RequiredDuringSchedulingRequiredDuringExecution...)
			//}
			for i := range terms {
				term := &terms[i]
				processedTerm, err := newWeightedAffinityTerm(existingPod, term, *pl.HardPodAffinityWeight)
				if err != nil {
					return err
				}
				topoScore.processTerm(processedTerm, incomingPod, existingPodNode, 1)
			}
		}
		// For every soft pod affinity term of <existingPod>, if <pod> matches the term,
		// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
		// value as that of <existingPod>'s node by the term's weight.
		terms, err := getWeightedAffinityTerms(existingPod, existingPodAffinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
		if err != nil {
			klog.Error(err)
			return nil
		}

		topoScore.processTerms(terms, incomingPod, existingPodNode, 1)
	}
	if existingHasAntiAffinityConstraints {
		// For every soft pod anti-affinity term of <existingPod>, if <pod> matches the term,
		// decrement <pm.counts> for every node in the cluster with the same <term.TopologyKey>
		// value as that of <existingPod>'s node by the term's weight.
		terms, err := getWeightedAffinityTerms(existingPod, existingPodAffinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
		if err != nil {
			return err
		}
		topoScore.processTerms(terms, incomingPod, existingPodNode, -1)
	}
	return nil
}

// PreScore builds and writes cycle state used by Score and NormalizeScore.
func (pl *InterPodAffinity) PreScore(
	pCtx context.Context,
	cycleState *framework.CycleState,
	pod *v1.Pod,
	nodes []*v1.Node,
) *framework.Status {
	if len(nodes) == 0 {
		// No nodes to score.
		return nil
	}

	if pl.sharedLister == nil {
		return framework.NewStatus(framework.Error, fmt.Sprintf("BuildTopologyPairToScore with empty shared lister"))
	}

	affinity := pod.Spec.Affinity
	hasAffinityConstraints := affinity != nil && affinity.PodAffinity != nil
	hasAntiAffinityConstraints := affinity != nil && affinity.PodAntiAffinity != nil

	// Unless the pod being scheduled has affinity terms, we only
	// need to process nodes hosting pods with affinity.
	allNodes, err := pl.sharedLister.NodeInfos().HavePodsWithAffinityList()
	if err != nil {
		framework.NewStatus(framework.Error, fmt.Sprintf("get pods with affinity list error, err: %v", err))
	}
	if hasAffinityConstraints || hasAntiAffinityConstraints {
		allNodes, err = pl.sharedLister.NodeInfos().List()
		if err != nil {
			framework.NewStatus(framework.Error, fmt.Sprintf("get all nodes from shared lister error, err: %v", err))
		}
	}

	var affinityTerms []*weightedAffinityTerm
	var antiAffinityTerms []*weightedAffinityTerm
	if hasAffinityConstraints {
		if affinityTerms, err = getWeightedAffinityTerms(pod, affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution); err != nil {
			klog.Error(err)
			return nil
		}
	}
	if hasAntiAffinityConstraints {
		if antiAffinityTerms, err = getWeightedAffinityTerms(pod, affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution); err != nil {
			klog.Error(err)
			return nil
		}
	}

	state := &preScoreState{
		topologyScore:     make(map[string]map[string]int64),
		affinityTerms:     affinityTerms,
		antiAffinityTerms: antiAffinityTerms,
	}

	errCh := schedutil.NewErrorChannel()
	ctx, cancel := context.WithCancel(pCtx)
	processNode := func(i int) {
		nodeInfo := allNodes[i]
		if nodeInfo.Node() == nil {
			return
		}
		// Unless the pod being scheduled has affinity terms, we only
		// need to process pods with affinity in the node.
		podsToProcess := nodeInfo.PodsWithAffinity()
		if hasAffinityConstraints || hasAntiAffinityConstraints {
			// We need to process all the pods.
			podsToProcess = nodeInfo.Pods()
		}

		topoScore := make(scoreMap)
		for _, existingPod := range podsToProcess {
			if err := pl.processExistingPod(state, existingPod, nodeInfo, pod, topoScore); err != nil {
				errCh.SendErrorWithCancel(err, cancel)
				return
			}
		}
		if len(topoScore) > 0 {
			pl.Lock()
			state.topologyScore.append(topoScore)
			pl.Unlock()
		}
	}
	workqueue.ParallelizeUntil(ctx, 16, len(allNodes), processNode)
	if err := errCh.ReceiveError(); err != nil {
		return framework.NewStatus(framework.Error, err.Error())
	}

	cycleState.Write(preScoreStateKey, state)
	return nil
}

func getPreScoreState(cycleState *framework.CycleState) (*preScoreState, error) {
	c, err := cycleState.Read(preScoreStateKey)
	if err != nil {
		return nil, fmt.Errorf("Error reading %q from cycleState: %v", preScoreStateKey, err)
	}

	s, ok := c.(*preScoreState)
	if !ok {
		return nil, fmt.Errorf("%+v  convert to interpodaffinity.preScoreState error", c)
	}
	return s, nil
}

// Score invoked at the Score extension point.
// The "score" returned in this function is the matching number of pods on the `nodeName`,
// it is normalized later.
func (pl *InterPodAffinity) Score(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
	nodeInfo, err := pl.sharedLister.NodeInfos().Get(nodeName)
	if err != nil || nodeInfo.Node() == nil {
		return 0, framework.NewStatus(framework.Error, fmt.Sprintf("getting node %q from Snapshot: %v, node is nil: %v", nodeName, err, nodeInfo.Node() == nil))
	}
	node := nodeInfo.Node()

	s, err := getPreScoreState(cycleState)
	if err != nil {
		return 0, framework.NewStatus(framework.Error, err.Error())
	}
	var score int64
	for tpKey, tpValues := range s.topologyScore {
		if v, exist := node.Labels[tpKey]; exist {
			score += tpValues[v]
		}
	}

	return score, nil
}

// NormalizeScore normalizes the score for each filteredNode.
// The basic rule is: the bigger the score(matching number of pods) is, the smaller the
// final normalized score will be.
func (pl *InterPodAffinity) NormalizeScore(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
	s, err := getPreScoreState(cycleState)
	if err != nil {
		return framework.NewStatus(framework.Error, err.Error())
	}
	if len(s.topologyScore) == 0 {
		return nil
	}

	var maxCount, minCount int64
	for i := range scores {
		score := scores[i].Score
		if score > maxCount {
			maxCount = score
		}
		if score < minCount {
			minCount = score
		}
	}

	maxMinDiff := maxCount - minCount
	for i := range scores {
		fScore := float64(0)
		if maxMinDiff > 0 {
			fScore = float64(framework.MaxNodeScore) * (float64(scores[i].Score-minCount) / float64(maxMinDiff))
		}

		scores[i].Score = int64(fScore)
	}

	return nil
}

// ScoreExtensions of the Score plugin.
func (pl *InterPodAffinity) ScoreExtensions() framework.ScoreExtensions {
	return pl
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build linux

package bpf

import (
	"fmt"
	"runtime"
	"unsafe"

	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/spanstat"

	"golang.org/x/sys/unix"
)

// attrProg holds values from the upstream struct union for BPF_*_GET_*_ID.
// From: https://github.com/torvalds/linux/blob/v4.19-rc2/include/uapi/linux/bpf.h#L358
type attrProg struct {
	progID    uint32 // union: start_id, prog_id, or map_id
	nextID    uint32
	openFlags uint32
}

// ProgInfo holds values from the upstream struct bpf_prog_info.
// From: https://github.com/torvalds/linux/blob/v4.19-rc2/include/uapi/linux/bpf.h#L2427
type ProgInfo struct {
	ProgType        uint32
	ID              uint32
	Tag             uint8
	JitedProgLen    uint32
	XlatedProgLen   uint32
	JitedProgInsns  uint64
	XlatedProgInsns uint64
	LoadTime        uint64 // ns since boottime
	CreatedByUID    uint32
	NRMapIDs        uint32
	MapIDs          []uint32
	Name            string
	IfIndex         uint32
	NetnsDev        uint64
	NetnsIno        uint64
	NrJitedKsyms    uint32
	NrJitedFuncLens uint32
	JitedKsyms      uint64
	JitedFuncLens   uint64
}

// attrObjInfo holds values from the upstream struct union for BPF_OBJ_GET_INFO_BY_FD.
// From: https://github.com/torvalds/linux/blob/v4.19-rc2/include/uapi/linux/bpf.h#L369
type attrObjInfo struct {
	bpfFD   uint32
	infoLen uint32
	info    uint64
}

// GetProgNextID takes a current program ID and returns the next program ID.
func GetProgNextID(current uint32) (uint32, error) {
	attr := attrProg{
		progID: current,
	}

	var duration *spanstat.SpanStat
	if option.Config.MetricsConfig.BPFSyscallDurationEnabled {
		duration = spanstat.Start()
	}
	ret, _, err := unix.Syscall(unix.SYS_BPF, BPF_PROG_GET_NEXT_ID, uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr))
	runtime.KeepAlive(&attr)
	if option.Config.MetricsConfig.BPFSyscallDurationEnabled {
		metrics.BPFSyscallDuration.WithLabelValues(metricOpProgGetNextID, metrics.Errno2Outcome(err)).Observe(duration.End(err == 0).Total().Seconds())
	}
	if ret != 0 || err != 0 {
		return 0, fmt.Errorf("Unable to get next id: %v", err)
	}

	return attr.nextID, nil
}

// GetProgFDByID returns the file descriptor for the program id.
func GetProgFDByID(id uint32) (int, error) {
	attr := attrProg{
		progID: uint32(uintptr(id)),
	}

	fd, _, err := unix.Syscall(unix.SYS_BPF, BPF_PROG_GET_FD_BY_ID, uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr))
	runtime.KeepAlive(&attr)
	if fd < 0 || err != 0 {
		return int(fd), fmt.Errorf("Unable to get fd for program id %d: %v", id, err)
	}

	return int(fd), nil
}

// GetProgInfoByFD gets the bpf program info from its file descriptor.
func GetProgInfoByFD(fd int) (ProgInfo, error) {
	info := ProgInfo{}
	attrInfo := attrObjInfo{
		bpfFD:   uint32(fd),
		infoLen: uint32(unsafe.Sizeof(info)),
		info:    uint64(uintptr(unsafe.Pointer(&info))),
	}
	// This struct must be in sync with union bpf_attr's anonymous struct
	attr := struct {
		info attrObjInfo
	}{
		info: attrInfo,
	}

	var duration *spanstat.SpanStat
	if option.Config.MetricsConfig.BPFSyscallDurationEnabled {
		duration = spanstat.Start()
	}
	ret, _, err := unix.Syscall(unix.SYS_BPF, BPF_OBJ_GET_INFO_BY_FD, uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr))
	runtime.KeepAlive(&info)
	runtime.KeepAlive(&attrInfo)
	if option.Config.MetricsConfig.BPFSyscallDurationEnabled {
		metrics.BPFSyscallDuration.WithLabelValues(metricOpObjGetInfoByFD, metrics.Errno2Outcome(err)).Observe(duration.End(err == 0).Total().Seconds())
	}
	if ret != 0 || err != 0 {
		return ProgInfo{}, fmt.Errorf("Unable to get object info: %v", err)
	}

	return info, nil
}
<commit_msg>bpf: remove unused GetProgNextID and GetProgFDByID<commit_after><|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"github.com/containers/image/manifest"
	"github.com/docker/docker/builder/dockerfile/instructions"
	"github.com/sirupsen/logrus"
	"strings"
)

type LabelCommand struct {
	cmd *instructions.LabelCommand
}

func (r *LabelCommand) ExecuteCommand(config *manifest.Schema2Config) error {
	return updateLabels(r.cmd.Labels, config)
}

func updateLabels(labels []instructions.KeyValuePair, config *manifest.Schema2Config) error {
	existingLabels := config.Labels

	for _, kvp := range labels {
		logrus.Infof("Applying label %s=%s", kvp.Key, kvp.Value)
		existingLabels[kvp.Key] = kvp.Value
	}

	config.Labels = existingLabels
	return nil

}

func (r *LabelCommand) FilesToSnapshot() []string {
	return []string{}
}

func (r *LabelCommand) CreatedBy() string {
	l := []string{r.cmd.Name()}
	for _, kvp := range r.cmd.Labels {
		l = append(l, kvp.String())
	}
	return strings.Join(l, " ")
}
<commit_msg>unescaping values when appropriate<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"github.com/containers/image/manifest"
	"github.com/docker/docker/builder/dockerfile/instructions"
	"github.com/docker/docker/builder/dockerfile/shell"
	"github.com/sirupsen/logrus"
	"strings"
)

type LabelCommand struct {
	cmd *instructions.LabelCommand
}

func (r *LabelCommand) ExecuteCommand(config *manifest.Schema2Config) error {
	return updateLabels(r.cmd.Labels, config)
}

func updateLabels(labels []instructions.KeyValuePair, config *manifest.Schema2Config) error {
	existingLabels := config.Labels

	// Let's unescape values before setting the label
	shlex := shell.NewLex('\\')
	for index, kvp := range labels {
		unescaped, err := shlex.ProcessWord(kvp.Value, []string{})
		if err != nil {
			return err
		}
		labels[index] = instructions.KeyValuePair{
			Key:   kvp.Key,
			Value: unescaped,
		}
	}
	for _, kvp := range labels {
		logrus.Infof("Applying label %s=%s", kvp.Key, kvp.Value)
		existingLabels[kvp.Key] = kvp.Value
	}

	config.Labels = existingLabels
	return nil

}

// No files have changed, this command only touches metadata.
func (r *LabelCommand) FilesToSnapshot() []string {
	return []string{}
}

// CreatedBy returns some information about the command for the image config history
func (r *LabelCommand) CreatedBy() string {
	l := []string{r.cmd.Name()}
	for _, kvp := range r.cmd.Labels {
		l = append(l, kvp.String())
	}
	return strings.Join(l, " ")
}
<|endoftext|>


<commit_before>package plugins

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
)

const (
	versionMimetype = "application/vnd.docker.plugins.v1+json"
	defaultTimeOut  = 30
)

func NewClient(addr string) *Client {
	tr := &http.Transport{}
	protoAndAddr := strings.Split(addr, "://")
	configureTCPTransport(tr, protoAndAddr[0], protoAndAddr[1])
	return &Client{&http.Client{Transport: tr}, protoAndAddr[1]}
}

type Client struct {
	http *http.Client
	addr string
}

func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) error {
	return c.callWithRetry(serviceMethod, args, ret, true)
}

func (c *Client) callWithRetry(serviceMethod string, args interface{}, ret interface{}, retry bool) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(args); err != nil {
		return err
	}

	req, err := http.NewRequest("POST", "/"+serviceMethod, &buf)
	if err != nil {
		return err
	}
	req.Header.Add("Accept", versionMimetype)
	req.URL.Scheme = "http"
	req.URL.Host = c.addr

	var retries int
	start := time.Now()

	for {
		resp, err := c.http.Do(req)
		if err != nil {
			if !retry {
				return err
			}

			timeOff := backoff(retries)
			if abort(start, timeOff) {
				return err
			}
			retries++
			logrus.Warnf("Unable to connect to plugin: %s, retrying in %v", c.addr, timeOff)
			time.Sleep(timeOff)
			continue
		}

		if resp.StatusCode != http.StatusOK {
			remoteErr, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return nil
			}
			return fmt.Errorf("Plugin Error: %s", remoteErr)
		}

		return json.NewDecoder(resp.Body).Decode(&ret)
	}
}

func backoff(retries int) time.Duration {
	b, max := 1, defaultTimeOut
	for b < max && retries > 0 {
		b *= 2
		retries--
	}
	if b > max {
		b = max
	}
	return time.Duration(b) * time.Second
}

func abort(start time.Time, timeOff time.Duration) bool {
	return timeOff+time.Since(start) > time.Duration(defaultTimeOut)*time.Second
}

func configureTCPTransport(tr *http.Transport, proto, addr string) {
	// Why 32? See https://github.com/docker/docker/pull/8035.
	timeout := 32 * time.Second
	if proto == "unix" {
		// No need for compression in local communications.
		tr.DisableCompression = true
		tr.Dial = func(_, _ string) (net.Conn, error) {
			return net.DialTimeout(proto, addr, timeout)
		}
	} else {
		tr.Proxy = http.ProxyFromEnvironment
		tr.Dial = (&net.Dialer{Timeout: timeout}).Dial
	}
}
<commit_msg>bug fix: close http response body no longer in use<commit_after>package plugins

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
)

const (
	versionMimetype = "application/vnd.docker.plugins.v1+json"
	defaultTimeOut  = 30
)

func NewClient(addr string) *Client {
	tr := &http.Transport{}
	protoAndAddr := strings.Split(addr, "://")
	configureTCPTransport(tr, protoAndAddr[0], protoAndAddr[1])
	return &Client{&http.Client{Transport: tr}, protoAndAddr[1]}
}

type Client struct {
	http *http.Client
	addr string
}

func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) error {
	return c.callWithRetry(serviceMethod, args, ret, true)
}

func (c *Client) callWithRetry(serviceMethod string, args interface{}, ret interface{}, retry bool) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(args); err != nil {
		return err
	}

	req, err := http.NewRequest("POST", "/"+serviceMethod, &buf)
	if err != nil {
		return err
	}
	req.Header.Add("Accept", versionMimetype)
	req.URL.Scheme = "http"
	req.URL.Host = c.addr

	var retries int
	start := time.Now()

	for {
		resp, err := c.http.Do(req)
		if err != nil {
			if !retry {
				return err
			}

			timeOff := backoff(retries)
			if abort(start, timeOff) {
				return err
			}
			retries++
			logrus.Warnf("Unable to connect to plugin: %s, retrying in %v", c.addr, timeOff)
			time.Sleep(timeOff)
			continue
		}

		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			remoteErr, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return nil
			}
			return fmt.Errorf("Plugin Error: %s", remoteErr)
		}

		return json.NewDecoder(resp.Body).Decode(&ret)
	}
}

func backoff(retries int) time.Duration {
	b, max := 1, defaultTimeOut
	for b < max && retries > 0 {
		b *= 2
		retries--
	}
	if b > max {
		b = max
	}
	return time.Duration(b) * time.Second
}

func abort(start time.Time, timeOff time.Duration) bool {
	return timeOff+time.Since(start) > time.Duration(defaultTimeOut)*time.Second
}

func configureTCPTransport(tr *http.Transport, proto, addr string) {
	// Why 32? See https://github.com/docker/docker/pull/8035.
	timeout := 32 * time.Second
	if proto == "unix" {
		// No need for compression in local communications.
		tr.DisableCompression = true
		tr.Dial = func(_, _ string) (net.Conn, error) {
			return net.DialTimeout(proto, addr, timeout)
		}
	} else {
		tr.Proxy = http.ProxyFromEnvironment
		tr.Dial = (&net.Dialer{Timeout: timeout}).Dial
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
	"strings"

	"github.com/kubernetes-incubator/cri-tools/pkg/framework"
	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var (
	// image name for test image api
	testImageName = "busybox"

	// name-tagged reference for test image
	testImageRef = testImageName + ":1.26.2"

	// Digested reference for test image
	busyboxDigestRef = testImageName + "@sha256:817a12c32a39bbe394944ba49de563e085f1d3c5266eb8e9723256bc4448680e"
)

var _ = framework.KubeDescribe("Image Manager", func() {
	f := framework.NewDefaultCRIFramework()

	var c internalapi.ImageManagerService

	BeforeEach(func() {
		c = f.CRIClient.CRIImageClient
	})

	It("public image with tag should be pulled and removed [Conformance]", func() {
		testPullPublicImage(c, testImageRef)
	})

	It("public image without tag should be pulled and removed [Conformance]", func() {
		testPullPublicImage(c, testImageName)
	})

	It("public image with digest should be pulled and removed [Conformance]", func() {
		testPullPublicImage(c, busyboxDigestRef)
	})

	It("image status get image fields should not be empty [Conformance]", func() {
		framework.PullPublicImage(c, testImageRef)

		defer removeImage(c, testImageRef)

		status := framework.ImageStatus(c, testImageRef)
		Expect(status.Id).NotTo(BeNil(), "Image Id should not be nil")
		Expect(len(status.RepoTags)).NotTo(Equal(0), "Should have repoTags in image")
		Expect(status.Size_).NotTo(BeNil(), "Image Size should not be nil")
	})

	It("listImage should get exactly 3 image in the result list [Conformance]", func() {
		// different tags refer to different images
		testImageList := []string{
			"busybox:1-uclibc",
			"busybox:1-musl",
			"busybox:1-glibc",
		}

		pullImageList(c, testImageList)

		defer removeImageList(c, testImageList)

		images := framework.ListImage(c, &runtimeapi.ImageFilter{})

		count := 0
		for _, imageName := range images {
			for _, imagesTag := range imageName.RepoTags {
				if strings.HasSuffix(imagesTag, "busybox:1-uclibc") {
					count = count + 1
				}
				if strings.HasSuffix(imagesTag, "busybox:1-musl") {
					count = count + 1
				}
				if strings.HasSuffix(imagesTag, "busybox:1-glibc") {
					count = count + 1
				}
			}
		}
		Expect(count).To(Equal(3), "Should have the specified three images in list")

	})

	It("listImage should get exactly 3 repoTags in the result image [Conformance]", func() {
		// different tags refer to the same image
		testImageList := []string{
			"busybox:1-uclibc",
			"busybox:1",
		}

		pullImageList(c, testImageList)

		defer removeImageList(c, testImageList)

		images := framework.ListImage(c, &runtimeapi.ImageFilter{})

		count := 0
		for _, imageName := range images {
			for _, imagesTag := range imageName.RepoTags {
				if strings.HasSuffix(imagesTag, "busybox:1-uclibc") {
					count = count + 1
				}
				if strings.HasSuffix(imagesTag, "busybox:1") {
					count = count + 1
				}
			}
			if count < 2 {
				count = 0
			}
		}
		Expect(count).To(Equal(2), "Should have three repoTags in single image in list")
	})
})

// testRemoveImage removes the image name imageName and check if it successes.
func testRemoveImage(c internalapi.ImageManagerService, imageName string) {
	By("Remove image : " + imageName)
	removeImage(c, imageName)

	By("Check image list empty")
	status := framework.ImageStatus(c, imageName)
	Expect(status).To(BeNil(), "Should have none image in list")
}

// testPullPublicImage pulls the image named imageName, make sure it success and remove the image.
func testPullPublicImage(c internalapi.ImageManagerService, imageName string) {
	if !strings.Contains(imageName, ":") {
		imageName = imageName + ":latest"
	}
	framework.PullPublicImage(c, imageName)

	By("Check image list to make sure pulling image success : " + imageName)
	status := framework.ImageStatus(c, imageName)
	Expect(status).NotTo(BeNil(), "Should have one image in list")

	testRemoveImage(c, imageName)
}

// pullImageList pulls the images listed in the imageList.
func pullImageList(c internalapi.ImageManagerService, imageList []string) {
	for _, imageName := range imageList {
		framework.PullPublicImage(c, imageName)
	}
}

// removeImageList removes the images listed in the imageList.
func removeImageList(c internalapi.ImageManagerService, imageList []string) {
	for _, imageName := range imageList {
		removeImage(c, imageName)
	}
}

// removeImage removes the image named imagesName.
func removeImage(c internalapi.ImageManagerService, imageName string) {
	By("Remove image : " + imageName)
	image, err := c.ImageStatus(&runtimeapi.ImageSpec{Image: imageName})
	framework.ExpectNoError(err, "failed to get image status: %v", err)

	if image != nil {
		By("Remove image by ID : " + image.Id)
		err = c.RemoveImage(&runtimeapi.ImageSpec{Image: image.Id})
		framework.ExpectNoError(err, "failed to remove image: %v", err)
	}
}
<commit_msg>Fixing the test description and validation message<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
	"strings"

	"github.com/kubernetes-incubator/cri-tools/pkg/framework"
	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var (
	// image name for test image api
	testImageName = "busybox"

	// name-tagged reference for test image
	testImageRef = testImageName + ":1.26.2"

	// Digested reference for test image
	busyboxDigestRef = testImageName + "@sha256:817a12c32a39bbe394944ba49de563e085f1d3c5266eb8e9723256bc4448680e"
)

var _ = framework.KubeDescribe("Image Manager", func() {
	f := framework.NewDefaultCRIFramework()

	var c internalapi.ImageManagerService

	BeforeEach(func() {
		c = f.CRIClient.CRIImageClient
	})

	It("public image with tag should be pulled and removed [Conformance]", func() {
		testPullPublicImage(c, testImageRef)
	})

	It("public image without tag should be pulled and removed [Conformance]", func() {
		testPullPublicImage(c, testImageName)
	})

	It("public image with digest should be pulled and removed [Conformance]", func() {
		testPullPublicImage(c, busyboxDigestRef)
	})

	It("image status get image fields should not be empty [Conformance]", func() {
		framework.PullPublicImage(c, testImageRef)

		defer removeImage(c, testImageRef)

		status := framework.ImageStatus(c, testImageRef)
		Expect(status.Id).NotTo(BeNil(), "Image Id should not be nil")
		Expect(len(status.RepoTags)).NotTo(Equal(0), "Should have repoTags in image")
		Expect(status.Size_).NotTo(BeNil(), "Image Size should not be nil")
	})

	It("listImage should get exactly 3 image in the result list [Conformance]", func() {
		// different tags refer to different images
		testImageList := []string{
			"busybox:1-uclibc",
			"busybox:1-musl",
			"busybox:1-glibc",
		}

		pullImageList(c, testImageList)

		defer removeImageList(c, testImageList)

		images := framework.ListImage(c, &runtimeapi.ImageFilter{})

		count := 0
		for _, imageName := range images {
			for _, imagesTag := range imageName.RepoTags {
				if strings.HasSuffix(imagesTag, "busybox:1-uclibc") {
					count = count + 1
				}
				if strings.HasSuffix(imagesTag, "busybox:1-musl") {
					count = count + 1
				}
				if strings.HasSuffix(imagesTag, "busybox:1-glibc") {
					count = count + 1
				}
			}
		}
		Expect(count).To(Equal(3), "Should have the specified three images in list")

	})

	It("listImage should get exactly 2 repoTags in the result image [Conformance]", func() {
		// different tags refer to the same image
		testImageList := []string{
			"busybox:1-uclibc",
			"busybox:1",
		}

		pullImageList(c, testImageList)

		defer removeImageList(c, testImageList)

		images := framework.ListImage(c, &runtimeapi.ImageFilter{})

		count := 0
		for _, imageName := range images {
			for _, imagesTag := range imageName.RepoTags {
				if strings.HasSuffix(imagesTag, "busybox:1-uclibc") {
					count = count + 1
				}
				if strings.HasSuffix(imagesTag, "busybox:1") {
					count = count + 1
				}
			}
			if count < 2 {
				count = 0
			}
		}
		Expect(count).To(Equal(2), "Should have two repoTags in single image in list")
	})
})

// testRemoveImage removes the image name imageName and check if it successes.
func testRemoveImage(c internalapi.ImageManagerService, imageName string) {
	By("Remove image : " + imageName)
	removeImage(c, imageName)

	By("Check image list empty")
	status := framework.ImageStatus(c, imageName)
	Expect(status).To(BeNil(), "Should have none image in list")
}

// testPullPublicImage pulls the image named imageName, make sure it success and remove the image.
func testPullPublicImage(c internalapi.ImageManagerService, imageName string) {
	if !strings.Contains(imageName, ":") {
		imageName = imageName + ":latest"
	}
	framework.PullPublicImage(c, imageName)

	By("Check image list to make sure pulling image success : " + imageName)
	status := framework.ImageStatus(c, imageName)
	Expect(status).NotTo(BeNil(), "Should have one image in list")

	testRemoveImage(c, imageName)
}

// pullImageList pulls the images listed in the imageList.
func pullImageList(c internalapi.ImageManagerService, imageList []string) {
	for _, imageName := range imageList {
		framework.PullPublicImage(c, imageName)
	}
}

// removeImageList removes the images listed in the imageList.
func removeImageList(c internalapi.ImageManagerService, imageList []string) {
	for _, imageName := range imageList {
		removeImage(c, imageName)
	}
}

// removeImage removes the image named imagesName.
func removeImage(c internalapi.ImageManagerService, imageName string) {
	By("Remove image : " + imageName)
	image, err := c.ImageStatus(&runtimeapi.ImageSpec{Image: imageName})
	framework.ExpectNoError(err, "failed to get image status: %v", err)

	if image != nil {
		By("Remove image by ID : " + image.Id)
		err = c.RemoveImage(&runtimeapi.ImageSpec{Image: image.Id})
		framework.ExpectNoError(err, "failed to remove image: %v", err)
	}
}
<|endoftext|>


<commit_before>package state

import (
	"fmt"

	"github.com/hashicorp/go-memdb"
)

// stateStoreSchema is used to return the schema for the state store
func stateStoreSchema() *memdb.DBSchema {
	// Create the root DB schema
	db := &memdb.DBSchema{
		Tables: make(map[string]*memdb.TableSchema),
	}

	// Collect all the schemas that are needed
	schemas := []func() *memdb.TableSchema{
		indexTableSchema,
		nodeTableSchema,
		jobTableSchema,
		evalTableSchema,
		allocTableSchema,
	}

	// Add each of the tables
	for _, schemaFn := range schemas {
		schema := schemaFn()
		if _, ok := db.Tables[schema.Name]; ok {
			panic(fmt.Sprintf("duplicate table name: %s", schema.Name))
		}
		db.Tables[schema.Name] = schema
	}
	return db
}

// indexTableSchema is used for
func indexTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "index",
		Indexes: map[string]*memdb.IndexSchema{
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.StringFieldIndex{
					Field:     "Key",
					Lowercase: true,
				},
			},
		},
	}
}

// nodeTableSchema returns the MemDB schema for the nodes table.
// This table is used to store all the client nodes that are registered.
func nodeTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "nodes",
		Indexes: map[string]*memdb.IndexSchema{
			// Primary index is used for node management
			// and simple direct lookup. ID is required to be
			// unique.
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.StringFieldIndex{
					Field:     "ID",
					Lowercase: true,
				},
			},
		},
	}
}

// jobTableSchema returns the MemDB schema for the jobs table.
// This table is used to store all the jobs that have been submitted.
func jobTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "jobs",
		Indexes: map[string]*memdb.IndexSchema{
			// Primary index is used for job management
			// and simple direct lookup. ID is required to be
			// unique.
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.StringFieldIndex{
					Field:     "ID",
					Lowercase: true,
				},
			},

			// Status is used to scan for jobs that are in need
			// of scheduling attention.
			"status": &memdb.IndexSchema{
				Name:         "status",
				AllowMissing: false,
				Unique:       false,
				Indexer: &memdb.StringFieldIndex{
					Field: "Status",
				},
			},
		},
	}
}

// evalTableSchema returns the MemDB schema for the eval table.
// This table is used to store all the evaluations that are pending
// or recently completed.
func evalTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "evals",
		Indexes: map[string]*memdb.IndexSchema{
			// Primary index is used for direct lookup.
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.UUIDFieldIndex{
					Field: "ID",
				},
			},

			// Job index is used to lookup allocations by job
			"job": &memdb.IndexSchema{
				Name:         "job",
				AllowMissing: false,
				Unique:       false,
				Indexer: &memdb.StringFieldIndex{
					Field:     "JobID",
					Lowercase: true,
				},
			},

			// Status is used to scan for evaluations that are in need
			// of scheduling attention.
			"status": &memdb.IndexSchema{
				Name:         "status",
				AllowMissing: false,
				Unique:       false,
				Indexer: &memdb.StringFieldIndex{
					Field: "Status",
				},
			},
		},
	}
}

// allocTableSchema returns the MemDB schema for the allocation table.
// This table is used to store all the task allocations between task groups
// and nodes.
func allocTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "allocs",
		Indexes: map[string]*memdb.IndexSchema{
			// Primary index is a UUID
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.UUIDFieldIndex{
					Field: "ID",
				},
			},

			// Node index is used to lookup allocations by node
			"node": &memdb.IndexSchema{
				Name:         "node",
				AllowMissing: true, // Missing is allow for failed allocations
				Unique:       false,
				Indexer: &memdb.StringFieldIndex{
					Field:     "NodeID",
					Lowercase: true,
				},
			},

			// Job index is used to lookup allocations by job
			"job": &memdb.IndexSchema{
				Name:         "job",
				AllowMissing: false,
				Unique:       false,
				Indexer: &memdb.StringFieldIndex{
					Field:     "JobID",
					Lowercase: true,
				},
			},

			// Eval index is used to lookup allocations by eval
			"eval": &memdb.IndexSchema{
				Name:         "eval",
				AllowMissing: false,
				Unique:       false,
				Indexer: &memdb.UUIDFieldIndex{
					Field: "EvalID",
				},
			},
		},
	}
}
<commit_msg>state: remove unused indexes<commit_after>package state

import (
	"fmt"

	"github.com/hashicorp/go-memdb"
)

// stateStoreSchema is used to return the schema for the state store
func stateStoreSchema() *memdb.DBSchema {
	// Create the root DB schema
	db := &memdb.DBSchema{
		Tables: make(map[string]*memdb.TableSchema),
	}

	// Collect all the schemas that are needed
	schemas := []func() *memdb.TableSchema{
		indexTableSchema,
		nodeTableSchema,
		jobTableSchema,
		evalTableSchema,
		allocTableSchema,
	}

	// Add each of the tables
	for _, schemaFn := range schemas {
		schema := schemaFn()
		if _, ok := db.Tables[schema.Name]; ok {
			panic(fmt.Sprintf("duplicate table name: %s", schema.Name))
		}
		db.Tables[schema.Name] = schema
	}
	return db
}

// indexTableSchema is used for
func indexTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "index",
		Indexes: map[string]*memdb.IndexSchema{
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.StringFieldIndex{
					Field:     "Key",
					Lowercase: true,
				},
			},
		},
	}
}

// nodeTableSchema returns the MemDB schema for the nodes table.
// This table is used to store all the client nodes that are registered.
func nodeTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "nodes",
		Indexes: map[string]*memdb.IndexSchema{
			// Primary index is used for node management
			// and simple direct lookup. ID is required to be
			// unique.
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.StringFieldIndex{
					Field:     "ID",
					Lowercase: true,
				},
			},
		},
	}
}

// jobTableSchema returns the MemDB schema for the jobs table.
// This table is used to store all the jobs that have been submitted.
func jobTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "jobs",
		Indexes: map[string]*memdb.IndexSchema{
			// Primary index is used for job management
			// and simple direct lookup. ID is required to be
			// unique.
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.StringFieldIndex{
					Field:     "ID",
					Lowercase: true,
				},
			},
		},
	}
}

// evalTableSchema returns the MemDB schema for the eval table.
// This table is used to store all the evaluations that are pending
// or recently completed.
func evalTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "evals",
		Indexes: map[string]*memdb.IndexSchema{
			// Primary index is used for direct lookup.
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.UUIDFieldIndex{
					Field: "ID",
				},
			},

			// Job index is used to lookup allocations by job
			"job": &memdb.IndexSchema{
				Name:         "job",
				AllowMissing: false,
				Unique:       false,
				Indexer: &memdb.StringFieldIndex{
					Field:     "JobID",
					Lowercase: true,
				},
			},
		},
	}
}

// allocTableSchema returns the MemDB schema for the allocation table.
// This table is used to store all the task allocations between task groups
// and nodes.
func allocTableSchema() *memdb.TableSchema {
	return &memdb.TableSchema{
		Name: "allocs",
		Indexes: map[string]*memdb.IndexSchema{
			// Primary index is a UUID
			"id": &memdb.IndexSchema{
				Name:         "id",
				AllowMissing: false,
				Unique:       true,
				Indexer: &memdb.UUIDFieldIndex{
					Field: "ID",
				},
			},

			// Node index is used to lookup allocations by node
			"node": &memdb.IndexSchema{
				Name:         "node",
				AllowMissing: true, // Missing is allow for failed allocations
				Unique:       false,
				Indexer: &memdb.StringFieldIndex{
					Field:     "NodeID",
					Lowercase: true,
				},
			},

			// Job index is used to lookup allocations by job
			"job": &memdb.IndexSchema{
				Name:         "job",
				AllowMissing: false,
				Unique:       false,
				Indexer: &memdb.StringFieldIndex{
					Field:     "JobID",
					Lowercase: true,
				},
			},

			// Eval index is used to lookup allocations by eval
			"eval": &memdb.IndexSchema{
				Name:         "eval",
				AllowMissing: false,
				Unique:       false,
				Indexer: &memdb.UUIDFieldIndex{
					Field: "EvalID",
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package openapi3

import (
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/grokify/gotilla/type/stringsutil"

	oas3 "github.com/getkin/kin-openapi/openapi3"
	"github.com/grokify/gocharts/data/table"
	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/grokify/gotilla/text"
)

type SpecMore struct {
	Spec *oas3.Swagger
}

func ReadSpecMore(path string, validate bool) (*SpecMore, error) {
	spec, err := ReadFile(path, validate)
	if err != nil {
		return nil, err
	}
	return &SpecMore{Spec: spec}, nil
}

func (s *SpecMore) SchemaCount() int {
	if s.Spec == nil {
		return -1
	} else if s.Spec.Components.Schemas == nil {
		return 0
	}
	return len(s.Spec.Components.Schemas)
}

func (s *SpecMore) OperationsTable(columns *text.TextSet) (*table.Table, error) {
	return operationsTable(s.Spec, columns)
}

func operationsTable(spec *oas3.Swagger, columns *text.TextSet) (*table.Table, error) {
	if columns == nil {
		columns = &text.TextSet{Texts: DefaultColumns()}
	}
	tbl := table.NewTable()
	tbl.Name = spec.Info.Title
	tbl.Columns = columns.DisplayTexts()

	tgs, err := SpecTagGroups(spec)
	if err != nil {
		return nil, err
	}

	VisitOperations(spec, func(path, method string, op *oas3.Operation) {
		row := []string{}

		for _, text := range columns.Texts {
			switch text.Slug {
			case "method":
				row = append(row, method)
			case "path":
				row = append(row, path)
			case "operationId":
				row = append(row, op.OperationID)
			case "summary":
				row = append(row, op.Summary)
			case "tags":
				row = append(row, strings.Join(op.Tags, ", "))
			case "x-tag-groups":
				row = append(row, strings.Join(
					tgs.GetTagGroupNamesForTagNames(op.Tags...), ", "))
			default:
				row = append(row, GetExtensionPropStringOrEmpty(op.ExtensionProps, text.Slug))
			}
		}

		tbl.Records = append(tbl.Records, row)
	})
	return &tbl, nil
}

func DefaultColumns() []text.Text {
	texts := []text.Text{
		{
			Display: "Method",
			Slug:    "method"},
		{
			Display: "Path",
			Slug:    "path"},
		{
			Display: "OperationID",
			Slug:    "operationId"},
		{
			Display: "Summary",
			Slug:    "summary"},
		{
			Display: "Tags",
			Slug:    "tags"},
	}
	return texts
}

/*
func (s *SpecMore) OperationsTableOld() (*table.TableData, error) {
	tbl := table.NewTableData()
	tbl.Name = "Operations"
	tgs, err := SpecTagGroups(s.Spec)
	if err != nil {
		return nil, err
	}
	addTagGroups := false
	if len(tgs.TagGroups) > 0 {
		addTagGroups = true
		tbl.Columns = []string{"OperationId", "Summary", "Path", "Method", "Tag Groups", "Tags"}
	} else {
		tbl.Columns = []string{"OperationId", "Summary", "Path", "Method", "Tags"}
	}
	ops := s.OperationMetas()
	for _, op := range ops {
		if addTagGroups {
			tagGroupNames := tgs.GetTagGroupNamesForTagNames(op.Tags...)
			tbl.Records = append(tbl.Records, []string{
				op.OperationID,
				op.Summary,
				op.Path,
				op.Method,
				strings.Join(tagGroupNames, ","),
				strings.Join(stringsutil.SliceCondenseSpace(op.Tags, true, true), ",")})
		} else {
			tbl.Records = append(tbl.Records, []string{
				op.OperationID,
				op.Summary,
				op.Path,
				op.Method,
				strings.Join(op.Tags, ",")})
		}
	}
	return &tbl, nil
}
*/

func (s *SpecMore) OperationMetas() []OperationMeta {
	ometas := []OperationMeta{}
	if s.Spec == nil {
		return ometas
	}
	for url, path := range s.Spec.Paths {
		if path.Connect != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodConnect, path.Connect))
		}
		if path.Delete != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodDelete, path.Delete))
		}
		if path.Get != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodGet, path.Get))
		}
		if path.Head != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodHead, path.Head))
		}
		if path.Options != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodOptions, path.Options))
		}
		if path.Patch != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodPatch, path.Patch))
		}
		if path.Post != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodPost, path.Post))
		}
		if path.Put != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodPut, path.Put))
		}
		if path.Trace != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodTrace, path.Trace))
		}
	}

	return ometas
}

func (s *SpecMore) OperationsCount() uint {
	return uint(len(s.OperationMetas()))
}

func (sm *SpecMore) SchemaNames() []string {
	schemaNames := []string{}
	for schemaName := range sm.Spec.Components.Schemas {
		schemaNames = append(schemaNames, schemaName)
	}
	return stringsutil.SliceCondenseSpace(schemaNames, true, true)
}

func (sm *SpecMore) SchemaNameExists(schemaName string, includeNil bool) bool {
	for schemaNameTry, schemaRef := range sm.Spec.Components.Schemas {
		if schemaNameTry == schemaName {
			if includeNil {
				return true
			} else if schemaRef == nil {
				return false
			}
			schemaRef.Ref = strings.TrimSpace(schemaRef.Ref)
			if len(schemaRef.Ref) > 0 {
				return true
			}
			if schemaRef.Value == nil {
				return false
			} else {
				return true
			}
		}
	}
	return false
}

func (s *SpecMore) WriteFileJSON(filename string, perm os.FileMode, prefix, indent string) error {
	jsonData, err := s.Spec.MarshalJSON()
	if err != nil {
		return err
	}
	pretty := false
	if len(prefix) > 0 || len(indent) > 0 {
		pretty = true
	}
	if pretty {
		jsonData = jsonutil.PrettyPrint(jsonData, "", "  ")
	}
	return ioutil.WriteFile(filename, jsonData, perm)
}

func (sm *SpecMore) WriteFileXLSX(filename string) error {
	tbl, err := sm.OperationsTable(nil)
	if err != nil {
		return err
	}
	return table.WriteXLSX(filename, tbl)
}

type TagsMore struct {
	Tags oas3.Tags
}

func (tg *TagsMore) Get(tagName string) *oas3.Tag {
	for _, tag := range tg.Tags {
		if tagName == tag.Name {
			return tag
		}
	}
	return nil
}
<commit_msg>feat: openapi3: add OpTableColumnsRingCentral()<commit_after>package openapi3

import (
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/grokify/gotilla/type/stringsutil"

	oas3 "github.com/getkin/kin-openapi/openapi3"
	"github.com/grokify/gocharts/data/table"
	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/grokify/gotilla/text"
)

type SpecMore struct {
	Spec *oas3.Swagger
}

func ReadSpecMore(path string, validate bool) (*SpecMore, error) {
	spec, err := ReadFile(path, validate)
	if err != nil {
		return nil, err
	}
	return &SpecMore{Spec: spec}, nil
}

func (s *SpecMore) SchemaCount() int {
	if s.Spec == nil {
		return -1
	} else if s.Spec.Components.Schemas == nil {
		return 0
	}
	return len(s.Spec.Components.Schemas)
}

func (s *SpecMore) OperationsTable(columns *text.TextSet) (*table.Table, error) {
	return operationsTable(s.Spec, columns)
}

func operationsTable(spec *oas3.Swagger, columns *text.TextSet) (*table.Table, error) {
	if columns == nil {
		columns = &text.TextSet{Texts: OpTableColumnsDefault()}
	}
	tbl := table.NewTable()
	tbl.Name = spec.Info.Title
	tbl.Columns = columns.DisplayTexts()

	tgs, err := SpecTagGroups(spec)
	if err != nil {
		return nil, err
	}

	VisitOperations(spec, func(path, method string, op *oas3.Operation) {
		row := []string{}

		for _, text := range columns.Texts {
			switch text.Slug {
			case "method":
				row = append(row, method)
			case "path":
				row = append(row, path)
			case "operationId":
				row = append(row, op.OperationID)
			case "summary":
				row = append(row, op.Summary)
			case "tags":
				row = append(row, strings.Join(op.Tags, ", "))
			case "x-tag-groups":
				row = append(row, strings.Join(
					tgs.GetTagGroupNamesForTagNames(op.Tags...), ", "))
			default:
				row = append(row, GetExtensionPropStringOrEmpty(op.ExtensionProps, text.Slug))
			}
		}

		tbl.Records = append(tbl.Records, row)
	})
	return &tbl, nil
}

func OpTableColumnsDefault() []text.Text {
	texts := []text.Text{
		{
			Display: "Method",
			Slug:    "method"},
		{
			Display: "Path",
			Slug:    "path"},
		{
			Display: "OperationID",
			Slug:    "operationId"},
		{
			Display: "Summary",
			Slug:    "summary"},
		{
			Display: "Tags",
			Slug:    "tags"},
	}
	return texts
}

func OpTableColumnsRingCentral() *text.TextSet {
	texts := []text.Text{
		{
			Display: "Method",
			Slug:    "method"},
		{
			Display: "Path",
			Slug:    "path"},
		{
			Display: "OperationID",
			Slug:    "operationId"},
		{
			Display: "Summary",
			Slug:    "summary"},
		{
			Display: "Tags",
			Slug:    "tags"},
		{
			Display: "API Group",
			Slug:    "x-api-group"},
		{
			Display: "Throttling",
			Slug:    "x-throttling-group"},
		{
			Display: "App Permission",
			Slug:    "x-app-permission"},
		{
			Display: "User Permissions",
			Slug:    "x-user-permission"},
	}
	return &text.TextSet{Texts: texts}
}

/*
func (s *SpecMore) OperationsTableOld() (*table.TableData, error) {
	tbl := table.NewTableData()
	tbl.Name = "Operations"
	tgs, err := SpecTagGroups(s.Spec)
	if err != nil {
		return nil, err
	}
	addTagGroups := false
	if len(tgs.TagGroups) > 0 {
		addTagGroups = true
		tbl.Columns = []string{"OperationId", "Summary", "Path", "Method", "Tag Groups", "Tags"}
	} else {
		tbl.Columns = []string{"OperationId", "Summary", "Path", "Method", "Tags"}
	}
	ops := s.OperationMetas()
	for _, op := range ops {
		if addTagGroups {
			tagGroupNames := tgs.GetTagGroupNamesForTagNames(op.Tags...)
			tbl.Records = append(tbl.Records, []string{
				op.OperationID,
				op.Summary,
				op.Path,
				op.Method,
				strings.Join(tagGroupNames, ","),
				strings.Join(stringsutil.SliceCondenseSpace(op.Tags, true, true), ",")})
		} else {
			tbl.Records = append(tbl.Records, []string{
				op.OperationID,
				op.Summary,
				op.Path,
				op.Method,
				strings.Join(op.Tags, ",")})
		}
	}
	return &tbl, nil
}
*/

func (s *SpecMore) OperationMetas() []OperationMeta {
	ometas := []OperationMeta{}
	if s.Spec == nil {
		return ometas
	}
	for url, path := range s.Spec.Paths {
		if path.Connect != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodConnect, path.Connect))
		}
		if path.Delete != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodDelete, path.Delete))
		}
		if path.Get != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodGet, path.Get))
		}
		if path.Head != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodHead, path.Head))
		}
		if path.Options != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodOptions, path.Options))
		}
		if path.Patch != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodPatch, path.Patch))
		}
		if path.Post != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodPost, path.Post))
		}
		if path.Put != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodPut, path.Put))
		}
		if path.Trace != nil {
			ometas = append(ometas, OperationToMeta(url, http.MethodTrace, path.Trace))
		}
	}

	return ometas
}

func (s *SpecMore) OperationsCount() uint {
	return uint(len(s.OperationMetas()))
}

func (sm *SpecMore) SchemaNames() []string {
	schemaNames := []string{}
	for schemaName := range sm.Spec.Components.Schemas {
		schemaNames = append(schemaNames, schemaName)
	}
	return stringsutil.SliceCondenseSpace(schemaNames, true, true)
}

func (sm *SpecMore) SchemaNameExists(schemaName string, includeNil bool) bool {
	for schemaNameTry, schemaRef := range sm.Spec.Components.Schemas {
		if schemaNameTry == schemaName {
			if includeNil {
				return true
			} else if schemaRef == nil {
				return false
			}
			schemaRef.Ref = strings.TrimSpace(schemaRef.Ref)
			if len(schemaRef.Ref) > 0 {
				return true
			}
			if schemaRef.Value == nil {
				return false
			} else {
				return true
			}
		}
	}
	return false
}

func (s *SpecMore) WriteFileJSON(filename string, perm os.FileMode, prefix, indent string) error {
	jsonData, err := s.Spec.MarshalJSON()
	if err != nil {
		return err
	}
	pretty := false
	if len(prefix) > 0 || len(indent) > 0 {
		pretty = true
	}
	if pretty {
		jsonData = jsonutil.PrettyPrint(jsonData, "", "  ")
	}
	return ioutil.WriteFile(filename, jsonData, perm)
}

func (sm *SpecMore) WriteFileXLSX(filename string) error {
	tbl, err := sm.OperationsTable(nil)
	if err != nil {
		return err
	}
	return table.WriteXLSX(filename, tbl)
}

type TagsMore struct {
	Tags oas3.Tags
}

func (tg *TagsMore) Get(tagName string) *oas3.Tag {
	for _, tag := range tg.Tags {
		if tagName == tag.Name {
			return tag
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package osmapi

import (
	"log"
	"os"
	"testing"
)

func Test_MyRequestSt(t *testing.T) {
	req := MyRequest()
	body, err := req.Get("")

	if err != nil || body == "" {
		log.Println("Test_MyRequestSt")
		t.Fatal(err)
	}
}

func init_req(t *testing.T, title string) *MyRequestSt {
	login := os.Getenv("OSM_USER")
	pass := os.Getenv("OSM_PASSWD")
	url := os.Getenv("OSM_URL")

	if login == "" || pass == "" {
		log.Println("Skip " + title + ". login and password are not found.")
		return nil
	}

	req := MyRequest()
	req.UserPass(login, pass)
	if url == "" {
		url = MainURLTest
	}
	req.SetUrl(url)

	req.SetDebug()

	return req
}

func _ChangeSetClose(t *testing.T, c *ChangeSetSt) {
	err := c.Close()
	if err != nil {
		log.Println("_ChangeSetClose. Close.")
		t.Fatal(err)
	}
}
<commit_msg>Log OSM_PASSWD & OSM_USER & OSM_URL when tests are wrong<commit_after>package osmapi

import (
	"log"
	"os"
	"testing"
)

func Test_MyRequestSt(t *testing.T) {
	req := MyRequest()
	body, err := req.Get("")

	if err != nil || body == "" {
		log.Println("Test_MyRequestSt")
		t.Fatal(err)
	}
}

func init_req(t *testing.T, title string) *MyRequestSt {
	login := os.Getenv("OSM_USER")
	pass := os.Getenv("OSM_PASSWD")
	url := os.Getenv("OSM_URL")

	if login == "" || pass == "" {
		log.Println("Skip " + title + ". login and password are not found.")
		return nil
	}

	log.Println("OSM_USER: " + login + ". OSM_PASSWD: " + pass + ". OSM_URL: " + url)

	req := MyRequest()
	req.UserPass(login, pass)
	if url == "" {
		url = MainURLTest
	}
	req.SetUrl(url)

	req.SetDebug()

	return req
}

func _ChangeSetClose(t *testing.T, c *ChangeSetSt) {
	err := c.Close()
	if err != nil {
		log.Println("_ChangeSetClose. Close.")
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"sync"
	"time"

	"go.chromium.org/luci/appengine/gaemiddleware"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/config/server/cfgmodule"

	"go.chromium.org/luci/server"
	"go.chromium.org/luci/server/gaeemulation"
	"go.chromium.org/luci/server/module"
	"go.chromium.org/luci/server/redisconn"
	"go.chromium.org/luci/server/router"
	"go.chromium.org/luci/server/tq"

	migrationpb "go.chromium.org/luci/cv/api/migration"
	"go.chromium.org/luci/cv/internal/config/configcron"
	"go.chromium.org/luci/cv/internal/gerrit"
	"go.chromium.org/luci/cv/internal/migration"
	"go.chromium.org/luci/cv/internal/servicecfg"

	// import all modules with server/tq handler additions in init() calls,
	// which are otherwise not imported directly or transitively via imports
	// above.
	_ "go.chromium.org/luci/cv/internal/prjmanager/impl"
	_ "go.chromium.org/luci/cv/internal/run/impl"
)

func main() {
	modules := []module.Module{
		cfgmodule.NewModuleFromFlags(),
		gaeemulation.NewModuleFromFlags(),
		redisconn.NewModuleFromFlags(),
		tq.NewModuleFromFlags(),
	}

	server.Main(nil, modules, func(srv *server.Server) error {
		// Register pRPC servers.
		migrationpb.RegisterMigrationServer(srv.PRPC, &migration.MigrationServer{})
		srv.Context = gerrit.UseProd(srv.Context)

		isDev := srv.Options.CloudProject == "luci-change-verifier-dev"

		srv.Routes.GET("/internal/cron/refresh-config",
			router.NewMiddlewareChain(gaemiddleware.RequireCron),
			func(rc *router.Context) {
				// The cron job interval is 1 minute.
				ctx, cancel := context.WithTimeout(rc.Context, 1*time.Minute)
				defer cancel()
				wg := sync.WaitGroup{}
				errs := make(errors.MultiError, 2)

				wg.Add(1)
				go func() {
					defer wg.Done()
					errs[0] = servicecfg.ImportConfig(ctx)
				}()

				wg.Add(1)
				go func() {
					defer wg.Done()
					errs[1] = configcron.SubmitRefreshTasks(ctx, isDev)
				}()

				wg.Wait()
				rc.Writer.Header().Set("Content-Type", "text/plain; charset=utf-8")
				code := 200
				for _, err := range errs {
					if err != nil {
						errors.Log(ctx, err)
						code = 500
					}
				}
				rc.Writer.WriteHeader(code)
			})

		return nil
	})
}
<commit_msg>cv: temporary flush Redis on start.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"sync"
	"time"

	"go.chromium.org/luci/appengine/gaemiddleware"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/config/server/cfgmodule"

	"go.chromium.org/luci/server"
	"go.chromium.org/luci/server/gaeemulation"
	"go.chromium.org/luci/server/module"
	"go.chromium.org/luci/server/redisconn"
	"go.chromium.org/luci/server/router"
	"go.chromium.org/luci/server/tq"

	migrationpb "go.chromium.org/luci/cv/api/migration"
	"go.chromium.org/luci/cv/internal/config/configcron"
	"go.chromium.org/luci/cv/internal/gerrit"
	"go.chromium.org/luci/cv/internal/migration"
	"go.chromium.org/luci/cv/internal/servicecfg"

	// import all modules with server/tq handler additions in init() calls,
	// which are otherwise not imported directly or transitively via imports
	// above.
	_ "go.chromium.org/luci/cv/internal/prjmanager/impl"
	_ "go.chromium.org/luci/cv/internal/run/impl"
)

func main() {
	modules := []module.Module{
		cfgmodule.NewModuleFromFlags(),
		gaeemulation.NewModuleFromFlags(),
		redisconn.NewModuleFromFlags(),
		tq.NewModuleFromFlags(),
	}

	server.Main(nil, modules, func(srv *server.Server) error {
		// Register pRPC servers.
		migrationpb.RegisterMigrationServer(srv.PRPC, &migration.MigrationServer{})
		srv.Context = gerrit.UseProd(srv.Context)

		isDev := srv.Options.CloudProject == "luci-change-verifier-dev"

		srv.Routes.GET("/internal/cron/refresh-config",
			router.NewMiddlewareChain(gaemiddleware.RequireCron),
			func(rc *router.Context) {
				if r, err := redisconn.Get(rc.Context); err == nil {
					res, err := r.Do("FLUSHDB")
					logging.Debugf(rc.Context, "FLUSH: %s | %s", res, err)
					r.Close()
				}
				// The cron job interval is 1 minute.
				ctx, cancel := context.WithTimeout(rc.Context, 1*time.Minute)
				defer cancel()
				wg := sync.WaitGroup{}
				errs := make(errors.MultiError, 2)

				wg.Add(1)
				go func() {
					defer wg.Done()
					errs[0] = servicecfg.ImportConfig(ctx)
				}()

				wg.Add(1)
				go func() {
					defer wg.Done()
					errs[1] = configcron.SubmitRefreshTasks(ctx, isDev)
				}()

				wg.Wait()
				rc.Writer.Header().Set("Content-Type", "text/plain; charset=utf-8")
				code := 200
				for _, err := range errs {
					if err != nil {
						errors.Log(ctx, err)
						code = 500
					}
				}
				rc.Writer.WriteHeader(code)
			})

		return nil
	})
}
<|endoftext|>


<commit_before>package testing // import "go.pachyderm.com/pachyderm/src/pps/persist/server/testing"

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"testing"

	"go.pachyderm.com/pachyderm/src/pkg/require"
	"go.pachyderm.com/pachyderm/src/pps/persist/server"

	"github.com/satori/go.uuid"
)

func RunTestWithRethinkAPIServer(t *testing.T, testFunc func(t *testing.T, persistAPIServer server.APIServer)) {
	apiServer, err := NewTestRethinkAPIServer()
	require.NoError(t, err)
	defer func() {
		require.NoError(t, apiServer.Close())
	}()
	testFunc(t, apiServer)
}

func NewTestRethinkAPIServer() (server.APIServer, error) {
	address, err := getTestRethinkAddress()
	if err != nil {
		return nil, err
	}
	databaseName := strings.Replace(uuid.NewV4().String(), "-", "", -1)
	if err := server.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	return server.NewRethinkAPIServer(address, databaseName)
}

func getTestRethinkAddress() (string, error) {
	rethinkAddr := os.Getenv("RETHINK_PORT_28015_TCP_ADDR")
	if rethinkAddr == "" {
		return "", errors.New("RETHINK_PORT_28015_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:28015", rethinkAddr), nil
}
<commit_msg>fix pps persist server tests<commit_after>package testing // import "go.pachyderm.com/pachyderm/src/pps/persist/server/testing"

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"testing"

	"go.pachyderm.com/pachyderm/src/pkg/require"
	"go.pachyderm.com/pachyderm/src/pps/persist"
	"go.pachyderm.com/pachyderm/src/pps/persist/server"

	"github.com/satori/go.uuid"
)

func RunTestWithRethinkAPIServer(t *testing.T, testFunc func(t *testing.T, persistAPIServer persist.APIServer)) {
	apiServer, err := NewTestRethinkAPIServer()
	require.NoError(t, err)
	defer func() {
		require.NoError(t, apiServer.Close())
	}()
	testFunc(t, apiServer)
}

func NewTestRethinkAPIServer() (server.APIServer, error) {
	address, err := getTestRethinkAddress()
	if err != nil {
		return nil, err
	}
	databaseName := strings.Replace(uuid.NewV4().String(), "-", "", -1)
	if err := server.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	return server.NewRethinkAPIServer(address, databaseName)
}

func getTestRethinkAddress() (string, error) {
	rethinkAddr := os.Getenv("RETHINK_PORT_28015_TCP_ADDR")
	if rethinkAddr == "" {
		return "", errors.New("RETHINK_PORT_28015_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:28015", rethinkAddr), nil
}
<|endoftext|>


<commit_before>package hostdb

import (
	"strconv"
	"testing"

	"github.com/NebulousLabs/Sia/consensus"
)

func TestWeightedList(t *testing.T) {
	// Create a hostdb and 3 equal entries to insert.
	hdb := New()

	// Create a bunch of host entries of equal weight.
	firstInsertions := 64
	for i := 0; i < firstInsertions; i++ {
		entry := HostEntry{
			ID:     strconv.Itoa(i),
			Burn:   10,
			Freeze: 10,
			Price:  10,
		}
		hdb.Insert(entry)
	}

	// Check that the length of activeHosts and the count of hostTree are
	// consistent.
	if len(hdb.activeHosts) != firstInsertions {
		t.Error("activeHosts should equal ", firstInsertions, "equals", len(hdb.activeHosts))
	}
	if hdb.hostTree.count != firstInsertions {
		t.Error("hostTree count is off")
	}

	// Check that the weight of the hostTree is what is expected.
	randomHost, err := hdb.RandomHost()
	if err != nil {
		t.Fatal(err)
	}
	expectedWeight := consensus.Currency(firstInsertions) * randomHost.Weight()
	if hdb.hostTree.weight != expectedWeight {
		t.Error("Expected weight is incorrect")
	}
}
<commit_msg>add RandomHost test after insertions<commit_after>package hostdb

import (
	"strconv"
	"testing"

	"github.com/NebulousLabs/Sia/consensus"
)

func TestWeightedList(t *testing.T) {
	// Create a hostdb and 3 equal entries to insert.
	hdb := New()

	// Create a bunch of host entries of equal weight.
	firstInsertions := 64
	for i := 0; i < firstInsertions; i++ {
		entry := HostEntry{
			ID:     strconv.Itoa(i),
			Burn:   10,
			Freeze: 10,
			Price:  10,
		}
		hdb.Insert(entry)
	}

	// Check that the length of activeHosts and the count of hostTree are
	// consistent.
	if len(hdb.activeHosts) != firstInsertions {
		t.Error("activeHosts should equal ", firstInsertions, "equals", len(hdb.activeHosts))
	}
	if hdb.hostTree.count != firstInsertions {
		t.Error("hostTree count is off")
	}

	// Check that the weight of the hostTree is what is expected.
	randomHost, err := hdb.RandomHost()
	if err != nil {
		t.Fatal(err)
	}
	expectedWeight := consensus.Currency(firstInsertions) * randomHost.Weight()
	if hdb.hostTree.weight != expectedWeight {
		t.Error("Expected weight is incorrect")
	}

	// Select many random hosts and do naive statistical analysis on the
	// results.
	if !testing.Short() {
		// Pull a bunch of random hosts and count how many times we pull each
		// host.
		selectionSlice := make([]int, firstInsertions)
		expected := 50
		for i := 0; i < expected*firstInsertions; i++ {
			entry, err := hdb.RandomHost()
			if err != nil {
				t.Fatal(err)
			}
			idInt, err := strconv.Atoi(entry.ID)
			if err != nil {
				t.Fatal(err)
			}
			selectionSlice[idInt]++
		}

		// See if each host was selected enough times.
		errorBound := 21
		for i, count := range selectionSlice {
			if count < expected-errorBound || count > expected+errorBound {
				t.Error(i, count)
			}
		}
	}

	// Remove a few hosts and check that the tree is still in order.
}
<|endoftext|>


<commit_before>package profile_test

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/cespare/xxhash"
	"github.com/tormoder/fit/cmd/fitgen/internal/profile"
)

const (
	testdata      = "testdata"
	fileExt       = ".xlsx"
	goldenSuffix  = ".golden"
	currentSuffix = ".current"
)

var update = flag.Bool("update", false, "update .golden output files")

var currentSDK = sdks[0]

var defGenOpts = []profile.GeneratorOption{
	profile.WithGenerationTimestamp(false),
}

func relPath(sdkVersion string) string {
	return filepath.Join(testdata, sdkVersion+fileExt)
}

func writeProfile(p *profile.Profile, w io.Writer) error {
	var err error
	write := func(buf []byte) {
		if err != nil {
			return
		}
		_, err = w.Write(buf)
	}
	write([]byte("// TYPES\n"))
	write(p.TypesSource)
	write([]byte("\n// MESSAGES\n"))
	write(p.MessagesSource)
	write([]byte("\n// PROFILE\n"))
	write(p.ProfileSource)
	write([]byte("\n// FITSTRINGER TYPE INPUT\n"))
	for _, t := range p.StringerInput {
		write([]byte(t))
		write([]byte{'\n'})
	}
	write([]byte("\n// MESSAGE NUMS WITHOUT MESSAGE\n"))
	for _, mn := range p.MesgNumsWithoutMessage {
		write([]byte(mn))
		write([]byte{'\n'})
	}
	return err
}

func writeProfileToFile(p *profile.Profile, path string) error {
	buf := new(bytes.Buffer)
	err := writeProfile(p, buf)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path, buf.Bytes(), 0644)
}

func profileFingerprint(p *profile.Profile) uint64 {
	h := xxhash.New()
	_ = writeProfile(p, h)
	return h.Sum64()
}

type sdk struct {
	majVer, minVer    int
	goldenFingerprint uint64
}

var sdks = []sdk{
	{16, 20, 18062740196559836474},
	{20, 14, 3924874714243287142},
	{20, 27, 7008931037264316719},
	{20, 43, 8070657066305942728},
}

func TestMain(m *testing.M) {
	flag.Parse()
	os.Exit(m.Run())
}

func TestGenerator(t *testing.T) {
	for _, sdk := range sdks {
		sdk := sdk // Capture range variable.
		sdkFullVer := fmt.Sprintf("%d.%d", sdk.majVer, sdk.minVer)
		t.Run(sdkFullVer, func(t *testing.T) {
			t.Parallel()
			if sdk == currentSDK && testing.Short() {
				t.Skip("skipping test in short mode")
			}
			path := relPath(sdkFullVer)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				t.Fatal(err)
			}
			g, err := profile.NewGenerator(sdk.majVer, sdk.minVer, data, defGenOpts...)
			if err != nil {
				t.Fatal(err)
			}
			p, err := g.GenerateProfile()
			if err != nil {
				t.Fatal(err)
			}
			gotFP := profileFingerprint(p)
			if gotFP == sdk.goldenFingerprint {
				return
			}
			t.Errorf("profile fingerprint differs: got: %d, want %d", gotFP, sdk.goldenFingerprint)
			if !*update {
				path = path + currentSuffix
			} else {
				path = path + goldenSuffix
			}
			err = writeProfileToFile(p, path)
			if err != nil {
				t.Fatalf("error writing output: %v", err)
			}
			if !*update {
				t.Logf("current output written to: %s", path)
			} else {
				t.Logf("%q has been updated", path)
				t.Logf("new fingerprint is: %d", gotFP)
			}
		})
	}
}

var profileSink *profile.Profile

func BenchmarkGenerator(b *testing.B) {
	for _, sdk := range sdks {
		sdkFullVer := fmt.Sprintf("%d.%d", sdk.majVer, sdk.minVer)
		b.Run(sdkFullVer, func(b *testing.B) {
			path := relPath(sdkFullVer)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				b.Fatalf("error reading profile workbook: %v", err)
			}
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				g, err := profile.NewGenerator(sdk.majVer, sdk.minVer, data, defGenOpts...)
				if err != nil {
					b.Fatal(err)
				}
				profileSink, err = g.GenerateProfile()
				if err != nil {
					b.Fatal(err)
				}
			}
		})
	}
}
<commit_msg>generator_test: Calculate golden fingerprints at runtime<commit_after>package profile_test

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/cespare/xxhash"
	"github.com/tormoder/fit/cmd/fitgen/internal/profile"
)

const (
	testdata      = "testdata"
	fileExt       = ".xlsx"
	goldenSuffix  = ".golden"
	currentSuffix = ".current"
)

var update = flag.Bool("update", false, "update .golden output files")

var currentSDK = sdks[0]

var defGenOpts = []profile.GeneratorOption{
	profile.WithGenerationTimestamp(false),
}

func relPath(sdkVersion string) string {
	return filepath.Join(testdata, sdkVersion+fileExt)
}

func writeProfile(p *profile.Profile, w io.Writer) error {
	var err error
	write := func(buf []byte) {
		if err != nil {
			return
		}
		_, err = w.Write(buf)
	}
	write([]byte("// TYPES\n"))
	write(p.TypesSource)
	write([]byte("\n// MESSAGES\n"))
	write(p.MessagesSource)
	write([]byte("\n// PROFILE\n"))
	write(p.ProfileSource)
	write([]byte("\n// FITSTRINGER TYPE INPUT\n"))
	for _, t := range p.StringerInput {
		write([]byte(t))
		write([]byte{'\n'})
	}
	write([]byte("\n// MESSAGE NUMS WITHOUT MESSAGE\n"))
	for _, mn := range p.MesgNumsWithoutMessage {
		write([]byte(mn))
		write([]byte{'\n'})
	}
	return err
}

func writeProfileToFile(p *profile.Profile, path string) error {
	buf := new(bytes.Buffer)
	err := writeProfile(p, buf)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path, buf.Bytes(), 0644)
}

func scanLinesPreserveEOL(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, data[0 : i+1], nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), data, nil
	}
	// Request more data.
	return 0, nil, nil
}

func readGoldenProfile(path string) (*profile.Profile, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer func() { _ = f.Close() }() // Sigh. To keep errcheck happy

	p := &profile.Profile{}
	headings := []string{
		"// TYPES",
		"// MESSAGES",
		"// PROFILE",
		"// FITSTRINGER TYPE INPUT",
		"// MESSAGE NUMS WITHOUT MESSAGE",
	}
	i := 0

	scanner := bufio.NewScanner(f)
	scanner.Split(scanLinesPreserveEOL)

	scanner.Scan()
	err = scanner.Err()
	if err != nil {
		return nil, err
	}

	if !strings.HasPrefix(scanner.Text(), headings[i]) {
		return nil, fmt.Errorf("first line should be '%s'", headings[i])
	}

	for scanner.Scan() {
		if strings.HasPrefix(scanner.Text(), headings[i+1]) {
			i++
			break
		}
		p.TypesSource = append(p.TypesSource, scanner.Bytes()...)
	}

	// Format
	p.TypesSource, err = format.Source(p.TypesSource)
	if err != nil {
		return nil, fmt.Errorf("format Types: %v", err)
	}

	for scanner.Scan() {
		if strings.HasPrefix(scanner.Text(), headings[i+1]) {
			i++
			break
		}
		p.MessagesSource = append(p.MessagesSource, scanner.Bytes()...)
	}

	// Format
	p.MessagesSource, err = format.Source(p.MessagesSource)
	if err != nil {
		return nil, fmt.Errorf("format Messages: %v", err)
	}

	for scanner.Scan() {
		if strings.HasPrefix(scanner.Text(), headings[i+1]) {
			i++
			break
		}
		p.ProfileSource = append(p.ProfileSource, scanner.Bytes()...)
	}

	// Format
	p.ProfileSource, err = format.Source(p.ProfileSource)
	if err != nil {
		return nil, fmt.Errorf("format Profile: %v", err)
	}

	for scanner.Scan() {
		if strings.HasPrefix(scanner.Text(), headings[i+1]) {
			break
		}
		s := strings.TrimSpace(scanner.Text())
		if len(s) > 0 {
			p.StringerInput = append(p.StringerInput, s)
		}
	}

	for scanner.Scan() {
		s := strings.TrimSpace(scanner.Text())
		if len(s) > 0 {
			p.MesgNumsWithoutMessage = append(p.MesgNumsWithoutMessage, s)
		}
	}

	return p, scanner.Err()
}

func profileFingerprint(p *profile.Profile) uint64 {
	h := xxhash.New()
	_ = writeProfile(p, h)
	return h.Sum64()
}

type sdk struct {
	majVer, minVer int
}

var sdks = []sdk{
	{16, 20},
	{20, 14},
	{20, 27},
	{20, 43},
}

func TestMain(m *testing.M) {
	flag.Parse()
	os.Exit(m.Run())
}

func TestGenerator(t *testing.T) {
	for _, sdk := range sdks {
		sdk := sdk // Capture range variable.
		sdkFullVer := fmt.Sprintf("%d.%d", sdk.majVer, sdk.minVer)
		t.Run(sdkFullVer, func(t *testing.T) {
			t.Parallel()
			if sdk == currentSDK && testing.Short() {
				t.Skip("skipping test in short mode")
			}
			path := relPath(sdkFullVer)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				t.Fatal(err)
			}
			g, err := profile.NewGenerator(sdk.majVer, sdk.minVer, data, defGenOpts...)
			if err != nil {
				t.Fatal(err)
			}
			p, err := g.GenerateProfile()
			if err != nil {
				t.Fatal(err)
			}
			gotFP := profileFingerprint(p)

			// Read in the golden profile, format it and fingerprint it
			// This makes the test robust against gofmt changes
			goldenProfile, err := readGoldenProfile(path + ".golden")
			if err != nil {
				t.Fatal(err)
			}
			goldenFingerprint := profileFingerprint(goldenProfile)

			if gotFP == goldenFingerprint {
				return
			}
			t.Errorf("profile fingerprint differs: got: %d, want %d", gotFP, goldenFingerprint)
			if !*update {
				path = path + currentSuffix
			} else {
				path = path + goldenSuffix
			}
			err = writeProfileToFile(p, path)
			if err != nil {
				t.Fatalf("error writing output: %v", err)
			}
			if !*update {
				t.Logf("current output written to: %s", path)
			} else {
				t.Logf("%q has been updated", path)
				t.Logf("new fingerprint is: %d", gotFP)
			}
		})
	}
}

var profileSink *profile.Profile

func BenchmarkGenerator(b *testing.B) {
	for _, sdk := range sdks {
		sdkFullVer := fmt.Sprintf("%d.%d", sdk.majVer, sdk.minVer)
		b.Run(sdkFullVer, func(b *testing.B) {
			path := relPath(sdkFullVer)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				b.Fatalf("error reading profile workbook: %v", err)
			}
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				g, err := profile.NewGenerator(sdk.majVer, sdk.minVer, data, defGenOpts...)
				if err != nil {
					b.Fatal(err)
				}
				profileSink, err = g.GenerateProfile()
				if err != nil {
					b.Fatal(err)
				}
			}
		})
	}
}
<|endoftext|>