<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kpa

import (
	"context"
	"fmt"

	"go.opencensus.io/stats"
	"go.uber.org/zap"

	nv1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1"
	"knative.dev/pkg/logging"
	pkgmetrics "knative.dev/pkg/metrics"
	"knative.dev/pkg/ptr"
	pkgreconciler "knative.dev/pkg/reconciler"
	pav1alpha1 "knative.dev/serving/pkg/apis/autoscaling/v1alpha1"
	"knative.dev/serving/pkg/apis/serving"
	"knative.dev/serving/pkg/autoscaler/scaling"
	pareconciler "knative.dev/serving/pkg/client/injection/reconciler/autoscaling/v1alpha1/podautoscaler"
	"knative.dev/serving/pkg/metrics"
	areconciler "knative.dev/serving/pkg/reconciler/autoscaling"
	"knative.dev/serving/pkg/reconciler/autoscaling/config"
	"knative.dev/serving/pkg/reconciler/autoscaling/kpa/resources"
	anames "knative.dev/serving/pkg/reconciler/autoscaling/resources/names"
	resourceutil "knative.dev/serving/pkg/resources"

	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	corev1listers "k8s.io/client-go/listers/core/v1"
)

const (
	noPrivateServiceName = "No Private Service Name"
	noTrafficReason      = "NoTraffic"
)

// podCounts keeps record of various numbers of pods
// for each revision.
type podCounts struct {
	want        int
	ready       int
	notReady    int
	pending     int
	terminating int
}

// Reconciler tracks PAs and right sizes the ScaleTargetRef based on the
// information from Deciders.
type Reconciler struct {
	*areconciler.Base

	podsLister corev1listers.PodLister
	deciders   resources.Deciders
	scaler     *scaler
}

// Check that our Reconciler implements pareconciler.Interface
var _ pareconciler.Interface = (*Reconciler)(nil)

func (c *Reconciler) ReconcileKind(ctx context.Context, pa *pav1alpha1.PodAutoscaler) pkgreconciler.Event {
	logger := logging.FromContext(ctx)

	// We need the SKS object in order to optimize scale to zero
	// performance. It is OK if SKS is nil at this point.
	sksName := anames.SKS(pa.Name)
	sks, err := c.SKSLister.ServerlessServices(pa.Namespace).Get(sksName)
	if err != nil && !errors.IsNotFound(err) {
		logger.Warnw("Error retrieving SKS for Scaler", zap.Error(err))
	}

	// Having an SKS and its PrivateServiceName is a prerequisite for all upcoming steps.
	if sks == nil || sks.Status.PrivateServiceName == "" {
		// Before we can reconcile decider and get real number of activators
		// we start with default of 2.
		if _, err = c.ReconcileSKS(ctx, pa, nv1alpha1.SKSOperationModeServe, 0 /*numActivators == all*/); err != nil {
			return fmt.Errorf("error reconciling SKS: %w", err)
		}
		pa.Status.MarkSKSNotReady(noPrivateServiceName) // In both cases this is true.
		return computeStatus(ctx, pa, podCounts{want: scaleUnknown}, logger)
	}

	pa.Status.MetricsServiceName = sks.Status.PrivateServiceName
	decider, err := c.reconcileDecider(ctx, pa)
	if err != nil {
		return fmt.Errorf("error reconciling Decider: %w", err)
	}

	if err := c.ReconcileMetric(ctx, pa, resolveScrapeTarget(ctx, pa)); err != nil {
		return fmt.Errorf("error reconciling Metric: %w", err)
	}

	// Get the appropriate current scale from the metric, and right size
	// the scaleTargetRef based on it.
	want, err := c.scaler.scale(ctx, pa, sks, decider.Status.DesiredScale)
	if err != nil {
		return fmt.Errorf("error scaling target: %w", err)
	}

	mode := nv1alpha1.SKSOperationModeServe
	// We put activator in the serving path in the following cases:
	// 1. The revision is scaled to 0:
	//   a. want == 0
	//   b. want == -1 && PA is inactive (Autoscaler has no previous knowledge of
	//			this revision, e.g. after a restart) but PA status is inactive (it was
	//			already scaled to 0).
	// 2. The excess burst capacity is negative.
	if want == 0 || decider.Status.ExcessBurstCapacity < 0 || want == scaleUnknown && pa.Status.IsInactive() {
		logger.Infof("SKS should be in proxy mode: want = %d, ebc = %d, #act's = %d PA Inactive? = %v",
			want, decider.Status.ExcessBurstCapacity, decider.Status.NumActivators,
			pa.Status.IsInactive())
		mode = nv1alpha1.SKSOperationModeProxy
	}

	// If we have not successfully reconciled Decider yet, NumActivators will be 0 and
	// we'll use all activators to back this revision.
	sks, err = c.ReconcileSKS(ctx, pa, mode, decider.Status.NumActivators)
	if err != nil {
		return fmt.Errorf("error reconciling SKS: %w", err)
	}
	// Propagate service name.
	pa.Status.ServiceName = sks.Status.ServiceName

	// Compare the desired and observed resources to determine our situation.
	podCounter := resourceutil.NewPodAccessor(c.podsLister, pa.Namespace, pa.Labels[serving.RevisionLabelKey])
	ready, notReady, pending, terminating, err := podCounter.PodCountsByState()
	if err != nil {
		return fmt.Errorf("error getting pod counts %s: %w", sks.Status.PrivateServiceName, err)
	}
	// If SKS is not ready — ensure we're not becoming ready.
	// TODO: see if we can perhaps propagate the SKS state to computing active status.
	if sks.IsReady() {
		logger.Debug("SKS is ready, marking SKS status ready")
		pa.Status.MarkSKSReady()
	} else {
		logger.Debug("SKS is not ready, marking SKS status not ready")
		pa.Status.MarkSKSNotReady(sks.Status.GetCondition(nv1alpha1.ServerlessServiceConditionReady).Message)
	}

	logger.Infof("PA scale got=%d, want=%d, desiredPods=%d ebc=%d", ready, want,
		decider.Status.DesiredScale, decider.Status.ExcessBurstCapacity)

	pc := podCounts{
		want:        int(want),
		ready:       ready,
		notReady:    notReady,
		pending:     pending,
		terminating: terminating,
	}
	logger.Infof("Observed pod counts=%#v", pc)
	return computeStatus(ctx, pa, pc, logger)
}

func (c *Reconciler) reconcileDecider(ctx context.Context, pa *pav1alpha1.PodAutoscaler) (*scaling.Decider, error) {
	desiredDecider := resources.MakeDecider(ctx, pa, config.FromContext(ctx).Autoscaler)
	decider, err := c.deciders.Get(ctx, desiredDecider.Namespace, desiredDecider.Name)
	if errors.IsNotFound(err) {
		decider, err = c.deciders.Create(ctx, desiredDecider)
		if err != nil {
			return nil, fmt.Errorf("error creating Decider: %w", err)
		}
	} else if err != nil {
		return nil, fmt.Errorf("error fetching Decider: %w", err)
	}

	// Ignore status when reconciling
	desiredDecider.Status = decider.Status
	if !equality.Semantic.DeepEqual(desiredDecider, decider) {
		decider, err = c.deciders.Update(ctx, desiredDecider)
		if err != nil {
			return nil, fmt.Errorf("error updating decider: %w", err)
		}
	}
	return decider, nil
}

func computeStatus(ctx context.Context, pa *pav1alpha1.PodAutoscaler, pc podCounts, logger *zap.SugaredLogger) error {
	pa.Status.DesiredScale, pa.Status.ActualScale = ptr.Int32(int32(pc.want)), ptr.Int32(int32(pc.ready))

	if err := reportMetrics(pa, pc); err != nil {
		return fmt.Errorf("error reporting metrics: %w", err)
	}

	computeActiveCondition(ctx, pa, pc)
	logger.Debugf("PA Status after reconcile: %#v", pa.Status.Status)

	return nil
}

func reportMetrics(pa *pav1alpha1.PodAutoscaler, pc podCounts) error {
	serviceLabel := pa.Labels[serving.ServiceLabelKey] // This might be empty.
	configLabel := pa.Labels[serving.ConfigurationLabelKey]

	ctx, err := metrics.RevisionContext(pa.Namespace, serviceLabel, configLabel, pa.Name)
	if err != nil {
		return err
	}

	stats := []stats.Measurement{
		actualPodCountM.M(int64(pc.ready)), notReadyPodCountM.M(int64(pc.notReady)),
		pendingPodCountM.M(int64(pc.pending)), terminatingPodCountM.M(int64(pc.terminating)),
	}
	// Negative "want" values represent an empty metrics pipeline and thus no specific request is being made.
	if pc.want >= 0 {
		stats = append(stats, requestedPodCountM.M(int64(pc.want)))
	}
	pkgmetrics.RecordBatch(ctx, stats...)
	return nil
}

// computeActiveCondition updates the status of a PA given the current scale (got), desired scale (want)
// active threshold (min), and the current status, as per the following table:
//
//    | Want | Got    | min   | Status     | New status |
//    | 0    | <any>  | <any> | <any>      | inactive   |
//    | >0   | < min  | <any> | <any>      | activating |
//    | >0   | >= min | <any> | <any>      | active     |
//    | -1   | < min  | <any> | inactive   | inactive   |
//    | -1   | < min  | <any> | activating | activating |
//    | -1   | < min  | <any> | active     | activating |
//    | -1   | >= min | <any> | inactive   | inactive   |
//    | -1   | >= min | 0     | activating | inactive   |
//    | -1   | >= min | 0     | active     | inactive   | <-- this case technically is impossible.
//    | -1   | >= min | >0    | activating | active     |
//    | -1   | >= min | >0    | active     | active     |
func computeActiveCondition(ctx context.Context, pa *pav1alpha1.PodAutoscaler, pc podCounts) {
	minReady := activeThreshold(ctx, pa)
	// In pre-0.17 we could have scaled down normally without ever setting ScaleTargetInitialized.
	// In this case we'll be in the NoTraffic/inactive state.
	// TODO(taragu): remove after 0.19
	alreadyScaledDownSuccessfully := minReady > 0 && pa.Status.GetCondition(pav1alpha1.PodAutoscalerConditionActive).Reason == noTrafficReason
	if pc.ready >= minReady || alreadyScaledDownSuccessfully {
		pa.Status.MarkScaleTargetInitialized()
	}

	switch {
	// Need to check for minReady = 0 because in the initialScale 0 case, pc.want will be -1.
	case pc.want == 0 || minReady == 0:
		if pa.Status.IsActivating() && minReady > 0 {
			// We only ever scale to zero while activating if we fail to activate within the progress deadline.
			pa.Status.MarkInactive("TimedOut", "The target could not be activated.")
		} else {
			pa.Status.MarkInactive(noTrafficReason, "The target is not receiving traffic.")
		}

	case pc.ready < minReady:
		if pc.want > 0 || !pa.Status.IsInactive() {
			pa.Status.MarkActivating(
				"Queued", "Requests to the target are being buffered as resources are provisioned.")
		} else {
			// This is for the initialScale 0 case. In the first iteration, minReady is 0,
			// but for the following iterations, minReady is 1. pc.want will continue being
			// -1 until we start receiving metrics, so we will end up here.
			// Even though PA is already been marked as inactive in the first iteration, we
			// still need to set it again. Otherwise reconciliation will fail with NewObservedGenFailure
			// because we cannot go through one iteration of reconciliation without setting
			// some status.
			pa.Status.MarkInactive(noTrafficReason, "The target is not receiving traffic.")
		}

	case pc.ready >= minReady:
		if pc.want > 0 || !pa.Status.IsInactive() {
			pa.Status.MarkActive()
		}
	}
}

// activeThreshold returns the scale required for the pa to be marked Active
func activeThreshold(ctx context.Context, pa *pav1alpha1.PodAutoscaler) int {
	asConfig := config.FromContext(ctx).Autoscaler
	min, _ := pa.ScaleBounds(asConfig)
	if !pa.Status.IsScaleTargetInitialized() {
		initialScale := resources.GetInitialScale(asConfig, pa)
		return int(intMax(min, initialScale))
	}
	return int(intMax(min, 1))
}

// resolveScrapeTarget returns metric service name to be scraped based on TBC configuration
// TBC == -1 => activator in path, don't scrape the service
func resolveScrapeTarget(ctx context.Context, pa *pav1alpha1.PodAutoscaler) string {
	tbc := resolveTBC(ctx, pa)
	if tbc == -1 {
		return ""
	}

	return pa.Status.MetricsServiceName
}

func resolveTBC(ctx context.Context, pa *pav1alpha1.PodAutoscaler) float64 {
	if v, ok := pa.TargetBC(); ok {
		return v
	}

	return config.FromContext(ctx).Autoscaler.TargetBurstCapacity
}

func intMax(a, b int32) int32 {
	if a < b {
		return b
	}
	return a
}
<commit_msg>Add log message when Serve mode happens (#9159)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kpa

import (
	"context"
	"fmt"

	"go.opencensus.io/stats"
	"go.uber.org/zap"

	nv1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1"
	"knative.dev/pkg/logging"
	pkgmetrics "knative.dev/pkg/metrics"
	"knative.dev/pkg/ptr"
	pkgreconciler "knative.dev/pkg/reconciler"
	pav1alpha1 "knative.dev/serving/pkg/apis/autoscaling/v1alpha1"
	"knative.dev/serving/pkg/apis/serving"
	"knative.dev/serving/pkg/autoscaler/scaling"
	pareconciler "knative.dev/serving/pkg/client/injection/reconciler/autoscaling/v1alpha1/podautoscaler"
	"knative.dev/serving/pkg/metrics"
	areconciler "knative.dev/serving/pkg/reconciler/autoscaling"
	"knative.dev/serving/pkg/reconciler/autoscaling/config"
	"knative.dev/serving/pkg/reconciler/autoscaling/kpa/resources"
	anames "knative.dev/serving/pkg/reconciler/autoscaling/resources/names"
	resourceutil "knative.dev/serving/pkg/resources"

	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	corev1listers "k8s.io/client-go/listers/core/v1"
)

const (
	noPrivateServiceName = "No Private Service Name"
	noTrafficReason      = "NoTraffic"
)

// podCounts keeps record of various numbers of pods
// for each revision.
type podCounts struct {
	want        int
	ready       int
	notReady    int
	pending     int
	terminating int
}

// Reconciler tracks PAs and right sizes the ScaleTargetRef based on the
// information from Deciders.
type Reconciler struct {
	*areconciler.Base

	podsLister corev1listers.PodLister
	deciders   resources.Deciders
	scaler     *scaler
}

// Check that our Reconciler implements pareconciler.Interface
var _ pareconciler.Interface = (*Reconciler)(nil)

func (c *Reconciler) ReconcileKind(ctx context.Context, pa *pav1alpha1.PodAutoscaler) pkgreconciler.Event {
	logger := logging.FromContext(ctx)

	// We need the SKS object in order to optimize scale to zero
	// performance. It is OK if SKS is nil at this point.
	sksName := anames.SKS(pa.Name)
	sks, err := c.SKSLister.ServerlessServices(pa.Namespace).Get(sksName)
	if err != nil && !errors.IsNotFound(err) {
		logger.Warnw("Error retrieving SKS for Scaler", zap.Error(err))
	}

	// Having an SKS and its PrivateServiceName is a prerequisite for all upcoming steps.
	if sks == nil || sks.Status.PrivateServiceName == "" {
		// Before we can reconcile decider and get real number of activators
		// we start with default of 2.
		if _, err = c.ReconcileSKS(ctx, pa, nv1alpha1.SKSOperationModeServe, 0 /*numActivators == all*/); err != nil {
			return fmt.Errorf("error reconciling SKS: %w", err)
		}
		pa.Status.MarkSKSNotReady(noPrivateServiceName) // In both cases this is true.
		return computeStatus(ctx, pa, podCounts{want: scaleUnknown}, logger)
	}

	pa.Status.MetricsServiceName = sks.Status.PrivateServiceName
	decider, err := c.reconcileDecider(ctx, pa)
	if err != nil {
		return fmt.Errorf("error reconciling Decider: %w", err)
	}

	if err := c.ReconcileMetric(ctx, pa, resolveScrapeTarget(ctx, pa)); err != nil {
		return fmt.Errorf("error reconciling Metric: %w", err)
	}

	// Get the appropriate current scale from the metric, and right size
	// the scaleTargetRef based on it.
	want, err := c.scaler.scale(ctx, pa, sks, decider.Status.DesiredScale)
	if err != nil {
		return fmt.Errorf("error scaling target: %w", err)
	}

	mode := nv1alpha1.SKSOperationModeServe
	// We put activator in the serving path in the following cases:
	// 1. The revision is scaled to 0:
	//   a. want == 0
	//   b. want == -1 && PA is inactive (Autoscaler has no previous knowledge of
	//			this revision, e.g. after a restart) but PA status is inactive (it was
	//			already scaled to 0).
	// 2. The excess burst capacity is negative.
	if want == 0 || decider.Status.ExcessBurstCapacity < 0 || want == scaleUnknown && pa.Status.IsInactive() {
		mode = nv1alpha1.SKSOperationModeProxy
	}
	logger.Infof("SKS should be in %s mode: want = %d, ebc = %d, #act's = %d PA Inactive? = %v",
		mode, want, decider.Status.ExcessBurstCapacity, decider.Status.NumActivators,
		pa.Status.IsInactive())

	// If we have not successfully reconciled Decider yet, NumActivators will be 0 and
	// we'll use all activators to back this revision.
	sks, err = c.ReconcileSKS(ctx, pa, mode, decider.Status.NumActivators)
	if err != nil {
		return fmt.Errorf("error reconciling SKS: %w", err)
	}
	// Propagate service name.
	pa.Status.ServiceName = sks.Status.ServiceName

	// Compare the desired and observed resources to determine our situation.
	podCounter := resourceutil.NewPodAccessor(c.podsLister, pa.Namespace, pa.Labels[serving.RevisionLabelKey])
	ready, notReady, pending, terminating, err := podCounter.PodCountsByState()
	if err != nil {
		return fmt.Errorf("error getting pod counts %s: %w", sks.Status.PrivateServiceName, err)
	}
	// If SKS is not ready — ensure we're not becoming ready.
	// TODO: see if we can perhaps propagate the SKS state to computing active status.
	if sks.IsReady() {
		logger.Debug("SKS is ready, marking SKS status ready")
		pa.Status.MarkSKSReady()
	} else {
		logger.Debug("SKS is not ready, marking SKS status not ready")
		pa.Status.MarkSKSNotReady(sks.Status.GetCondition(nv1alpha1.ServerlessServiceConditionReady).Message)
	}

	logger.Infof("PA scale got=%d, want=%d, desiredPods=%d ebc=%d", ready, want,
		decider.Status.DesiredScale, decider.Status.ExcessBurstCapacity)

	pc := podCounts{
		want:        int(want),
		ready:       ready,
		notReady:    notReady,
		pending:     pending,
		terminating: terminating,
	}
	logger.Infof("Observed pod counts=%#v", pc)
	return computeStatus(ctx, pa, pc, logger)
}

func (c *Reconciler) reconcileDecider(ctx context.Context, pa *pav1alpha1.PodAutoscaler) (*scaling.Decider, error) {
	desiredDecider := resources.MakeDecider(ctx, pa, config.FromContext(ctx).Autoscaler)
	decider, err := c.deciders.Get(ctx, desiredDecider.Namespace, desiredDecider.Name)
	if errors.IsNotFound(err) {
		decider, err = c.deciders.Create(ctx, desiredDecider)
		if err != nil {
			return nil, fmt.Errorf("error creating Decider: %w", err)
		}
	} else if err != nil {
		return nil, fmt.Errorf("error fetching Decider: %w", err)
	}

	// Ignore status when reconciling
	desiredDecider.Status = decider.Status
	if !equality.Semantic.DeepEqual(desiredDecider, decider) {
		decider, err = c.deciders.Update(ctx, desiredDecider)
		if err != nil {
			return nil, fmt.Errorf("error updating decider: %w", err)
		}
	}
	return decider, nil
}

func computeStatus(ctx context.Context, pa *pav1alpha1.PodAutoscaler, pc podCounts, logger *zap.SugaredLogger) error {
	pa.Status.DesiredScale, pa.Status.ActualScale = ptr.Int32(int32(pc.want)), ptr.Int32(int32(pc.ready))

	if err := reportMetrics(pa, pc); err != nil {
		return fmt.Errorf("error reporting metrics: %w", err)
	}

	computeActiveCondition(ctx, pa, pc)
	logger.Debugf("PA Status after reconcile: %#v", pa.Status.Status)

	return nil
}

func reportMetrics(pa *pav1alpha1.PodAutoscaler, pc podCounts) error {
	serviceLabel := pa.Labels[serving.ServiceLabelKey] // This might be empty.
	configLabel := pa.Labels[serving.ConfigurationLabelKey]

	ctx, err := metrics.RevisionContext(pa.Namespace, serviceLabel, configLabel, pa.Name)
	if err != nil {
		return err
	}

	stats := []stats.Measurement{
		actualPodCountM.M(int64(pc.ready)), notReadyPodCountM.M(int64(pc.notReady)),
		pendingPodCountM.M(int64(pc.pending)), terminatingPodCountM.M(int64(pc.terminating)),
	}
	// Negative "want" values represent an empty metrics pipeline and thus no specific request is being made.
	if pc.want >= 0 {
		stats = append(stats, requestedPodCountM.M(int64(pc.want)))
	}
	pkgmetrics.RecordBatch(ctx, stats...)
	return nil
}

// computeActiveCondition updates the status of a PA given the current scale (got), desired scale (want)
// active threshold (min), and the current status, as per the following table:
//
//    | Want | Got    | min   | Status     | New status |
//    | 0    | <any>  | <any> | <any>      | inactive   |
//    | >0   | < min  | <any> | <any>      | activating |
//    | >0   | >= min | <any> | <any>      | active     |
//    | -1   | < min  | <any> | inactive   | inactive   |
//    | -1   | < min  | <any> | activating | activating |
//    | -1   | < min  | <any> | active     | activating |
//    | -1   | >= min | <any> | inactive   | inactive   |
//    | -1   | >= min | 0     | activating | inactive   |
//    | -1   | >= min | 0     | active     | inactive   | <-- this case technically is impossible.
//    | -1   | >= min | >0    | activating | active     |
//    | -1   | >= min | >0    | active     | active     |
func computeActiveCondition(ctx context.Context, pa *pav1alpha1.PodAutoscaler, pc podCounts) {
	minReady := activeThreshold(ctx, pa)
	// In pre-0.17 we could have scaled down normally without ever setting ScaleTargetInitialized.
	// In this case we'll be in the NoTraffic/inactive state.
	// TODO(taragu): remove after 0.19
	alreadyScaledDownSuccessfully := minReady > 0 && pa.Status.GetCondition(pav1alpha1.PodAutoscalerConditionActive).Reason == noTrafficReason
	if pc.ready >= minReady || alreadyScaledDownSuccessfully {
		pa.Status.MarkScaleTargetInitialized()
	}

	switch {
	// Need to check for minReady = 0 because in the initialScale 0 case, pc.want will be -1.
	case pc.want == 0 || minReady == 0:
		if pa.Status.IsActivating() && minReady > 0 {
			// We only ever scale to zero while activating if we fail to activate within the progress deadline.
			pa.Status.MarkInactive("TimedOut", "The target could not be activated.")
		} else {
			pa.Status.MarkInactive(noTrafficReason, "The target is not receiving traffic.")
		}

	case pc.ready < minReady:
		if pc.want > 0 || !pa.Status.IsInactive() {
			pa.Status.MarkActivating(
				"Queued", "Requests to the target are being buffered as resources are provisioned.")
		} else {
			// This is for the initialScale 0 case. In the first iteration, minReady is 0,
			// but for the following iterations, minReady is 1. pc.want will continue being
			// -1 until we start receiving metrics, so we will end up here.
			// Even though PA is already been marked as inactive in the first iteration, we
			// still need to set it again. Otherwise reconciliation will fail with NewObservedGenFailure
			// because we cannot go through one iteration of reconciliation without setting
			// some status.
			pa.Status.MarkInactive(noTrafficReason, "The target is not receiving traffic.")
		}

	case pc.ready >= minReady:
		if pc.want > 0 || !pa.Status.IsInactive() {
			pa.Status.MarkActive()
		}
	}
}

// activeThreshold returns the scale required for the pa to be marked Active
func activeThreshold(ctx context.Context, pa *pav1alpha1.PodAutoscaler) int {
	asConfig := config.FromContext(ctx).Autoscaler
	min, _ := pa.ScaleBounds(asConfig)
	if !pa.Status.IsScaleTargetInitialized() {
		initialScale := resources.GetInitialScale(asConfig, pa)
		return int(intMax(min, initialScale))
	}
	return int(intMax(min, 1))
}

// resolveScrapeTarget returns metric service name to be scraped based on TBC configuration
// TBC == -1 => activator in path, don't scrape the service
func resolveScrapeTarget(ctx context.Context, pa *pav1alpha1.PodAutoscaler) string {
	tbc := resolveTBC(ctx, pa)
	if tbc == -1 {
		return ""
	}

	return pa.Status.MetricsServiceName
}

func resolveTBC(ctx context.Context, pa *pav1alpha1.PodAutoscaler) float64 {
	if v, ok := pa.TargetBC(); ok {
		return v
	}

	return config.FromContext(ctx).Autoscaler.TargetBurstCapacity
}

func intMax(a, b int32) int32 {
	if a < b {
		return b
	}
	return a
}
<|endoftext|>


<commit_before>// Package ucloud provides ...
package ucloud

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"net/http"
	_ "reflect"
)

type UcloudApiClient struct {
	baseURL    string
	publicKey  string
	privateKey string
	regionId   string
	zoneId     string
	conn       *http.Client
}

func NewUcloudApiClient(baseURL, publicKey, privateKey, regionId, zoneId string) *UcloudApiClient {

	conn := &http.Client{}
	return &UcloudApiClient{baseURL, publicKey, privateKey, regionId, zoneId, conn}
}

func (u *UcloudApiClient) verify_ac(params interface{}) ([]byte, error) {

	var buf bytes.Buffer
	b, err := json.Marshal(params)
	if err != nil {
		return []byte{}, err
	}
	buf.Write(b)
	buf.WriteString(u.privateKey)

	h := sha1.New()

	return h.Sum(buf.Bytes()), nil
}

func (u *UcloudApiClient) MakeReqParams(origin_req []byte) []byte {

	return []byte{}

}

func (u *UcloudApiClient) Get(url string, params interface{}) {

}

func (u *UcloudApiClient) Post(url string, params interface{}) {
}

func (u *UcloudApiClient) Put(url string, params interface{}) {

}
func (u *UcloudApiClient) Delete(url string, params interface{}) {

}
<commit_msg>API Get complete<commit_after>// Package ucloud provides ...
package ucloud

import (
	"bytes"
	"crypto/sha1"
	_ "encoding/json"
	"net/http"
	URL "net/url"
	"sort"

	"github.com/mengzhuo/gopystr"
)

type UcloudApiClient struct {
	baseURL    string
	publicKey  string
	privateKey string
	regionId   string
	zoneId     string
	conn       *http.Client
}

func NewUcloudApiClient(baseURL, publicKey, privateKey, regionId, zoneId string) *UcloudApiClient {

	conn := &http.Client{}
	return &UcloudApiClient{baseURL, publicKey, privateKey, regionId, zoneId, conn}
}

func (u *UcloudApiClient) verify_ac(params map[string]interface{}) []byte {

	var buf bytes.Buffer

	keys := make([]string, len(params))
	i := 0
	for k, _ := range params {
		keys[i] = k
	}
	sort.Strings(keys)

	for _, k := range keys {
		buf.WriteString(k)
		buf.WriteString(gopystr.Str(params[k]))
	}
	buf.WriteString(u.privateKey)

	h := sha1.New()

	return h.Sum(buf.Bytes())
}

func (u *UcloudApiClient) Get(url string, params map[string]interface{}) (*http.Response, error) {

	data := URL.Values{}
	for k, v := range params {
		data.Set(k, gopystr.Str(v))
	}
	data.Set("Signature", string(u.verify_ac(params)))
	r, _ := http.NewRequest("GET", url, bytes.NewBufferString(data.Encode()))
	return u.conn.Do(r)
}
<|endoftext|>


<commit_before>package rbac

import (
	"context"
	"regexp"

	v1 "k8s.io/client-go/kubernetes/typed/core/v1"

	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"

	mf "github.com/manifestival/manifestival"
	"github.com/tektoncd/operator/pkg/apis/operator/v1alpha1"
	clientset "github.com/tektoncd/operator/pkg/client/clientset/versioned"
	"github.com/tektoncd/operator/pkg/reconciler/common"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	nsreconciler "knative.dev/pkg/client/injection/kube/reconciler/core/v1/namespace"
	"knative.dev/pkg/logging"
	pkgreconciler "knative.dev/pkg/reconciler"
)

// Reconciler implements controller.Reconciler for TektonPipeline resources.
type Reconciler struct {
	// kubeClientSet allows us to talk to the k8s for core APIs
	kubeClientSet kubernetes.Interface
	// operatorClientSet allows us to configure operator objects
	operatorClientSet clientset.Interface
	// manifest is empty, but with a valid client and logger. all
	// manifests are immutable, and any created during reconcile are
	// expected to be appended to this one, obviating the passing of
	// client & logger
	manifest mf.Manifest
	// Platform-specific behavior to affect the transform
	extension common.Extension
}

// Check that our Reconciler implements controller.Reconciler
var _ nsreconciler.Interface = (*Reconciler)(nil)

const (
	pipelineAnyuid           = "pipeline-anyuid"
	pipelineSA               = "pipeline"
	serviceCABundleCofigMap  = "config-service-cabundle"
	trustedCABundleConfigMap = "config-trusted-cabundle"
)

// FinalizeKind removes all resources after deletion of a TektonPipelines.
func (r *Reconciler) FinalizeKind(ctx context.Context, original *v1alpha1.TektonPipeline) pkgreconciler.Event {
	return nil
}

// ReconcileKind compares the actual state with the desired, and attempts to
// converge the two.
func (r *Reconciler) ReconcileKind(ctx context.Context, ns *corev1.Namespace) pkgreconciler.Event {
	logger := logging.FromContext(ctx)
	logger.Infow("Reconciling Namespace: Platform Openshift", "status", ns.GetName())

	ignorePattern := "^(openshift|kube)-"
	if ignore, _ := regexp.MatchString(ignorePattern, ns.GetName()); ignore {
		logger.Infow("Reconciling Namespace: IGNORE", "status", ns.GetName())
		return nil
	}

	logger.Infow("Reconciling inject CA bundle configmap in ", "Namespace", ns.GetName())
	if err := r.ensureCABundles(ctx, ns); err != nil {
		return err
	}

	logger.Infow("Reconciling Default SA in ", "Namespace", ns.GetName())

	sa, err := r.ensureSA(ctx, ns)
	if err != nil {
		return err
	}

	// Maintaining a separate cluster role for the scc declaration.
	// to assist us in managing this the scc association in a
	// granular way.
	if err := r.ensureSCClusterRole(ctx); err != nil {
		return err
	}

	if err := r.ensureSCCRoleBinding(ctx, sa); err != nil {
		return err
	}

	if err := r.ensureRoleBindings(ctx, sa); err != nil {
		return err
	}

	return nil
}

func (r *Reconciler) ensureCABundles(ctx context.Context, ns *corev1.Namespace) error {
	logger := logging.FromContext(ctx)
	cfgInterface := r.kubeClientSet.CoreV1().ConfigMaps(ns.Name)

	// Ensure trusted CA bundle
	logger.Info("finding configmap: %s/%s", ns.Name, trustedCABundleConfigMap)
	_, err := cfgInterface.Get(ctx, trustedCABundleConfigMap, metav1.GetOptions{})
	if err != nil && !errors.IsNotFound(err) {
		return err
	}
	if err != nil && errors.IsNotFound(err) {
		logger.Info("creating configmap", trustedCABundleConfigMap, "ns", ns.Name)
		if err := createTrustedCABundleConfigMap(ctx, cfgInterface, trustedCABundleConfigMap, ns.Name); err != nil {
			return err
		}
	}

	// Ensure service CA bundle
	logger.Info("finding configmap: %s/%s", ns.Name, serviceCABundleCofigMap)
	_, err = cfgInterface.Get(ctx, serviceCABundleCofigMap, metav1.GetOptions{})
	if err != nil && !errors.IsNotFound(err) {
		return err
	}
	if err != nil && errors.IsNotFound(err) {
		logger.Info("creating configmap", serviceCABundleCofigMap, "ns", ns.Name)
		if err := createServiceCABundleConfigMap(ctx, cfgInterface, serviceCABundleCofigMap, ns.Name); err != nil {
			return err
		}
	}

	return nil
}

func (r *Reconciler) ensureSA(ctx context.Context, ns *corev1.Namespace) (*corev1.ServiceAccount, error) {
	logger := logging.FromContext(ctx)
	logger.Info("finding sa: %s/%s", ns.Name, "pipeline")
	saInterface := r.kubeClientSet.CoreV1().ServiceAccounts(ns.Name)
	sa, err := saInterface.Get(ctx, pipelineSA, metav1.GetOptions{})
	if err != nil && !errors.IsNotFound(err) {
		return nil, err
	}
	if err != nil && errors.IsNotFound(err) {
		logger.Info("creating sa", "sa", pipelineSA, "ns", ns.Name)
		return createSA(ctx, saInterface, ns.Name)
	}

	return sa, nil
}

func (r *Reconciler) ensureRoleBindings(ctx context.Context, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	logger.Info("finding role-binding edit")
	rbacClient := r.kubeClientSet.RbacV1()

	editRB, err := rbacClient.RoleBindings(sa.Namespace).Get(ctx, "edit", metav1.GetOptions{})

	if err == nil {
		logger.Infof("found rolebinding %s/%s", editRB.Namespace, editRB.Name)
		return r.updateRoleBinding(ctx, editRB, sa)
	}

	if errors.IsNotFound(err) {
		return r.createRoleBinding(ctx, sa)
	}

	return err
}

func (r *Reconciler) createRoleBinding(ctx context.Context, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	logger.Info("create new rolebinding edit, in Namespace", sa.GetNamespace())
	rbacClient := r.kubeClientSet.RbacV1()

	logger.Info("finding clusterrole edit")
	_, err := rbacClient.ClusterRoles().Get(ctx, "edit", metav1.GetOptions{})
	if err != nil {
		logger.Error(err, "getting clusterRole 'edit' failed")
		return err
	}

	rb := &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: "edit", Namespace: sa.Namespace},
		RoleRef:    rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "edit"},
		Subjects:   []rbacv1.Subject{{Kind: rbacv1.ServiceAccountKind, Name: sa.Name, Namespace: sa.Namespace}},
	}

	_, err = rbacClient.RoleBindings(sa.Namespace).Create(ctx, rb, metav1.CreateOptions{})
	if err != nil {
		logger.Error(err, "creation of 'edit' rolebinding failed, in Namespace", sa.GetNamespace())
	}
	return err
}

func (r *Reconciler) ensureSCClusterRole(ctx context.Context) error {
	logger := logging.FromContext(ctx)

	logger.Info("finding cluster role pipeline-anyuid")

	clusterRole := &rbacv1.ClusterRole{
		ObjectMeta: metav1.ObjectMeta{Name: pipelineAnyuid},
		Rules: []rbacv1.PolicyRule{
			{
				APIGroups: []string{
					"security.openshift.io",
				},
				ResourceNames: []string{
					"anyuid",
				},
				Resources: []string{
					"securitycontextconstraints",
				},
				Verbs: []string{
					"use",
				},
			},
		},
	}

	rbacClient := r.kubeClientSet.RbacV1()
	_, err := rbacClient.ClusterRoles().Get(ctx, pipelineAnyuid, metav1.GetOptions{})

	if err != nil {
		if errors.IsNotFound(err) {
			_, err = rbacClient.ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{})
		}
		return err
	}
	_, err = rbacClient.ClusterRoles().Update(ctx, clusterRole, metav1.UpdateOptions{})
	return err
}

func (r *Reconciler) ensureSCCRoleBinding(ctx context.Context, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	logger.Info("finding role-binding pipeline-anyuid")
	rbacClient := r.kubeClientSet.RbacV1()
	pipelineRB, rbErr := rbacClient.RoleBindings(sa.Namespace).Get(ctx, pipelineAnyuid, metav1.GetOptions{})
	if rbErr != nil && !errors.IsNotFound(rbErr) {
		logger.Error(rbErr, "rbac pipeline-anyuid get error")
		return rbErr
	}

	logger.Info("finding cluster role pipeline-anyuid")
	if _, err := rbacClient.ClusterRoles().Get(ctx, pipelineAnyuid, metav1.GetOptions{}); err != nil {
		logger.Error(err, "finding pipeline-anyuid cluster role failed")
		return err
	}

	if rbErr != nil && errors.IsNotFound(rbErr) {
		return r.createSCCRoleBinding(ctx, sa)
	}

	logger.Info("found rbac", "subjects", pipelineRB.Subjects)
	return r.updateRoleBinding(ctx, pipelineRB, sa)
}

func (r *Reconciler) createSCCRoleBinding(ctx context.Context, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	logger.Info("create new rolebinding pipeline-anyuid")
	rbacClient := r.kubeClientSet.RbacV1()
	rb := &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: pipelineAnyuid, Namespace: sa.Namespace},
		RoleRef:    rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: pipelineAnyuid},
		Subjects:   []rbacv1.Subject{{Kind: rbacv1.ServiceAccountKind, Name: sa.Name, Namespace: sa.Namespace}},
	}

	_, err := rbacClient.RoleBindings(sa.Namespace).Create(ctx, rb, metav1.CreateOptions{})
	if err != nil {
		logger.Error(err, "creation of pipeline-anyuid rb failed")
	}
	return err
}

func hasSubject(subjects []rbacv1.Subject, x rbacv1.Subject) bool {
	for _, v := range subjects {
		if v.Name == x.Name && v.Kind == x.Kind && v.Namespace == x.Namespace {
			return true
		}
	}
	return false
}

func (r *Reconciler) updateRoleBinding(ctx context.Context, rb *rbacv1.RoleBinding, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	subject := rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Name: sa.Name, Namespace: sa.Namespace}

	if hasSubject(rb.Subjects, subject) {
		logger.Info("rolebinding is up to date", "action", "none")
		return nil
	}

	logger.Info("update existing rolebinding edit")
	rbacClient := r.kubeClientSet.RbacV1()
	rb.Subjects = append(rb.Subjects, subject)
	_, err := rbacClient.RoleBindings(sa.Namespace).Update(ctx, rb, metav1.UpdateOptions{})
	if err != nil {
		logger.Error(err, "updation of edit rb failed")
		return err
	}
	logger.Error(err, "successfully updated edit rb")
	return nil
}

func createSA(ctx context.Context, saInterface v1.ServiceAccountInterface, ns string) (*corev1.ServiceAccount, error) {
	sa := &corev1.ServiceAccount{
		ObjectMeta: metav1.ObjectMeta{
			Name:      pipelineSA,
			Namespace: ns,
		},
	}

	sa, err := saInterface.Create(ctx, sa, metav1.CreateOptions{})
	if err != nil && !errors.IsAlreadyExists(err) {
		return nil, err
	}

	return sa, nil
}

func createTrustedCABundleConfigMap(ctx context.Context, cfgInterface v1.ConfigMapInterface, name, ns string) error {
	c := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: ns,
			Labels: map[string]string{
				"app.kubernetes.io/part-of": "tekton-pipelines",
				// user-provided and system CA certificates
				"config.openshift.io/inject-trusted-cabundle": "true",
			},
		},
	}

	_, err := cfgInterface.Create(ctx, c, metav1.CreateOptions{})
	if err != nil && !errors.IsAlreadyExists(err) {
		return err
	}
	return nil
}

func createServiceCABundleConfigMap(ctx context.Context, cfgInterface v1.ConfigMapInterface, name, ns string) error {
	c := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: ns,
			Labels: map[string]string{
				"app.kubernetes.io/part-of": "tekton-pipelines",
			},
			Annotations: map[string]string{
				// service serving certificates (required to talk to the internal registry)
				"service.beta.openshift.io/inject-cabundle": "true",
			},
		},
	}

	_, err := cfgInterface.Create(ctx, c, metav1.CreateOptions{})
	if err != nil && !errors.IsAlreadyExists(err) {
		return err
	}
	return nil
}
<commit_msg>openshift: move pipeline sa to restricted instead of anyuid<commit_after>package rbac

import (
	"context"
	"regexp"

	v1 "k8s.io/client-go/kubernetes/typed/core/v1"

	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"

	mf "github.com/manifestival/manifestival"
	"github.com/tektoncd/operator/pkg/apis/operator/v1alpha1"
	clientset "github.com/tektoncd/operator/pkg/client/clientset/versioned"
	"github.com/tektoncd/operator/pkg/reconciler/common"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	nsreconciler "knative.dev/pkg/client/injection/kube/reconciler/core/v1/namespace"
	"knative.dev/pkg/logging"
	pkgreconciler "knative.dev/pkg/reconciler"
)

// Reconciler implements controller.Reconciler for TektonPipeline resources.
type Reconciler struct {
	// kubeClientSet allows us to talk to the k8s for core APIs
	kubeClientSet kubernetes.Interface
	// operatorClientSet allows us to configure operator objects
	operatorClientSet clientset.Interface
	// manifest is empty, but with a valid client and logger. all
	// manifests are immutable, and any created during reconcile are
	// expected to be appended to this one, obviating the passing of
	// client & logger
	manifest mf.Manifest
	// Platform-specific behavior to affect the transform
	extension common.Extension
}

// Check that our Reconciler implements controller.Reconciler
var _ nsreconciler.Interface = (*Reconciler)(nil)

const (
	pipelineRestricted       = "pipeline-restricted"
	pipelineSA               = "pipeline"
	serviceCABundleCofigMap  = "config-service-cabundle"
	trustedCABundleConfigMap = "config-trusted-cabundle"
)

// FinalizeKind removes all resources after deletion of a TektonPipelines.
func (r *Reconciler) FinalizeKind(ctx context.Context, original *v1alpha1.TektonPipeline) pkgreconciler.Event {
	return nil
}

// ReconcileKind compares the actual state with the desired, and attempts to
// converge the two.
func (r *Reconciler) ReconcileKind(ctx context.Context, ns *corev1.Namespace) pkgreconciler.Event {
	logger := logging.FromContext(ctx)
	logger.Infow("Reconciling Namespace: Platform Openshift", "status", ns.GetName())

	ignorePattern := "^(openshift|kube)-"
	if ignore, _ := regexp.MatchString(ignorePattern, ns.GetName()); ignore {
		logger.Infow("Reconciling Namespace: IGNORE", "status", ns.GetName())
		return nil
	}

	logger.Infow("Reconciling inject CA bundle configmap in ", "Namespace", ns.GetName())
	if err := r.ensureCABundles(ctx, ns); err != nil {
		return err
	}

	logger.Infow("Reconciling Default SA in ", "Namespace", ns.GetName())

	sa, err := r.ensureSA(ctx, ns)
	if err != nil {
		return err
	}

	// Maintaining a separate cluster role for the scc declaration.
	// to assist us in managing this the scc association in a
	// granular way.
	if err := r.ensureSCClusterRole(ctx); err != nil {
		return err
	}

	if err := r.ensureSCCRoleBinding(ctx, sa); err != nil {
		return err
	}

	if err := r.ensureRoleBindings(ctx, sa); err != nil {
		return err
	}

	return nil
}

func (r *Reconciler) ensureCABundles(ctx context.Context, ns *corev1.Namespace) error {
	logger := logging.FromContext(ctx)
	cfgInterface := r.kubeClientSet.CoreV1().ConfigMaps(ns.Name)

	// Ensure trusted CA bundle
	logger.Info("finding configmap: %s/%s", ns.Name, trustedCABundleConfigMap)
	_, err := cfgInterface.Get(ctx, trustedCABundleConfigMap, metav1.GetOptions{})
	if err != nil && !errors.IsNotFound(err) {
		return err
	}
	if err != nil && errors.IsNotFound(err) {
		logger.Info("creating configmap", trustedCABundleConfigMap, "ns", ns.Name)
		if err := createTrustedCABundleConfigMap(ctx, cfgInterface, trustedCABundleConfigMap, ns.Name); err != nil {
			return err
		}
	}

	// Ensure service CA bundle
	logger.Info("finding configmap: %s/%s", ns.Name, serviceCABundleCofigMap)
	_, err = cfgInterface.Get(ctx, serviceCABundleCofigMap, metav1.GetOptions{})
	if err != nil && !errors.IsNotFound(err) {
		return err
	}
	if err != nil && errors.IsNotFound(err) {
		logger.Info("creating configmap", serviceCABundleCofigMap, "ns", ns.Name)
		if err := createServiceCABundleConfigMap(ctx, cfgInterface, serviceCABundleCofigMap, ns.Name); err != nil {
			return err
		}
	}

	return nil
}

func (r *Reconciler) ensureSA(ctx context.Context, ns *corev1.Namespace) (*corev1.ServiceAccount, error) {
	logger := logging.FromContext(ctx)
	logger.Info("finding sa: %s/%s", ns.Name, "pipeline")
	saInterface := r.kubeClientSet.CoreV1().ServiceAccounts(ns.Name)
	sa, err := saInterface.Get(ctx, pipelineSA, metav1.GetOptions{})
	if err != nil && !errors.IsNotFound(err) {
		return nil, err
	}
	if err != nil && errors.IsNotFound(err) {
		logger.Info("creating sa", "sa", pipelineSA, "ns", ns.Name)
		return createSA(ctx, saInterface, ns.Name)
	}

	return sa, nil
}

func (r *Reconciler) ensureRoleBindings(ctx context.Context, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	logger.Info("finding role-binding edit")
	rbacClient := r.kubeClientSet.RbacV1()

	editRB, err := rbacClient.RoleBindings(sa.Namespace).Get(ctx, "edit", metav1.GetOptions{})

	if err == nil {
		logger.Infof("found rolebinding %s/%s", editRB.Namespace, editRB.Name)
		return r.updateRoleBinding(ctx, editRB, sa)
	}

	if errors.IsNotFound(err) {
		return r.createRoleBinding(ctx, sa)
	}

	return err
}

func (r *Reconciler) createRoleBinding(ctx context.Context, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	logger.Info("create new rolebinding edit, in Namespace", sa.GetNamespace())
	rbacClient := r.kubeClientSet.RbacV1()

	logger.Info("finding clusterrole edit")
	_, err := rbacClient.ClusterRoles().Get(ctx, "edit", metav1.GetOptions{})
	if err != nil {
		logger.Error(err, "getting clusterRole 'edit' failed")
		return err
	}

	rb := &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: "edit", Namespace: sa.Namespace},
		RoleRef:    rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "edit"},
		Subjects:   []rbacv1.Subject{{Kind: rbacv1.ServiceAccountKind, Name: sa.Name, Namespace: sa.Namespace}},
	}

	_, err = rbacClient.RoleBindings(sa.Namespace).Create(ctx, rb, metav1.CreateOptions{})
	if err != nil {
		logger.Error(err, "creation of 'edit' rolebinding failed, in Namespace", sa.GetNamespace())
	}
	return err
}

func (r *Reconciler) ensureSCClusterRole(ctx context.Context) error {
	logger := logging.FromContext(ctx)

	logger.Info("finding cluster role pipeline-restricted")

	clusterRole := &rbacv1.ClusterRole{
		ObjectMeta: metav1.ObjectMeta{Name: pipelineRestricted},
		Rules: []rbacv1.PolicyRule{
			{
				APIGroups: []string{
					"security.openshift.io",
				},
				ResourceNames: []string{
					"restricted",
				},
				Resources: []string{
					"securitycontextconstraints",
				},
				Verbs: []string{
					"use",
				},
			},
		},
	}

	rbacClient := r.kubeClientSet.RbacV1()
	_, err := rbacClient.ClusterRoles().Get(ctx, pipelineRestricted, metav1.GetOptions{})

	if err != nil {
		if errors.IsNotFound(err) {
			_, err = rbacClient.ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{})
		}
		return err
	}
	_, err = rbacClient.ClusterRoles().Update(ctx, clusterRole, metav1.UpdateOptions{})
	return err
}

func (r *Reconciler) ensureSCCRoleBinding(ctx context.Context, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	logger.Info("finding role-binding pipeline-restricted")
	rbacClient := r.kubeClientSet.RbacV1()
	pipelineRB, rbErr := rbacClient.RoleBindings(sa.Namespace).Get(ctx, pipelineRestricted, metav1.GetOptions{})
	if rbErr != nil && !errors.IsNotFound(rbErr) {
		logger.Error(rbErr, "rbac pipeline-restricted get error")
		return rbErr
	}

	logger.Info("finding cluster role pipeline-restricted")
	if _, err := rbacClient.ClusterRoles().Get(ctx, pipelineRestricted, metav1.GetOptions{}); err != nil {
		logger.Error(err, "finding pipeline-restricted cluster role failed")
		return err
	}

	if rbErr != nil && errors.IsNotFound(rbErr) {
		return r.createSCCRoleBinding(ctx, sa)
	}

	logger.Info("found rbac", "subjects", pipelineRB.Subjects)
	return r.updateRoleBinding(ctx, pipelineRB, sa)
}

func (r *Reconciler) createSCCRoleBinding(ctx context.Context, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	logger.Info("create new rolebinding pipeline-restricted")
	rbacClient := r.kubeClientSet.RbacV1()
	rb := &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: pipelineRestricted, Namespace: sa.Namespace},
		RoleRef:    rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: pipelineRestricted},
		Subjects:   []rbacv1.Subject{{Kind: rbacv1.ServiceAccountKind, Name: sa.Name, Namespace: sa.Namespace}},
	}

	_, err := rbacClient.RoleBindings(sa.Namespace).Create(ctx, rb, metav1.CreateOptions{})
	if err != nil {
		logger.Error(err, "creation of pipeline-restricted rb failed")
	}
	return err
}

func hasSubject(subjects []rbacv1.Subject, x rbacv1.Subject) bool {
	for _, v := range subjects {
		if v.Name == x.Name && v.Kind == x.Kind && v.Namespace == x.Namespace {
			return true
		}
	}
	return false
}

func (r *Reconciler) updateRoleBinding(ctx context.Context, rb *rbacv1.RoleBinding, sa *corev1.ServiceAccount) error {
	logger := logging.FromContext(ctx)

	subject := rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Name: sa.Name, Namespace: sa.Namespace}

	if hasSubject(rb.Subjects, subject) {
		logger.Info("rolebinding is up to date", "action", "none")
		return nil
	}

	logger.Info("update existing rolebinding edit")
	rbacClient := r.kubeClientSet.RbacV1()
	rb.Subjects = append(rb.Subjects, subject)
	_, err := rbacClient.RoleBindings(sa.Namespace).Update(ctx, rb, metav1.UpdateOptions{})
	if err != nil {
		logger.Error(err, "updation of edit rb failed")
		return err
	}
	logger.Error(err, "successfully updated edit rb")
	return nil
}

func createSA(ctx context.Context, saInterface v1.ServiceAccountInterface, ns string) (*corev1.ServiceAccount, error) {
	sa := &corev1.ServiceAccount{
		ObjectMeta: metav1.ObjectMeta{
			Name:      pipelineSA,
			Namespace: ns,
		},
	}

	sa, err := saInterface.Create(ctx, sa, metav1.CreateOptions{})
	if err != nil && !errors.IsAlreadyExists(err) {
		return nil, err
	}

	return sa, nil
}

func createTrustedCABundleConfigMap(ctx context.Context, cfgInterface v1.ConfigMapInterface, name, ns string) error {
	c := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: ns,
			Labels: map[string]string{
				"app.kubernetes.io/part-of": "tekton-pipelines",
				// user-provided and system CA certificates
				"config.openshift.io/inject-trusted-cabundle": "true",
			},
		},
	}

	_, err := cfgInterface.Create(ctx, c, metav1.CreateOptions{})
	if err != nil && !errors.IsAlreadyExists(err) {
		return err
	}
	return nil
}

func createServiceCABundleConfigMap(ctx context.Context, cfgInterface v1.ConfigMapInterface, name, ns string) error {
	c := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: ns,
			Labels: map[string]string{
				"app.kubernetes.io/part-of": "tekton-pipelines",
			},
			Annotations: map[string]string{
				// service serving certificates (required to talk to the internal registry)
				"service.beta.openshift.io/inject-cabundle": "true",
			},
		},
	}

	_, err := cfgInterface.Create(ctx, c, metav1.CreateOptions{})
	if err != nil && !errors.IsAlreadyExists(err) {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package url

import (
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL             string   `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
	Credentials     string   `description:"credentials file"`                                          //name of credential file or credential key depending on implementation
	ParsedURL       *url.URL `json:"-"`                                                                //parsed URL resource
	Cache           string   `description:"local cache path"`                                          //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs   int      //CacheExpiryMs expiry time in ms
	modificationTag int64
	init            bool
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		init:          r.init,
		URL:           r.URL,
		Credentials:   r.Credentials,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"scp":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
	var urlCredential = ""
	if username != "" {
		urlCredential = username
		if password != "" {
			urlCredential += ":" + password
		}
		urlCredential += "@"
	}
	result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
	if r.ParsedURL.RawQuery != "" {
		result += "?" + r.ParsedURL.RawQuery
	}

	return result
}

//Path returns url's path  directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
	if r.ParsedURL == nil {
		return ""
	}
	var result = r.ParsedURL.Path

	parent, name := path.Split(result)
	if path.Ext(name) != "" {
		result = parent
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}
	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}
	service, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
		}
	}()
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		err = r.YAMLDecode(target)
	default:
		err = r.JSONDecode(target)
	}
	return err
}

//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		return toolbox.NewYamlDecoderFactory()
	default:
		return toolbox.NewJSONDecoderFactory()
	}
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
	}
	if decoderFactory == nil {
		return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}

	text := string(content)
	if toolbox.IsNewLineDelimitedJSON(text) {
		if aSlice, err := toolbox.NewLineDelimitedJSON(text); err == nil {
			return toolbox.DefaultConverter.AssignConverted(target, aSlice)
		}
	}
	err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
	if err != nil {
		return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
	}
	return err
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	if currentName == "" && strings.HasSuffix(r.URL, "/") {
		_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
		currentName += "/"
	}

	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	if interfacePrt, ok := target.(*interface{}); ok {
		var data interface{}
		if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
			return err
		}
		if toolbox.IsSlice(data) {
			*interfacePrt = data
			return nil
		}
	}
	var mapSlice = yaml.MapSlice{}
	if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
		return err
	}
	if !toolbox.IsMap(target) {
		return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
	}
	resultMap := toolbox.AsMap(target)
	for _, v := range mapSlice {
		resultMap[toolbox.AsString(v.Key)] = v.Value
	}
	return nil
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func computeResourceModificationTag(resource *Resource) (int64, error) {
	service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
	if err != nil {
		return 0, err
	}
	object, err := service.StorageObject(resource.URL)
	if err != nil {
		return 0, err
	}
	var fileInfo = object.FileInfo()

	if object.IsContent() {
		return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
	}
	var result int64 = 0
	objects, err := service.List(resource.URL)
	if err != nil {
		return 0, err
	}
	for _, object := range objects {
		objectResource := NewResource(object.URL())
		if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
			continue
		}
		modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
		if err != nil {
			return 0, err
		}
		result += modificationTag

	}
	return result, nil
}

func (r *Resource) HasChanged() (changed bool, err error) {
	if r.modificationTag == 0 {
		r.modificationTag, err = computeResourceModificationTag(r)
		return false, err
	}
	var recentModificationTag int64
	recentModificationTag, err = computeResourceModificationTag(r)
	if err != nil {
		return false, err
	}
	if recentModificationTag != r.modificationTag {
		changed = true
		r.modificationTag = recentModificationTag
	}
	return changed, err
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		var protoPosition = strings.Index(URL, "://")
		if protoPosition != -1 {
			var urlSuffix = string(URL[protoPosition+3:])
			urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
			URL = string(URL[:protoPosition+3]) + urlSuffix
		}
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, _ := os.Getwd()

		if strings.Contains(URL, "..") {
			fragments := strings.Split(URL, "/")
			var index = 0
			var offset = 0
			if fragments[0] == "." {
				offset = 1
			}

			for index = offset; index < len(fragments); index++ {
				var fragment = fragments[index]
				if fragment == ".." {
					currentDirectory, _ = path.Split(currentDirectory)
					if strings.HasSuffix(currentDirectory, "/") {
						currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
					}
					continue
				}
				break
			}
			return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
		}

		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			URL = candidate
		}
	}
	return toolbox.FileSchema + URL
}

func (r *Resource) Init() (err error) {
	if r.init {
		return nil
	}
	r.init = true
	r.URL = normalizeURL(r.URL)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		init:          true,
		ParsedURL:     parsedURL,
		URL:           URL,
		Credentials:   credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<commit_msg>added url check<commit_after>package url

import (
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL             string   `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
	Credentials     string   `description:"credentials file"`                                          //name of credential file or credential key depending on implementation
	ParsedURL       *url.URL `json:"-"`                                                                //parsed URL resource
	Cache           string   `description:"local cache path"`                                          //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs   int      //CacheExpiryMs expiry time in ms
	modificationTag int64
	init            bool
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		init:          r.init,
		URL:           r.URL,
		Credentials:   r.Credentials,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"scp":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
	var urlCredential = ""
	if username != "" {
		urlCredential = username
		if password != "" {
			urlCredential += ":" + password
		}
		urlCredential += "@"
	}
	result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
	if r.ParsedURL.RawQuery != "" {
		result += "?" + r.ParsedURL.RawQuery
	}

	return result
}

//Path returns url's path  directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
	if r.ParsedURL == nil {
		return ""
	}
	var result = r.ParsedURL.Path

	parent, name := path.Split(result)
	if path.Ext(name) != "" {
		result = parent
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}
	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}
	service, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
		}
	}()
	if r.ParsedURL == nil {
		if r.ParsedURL, err = url.Parse(r.URL); err != nil {
			return err
		}
	}
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		err = r.YAMLDecode(target)
	default:
		err = r.JSONDecode(target)
	}
	return err
}

//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		return toolbox.NewYamlDecoderFactory()
	default:
		return toolbox.NewJSONDecoderFactory()
	}
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
	}
	if decoderFactory == nil {
		return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}

	text := string(content)
	if toolbox.IsNewLineDelimitedJSON(text) {
		if aSlice, err := toolbox.NewLineDelimitedJSON(text); err == nil {
			return toolbox.DefaultConverter.AssignConverted(target, aSlice)
		}
	}
	err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
	if err != nil {
		return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
	}
	return err
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	if currentName == "" && strings.HasSuffix(r.URL, "/") {
		_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
		currentName += "/"
	}

	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	if interfacePrt, ok := target.(*interface{}); ok {
		var data interface{}
		if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
			return err
		}
		if toolbox.IsSlice(data) {
			*interfacePrt = data
			return nil
		}
	}
	var mapSlice = yaml.MapSlice{}
	if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
		return err
	}
	if !toolbox.IsMap(target) {
		return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
	}
	resultMap := toolbox.AsMap(target)
	for _, v := range mapSlice {
		resultMap[toolbox.AsString(v.Key)] = v.Value
	}
	return nil
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func computeResourceModificationTag(resource *Resource) (int64, error) {
	service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
	if err != nil {
		return 0, err
	}
	object, err := service.StorageObject(resource.URL)
	if err != nil {
		return 0, err
	}
	var fileInfo = object.FileInfo()

	if object.IsContent() {
		return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
	}
	var result int64 = 0
	objects, err := service.List(resource.URL)
	if err != nil {
		return 0, err
	}
	for _, object := range objects {
		objectResource := NewResource(object.URL())
		if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
			continue
		}
		modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
		if err != nil {
			return 0, err
		}
		result += modificationTag

	}
	return result, nil
}

func (r *Resource) HasChanged() (changed bool, err error) {
	if r.modificationTag == 0 {
		r.modificationTag, err = computeResourceModificationTag(r)
		return false, err
	}
	var recentModificationTag int64
	recentModificationTag, err = computeResourceModificationTag(r)
	if err != nil {
		return false, err
	}
	if recentModificationTag != r.modificationTag {
		changed = true
		r.modificationTag = recentModificationTag
	}
	return changed, err
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		var protoPosition = strings.Index(URL, "://")
		if protoPosition != -1 {
			var urlSuffix = string(URL[protoPosition+3:])
			urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
			URL = string(URL[:protoPosition+3]) + urlSuffix
		}
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, _ := os.Getwd()

		if strings.Contains(URL, "..") {
			fragments := strings.Split(URL, "/")
			var index = 0
			var offset = 0
			if fragments[0] == "." {
				offset = 1
			}

			for index = offset; index < len(fragments); index++ {
				var fragment = fragments[index]
				if fragment == ".." {
					currentDirectory, _ = path.Split(currentDirectory)
					if strings.HasSuffix(currentDirectory, "/") {
						currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
					}
					continue
				}
				break
			}
			return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
		}

		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			URL = candidate
		}
	}
	return toolbox.FileSchema + URL
}

func (r *Resource) Init() (err error) {
	if r.init {
		return nil
	}
	r.init = true
	r.URL = normalizeURL(r.URL)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		init:          true,
		ParsedURL:     parsedURL,
		URL:           URL,
		Credentials:   credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sidecar

import (
	"time"

	"github.com/golang/glog"
	"k8s.io/dns/pkg/dnsmasq"
)

// Server that runs the dnsmasq-metrics daemon.
type Server interface {
	Run(options *Options)
}

type server struct {
	options       *Options
	metricsClient dnsmasq.MetricsClient
	probes        []*dnsProbe
}

// NewServer creates a new server instance
func NewServer() Server {
	return &server{}
}

// Run the server (does not return)
func (s *server) Run(options *Options) {
	s.options = options
	glog.Infof("Starting server (options %+v)", *s.options)

	for _, probeOption := range options.Probes {
		probe := &dnsProbe{DNSProbeOption: probeOption}
		s.probes = append(s.probes, probe)
		probe.Start(options)
	}

	s.runMetrics(options)
}

func (s *server) runMetrics(options *Options) {
	InitializeMetrics(options)

	s.metricsClient = dnsmasq.NewMetricsClient(options.DnsMasqAddr, options.DnsMasqPort)

	for {
		metrics, err := s.metricsClient.GetMetrics()
		if err != nil {
			glog.Warningf("Error getting metrics from dnsmasq: %v", err)
			errorsCounter.Add(1)
		} else {
			glog.V(3).Infof("DnsMasq metrics %+v", metrics)
			exportMetrics(metrics)
		}

		time.Sleep(time.Duration(options.DnsMasqPollIntervalMs) * time.Millisecond)
	}
}

func exportMetrics(metrics *dnsmasq.Metrics) {
	for key := range *metrics {
		// Retrieve the previous value of the metric and get the delta
		// between the previous and current values. Add the delta to the
		// previous to get the proper value. This is needed because the
		// Counter API does not allow us to set the counter to a value.
		previousValue := countersCache[key]
		delta := float64((*metrics)[key]) - previousValue
		newValue := previousValue + delta
		// Update cache to new value.
		countersCache[key] = newValue
		counters[key].Add(newValue)
	}
}
<commit_msg>Update server.go<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sidecar

import (
	"time"

	"github.com/golang/glog"
	"k8s.io/dns/pkg/dnsmasq"
)

// Server that runs the dnsmasq-metrics daemon.
type Server interface {
	Run(options *Options)
}

type server struct {
	options       *Options
	metricsClient dnsmasq.MetricsClient
	probes        []*dnsProbe
}

// NewServer creates a new server instance
func NewServer() Server {
	return &server{}
}

// Run the server (does not return)
func (s *server) Run(options *Options) {
	s.options = options
	glog.Infof("Starting server (options %+v)", *s.options)

	for _, probeOption := range options.Probes {
		probe := &dnsProbe{DNSProbeOption: probeOption}
		s.probes = append(s.probes, probe)
		probe.Start(options)
	}

	s.runMetrics(options)
}

func (s *server) runMetrics(options *Options) {
	InitializeMetrics(options)

	s.metricsClient = dnsmasq.NewMetricsClient(options.DnsMasqAddr, options.DnsMasqPort)

	for {
		metrics, err := s.metricsClient.GetMetrics()
		if err != nil {
			glog.Warningf("Error getting metrics from dnsmasq: %v", err)
			errorsCounter.Add(1)
		} else {
			glog.V(3).Infof("DnsMasq metrics %+v", metrics)
			exportMetrics(metrics)
		}

		time.Sleep(time.Duration(options.DnsMasqPollIntervalMs) * time.Millisecond)
	}
}

func exportMetrics(metrics *dnsmasq.Metrics) {
	for key := range *metrics {
		// Retrieve the previous value of the metric and get the delta
		// between the previous and current values. Add the delta to the
		// previous to get the proper value. This is needed because the
		// Counter API does not allow us to set the counter to a value.
		previousValue := countersCache[key]
		delta := float64((*metrics)[key]) - previousValue
		newValue := previousValue + delta
		// Update cache to new value.
		countersCache[key] = newValue
		counters[key].Add(delta)
	}
}
<|endoftext|>


<commit_before>package stackit

import (
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"time"
)

type TailStackEvent struct {
	cloudformation.StackEvent
	StackitError error
}

func (s *Stackit) PollStackEvents(token string, callback func(event TailStackEvent)) TailStackEvent {
	lastSentEventId := ""

	for {
		time.Sleep(3 * time.Second)

		events := []*cloudformation.StackEvent{}

		s.api.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{
			StackName: &s.stackId,
		}, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool {
			for _, event := range page.StackEvents {
				crt := "nil"
				if event.ClientRequestToken != nil {
					crt = *event.ClientRequestToken
				}

				if token == "" {
					token = crt
				}

				if *event.EventId == lastSentEventId || crt != token {
					return false
				}

				events = append(events, event)
			}
			return true
		})

		if len(events) == 0 {
			continue
		}

		lastSentEventId = *events[0].EventId
		stack, err := s.Describe()
		if err != nil {
			callback(TailStackEvent{cloudformation.StackEvent{}, err})
		}
		terminal := IsTerminalStatus(*stack.StackStatus)

		for ev_i := len(events) - 1; ev_i >= 0; ev_i-- {
			event := events[ev_i]
			tailEvent := TailStackEvent{*event, nil}

			done := terminal && ev_i == 0
			if done {
				return tailEvent
			}

			callback(tailEvent)
		}
	}
}

func IsTerminalStatus(status string) bool {
	switch status {
	case
		"CREATE_COMPLETE",
		"DELETE_COMPLETE",
		"CREATE_FAILED",
		"DELETE_FAILED",
		"ROLLBACK_COMPLETE",
		"ROLLBACK_FAILED",
		"UPDATE_COMPLETE",
		"UPDATE_FAILED",
		"UPDATE_ROLLBACK_COMPLETE",
		"UPDATE_ROLLBACK_FAILED":
		return true
	default:
		return false
	}
}
<commit_msg>handle throttling in poller<commit_after>package stackit

import (
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"time"
)

type TailStackEvent struct {
	cloudformation.StackEvent
	StackitError error
}

func (s *Stackit) PollStackEvents(token string, callback func(event TailStackEvent)) TailStackEvent {
	lastSentEventId := ""

	for {
		time.Sleep(3 * time.Second)

		events := []*cloudformation.StackEvent{}

		err := s.api.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{
			StackName: &s.stackId,
		}, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool {
			for _, event := range page.StackEvents {
				crt := "nil"
				if event.ClientRequestToken != nil {
					crt = *event.ClientRequestToken
				}

				if token == "" {
					token = crt
				}

				if *event.EventId == lastSentEventId || crt != token {
					return false
				}

				events = append(events, event)
			}
			return true
		})

		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				code := awsErr.Code()
				if code == "ThrottlingException" {
					continue
				}
			}
			event := TailStackEvent{cloudformation.StackEvent{}, err}
			callback(event)
			return event
		}

		if len(events) == 0 {
			continue
		}

		lastSentEventId = *events[0].EventId
		stack, err := s.Describe()
		if err != nil {
			event := TailStackEvent{cloudformation.StackEvent{}, err}
			callback(event)
			return event
		}
		terminal := IsTerminalStatus(*stack.StackStatus)

		for ev_i := len(events) - 1; ev_i >= 0; ev_i-- {
			event := events[ev_i]
			tailEvent := TailStackEvent{*event, nil}

			done := terminal && ev_i == 0
			if done {
				return tailEvent
			}

			callback(tailEvent)
		}
	}
}

func IsTerminalStatus(status string) bool {
	switch status {
	case
		"CREATE_COMPLETE",
		"DELETE_COMPLETE",
		"CREATE_FAILED",
		"DELETE_FAILED",
		"ROLLBACK_COMPLETE",
		"ROLLBACK_FAILED",
		"UPDATE_COMPLETE",
		"UPDATE_FAILED",
		"UPDATE_ROLLBACK_COMPLETE",
		"UPDATE_ROLLBACK_FAILED":
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/util/clock"

	"github.com/golang/groupcache/lru"
)

func expectEntry(t *testing.T, c *LRUExpireCache, key lru.Key, value interface{}) {
	result, ok := c.Get(key)
	if !ok || result != value {
		t.Errorf("Expected cache[%v]: %v, got %v", key, value, result)
	}
}

func expectNotEntry(t *testing.T, c *LRUExpireCache, key lru.Key) {
	if result, ok := c.Get(key); ok {
		t.Errorf("Expected cache[%v] to be empty, got %v", key, result)
	}
}

func TestSimpleGet(t *testing.T) {
	c := NewLRUExpireCache(10)
	c.Add("long-lived", "12345", 10*time.Hour)
	expectEntry(t, c, "long-lived", "12345")
}

func TestExpiredGet(t *testing.T) {
	fakeClock := clock.NewFakeClock(time.Now())
	c := NewLRUExpireCacheWithClock(10, fakeClock)
	c.Add("short-lived", "12345", 1*time.Millisecond)
	// ensure the entry expired
	fakeClock.Step(2 * time.Millisecond)
	expectNotEntry(t, c, "short-lived")
}

func TestLRUOverflow(t *testing.T) {
	c := NewLRUExpireCache(4)
	c.Add("elem1", "1", 10*time.Hour)
	c.Add("elem2", "2", 10*time.Hour)
	c.Add("elem3", "3", 10*time.Hour)
	c.Add("elem4", "4", 10*time.Hour)
	c.Add("elem5", "5", 10*time.Hour)
	expectNotEntry(t, c, "elem1")
	expectEntry(t, c, "elem2", "2")
	expectEntry(t, c, "elem3", "3")
	expectEntry(t, c, "elem4", "4")
	expectEntry(t, c, "elem5", "5")
}
<commit_msg>move utils used in restclient to client-go<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"testing"
	"time"

	"k8s.io/client-go/pkg/util/clock"

	"github.com/golang/groupcache/lru"
)

func expectEntry(t *testing.T, c *LRUExpireCache, key lru.Key, value interface{}) {
	result, ok := c.Get(key)
	if !ok || result != value {
		t.Errorf("Expected cache[%v]: %v, got %v", key, value, result)
	}
}

func expectNotEntry(t *testing.T, c *LRUExpireCache, key lru.Key) {
	if result, ok := c.Get(key); ok {
		t.Errorf("Expected cache[%v] to be empty, got %v", key, result)
	}
}

func TestSimpleGet(t *testing.T) {
	c := NewLRUExpireCache(10)
	c.Add("long-lived", "12345", 10*time.Hour)
	expectEntry(t, c, "long-lived", "12345")
}

func TestExpiredGet(t *testing.T) {
	fakeClock := clock.NewFakeClock(time.Now())
	c := NewLRUExpireCacheWithClock(10, fakeClock)
	c.Add("short-lived", "12345", 1*time.Millisecond)
	// ensure the entry expired
	fakeClock.Step(2 * time.Millisecond)
	expectNotEntry(t, c, "short-lived")
}

func TestLRUOverflow(t *testing.T) {
	c := NewLRUExpireCache(4)
	c.Add("elem1", "1", 10*time.Hour)
	c.Add("elem2", "2", 10*time.Hour)
	c.Add("elem3", "3", 10*time.Hour)
	c.Add("elem4", "4", 10*time.Hour)
	c.Add("elem5", "5", 10*time.Hour)
	expectNotEntry(t, c, "elem1")
	expectEntry(t, c, "elem2", "2")
	expectEntry(t, c, "elem3", "3")
	expectEntry(t, c, "elem4", "4")
	expectEntry(t, c, "elem5", "5")
}
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"strings"

	"github.com/pingcap/errors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// TLS saves some information about tls
type TLS struct {
	inner  *tls.Config
	client *http.Client
	url    string
}

// ToTLSConfig generates tls's config.
func ToTLSConfig(caPath, certPath, keyPath string) (*tls.Config, error) {
	return ToTLSConfigWithVerify(caPath, certPath, keyPath, nil)
}

// ToTLSConfigWithVerify constructs a `*tls.Config` from the CA, certification and key
// paths, and add verify for CN.
//
// If the CA path is empty, returns nil.
func ToTLSConfigWithVerify(caPath, certPath, keyPath string, verifyCN []string) (*tls.Config, error) {
	if len(caPath) == 0 {
		return nil, nil
	}

	// Load the client certificates from disk
	var certificates []tls.Certificate
	if len(certPath) != 0 && len(keyPath) != 0 {
		cert, err := tls.LoadX509KeyPair(certPath, keyPath)
		if err != nil {
			return nil, errors.Annotate(err, "could not load client key pair")
		}
		certificates = []tls.Certificate{cert}
	}

	// Create a certificate pool from CA
	certPool := x509.NewCertPool()
	ca, err := ioutil.ReadFile(caPath)
	if err != nil {
		return nil, errors.Annotate(err, "could not read ca certificate")
	}

	// Append the certificates from the CA
	if !certPool.AppendCertsFromPEM(ca) {
		return nil, errors.New("failed to append ca certs")
	}

	tlsCfg := &tls.Config{
		Certificates: certificates,
		RootCAs:      certPool,
		ClientCAs:    certPool,
		NextProtos:   []string{"h2", "http/1.1"}, // specify `h2` to let Go use HTTP/2.
	}

	if len(verifyCN) != 0 {
		checkCN := make(map[string]struct{})
		for _, cn := range verifyCN {
			cn = strings.TrimSpace(cn)
			checkCN[cn] = struct{}{}
		}

		tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert

		tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
			cns := make([]string, 0, len(verifiedChains))
			for _, chains := range verifiedChains {
				for _, chain := range chains {
					cns = append(cns, chain.Subject.CommonName)
					if _, match := checkCN[chain.Subject.CommonName]; match {
						return nil
					}
				}
			}
			return errors.Errorf("client certificate authentication failed. The Common Name from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN)
		}
	}

	return tlsCfg, nil
}

// NewTLS constructs a new HTTP client with TLS configured with the CA,
// certificate and key paths.
//
// If the CA path is empty, returns an instance where TLS is disabled.
func NewTLS(caPath, certPath, keyPath, host string, verifyCN []string) (*TLS, error) {
	if len(caPath) == 0 {
		return &TLS{
			inner:  nil,
			client: &http.Client{},
			url:    "http://" + host,
		}, nil
	}
	inner, err := ToTLSConfigWithVerify(caPath, certPath, keyPath, verifyCN)
	if err != nil {
		return nil, err
	}

	client := ClientWithTLS(inner)

	return &TLS{
		inner:  inner,
		client: client,
		url:    "https://" + host,
	}, nil
}

// ClientWithTLS creates a http client wit tls
func ClientWithTLS(tlsCfg *tls.Config) *http.Client {
	transport := http.DefaultTransport.(*http.Transport).Clone()
	transport.TLSClientConfig = tlsCfg
	return &http.Client{Transport: transport}
}

// NewTLSFromMockServer constructs a new TLS instance from the certificates of
// an *httptest.Server.
func NewTLSFromMockServer(server *httptest.Server) *TLS {
	return &TLS{
		inner:  server.TLS,
		client: server.Client(),
		url:    server.URL,
	}
}

// WithHost creates a new TLS instance with the host replaced.
func (tc *TLS) WithHost(host string) *TLS {
	var url string
	if tc.inner != nil {
		url = "https://" + host
	} else {
		url = "http://" + host
	}
	return &TLS{
		inner:  tc.inner,
		client: tc.client,
		url:    url,
	}
}

// ToGRPCDialOption constructs a gRPC dial option.
func (tc *TLS) ToGRPCDialOption() grpc.DialOption {
	if tc.inner != nil {
		return grpc.WithTransportCredentials(credentials.NewTLS(tc.inner))
	}
	return grpc.WithInsecure()
}

// WrapListener places a TLS layer on top of the existing listener.
func (tc *TLS) WrapListener(l net.Listener) net.Listener {
	if tc.inner == nil {
		return l
	}
	return tls.NewListener(l, tc.inner)
}

// GetJSON obtains JSON result with the HTTP GET method.
func (tc *TLS) GetJSON(path string, v interface{}) error {
	return GetJSON(tc.client, tc.url+path, v)
}
<commit_msg>pkg: update tls (#329)<commit_after>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"strings"

	"github.com/pingcap/errors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// TLS saves some information about tls
type TLS struct {
	inner  *tls.Config
	client *http.Client
	url    string
}

// ToTLSConfig generates tls's config.
func ToTLSConfig(caPath, certPath, keyPath string) (*tls.Config, error) {
	return ToTLSConfigWithVerify(caPath, certPath, keyPath, nil)
}

// ToTLSConfigWithVerify constructs a `*tls.Config` from the CA, certification and key
// paths, and add verify for CN.
//
// If the CA path is empty, returns nil.
func ToTLSConfigWithVerify(caPath, certPath, keyPath string, verifyCN []string) (*tls.Config, error) {
	if len(caPath) == 0 {
		return nil, nil
	}

	// Load the client certificates from disk
	var certificates []tls.Certificate
	if len(certPath) != 0 && len(keyPath) != 0 {
		cert, err := tls.LoadX509KeyPair(certPath, keyPath)
		if err != nil {
			return nil, errors.Annotate(err, "could not load client key pair")
		}
		certificates = []tls.Certificate{cert}
	}

	// Create a certificate pool from CA
	certPool := x509.NewCertPool()
	ca, err := ioutil.ReadFile(caPath)
	if err != nil {
		return nil, errors.Annotate(err, "could not read ca certificate")
	}

	// Append the certificates from the CA
	if !certPool.AppendCertsFromPEM(ca) {
		return nil, errors.New("failed to append ca certs")
	}

	tlsCfg := &tls.Config{
		Certificates: certificates,
		RootCAs:      certPool,
		ClientCAs:    certPool,
		NextProtos:   []string{"h2", "http/1.1"}, // specify `h2` to let Go use HTTP/2.
	}

	if len(verifyCN) != 0 {
		checkCN := make(map[string]struct{})
		for _, cn := range verifyCN {
			cn = strings.TrimSpace(cn)
			checkCN[cn] = struct{}{}
		}

		tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert

		tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
			cns := make([]string, 0, len(verifiedChains))
			for _, chains := range verifiedChains {
				for _, chain := range chains {
					cns = append(cns, chain.Subject.CommonName)
					if _, match := checkCN[chain.Subject.CommonName]; match {
						return nil
					}
				}
			}
			return errors.Errorf("client certificate authentication failed. The Common Name from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN)
		}
	}

	return tlsCfg, nil
}

// NewTLS constructs a new HTTP client with TLS configured with the CA,
// certificate and key paths.
//
// If the CA path is empty, returns an instance where TLS is disabled.
func NewTLS(caPath, certPath, keyPath, host string, verifyCN []string) (*TLS, error) {
	if len(caPath) == 0 {
		return &TLS{
			inner:  nil,
			client: &http.Client{},
			url:    "http://" + host,
		}, nil
	}
	inner, err := ToTLSConfigWithVerify(caPath, certPath, keyPath, verifyCN)
	if err != nil {
		return nil, err
	}

	client := ClientWithTLS(inner)

	return &TLS{
		inner:  inner,
		client: client,
		url:    "https://" + host,
	}, nil
}

// ClientWithTLS creates a http client wit tls
func ClientWithTLS(tlsCfg *tls.Config) *http.Client {
	transport := http.DefaultTransport.(*http.Transport).Clone()
	transport.TLSClientConfig = tlsCfg
	return &http.Client{Transport: transport}
}

// NewTLSFromMockServer constructs a new TLS instance from the certificates of
// an *httptest.Server.
func NewTLSFromMockServer(server *httptest.Server) *TLS {
	return &TLS{
		inner:  server.TLS,
		client: server.Client(),
		url:    server.URL,
	}
}

// WithHost creates a new TLS instance with the host replaced.
func (tc *TLS) WithHost(host string) *TLS {
	var url string
	if tc.inner != nil {
		url = "https://" + host
	} else {
		url = "http://" + host
	}
	return &TLS{
		inner:  tc.inner,
		client: tc.client,
		url:    url,
	}
}

// TLSConfig returns tls config
func (tc *TLS) TLSConfig() *tls.Config {
	return tc.inner
}

// ToGRPCDialOption constructs a gRPC dial option.
func (tc *TLS) ToGRPCDialOption() grpc.DialOption {
	if tc.inner != nil {
		return grpc.WithTransportCredentials(credentials.NewTLS(tc.inner))
	}
	return grpc.WithInsecure()
}

// ToGRPCServerOption constructs a gRPC server option.
func (tc *TLS) ToGRPCServerOption() grpc.ServerOption {
	if tc.inner != nil {
		return grpc.Creds(credentials.NewTLS(tc.inner))
	}

	return grpc.Creds(nil)
}

// WrapListener places a TLS layer on top of the existing listener.
func (tc *TLS) WrapListener(l net.Listener) net.Listener {
	if tc.inner == nil {
		return l
	}
	return tls.NewListener(l, tc.inner)
}

// GetJSON obtains JSON result with the HTTP GET method.
func (tc *TLS) GetJSON(path string, v interface{}) error {
	return GetJSON(tc.client, tc.url+path, v)
}
<|endoftext|>


<commit_before>package plugin

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/labstack/echo"
	"github.com/stretchr/testify/assert"
)

func TestHeader(t *testing.T) {
	e := echo.New()
	req := httptest.NewRequest(echo.GET, "/", nil)
	rec := httptest.NewRecorder()
	c := e.NewContext(req, rec)
	ok := func(c echo.Context) error {
		return c.String(http.StatusOK, "OK")
	}
	h := &Header{
		Set: map[string]string{"Name": "Jon"},
		Add: map[string]string{"Name": "Joe"},
		Del: []string{"Delete"},
	}
	rec.Header().Set("Delete", "me")

	h.Init()
	h.Process(ok)(c)

	assert.Equal(t, "Jon", rec.Header().Get("Name"))                    // Set
	assert.EqualValues(t, []string{"Jon", "Joe"}, rec.Header()["Name"]) // Add
	assert.Equal(t, "", rec.Header().Get("Delete"))                     // Del
}
<commit_msg>Fixed build<commit_after>package plugin

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/labstack/echo"
	"github.com/stretchr/testify/assert"
)

func TestHeader(t *testing.T) {
	e := echo.New()
	req := httptest.NewRequest(echo.GET, "/", nil)
	rec := httptest.NewRecorder()
	c := e.NewContext(req, rec)
	ok := func(c echo.Context) error {
		return c.String(http.StatusOK, "OK")
	}
	h := new(Header)
	h.Set = map[string]string{"Name": "Jon"}
	h.Add = map[string]string{"Name": "Joe"}
	h.Del = []string{"Delete"}
	rec.Header().Set("Delete", "me")

	h.Initialize()
	h.Process(ok)(c)

	assert.Equal(t, "Jon", rec.Header().Get("Name"))                    // Set
	assert.EqualValues(t, []string{"Jon", "Joe"}, rec.Header()["Name"]) // Add
	assert.Equal(t, "", rec.Header().Get("Delete"))                     // Del
}
<|endoftext|>


<commit_before>// Package ready is used to signal readiness of the CoreDNS process. Once all
// plugins have called in the plugin will signal readiness by returning a 200
// OK on the HTTP handler (on port 8181). If not ready yet, the handler will
// return a 503.
package ready

import (
	"io"
	"net"
	"net/http"
	"sync"

	clog "github.com/coredns/coredns/plugin/pkg/log"
	"github.com/coredns/coredns/plugin/pkg/reuseport"
	"github.com/coredns/coredns/plugin/pkg/uniq"
)

var (
	log      = clog.NewWithPlugin("ready")
	plugins  = &list{}
	uniqAddr = uniq.New()
)

type ready struct {
	Addr string

	sync.RWMutex
	ln   net.Listener
	done bool
	mux  *http.ServeMux
}

func (rd *ready) onStartup() error {
	ln, err := reuseport.Listen("tcp", rd.Addr)
	if err != nil {
		return err
	}

	rd.Lock()
	rd.ln = ln
	rd.mux = http.NewServeMux()
	rd.done = true
	rd.Unlock()

	rd.mux.HandleFunc("/ready", func(w http.ResponseWriter, _ *http.Request) {
		ok, todo := plugins.Ready()
		if ok {
			w.WriteHeader(http.StatusOK)
			io.WriteString(w, http.StatusText(http.StatusOK))
			return
		}
		log.Infof("Still waiting on: %q", todo)
		w.WriteHeader(http.StatusServiceUnavailable)
		io.WriteString(w, todo)
	})

	go func() { http.Serve(rd.ln, rd.mux) }()

	return nil
}

func (rd *ready) onFinalShutdown() error {
	rd.Lock()
	defer rd.Unlock()
	if !rd.done {
		return nil
	}

	uniqAddr.Unset(rd.Addr)

	rd.ln.Close()
	rd.done = false
	return nil
}
<commit_msg>dont return 200 during shutdown (#4167)<commit_after>// Package ready is used to signal readiness of the CoreDNS process. Once all
// plugins have called in the plugin will signal readiness by returning a 200
// OK on the HTTP handler (on port 8181). If not ready yet, the handler will
// return a 503.
package ready

import (
	"io"
	"net"
	"net/http"
	"sync"

	clog "github.com/coredns/coredns/plugin/pkg/log"
	"github.com/coredns/coredns/plugin/pkg/reuseport"
	"github.com/coredns/coredns/plugin/pkg/uniq"
)

var (
	log      = clog.NewWithPlugin("ready")
	plugins  = &list{}
	uniqAddr = uniq.New()
)

type ready struct {
	Addr string

	sync.RWMutex
	ln   net.Listener
	done bool
	mux  *http.ServeMux
}

func (rd *ready) onStartup() error {
	ln, err := reuseport.Listen("tcp", rd.Addr)
	if err != nil {
		return err
	}

	rd.Lock()
	rd.ln = ln
	rd.mux = http.NewServeMux()
	rd.done = true
	rd.Unlock()

	rd.mux.HandleFunc("/ready", func(w http.ResponseWriter, _ *http.Request) {
		rd.Lock()
		defer rd.Unlock()
		if !rd.done {
			w.WriteHeader(http.StatusServiceUnavailable)
			io.WriteString(w, "Shutting down")
			return
		}
		ok, todo := plugins.Ready()
		if ok {
			w.WriteHeader(http.StatusOK)
			io.WriteString(w, http.StatusText(http.StatusOK))
			return
		}
		log.Infof("Still waiting on: %q", todo)
		w.WriteHeader(http.StatusServiceUnavailable)
		io.WriteString(w, todo)
	})

	go func() { http.Serve(rd.ln, rd.mux) }()

	return nil
}

func (rd *ready) onFinalShutdown() error {
	rd.Lock()
	defer rd.Unlock()
	if !rd.done {
		return nil
	}

	uniqAddr.Unset(rd.Addr)

	rd.ln.Close()
	rd.done = false
	return nil
}
<|endoftext|>