<commit_before>package proxy

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/markbates/inflect"
	"go.uber.org/zap"
	"gomodules.xyz/jsonpatch/v2"
	admissionv1 "k8s.io/api/admission/v1"
	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	admissionlisters "k8s.io/client-go/listers/admissionregistration/v1"
	corelisters "k8s.io/client-go/listers/core/v1"
	"knative.dev/pkg/apis"
	"knative.dev/pkg/apis/duck"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/kmp"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/ptr"
	pkgreconciler "knative.dev/pkg/reconciler"
	"knative.dev/pkg/system"
	"knative.dev/pkg/webhook"
	certresources "knative.dev/pkg/webhook/certificates/resources"
)

// reconciler implements the AdmissionController for resources
type reconciler struct {
	webhook.StatelessAdmissionImpl
	pkgreconciler.LeaderAwareFuncs

	key  types.NamespacedName
	path string

	withContext func(context.Context) context.Context

	client       kubernetes.Interface
	mwhlister    admissionlisters.MutatingWebhookConfigurationLister
	secretlister corelisters.SecretLister

	disallowUnknownFields bool
	secretName            string
}

var _ controller.Reconciler = (*reconciler)(nil)
var _ pkgreconciler.LeaderAware = (*reconciler)(nil)
var _ webhook.AdmissionController = (*reconciler)(nil)
var _ webhook.StatelessAdmissionController = (*reconciler)(nil)

// Reconcile implements controller.Reconciler
func (ac *reconciler) Reconcile(ctx context.Context, key string) error {
	logger := logging.FromContext(ctx)

	if !ac.IsLeaderFor(ac.key) {
		logger.Debugf("Skipping key %q, not the leader.", ac.key)
		return nil
	}

	// Look up the webhook secret, and fetch the CA cert bundle.
	secret, err := ac.secretlister.Secrets(system.Namespace()).Get(ac.secretName)
	if err != nil {
		logger.Errorw("Error fetching secret", zap.Error(err))
		return err
	}
	caCert, ok := secret.Data[certresources.CACert]
	if !ok {
		return fmt.Errorf("secret %q is missing %q key", ac.secretName, certresources.CACert)
	}

	// Reconcile the webhook configuration.
	return ac.reconcileMutatingWebhook(ctx, caCert)
}

// Path implements AdmissionController
func (ac *reconciler) Path() string {
	return ac.path
}

// Admit implements AdmissionController
func (ac *reconciler) Admit(ctx context.Context, request *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse {
	if ac.withContext != nil {
		ctx = ac.withContext(ctx)
	}

	logger := logging.FromContext(ctx)
	switch request.Operation {
	case admissionv1.Create:
	default:
		logger.Info("Unhandled webhook operation, letting it through ", request.Operation)
		return &admissionv1.AdmissionResponse{Allowed: true}
	}

	patchBytes, err := ac.mutate(ctx, request)
	if err != nil {
		return webhook.MakeErrorStatus("mutation failed: %v", err)
	}
	logger.Infof("Kind: %q PatchBytes: %v", request.Kind, string(patchBytes))

	return &admissionv1.AdmissionResponse{
		Patch:   patchBytes,
		Allowed: true,
		PatchType: func() *admissionv1.PatchType {
			pt := admissionv1.PatchTypeJSONPatch
			return &pt
		}(),
	}
}

func (ac *reconciler) reconcileMutatingWebhook(ctx context.Context, caCert []byte) error {
	logger := logging.FromContext(ctx)

	plural := strings.ToLower(inflect.Pluralize("Pod"))
	rules := []admissionregistrationv1.RuleWithOperations{
		{
			Operations: []admissionregistrationv1.OperationType{
				admissionregistrationv1.Create,
			},
			Rule: admissionregistrationv1.Rule{
				APIGroups:   []string{""},
				APIVersions: []string{"v1"},
				Resources:   []string{plural, plural + "/status"},
			},
		},
	}

	configuredWebhook, err := ac.mwhlister.Get(ac.key.Name)
	if err != nil {
		return fmt.Errorf("error retrieving webhook: %w", err)
	}

	webhook := configuredWebhook.DeepCopy()

	// Clear out any previous (bad) OwnerReferences.
	// See: https://github.com/knative/serving/issues/5845
	webhook.OwnerReferences = nil

	for i, wh := range webhook.Webhooks {
		if wh.Name != webhook.Name {
			continue
		}
		webhook.Webhooks[i].Rules = rules
		webhook.Webhooks[i].NamespaceSelector = &metav1.LabelSelector{
			MatchExpressions: []metav1.LabelSelectorRequirement{{
				Key:      "operator.tekton.dev/disable-proxy",
				Operator: metav1.LabelSelectorOpDoesNotExist,
			}, {
				// "control-plane" is added to support Azure's AKS, otherwise the controllers fight.
				// See knative/pkg#1590 for details.
				Key:      "control-plane",
				Operator: metav1.LabelSelectorOpDoesNotExist,
			}},
		}
		webhook.Webhooks[i].ClientConfig.CABundle = caCert
		if webhook.Webhooks[i].ClientConfig.Service == nil {
			return fmt.Errorf("missing service reference for webhook: %s", wh.Name)
		}
		webhook.Webhooks[i].ClientConfig.Service.Path = ptr.String(ac.Path())
	}

	if ok, err := kmp.SafeEqual(configuredWebhook, webhook); err != nil {
		return fmt.Errorf("error diffing webhooks: %w", err)
	} else if !ok {
		logger.Info("Updating webhook")
		mwhclient := ac.client.AdmissionregistrationV1().MutatingWebhookConfigurations()
		if _, err := mwhclient.Update(ctx, webhook, metav1.UpdateOptions{}); err != nil {
			return fmt.Errorf("failed to update webhook: %w", err)
		}
	} else {
		logger.Info("Webhook is valid")
	}
	return nil
}

func (ac *reconciler) mutate(ctx context.Context, req *admissionv1.AdmissionRequest) ([]byte, error) {
	kind := req.Kind
	newBytes := req.Object.Raw
	oldBytes := req.OldObject.Raw
	// Why, oh why are these different types...
	gvk := schema.GroupVersionKind{
		Group:   kind.Group,
		Version: kind.Version,
		Kind:    kind.Kind,
	}

	logger := logging.FromContext(ctx)
	if gvk.Group != "" || gvk.Version != "v1" || gvk.Kind != "Pod" {
		logger.Error("Unhandled kind: ", gvk)
		return nil, fmt.Errorf("unhandled kind: %v", gvk)
	}

	// nil values denote absence of `old` (create) or `new` (delete) objects.
	var oldObj, newObj corev1.Pod

	if len(newBytes) != 0 {
		newDecoder := json.NewDecoder(bytes.NewBuffer(newBytes))
		if ac.disallowUnknownFields {
			newDecoder.DisallowUnknownFields()
		}
		if err := newDecoder.Decode(&newObj); err != nil {
			return nil, fmt.Errorf("cannot decode incoming new object: %w", err)
		}
	}
	if len(oldBytes) != 0 {
		oldDecoder := json.NewDecoder(bytes.NewBuffer(oldBytes))
		if ac.disallowUnknownFields {
			oldDecoder.DisallowUnknownFields()
		}
		if err := oldDecoder.Decode(&oldObj); err != nil {
			return nil, fmt.Errorf("cannot decode incoming old object: %w", err)
		}
	}
	var patches duck.JSONPatch

	var err error
	// Skip this step if the type we're dealing with is a duck type, since it is inherently
	// incomplete and this will patch away all of the unspecified fields.
	// Add these before defaulting fields, otherwise defaulting may cause an illegal patch
	// because it expects the round tripped through Golang fields to be present already.
	rtp, err := roundTripPatch(newBytes, newObj)
	if err != nil {
		return nil, fmt.Errorf("cannot create patch for round tripped newBytes: %w", err)
	}
	patches = append(patches, rtp...)

	ctx = apis.WithinCreate(ctx)
	ctx = apis.WithUserInfo(ctx, &req.UserInfo)

	// Default the new object.
	if patches, err = setDefaults(ctx, patches, newObj); err != nil {
		logger.Errorw("Failed the resource specific defaulter", zap.Error(err))
		// Return the error message as-is to give the defaulter callback
		// discretion over (our portion of) the message that the user sees.
		return nil, err
	}

	return json.Marshal(patches)
}

// roundTripPatch generates the JSONPatch that corresponds to round tripping the given bytes through
// the Golang type (JSON -> Golang type -> JSON). Because it is not always true that
// bytes == json.Marshal(json.Unmarshal(bytes)).
//
// For example, if bytes did not contain a 'spec' field and the Golang type specifies its 'spec'
// field without omitempty, then by round tripping through the Golang type, we would have added
// `'spec': {}`.
func roundTripPatch(bytes []byte, unmarshalled interface{}) (duck.JSONPatch, error) {
	if unmarshalled == nil {
		return duck.JSONPatch{}, nil
	}
	marshaledBytes, err := json.Marshal(unmarshalled)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal interface: %w", err)
	}
	return jsonpatch.CreatePatch(bytes, marshaledBytes)
}

// setDefaults simply leverages apis.Defaultable to set defaults.
func setDefaults(ctx context.Context, patches duck.JSONPatch, pod corev1.Pod) (duck.JSONPatch, error) {
	before, after := pod.DeepCopyObject(), pod

	if value, ok := after.GetLabels()["app.kubernetes.io/managed-by"]; ok && value == "tekton-pipelines" {
		var proxyEnv = []corev1.EnvVar{{
			Name:  "HTTPS_PROXY",
			Value: os.Getenv("HTTPS_PROXY"),
		}, {
			Name:  "HTTP_PROXY",
			Value: os.Getenv("HTTP_PROXY"),
		}, {
			Name:  "NO_PROXY",
			Value: os.Getenv("NO_PROXY"),
		}}

		if after.Spec.Containers != nil {
			for i, container := range after.Spec.Containers {
				newEnvs := updateAndMergeEnv(container.Env, proxyEnv)
				after.Spec.Containers[i].Env = newEnvs
			}
		}
	}

	patch, err := duck.CreatePatch(before, after)
	if err != nil {
		return nil, err
	}

	return append(patches, patch...), nil
}

// updateAndMergeEnv will merge two slices of env
// precedence will be given to second input if exist with same name key
func updateAndMergeEnv(containerenvs []corev1.EnvVar, proxyEnv []corev1.EnvVar) []corev1.EnvVar {
	for _, env := range proxyEnv {
		if env.Value == "" {
			// If value is empty then remove that key from container
			containerenvs = remove(containerenvs, env.Name)
		} else {
			var updated bool
			for i := range containerenvs {
				if env.Name == containerenvs[i].Name {
					containerenvs[i].Value = env.Value
					updated = true
				}
			}
			if !updated {
				containerenvs = append(containerenvs, corev1.EnvVar{
					Name:  env.Name,
					Value: env.Value,
				})
			}
		}
	}
	return containerenvs
}

func remove(env []corev1.EnvVar, key string) []corev1.EnvVar {
	for i := range env {
		if env[i].Name == key {
			return append(env[:i], env[i+1:]...)
		}
	}
	return env
}
<commit_msg>Filter objects on proxy webhook<commit_after>package proxy

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/markbates/inflect"
	"go.uber.org/zap"
	"gomodules.xyz/jsonpatch/v2"
	admissionv1 "k8s.io/api/admission/v1"
	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	admissionlisters "k8s.io/client-go/listers/admissionregistration/v1"
	corelisters "k8s.io/client-go/listers/core/v1"
	"knative.dev/pkg/apis"
	"knative.dev/pkg/apis/duck"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/kmp"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/ptr"
	pkgreconciler "knative.dev/pkg/reconciler"
	"knative.dev/pkg/system"
	"knative.dev/pkg/webhook"
	certresources "knative.dev/pkg/webhook/certificates/resources"
)

// reconciler implements the AdmissionController for resources
type reconciler struct {
	webhook.StatelessAdmissionImpl
	pkgreconciler.LeaderAwareFuncs

	key  types.NamespacedName
	path string

	withContext func(context.Context) context.Context

	client       kubernetes.Interface
	mwhlister    admissionlisters.MutatingWebhookConfigurationLister
	secretlister corelisters.SecretLister

	disallowUnknownFields bool
	secretName            string
}

var _ controller.Reconciler = (*reconciler)(nil)
var _ pkgreconciler.LeaderAware = (*reconciler)(nil)
var _ webhook.AdmissionController = (*reconciler)(nil)
var _ webhook.StatelessAdmissionController = (*reconciler)(nil)

// Reconcile implements controller.Reconciler
func (ac *reconciler) Reconcile(ctx context.Context, key string) error {
	logger := logging.FromContext(ctx)

	if !ac.IsLeaderFor(ac.key) {
		logger.Debugf("Skipping key %q, not the leader.", ac.key)
		return nil
	}

	// Look up the webhook secret, and fetch the CA cert bundle.
	secret, err := ac.secretlister.Secrets(system.Namespace()).Get(ac.secretName)
	if err != nil {
		logger.Errorw("Error fetching secret", zap.Error(err))
		return err
	}
	caCert, ok := secret.Data[certresources.CACert]
	if !ok {
		return fmt.Errorf("secret %q is missing %q key", ac.secretName, certresources.CACert)
	}

	// Reconcile the webhook configuration.
	return ac.reconcileMutatingWebhook(ctx, caCert)
}

// Path implements AdmissionController
func (ac *reconciler) Path() string {
	return ac.path
}

// Admit implements AdmissionController
func (ac *reconciler) Admit(ctx context.Context, request *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse {
	if ac.withContext != nil {
		ctx = ac.withContext(ctx)
	}

	logger := logging.FromContext(ctx)
	switch request.Operation {
	case admissionv1.Create:
	default:
		logger.Info("Unhandled webhook operation, letting it through ", request.Operation)
		return &admissionv1.AdmissionResponse{Allowed: true}
	}

	patchBytes, err := ac.mutate(ctx, request)
	if err != nil {
		return webhook.MakeErrorStatus("mutation failed: %v", err)
	}
	logger.Infof("Kind: %q PatchBytes: %v", request.Kind, string(patchBytes))

	return &admissionv1.AdmissionResponse{
		Patch:   patchBytes,
		Allowed: true,
		PatchType: func() *admissionv1.PatchType {
			pt := admissionv1.PatchTypeJSONPatch
			return &pt
		}(),
	}
}

func (ac *reconciler) reconcileMutatingWebhook(ctx context.Context, caCert []byte) error {
	logger := logging.FromContext(ctx)

	plural := strings.ToLower(inflect.Pluralize("Pod"))
	rules := []admissionregistrationv1.RuleWithOperations{
		{
			Operations: []admissionregistrationv1.OperationType{
				admissionregistrationv1.Create,
			},
			Rule: admissionregistrationv1.Rule{
				APIGroups:   []string{""},
				APIVersions: []string{"v1"},
				Resources:   []string{plural, plural + "/status"},
			},
		},
	}

	configuredWebhook, err := ac.mwhlister.Get(ac.key.Name)
	if err != nil {
		return fmt.Errorf("error retrieving webhook: %w", err)
	}

	webhook := configuredWebhook.DeepCopy()

	// Clear out any previous (bad) OwnerReferences.
	// See: https://github.com/knative/serving/issues/5845
	webhook.OwnerReferences = nil

	for i, wh := range webhook.Webhooks {
		if wh.Name != webhook.Name {
			continue
		}
		webhook.Webhooks[i].Rules = rules
		webhook.Webhooks[i].NamespaceSelector = &metav1.LabelSelector{
			MatchExpressions: []metav1.LabelSelectorRequirement{{
				Key:      "operator.tekton.dev/disable-proxy",
				Operator: metav1.LabelSelectorOpDoesNotExist,
			}, {
				// "control-plane" is added to support Azure's AKS, otherwise the controllers fight.
				// See knative/pkg#1590 for details.
				Key:      "control-plane",
				Operator: metav1.LabelSelectorOpDoesNotExist,
			}},
		}
		webhook.Webhooks[i].ObjectSelector = &metav1.LabelSelector{
			MatchLabels: map[string]string{
				"app.kubernetes.io/managed-by": "tekton-pipelines",
			},
		}
		webhook.Webhooks[i].ClientConfig.CABundle = caCert
		if webhook.Webhooks[i].ClientConfig.Service == nil {
			return fmt.Errorf("missing service reference for webhook: %s", wh.Name)
		}
		webhook.Webhooks[i].ClientConfig.Service.Path = ptr.String(ac.Path())
	}

	if ok, err := kmp.SafeEqual(configuredWebhook, webhook); err != nil {
		return fmt.Errorf("error diffing webhooks: %w", err)
	} else if !ok {
		logger.Info("Updating webhook")
		mwhclient := ac.client.AdmissionregistrationV1().MutatingWebhookConfigurations()
		if _, err := mwhclient.Update(ctx, webhook, metav1.UpdateOptions{}); err != nil {
			return fmt.Errorf("failed to update webhook: %w", err)
		}
	} else {
		logger.Info("Webhook is valid")
	}
	return nil
}

func (ac *reconciler) mutate(ctx context.Context, req *admissionv1.AdmissionRequest) ([]byte, error) {
	kind := req.Kind
	newBytes := req.Object.Raw
	oldBytes := req.OldObject.Raw
	// Why, oh why are these different types...
	gvk := schema.GroupVersionKind{
		Group:   kind.Group,
		Version: kind.Version,
		Kind:    kind.Kind,
	}

	logger := logging.FromContext(ctx)
	if gvk.Group != "" || gvk.Version != "v1" || gvk.Kind != "Pod" {
		logger.Error("Unhandled kind: ", gvk)
		return nil, fmt.Errorf("unhandled kind: %v", gvk)
	}

	// nil values denote absence of `old` (create) or `new` (delete) objects.
	var oldObj, newObj corev1.Pod

	if len(newBytes) != 0 {
		newDecoder := json.NewDecoder(bytes.NewBuffer(newBytes))
		if ac.disallowUnknownFields {
			newDecoder.DisallowUnknownFields()
		}
		if err := newDecoder.Decode(&newObj); err != nil {
			return nil, fmt.Errorf("cannot decode incoming new object: %w", err)
		}
	}
	if len(oldBytes) != 0 {
		oldDecoder := json.NewDecoder(bytes.NewBuffer(oldBytes))
		if ac.disallowUnknownFields {
			oldDecoder.DisallowUnknownFields()
		}
		if err := oldDecoder.Decode(&oldObj); err != nil {
			return nil, fmt.Errorf("cannot decode incoming old object: %w", err)
		}
	}
	var patches duck.JSONPatch

	var err error
	// Skip this step if the type we're dealing with is a duck type, since it is inherently
	// incomplete and this will patch away all of the unspecified fields.
	// Add these before defaulting fields, otherwise defaulting may cause an illegal patch
	// because it expects the round tripped through Golang fields to be present already.
	rtp, err := roundTripPatch(newBytes, newObj)
	if err != nil {
		return nil, fmt.Errorf("cannot create patch for round tripped newBytes: %w", err)
	}
	patches = append(patches, rtp...)

	ctx = apis.WithinCreate(ctx)
	ctx = apis.WithUserInfo(ctx, &req.UserInfo)

	// Default the new object.
	if patches, err = setDefaults(ctx, patches, newObj); err != nil {
		logger.Errorw("Failed the resource specific defaulter", zap.Error(err))
		// Return the error message as-is to give the defaulter callback
		// discretion over (our portion of) the message that the user sees.
		return nil, err
	}

	return json.Marshal(patches)
}

// roundTripPatch generates the JSONPatch that corresponds to round tripping the given bytes through
// the Golang type (JSON -> Golang type -> JSON). Because it is not always true that
// bytes == json.Marshal(json.Unmarshal(bytes)).
//
// For example, if bytes did not contain a 'spec' field and the Golang type specifies its 'spec'
// field without omitempty, then by round tripping through the Golang type, we would have added
// `'spec': {}`.
func roundTripPatch(bytes []byte, unmarshalled interface{}) (duck.JSONPatch, error) {
	if unmarshalled == nil {
		return duck.JSONPatch{}, nil
	}
	marshaledBytes, err := json.Marshal(unmarshalled)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal interface: %w", err)
	}
	return jsonpatch.CreatePatch(bytes, marshaledBytes)
}

// setDefaults simply leverages apis.Defaultable to set defaults.
func setDefaults(ctx context.Context, patches duck.JSONPatch, pod corev1.Pod) (duck.JSONPatch, error) {
	before, after := pod.DeepCopyObject(), pod

	var proxyEnv = []corev1.EnvVar{{
		Name:  "HTTPS_PROXY",
		Value: os.Getenv("HTTPS_PROXY"),
	}, {
		Name:  "HTTP_PROXY",
		Value: os.Getenv("HTTP_PROXY"),
	}, {
		Name:  "NO_PROXY",
		Value: os.Getenv("NO_PROXY"),
	}}

	if after.Spec.Containers != nil {
		for i, container := range after.Spec.Containers {
			newEnvs := updateAndMergeEnv(container.Env, proxyEnv)
			after.Spec.Containers[i].Env = newEnvs
		}
	}

	patch, err := duck.CreatePatch(before, after)
	if err != nil {
		return nil, err
	}

	return append(patches, patch...), nil
}

// updateAndMergeEnv will merge two slices of env
// precedence will be given to second input if exist with same name key
func updateAndMergeEnv(containerenvs []corev1.EnvVar, proxyEnv []corev1.EnvVar) []corev1.EnvVar {
	for _, env := range proxyEnv {
		if env.Value == "" {
			// If value is empty then remove that key from container
			containerenvs = remove(containerenvs, env.Name)
		} else {
			var updated bool
			for i := range containerenvs {
				if env.Name == containerenvs[i].Name {
					containerenvs[i].Value = env.Value
					updated = true
				}
			}
			if !updated {
				containerenvs = append(containerenvs, corev1.EnvVar{
					Name:  env.Name,
					Value: env.Value,
				})
			}
		}
	}
	return containerenvs
}

func remove(env []corev1.EnvVar, key string) []corev1.EnvVar {
	for i := range env {
		if env[i].Name == key {
			return append(env[:i], env[i+1:]...)
		}
	}
	return env
}
<|endoftext|>


<commit_before>package flags_test

import (
	"github.com/jessevdk/go-flags"
	"testing"
)

func TestShort(t *testing.T) {
	var opts = struct {
		Value bool `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-v")

	assertStringArray(t, ret, []string{})

	if !opts.Value {
		t.Errorf("Expected Value to be true")
	}
}

func TestShortMultiConcat(t *testing.T) {
	var opts = struct {
		V bool `short:"v"`
		O bool `short:"o"`
		F bool `short:"f"`
	}{}

	ret := assertParseSuccess(t, &opts, "-vo", "-f")

	assertStringArray(t, ret, []string{})

	if !opts.V {
		t.Errorf("Expected V to be true")
	}

	if !opts.O {
		t.Errorf("Expected O to be true")
	}

	if !opts.F {
		t.Errorf("Expected F to be true")
	}
}

func TestShortMultiSlice(t *testing.T) {
	var opts = struct {
		Values []bool `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-v", "-v")

	assertStringArray(t, ret, []string{})
	assertBoolArray(t, opts.Values, []bool{true, true})
}

func TestShortMultiSliceConcat(t *testing.T) {
	var opts = struct {
		Values []bool `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-vvv")

	assertStringArray(t, ret, []string{})
	assertBoolArray(t, opts.Values, []bool{true, true, true})
}

func TestShortWithEqualArg(t *testing.T) {
	var opts = struct {
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-v=value")

	assertStringArray(t, ret, []string{})
	assertString(t, opts.Value, "value")
}

func TestShortWithArg(t *testing.T) {
	var opts = struct {
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-vvalue")

	assertStringArray(t, ret, []string{})
	assertString(t, opts.Value, "value")
}

func TestShortArg(t *testing.T) {
	var opts = struct {
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-v", "value")

	assertStringArray(t, ret, []string{})
	assertString(t, opts.Value, "value")
}

func TestShortMultiWithEqualArg(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v"`
	}{}

	assertParseFail(t, flags.ErrExpectedArgument, "expected argument for flag `-v'", &opts, "-ffv=value")
}

func TestShortMultiArg(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-ffv", "value")

	assertStringArray(t, ret, []string{})
	assertBoolArray(t, opts.F, []bool{true, true})
	assertString(t, opts.Value, "value")
}

func TestShortMultiArgConcatFail(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v"`
	}{}

	assertParseFail(t, flags.ErrExpectedArgument, "expected argument for flag `-v'", &opts, "-ffvvalue")
}

func TestShortMultiArgConcat(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-vff")

	assertStringArray(t, ret, []string{})
	assertString(t, opts.Value, "ff")
}

func TestShortOptional(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v" optional:"yes" optional-value:"value"`
	}{}

	ret := assertParseSuccess(t, &opts, "-fv", "f")

	assertStringArray(t, ret, []string{"f"})
	assertString(t, opts.Value, "value")
}
<commit_msg>Added test for ErrShortNameTooLong<commit_after>package flags_test

import (
	"github.com/jessevdk/go-flags"
	"testing"
)

func TestShort(t *testing.T) {
	var opts = struct {
		Value bool `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-v")

	assertStringArray(t, ret, []string{})

	if !opts.Value {
		t.Errorf("Expected Value to be true")
	}
}

func TestShortTooLong(t *testing.T) {
	var opts = struct {
		Value bool `short:"vv"`
	}{}

	assertParseFail(t, flags.ErrShortNameTooLong, "short names can only be 1 character long, not `vv'", &opts)
}

func TestShortMultiConcat(t *testing.T) {
	var opts = struct {
		V bool `short:"v"`
		O bool `short:"o"`
		F bool `short:"f"`
	}{}

	ret := assertParseSuccess(t, &opts, "-vo", "-f")

	assertStringArray(t, ret, []string{})

	if !opts.V {
		t.Errorf("Expected V to be true")
	}

	if !opts.O {
		t.Errorf("Expected O to be true")
	}

	if !opts.F {
		t.Errorf("Expected F to be true")
	}
}

func TestShortMultiSlice(t *testing.T) {
	var opts = struct {
		Values []bool `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-v", "-v")

	assertStringArray(t, ret, []string{})
	assertBoolArray(t, opts.Values, []bool{true, true})
}

func TestShortMultiSliceConcat(t *testing.T) {
	var opts = struct {
		Values []bool `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-vvv")

	assertStringArray(t, ret, []string{})
	assertBoolArray(t, opts.Values, []bool{true, true, true})
}

func TestShortWithEqualArg(t *testing.T) {
	var opts = struct {
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-v=value")

	assertStringArray(t, ret, []string{})
	assertString(t, opts.Value, "value")
}

func TestShortWithArg(t *testing.T) {
	var opts = struct {
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-vvalue")

	assertStringArray(t, ret, []string{})
	assertString(t, opts.Value, "value")
}

func TestShortArg(t *testing.T) {
	var opts = struct {
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-v", "value")

	assertStringArray(t, ret, []string{})
	assertString(t, opts.Value, "value")
}

func TestShortMultiWithEqualArg(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v"`
	}{}

	assertParseFail(t, flags.ErrExpectedArgument, "expected argument for flag `-v'", &opts, "-ffv=value")
}

func TestShortMultiArg(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-ffv", "value")

	assertStringArray(t, ret, []string{})
	assertBoolArray(t, opts.F, []bool{true, true})
	assertString(t, opts.Value, "value")
}

func TestShortMultiArgConcatFail(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v"`
	}{}

	assertParseFail(t, flags.ErrExpectedArgument, "expected argument for flag `-v'", &opts, "-ffvvalue")
}

func TestShortMultiArgConcat(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v"`
	}{}

	ret := assertParseSuccess(t, &opts, "-vff")

	assertStringArray(t, ret, []string{})
	assertString(t, opts.Value, "ff")
}

func TestShortOptional(t *testing.T) {
	var opts = struct {
		F     []bool `short:"f"`
		Value string `short:"v" optional:"yes" optional-value:"value"`
	}{}

	ret := assertParseSuccess(t, &opts, "-fv", "f")

	assertStringArray(t, ret, []string{"f"})
	assertString(t, opts.Value, "value")
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"net/http"

	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func AddTag(src, target string) error {
	logrus.Debugf("attempting to add tag %s to src %s", target, src)
	srcRef, err := name.ParseReference(src, name.StrictValidation)
	if err != nil {
		return errors.Wrap(err, "getting source reference")
	}

	auth, err := authn.DefaultKeychain.Resolve(srcRef.Context().Registry)
	if err != nil {
		return err
	}

	targetRef, err := name.ParseReference(target, name.StrictValidation)
	if err != nil {
		return errors.Wrap(err, "getting target reference")
	}

	return addTag(srcRef, targetRef, auth, http.DefaultTransport)
}

func addTag(ref name.Reference, targetRef name.Reference, auth authn.Authenticator, t http.RoundTripper) error {
	tr, err := transport.New(ref.Context().Registry, auth, t, []string{targetRef.Scope(transport.PushScope)})
	if err != nil {
		return err
	}

	img, err := remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(tr))
	if err != nil {
		return err
	}

	return remote.Write(targetRef, img, auth, t)
}

func RemoteDigest(identifier string) (string, error) {
	img, err := remoteImage(identifier)
	if err != nil {
		return "", errors.Wrap(err, "getting image")
	}

	h, err := img.Digest()
	if err != nil {
		return "", errors.Wrap(err, "getting digest")
	}

	return h.String(), nil
}

func retrieveRemoteConfig(identifier string) (*v1.ConfigFile, error) {
	img, err := remoteImage(identifier)
	if err != nil {
		return nil, errors.Wrap(err, "getting image")
	}

	return img.ConfigFile()
}

func remoteImage(identifier string) (v1.Image, error) {
	ref, err := name.ParseReference(identifier, name.StrictValidation)
	if err != nil {
		return nil, errors.Wrap(err, "parsing initial ref")
	}

	auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry)
	if err != nil {
		return nil, errors.Wrap(err, "getting default keychain auth")
	}

	return remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport))
}
<commit_msg>Fix tagging image built by Jib<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"net/http"

	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func AddTag(src, target string) error {
	logrus.Debugf("attempting to add tag %s to src %s", target, src)
	img, err := remoteImage(src)
	if err != nil {
		return errors.Wrap(err, "getting image")
	}

	targetRef, err := name.ParseReference(target, name.StrictValidation)
	if err != nil {
		return errors.Wrap(err, "getting target reference")
	}

	auth, err := authn.DefaultKeychain.Resolve(targetRef.Context().Registry)
	if err != nil {
		return err
	}

	return remote.Write(targetRef, img, auth, http.DefaultTransport)
}

func RemoteDigest(identifier string) (string, error) {
	img, err := remoteImage(identifier)
	if err != nil {
		return "", errors.Wrap(err, "getting image")
	}

	h, err := img.Digest()
	if err != nil {
		return "", errors.Wrap(err, "getting digest")
	}

	return h.String(), nil
}

func retrieveRemoteConfig(identifier string) (*v1.ConfigFile, error) {
	img, err := remoteImage(identifier)
	if err != nil {
		return nil, errors.Wrap(err, "getting image")
	}

	return img.ConfigFile()
}

func remoteImage(identifier string) (v1.Image, error) {
	ref, err := name.ParseReference(identifier, name.StrictValidation)
	if err != nil {
		return nil, errors.Wrap(err, "parsing initial ref")
	}

	auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry)
	if err != nil {
		return nil, errors.Wrap(err, "getting default keychain auth")
	}

	return remote.Image(ref, remote.WithAuth(auth))
}
<|endoftext|>


<commit_before>package opentsdb

import (
	"context"
	"fmt"
	"path"
	"strconv"
	"strings"

	"golang.org/x/net/context/ctxhttp"

	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/tsdb"
)

type OpenTsdbExecutor struct {
	//*models.DataSource
	//httpClient *http.Client
}

func NewOpenTsdbExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	/*
		httpClient, err := datasource.GetHttpClient()

		if err != nil {
			return nil, err
		}
	*/

	return &OpenTsdbExecutor{
	//DataSource: datasource,
	//httpClient: httpClient,
	}, nil
}

var (
	plog log.Logger
)

func init() {
	plog = log.New("tsdb.opentsdb")
	tsdb.RegisterTsdbQueryEndpoint("opentsdb", NewOpenTsdbExecutor)
}

func (e *OpenTsdbExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	result := &tsdb.Response{}

	var tsdbQuery OpenTsdbQuery

	tsdbQuery.Start = queryContext.TimeRange.GetFromAsMsEpoch()
	tsdbQuery.End = queryContext.TimeRange.GetToAsMsEpoch()

	for _, query := range queryContext.Queries {
		metric := e.buildMetric(query)
		tsdbQuery.Queries = append(tsdbQuery.Queries, metric)
	}

	if setting.Env == setting.DEV {
		plog.Debug("OpenTsdb request", "params", tsdbQuery)
	}

	req, err := e.createRequest(dsInfo, tsdbQuery)
	if err != nil {
		return nil, err
	}

	httpClient, err := dsInfo.GetHttpClient()
	if err != nil {
		return nil, err
	}

	res, err := ctxhttp.Do(ctx, httpClient, req)
	if err != nil {
		return nil, err
	}

	queryResult, err := e.parseResponse(tsdbQuery, res)
	if err != nil {
		return nil, err
	}

	result.Results = queryResult
	return result, nil
}

func (e *OpenTsdbExecutor) createRequest(dsInfo *models.DataSource, data OpenTsdbQuery) (*http.Request, error) {
	u, _ := url.Parse(dsInfo.Url)
	u.Path = path.Join(u.Path, "api/query")

	postData, err := json.Marshal(data)

	req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(string(postData)))
	if err != nil {
		plog.Info("Failed to create request", "error", err)
		return nil, fmt.Errorf("Failed to create request. error: %v", err)
	}

	req.Header.Set("Content-Type", "application/json")
	if dsInfo.BasicAuth {
		req.SetBasicAuth(dsInfo.BasicAuthUser, dsInfo.BasicAuthPassword)
	}

	return req, err
}

func (e *OpenTsdbExecutor) parseResponse(query OpenTsdbQuery, res *http.Response) (map[string]*tsdb.QueryResult, error) {

	queryResults := make(map[string]*tsdb.QueryResult)
	queryRes := tsdb.NewQueryResult()

	body, err := ioutil.ReadAll(res.Body)
	defer res.Body.Close()
	if err != nil {
		return nil, err
	}

	if res.StatusCode/100 != 2 {
		plog.Info("Request failed", "status", res.Status, "body", string(body))
		return nil, fmt.Errorf("Request failed status: %v", res.Status)
	}

	var data []OpenTsdbResponse
	err = json.Unmarshal(body, &data)
	if err != nil {
		plog.Info("Failed to unmarshal opentsdb response", "error", err, "status", res.Status, "body", string(body))
		return nil, err
	}

	for _, val := range data {
		series := tsdb.TimeSeries{
			Name: val.Metric,
		}

		for timeString, value := range val.DataPoints {
			timestamp, err := strconv.ParseFloat(timeString, 64)
			if err != nil {
				plog.Info("Failed to unmarshal opentsdb timestamp", "timestamp", timeString)
				return nil, err
			}
			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), timestamp))
		}

		queryRes.Series = append(queryRes.Series, &series)
	}

	queryResults["A"] = queryRes
	return queryResults, nil
}

func (e *OpenTsdbExecutor) buildMetric(query *tsdb.Query) map[string]interface{} {

	metric := make(map[string]interface{})

	// Setting metric and aggregator
	metric["metric"] = query.Model.Get("metric").MustString()
	metric["aggregator"] = query.Model.Get("aggregator").MustString()

	// Setting downsampling options
	disableDownsampling := query.Model.Get("disableDownsampling").MustBool()
	if !disableDownsampling {
		downsampleInterval := query.Model.Get("downsampleInterval").MustString()
		if downsampleInterval == "" {
			downsampleInterval = "1m" //default value for blank
		}
		downsample := downsampleInterval + "-" + query.Model.Get("downsampleAggregator").MustString()
		if query.Model.Get("downsampleFillPolicy").MustString() != "none" {
			metric["downsample"] = downsample + "-" + query.Model.Get("downsampleFillPolicy").MustString()
		} else {
			metric["downsample"] = downsample
		}
	}

	// Setting rate options
	if query.Model.Get("shouldComputeRate").MustBool() {

		metric["rate"] = true
		rateOptions := make(map[string]interface{})
		rateOptions["counter"] = query.Model.Get("isCounter").MustBool()

		counterMax, counterMaxCheck := query.Model.CheckGet("counterMax")
		if counterMaxCheck {
			rateOptions["counterMax"] = counterMax.MustFloat64()
		}

		resetValue, resetValueCheck := query.Model.CheckGet("counterResetValue")
		if resetValueCheck {
			rateOptions["resetValue"] = resetValue.MustFloat64()
		}

		if !counterMaxCheck && (!resetValueCheck || resetValue.MustFloat64() == 0) {
			rateOptions["dropResets"] = true
		}

		metric["rateOptions"] = rateOptions
	}

	// Setting tags
	tags, tagsCheck := query.Model.CheckGet("tags")
	if tagsCheck && len(tags.MustMap()) > 0 {
		metric["tags"] = tags.MustMap()
	}

	// Setting filters
	filters, filtersCheck := query.Model.CheckGet("filters")
	if filtersCheck && len(filters.MustArray()) > 0 {
		metric["filters"] = filters.MustArray()
	}

	return metric

}
<commit_msg>removes commented code<commit_after>package opentsdb

import (
	"context"
	"fmt"
	"path"
	"strconv"
	"strings"

	"golang.org/x/net/context/ctxhttp"

	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/tsdb"
)

type OpenTsdbExecutor struct {
}

func NewOpenTsdbExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &OpenTsdbExecutor{}, nil
}

var (
	plog log.Logger
)

func init() {
	plog = log.New("tsdb.opentsdb")
	tsdb.RegisterTsdbQueryEndpoint("opentsdb", NewOpenTsdbExecutor)
}

func (e *OpenTsdbExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	result := &tsdb.Response{}

	var tsdbQuery OpenTsdbQuery

	tsdbQuery.Start = queryContext.TimeRange.GetFromAsMsEpoch()
	tsdbQuery.End = queryContext.TimeRange.GetToAsMsEpoch()

	for _, query := range queryContext.Queries {
		metric := e.buildMetric(query)
		tsdbQuery.Queries = append(tsdbQuery.Queries, metric)
	}

	if setting.Env == setting.DEV {
		plog.Debug("OpenTsdb request", "params", tsdbQuery)
	}

	req, err := e.createRequest(dsInfo, tsdbQuery)
	if err != nil {
		return nil, err
	}

	httpClient, err := dsInfo.GetHttpClient()
	if err != nil {
		return nil, err
	}

	res, err := ctxhttp.Do(ctx, httpClient, req)
	if err != nil {
		return nil, err
	}

	queryResult, err := e.parseResponse(tsdbQuery, res)
	if err != nil {
		return nil, err
	}

	result.Results = queryResult
	return result, nil
}

func (e *OpenTsdbExecutor) createRequest(dsInfo *models.DataSource, data OpenTsdbQuery) (*http.Request, error) {
	u, _ := url.Parse(dsInfo.Url)
	u.Path = path.Join(u.Path, "api/query")

	postData, err := json.Marshal(data)

	req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(string(postData)))
	if err != nil {
		plog.Info("Failed to create request", "error", err)
		return nil, fmt.Errorf("Failed to create request. error: %v", err)
	}

	req.Header.Set("Content-Type", "application/json")
	if dsInfo.BasicAuth {
		req.SetBasicAuth(dsInfo.BasicAuthUser, dsInfo.BasicAuthPassword)
	}

	return req, err
}

func (e *OpenTsdbExecutor) parseResponse(query OpenTsdbQuery, res *http.Response) (map[string]*tsdb.QueryResult, error) {

	queryResults := make(map[string]*tsdb.QueryResult)
	queryRes := tsdb.NewQueryResult()

	body, err := ioutil.ReadAll(res.Body)
	defer res.Body.Close()
	if err != nil {
		return nil, err
	}

	if res.StatusCode/100 != 2 {
		plog.Info("Request failed", "status", res.Status, "body", string(body))
		return nil, fmt.Errorf("Request failed status: %v", res.Status)
	}

	var data []OpenTsdbResponse
	err = json.Unmarshal(body, &data)
	if err != nil {
		plog.Info("Failed to unmarshal opentsdb response", "error", err, "status", res.Status, "body", string(body))
		return nil, err
	}

	for _, val := range data {
		series := tsdb.TimeSeries{
			Name: val.Metric,
		}

		for timeString, value := range val.DataPoints {
			timestamp, err := strconv.ParseFloat(timeString, 64)
			if err != nil {
				plog.Info("Failed to unmarshal opentsdb timestamp", "timestamp", timeString)
				return nil, err
			}
			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), timestamp))
		}

		queryRes.Series = append(queryRes.Series, &series)
	}

	queryResults["A"] = queryRes
	return queryResults, nil
}

func (e *OpenTsdbExecutor) buildMetric(query *tsdb.Query) map[string]interface{} {

	metric := make(map[string]interface{})

	// Setting metric and aggregator
	metric["metric"] = query.Model.Get("metric").MustString()
	metric["aggregator"] = query.Model.Get("aggregator").MustString()

	// Setting downsampling options
	disableDownsampling := query.Model.Get("disableDownsampling").MustBool()
	if !disableDownsampling {
		downsampleInterval := query.Model.Get("downsampleInterval").MustString()
		if downsampleInterval == "" {
			downsampleInterval = "1m" //default value for blank
		}
		downsample := downsampleInterval + "-" + query.Model.Get("downsampleAggregator").MustString()
		if query.Model.Get("downsampleFillPolicy").MustString() != "none" {
			metric["downsample"] = downsample + "-" + query.Model.Get("downsampleFillPolicy").MustString()
		} else {
			metric["downsample"] = downsample
		}
	}

	// Setting rate options
	if query.Model.Get("shouldComputeRate").MustBool() {

		metric["rate"] = true
		rateOptions := make(map[string]interface{})
		rateOptions["counter"] = query.Model.Get("isCounter").MustBool()

		counterMax, counterMaxCheck := query.Model.CheckGet("counterMax")
		if counterMaxCheck {
			rateOptions["counterMax"] = counterMax.MustFloat64()
		}

		resetValue, resetValueCheck := query.Model.CheckGet("counterResetValue")
		if resetValueCheck {
			rateOptions["resetValue"] = resetValue.MustFloat64()
		}

		if !counterMaxCheck && (!resetValueCheck || resetValue.MustFloat64() == 0) {
			rateOptions["dropResets"] = true
		}

		metric["rateOptions"] = rateOptions
	}

	// Setting tags
	tags, tagsCheck := query.Model.CheckGet("tags")
	if tagsCheck && len(tags.MustMap()) > 0 {
		metric["tags"] = tags.MustMap()
	}

	// Setting filters
	filters, filtersCheck := query.Model.CheckGet("filters")
	if filtersCheck && len(filters.MustArray()) > 0 {
		metric["filters"] = filters.MustArray()
	}

	return metric

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"

	"github.com/mitchellh/go-wordwrap"
)

type RoomInfo struct {
	Title    string `json:"title"`
	ReadCode string `json:"readCode"`
}

// SlugInfo describes what URL corresponds to what RP
type SlugInfo struct {
	Rpid   string `json:"rpid"`
	Access string `json:"access"`
}

type Doc interface {
	Meta() *DocMeta
	ParseBody(io.Reader) error
	Validate() error
	CheckRelations() error
}

type DocMeta struct {
	ID        string    `json:"_id"`
	Revision  int       `json:"revision"`
	Timestamp time.Time `json:"timestamp"`
	Userid    string    `json:"userid"`
}

type RpMessageBody struct {
	Type    string `json:"type"`
	Content string `json:"content,omitempty"`
	URL     string `json:"url,omitempty"`
	CharaID string `json:"charaId,omitempty"`
}
type RpMessage struct {
	*RpMessageBody
	*DocMeta
}

func NewRpMessage() RpMessage {
	return RpMessage{&RpMessageBody{}, &DocMeta{}}
}

func (x RpMessage) ParseBody(j io.Reader) error {
	return json.NewDecoder(j).Decode(x.RpMessageBody)
}

func (x RpMessage) Meta() *DocMeta {
	return x.DocMeta
}

type RpCharaBody struct {
	Name  string `json:"name"`
	Color string `json:"color"`
}
type RpChara struct {
	*RpCharaBody
	*DocMeta
}

func NewRpChara() RpChara {
	return RpChara{&RpCharaBody{}, &DocMeta{}}
}

func (x RpChara) ParseBody(j io.Reader) error {
	return json.NewDecoder(j).Decode(x.RpCharaBody)
}

func (x RpChara) Meta() *DocMeta {
	return x.DocMeta
}

func (m RpMessageBody) Validate() error {
	if m.Type == "image" {
		if m.Content != "" {
			return fmt.Errorf("Msg: image should not have 'content'")
		}
		if m.CharaID != "" {
			return fmt.Errorf("Msg: image should not have 'charaId'")
		}
		if len(m.URL) > 1000 {
			return fmt.Errorf("Msg: url is too long (%d characters)", len(m.Content))
		}
		urlRegexp := regexp.MustCompile("^https?://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+$")
		if !urlRegexp.MatchString(m.URL) {
			return fmt.Errorf("Msg: url is invalid")
		}
		return nil
	} else if m.Type == "narrator" || m.Type == "chara" || m.Type == "ooc" {
		if m.URL != "" {
			return fmt.Errorf("Msg: non-image should not have 'url'")
		}
		if m.Content == "" {
			return fmt.Errorf("Msg: content is empty")
		}
		if len(m.Content) > 10000 {
			return fmt.Errorf("Msg: content is too long (%d characters)", len(m.Content))
		}
		if m.Type == "chara" {
			if m.CharaID == "" {
				return fmt.Errorf("Msg: charaId is empty")
			}
		} else {
			if m.CharaID != "" {
				return fmt.Errorf("Msg: non-chara msg should not have 'charaId'")
			}
		}
		return nil
	} else {
		return fmt.Errorf("Msg: invalid type")
	}
}

func (m RpMessageBody) CheckRelations() error {
	if m.Type == "chara" {
		// TODO check if the charaId is in the db
	}
	return nil
}

func (c RpCharaBody) Validate() error {
	if len(c.Name) == 0 {
		return fmt.Errorf("Chara: name is empty")
	}
	if len(c.Name) > 30 {
		return fmt.Errorf("Chara: name is too long")
	}
	colorRegexp := regexp.MustCompile("^#[0-9a-f]{6}$")
	if !colorRegexp.MatchString(c.Color) {
		return fmt.Errorf("Chara: color is invalid")
	}
	return nil
}

func (c RpCharaBody) CheckRelations() error {
	return nil
}

func (m RpMessage) ToTxt(chara *RpChara) string {
	if m.Type == "chara" {
		return strings.ToUpper(chara.Name) + ":\r\n  " + strings.Replace(wordwrap.WrapString(m.Content, 70), "\n", "\r\n  ", 0)
	}
	if m.Type == "ooc" {
		return strings.Replace(wordwrap.WrapString("(( OOC: "+m.Content+" ))", 72), "\n", "\r\n", 0)
	}
	if m.Type == "narrator" {
		return strings.Replace(wordwrap.WrapString(m.Content, 72), "\n", "\r\n", 0)
	}
	if m.Type == "image" {
		return "--- IMAGE ---\r\n" + m.URL + "\r\n-------------"
		return m.URL
	}
	panic("What kind of message is this?")
}
<commit_msg>Increase max msg length<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"

	"github.com/mitchellh/go-wordwrap"
)

type RoomInfo struct {
	Title    string `json:"title"`
	ReadCode string `json:"readCode"`
}

// SlugInfo describes what URL corresponds to what RP
type SlugInfo struct {
	Rpid   string `json:"rpid"`
	Access string `json:"access"`
}

type Doc interface {
	Meta() *DocMeta
	ParseBody(io.Reader) error
	Validate() error
	CheckRelations() error
}

type DocMeta struct {
	ID        string    `json:"_id"`
	Revision  int       `json:"revision"`
	Timestamp time.Time `json:"timestamp"`
	Userid    string    `json:"userid"`
}

type RpMessageBody struct {
	Type    string `json:"type"`
	Content string `json:"content,omitempty"`
	URL     string `json:"url,omitempty"`
	CharaID string `json:"charaId,omitempty"`
}
type RpMessage struct {
	*RpMessageBody
	*DocMeta
}

func NewRpMessage() RpMessage {
	return RpMessage{&RpMessageBody{}, &DocMeta{}}
}

func (x RpMessage) ParseBody(j io.Reader) error {
	return json.NewDecoder(j).Decode(x.RpMessageBody)
}

func (x RpMessage) Meta() *DocMeta {
	return x.DocMeta
}

type RpCharaBody struct {
	Name  string `json:"name"`
	Color string `json:"color"`
}
type RpChara struct {
	*RpCharaBody
	*DocMeta
}

func NewRpChara() RpChara {
	return RpChara{&RpCharaBody{}, &DocMeta{}}
}

func (x RpChara) ParseBody(j io.Reader) error {
	return json.NewDecoder(j).Decode(x.RpCharaBody)
}

func (x RpChara) Meta() *DocMeta {
	return x.DocMeta
}

func (m RpMessageBody) Validate() error {
	if m.Type == "image" {
		if m.Content != "" {
			return fmt.Errorf("Msg: image should not have 'content'")
		}
		if m.CharaID != "" {
			return fmt.Errorf("Msg: image should not have 'charaId'")
		}
		if len(m.URL) > 1000 {
			return fmt.Errorf("Msg: url is too long (%d characters)", len(m.Content))
		}
		urlRegexp := regexp.MustCompile("^https?://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+$")
		if !urlRegexp.MatchString(m.URL) {
			return fmt.Errorf("Msg: url is invalid")
		}
		return nil
	} else if m.Type == "narrator" || m.Type == "chara" || m.Type == "ooc" {
		if m.URL != "" {
			return fmt.Errorf("Msg: non-image should not have 'url'")
		}
		if m.Content == "" {
			return fmt.Errorf("Msg: content is empty")
		}
		if len(m.Content) > 50000 {
			return fmt.Errorf("Msg: content is too long (%d characters)", len(m.Content))
		}
		if m.Type == "chara" {
			if m.CharaID == "" {
				return fmt.Errorf("Msg: charaId is empty")
			}
		} else {
			if m.CharaID != "" {
				return fmt.Errorf("Msg: non-chara msg should not have 'charaId'")
			}
		}
		return nil
	} else {
		return fmt.Errorf("Msg: invalid type")
	}
}

func (m RpMessageBody) CheckRelations() error {
	if m.Type == "chara" {
		// TODO check if the charaId is in the db
	}
	return nil
}

func (c RpCharaBody) Validate() error {
	if len(c.Name) == 0 {
		return fmt.Errorf("Chara: name is empty")
	}
	if len(c.Name) > 30 {
		return fmt.Errorf("Chara: name is too long")
	}
	colorRegexp := regexp.MustCompile("^#[0-9a-f]{6}$")
	if !colorRegexp.MatchString(c.Color) {
		return fmt.Errorf("Chara: color is invalid")
	}
	return nil
}

func (c RpCharaBody) CheckRelations() error {
	return nil
}

func (m RpMessage) ToTxt(chara *RpChara) string {
	if m.Type == "chara" {
		return strings.ToUpper(chara.Name) + ":\r\n  " + strings.Replace(wordwrap.WrapString(m.Content, 70), "\n", "\r\n  ", 0)
	}
	if m.Type == "ooc" {
		return strings.Replace(wordwrap.WrapString("(( OOC: "+m.Content+" ))", 72), "\n", "\r\n", 0)
	}
	if m.Type == "narrator" {
		return strings.Replace(wordwrap.WrapString(m.Content, 72), "\n", "\r\n", 0)
	}
	if m.Type == "image" {
		return "--- IMAGE ---\r\n" + m.URL + "\r\n-------------"
		return m.URL
	}
	panic("What kind of message is this?")
}
<|endoftext|>


<commit_before>// Copyright 2012, The gohg Authors. All rights reserved.
// Use of this source code is governed by a BSD style license
// that can be found in the LICENSE.md file.

package gohg

type showconfigOpts struct {
	Config
	Cwd
	Hidden
	NonInteractive
	Quiet
	Repository
	Verbose

	// Mq
	Untrusted

	Debug
	Profile
	Time
	Traceback
}

func (cmdOpts *showconfigOpts) String() string {
	return sprintfOpts(*cmdOpts)
}

func (hgcl *HgClient) DebugConfig(sections []string, opts ...optionAdder) ([]byte, error) {
	return hgcl.ShowConfig(sections, opts...)
}

// ShowConfig provides the 'hg showconfig' command.
func (hgcl *HgClient) ShowConfig(sections []string, opts ...optionAdder) ([]byte, error) {
	cmdOpts := new(showconfigOpts)

	hgcmd, err := hgcl.buildCommand("showconfig", cmdOpts, opts, sections)
	if err != nil {
		return nil, err
	}
	return hgcl.runcommand(hgcmd)
}
<commit_msg>showconfig: renamed first argument for more general use<commit_after>// Copyright 2012, The gohg Authors. All rights reserved.
// Use of this source code is governed by a BSD style license
// that can be found in the LICENSE.md file.

package gohg

type showconfigOpts struct {
	Config
	Cwd
	Hidden
	NonInteractive
	Quiet
	Repository
	Verbose

	// Mq
	Untrusted

	Debug
	Profile
	Time
	Traceback
}

func (cmdOpts *showconfigOpts) String() string {
	return sprintfOpts(*cmdOpts)
}

func (hgcl *HgClient) DebugConfig(configitems []string, opts ...optionAdder) ([]byte, error) {
	return hgcl.ShowConfig(configitems, opts...)
}

func (hgcl *HgClient) ShowConfig(configitems []string, opts ...optionAdder) ([]byte, error) {
	cmdOpts := new(showconfigOpts)

	hgcmd, err := hgcl.buildCommand("showconfig", cmdOpts, opts, configitems)
	if err != nil {
		return nil, err
	}
	return hgcl.runcommand(hgcmd)
}
<|endoftext|>


<commit_before>package helm

import (
	"fmt"
	"strings"
	"time"

	"github.com/keel-hq/keel/approvals"
	"github.com/keel-hq/keel/internal/policy"
	"github.com/keel-hq/keel/types"
	"github.com/keel-hq/keel/util/image"

	hapi_chart "k8s.io/helm/pkg/proto/hapi/chart"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/keel-hq/keel/extension/notification"

	"github.com/ghodss/yaml"
	log "github.com/sirupsen/logrus"
	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/helm"
	"k8s.io/helm/pkg/strvals"
)

var helmVersionedUpdatesCounter = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "helm_versioned_updates_total",
		Help: "How many versioned helm charts were updated, partitioned by chart name.",
	},
	[]string{"chart"},
)

var helmUnversionedUpdatesCounter = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "helm_unversioned_updates_total",
		Help: "How many unversioned helm charts were updated, partitioned by chart name.",
	},
	[]string{"chart"},
)

func init() {
	prometheus.MustRegister(helmVersionedUpdatesCounter)
	prometheus.MustRegister(helmUnversionedUpdatesCounter)
}

// Manager - high level interface into helm provider related data used by
// triggers
type Manager interface {
	Images() ([]*image.Reference, error)
}

// ProviderName - helm provider name
const ProviderName = "helm"

// DefaultUpdateTimeout - update timeout in seconds
const DefaultUpdateTimeout = 300

// UpdatePlan - release update plan
type UpdatePlan struct {
	Namespace string
	Name      string

	Config *KeelChartConfig

	// chart
	Chart *hapi_chart.Chart

	// values to update path=value
	Values map[string]string

	// Current (last seen cluster version)
	CurrentVersion string
	// New version that's already in the deployment
	NewVersion string

	// ReleaseNotes is a slice of combined release notes.
	ReleaseNotes []string
}

// keel:
//   # keel policy (all/major/minor/patch/force)
//   policy: all
//   # trigger type, defaults to events such as pubsub, webhooks
//   trigger: poll
//   pollSchedule: "@every 2m"
//   # images to track and update
//   images:
//     - repository: image.repository
//       tag: image.tag

// Root - root element of the values yaml
type Root struct {
	Keel KeelChartConfig `json:"keel"`
}

// KeelChartConfig - keel related configuration taken from values.yaml
type KeelChartConfig struct {
	Policy               string            `json:"policy"`
	MatchTag             bool              `json:"matchTag"`
	Trigger              types.TriggerType `json:"trigger"`
	PollSchedule         string            `json:"pollSchedule"`
	Approvals            int               `json:"approvals"`        // Minimum required approvals
	ApprovalDeadline     int               `json:"approvalDeadline"` // Deadline in hours
	Images               []ImageDetails    `json:"images"`
	NotificationChannels []string          `json:"notificationChannels"` // optional notification channels

	Plc policy.Policy `json:"-"`
}

// ImageDetails - image details
type ImageDetails struct {
	RepositoryPath string `json:"repository"`
	TagPath        string `json:"tag"`
	DigestPath     string `json:"digest"`
	ReleaseNotes   string `json:"releaseNotes"`
}

// Provider - helm provider, responsible for managing release updates
type Provider struct {
	implementer Implementer

	sender notification.Sender

	approvalManager approvals.Manager

	events chan *types.Event
	stop   chan struct{}
}

// NewProvider - create new Helm provider
func NewProvider(implementer Implementer, sender notification.Sender, approvalManager approvals.Manager) *Provider {
	return &Provider{
		implementer:     implementer,
		approvalManager: approvalManager,
		sender:          sender,
		events:          make(chan *types.Event, 100),
		stop:            make(chan struct{}),
	}
}

// GetName - get provider name
func (p *Provider) GetName() string {
	return ProviderName
}

// Submit - submit event to provider
func (p *Provider) Submit(event types.Event) error {
	p.events <- &event
	return nil
}

// Start - starts kubernetes provider, waits for events
func (p *Provider) Start() error {
	return p.startInternal()
}

// Stop - stops kubernetes provider
func (p *Provider) Stop() {
	close(p.stop)
}

// TrackedImages - returns tracked images from all releases that have keel configuration
func (p *Provider) TrackedImages() ([]*types.TrackedImage, error) {
	var trackedImages []*types.TrackedImage

	releaseList, err := p.implementer.ListReleases()
	if err != nil {
		return nil, err
	}

	for _, release := range releaseList.Releases {
		// getting configuration
		vals, err := values(release.Chart, release.Config)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"release":   release.Name,
				"namespace": release.Namespace,
			}).Error("provider.helm: failed to get values.yaml for release")
			continue
		}

		cfg, err := getKeelConfig(vals)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"release":   release.Name,
				"namespace": release.Namespace,
			}).Debug("provider.helm: failed to get config for release")
			continue
		}

		if cfg.PollSchedule == "" {
			cfg.PollSchedule = types.KeelPollDefaultSchedule
		}
		// used to check pod secrets
		selector := fmt.Sprintf("app=%s,release=%s", release.Chart.Metadata.Name, release.Name)

		releaseImages, err := getImages(vals)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"release":   release.Name,
				"namespace": release.Namespace,
			}).Error("provider.helm: failed to get images for release")
			continue
		}

		for _, img := range releaseImages {
			img.Meta = map[string]string{
				"selector": selector,
			}
			img.Namespace = release.Namespace
			img.Provider = ProviderName
			trackedImages = append(trackedImages, img)
		}

	}

	return trackedImages, nil
}

func (p *Provider) startInternal() error {
	for {
		select {
		case event := <-p.events:
			log.WithFields(log.Fields{
				"repository": event.Repository.Name,
				"tag":        event.Repository.Tag,
				"registry":   event.Repository.Host,
			}).Info("provider.helm: processing event")
			err := p.processEvent(event)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err,
					"image": event.Repository.Name,
					"tag":   event.Repository.Tag,
				}).Error("provider.helm: failed to process event")
			}
		case <-p.stop:
			log.Info("provider.helm: got shutdown signal, stopping...")
			return nil
		}
	}
}

func (p *Provider) processEvent(event *types.Event) (err error) {
	plans, err := p.createUpdatePlans(event)
	if err != nil {
		return err
	}

	approved := p.checkForApprovals(event, plans)

	return p.applyPlans(approved)
}

func (p *Provider) createUpdatePlans(event *types.Event) ([]*UpdatePlan, error) {
	var plans []*UpdatePlan

	releaseList, err := p.implementer.ListReleases()
	if err != nil {
		return nil, err
	}

	for _, release := range releaseList.Releases {

		// plan, update, err := checkRelease(newVersion, &event.Repository, release.Namespace, release.Name, release.Chart, release.Config)
		plan, update, err := checkRelease(&event.Repository, release.Namespace, release.Name, release.Chart, release.Config)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"name":      release.Name,
				"namespace": release.Namespace,
			}).Error("provider.helm: failed to process versioned release")
			continue
		}
		if update {
			helmVersionedUpdatesCounter.With(prometheus.Labels{"chart": fmt.Sprintf("%s/%s", release.Namespace, release.Name)}).Inc()
			plans = append(plans, plan)
		}
	}

	return plans, nil
}

func (p *Provider) applyPlans(plans []*UpdatePlan) error {
	for _, plan := range plans {

		p.sender.Send(types.EventNotification{
			Name:      "update release",
			Message:   fmt.Sprintf("Preparing to update release %s/%s %s->%s (%s)", plan.Namespace, plan.Name, plan.CurrentVersion, plan.NewVersion, strings.Join(mapToSlice(plan.Values), ", ")),
			CreatedAt: time.Now(),
			Type:      types.NotificationPreReleaseUpdate,
			Level:     types.LevelDebug,
			Channels:  plan.Config.NotificationChannels,
		})

		err := updateHelmRelease(p.implementer, plan.Name, plan.Chart, plan.Values)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"name":      plan.Name,
				"namespace": plan.Namespace,
			}).Error("provider.helm: failed to apply plan")

			p.sender.Send(types.EventNotification{
				Name:      "update release",
				Message:   fmt.Sprintf("Release update failed %s/%s %s->%s (%s), error: %s", plan.Namespace, plan.Name, plan.CurrentVersion, plan.NewVersion, strings.Join(mapToSlice(plan.Values), ", "), err),
				CreatedAt: time.Now(),
				Type:      types.NotificationReleaseUpdate,
				Level:     types.LevelError,
				Channels:  plan.Config.NotificationChannels,
			})
			continue
		}

		err = p.updateComplete(plan)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"name":      plan.Name,
				"namespace": plan.Namespace,
			}).Warn("provider.helm: got error while resetting approvals counter after successful update")
		}

		var msg string
		if len(plan.ReleaseNotes) == 0 {
			msg = fmt.Sprintf("Successfully updated release %s/%s %s->%s (%s)", plan.Namespace, plan.Name, plan.CurrentVersion, plan.NewVersion, strings.Join(mapToSlice(plan.Values), ", "))
		} else {
			msg = fmt.Sprintf("Successfully updated release %s/%s %s->%s (%s). Release notes: %s", plan.Namespace, plan.Name, plan.CurrentVersion, plan.NewVersion, strings.Join(mapToSlice(plan.Values), ", "), strings.Join(plan.ReleaseNotes, ", "))
		}

		p.sender.Send(types.EventNotification{
			Name:      "update release",
			Message:   msg,
			CreatedAt: time.Now(),
			Type:      types.NotificationReleaseUpdate,
			Level:     types.LevelSuccess,
			Channels:  plan.Config.NotificationChannels,
		})

	}

	return nil
}

func updateHelmRelease(implementer Implementer, releaseName string, chart *hapi_chart.Chart, overrideValues map[string]string) error {

	overrideBts, err := convertToYaml(mapToSlice(overrideValues))
	if err != nil {
		return err
	}

	resp, err := implementer.UpdateReleaseFromChart(releaseName, chart,
		helm.UpdateValueOverrides(overrideBts),
		helm.UpgradeDryRun(false),
		helm.UpgradeRecreate(false),
		helm.UpgradeForce(true),
		helm.UpgradeDisableHooks(false),
		helm.UpgradeTimeout(DefaultUpdateTimeout),
		helm.ResetValues(false),
		helm.ReuseValues(true),
		helm.UpgradeWait(true))

	if err != nil {
		return err
	}

	log.WithFields(log.Fields{
		"version": resp.Release.Version,
		"release": releaseName,
	}).Info("provider.helm: release updated")
	return nil
}

func mapToSlice(values map[string]string) []string {
	converted := []string{}
	for k, v := range values {
		concat := k + "=" + v
		converted = append(converted, concat)
	}
	return converted
}

// parse
func convertToYaml(values []string) ([]byte, error) {
	base := map[string]interface{}{}
	for _, value := range values {
		if err := strvals.ParseInto(value, base); err != nil {
			return []byte{}, fmt.Errorf("failed parsing --set data: %s", err)
		}
	}

	return yaml.Marshal(base)
}

func getValueAsString(vals chartutil.Values, path string) (string, error) {
	valinterface, err := vals.PathValue(path)
	if err != nil {
		return "", err
	}
	valString, ok := valinterface.(string)
	if !ok {
		return "", fmt.Errorf("failed to convert value  to string")
	}

	return valString, nil
}

func values(chart *hapi_chart.Chart, config *hapi_chart.Config) (chartutil.Values, error) {
	return chartutil.CoalesceValues(chart, config)
}

func getKeelConfig(vals chartutil.Values) (*KeelChartConfig, error) {
	yamlFull, err := vals.YAML()
	if err != nil {
		return nil, fmt.Errorf("failed to get vals config, error: %s", err)
	}

	var r Root
	err = yaml.Unmarshal([]byte(yamlFull), &r)
	if err != nil {
		return nil, fmt.Errorf("failed to parse keel config: %s", err)
	}

	if r.Keel.Policy == "" {
		return nil, fmt.Errorf("policy not specified")
	}

	cfg := r.Keel

	cfg.Plc = policy.GetPolicy(cfg.Policy, &policy.Options{MatchTag: cfg.MatchTag})

	return &cfg, nil
}
<commit_msg>stop complaining if keel cfg is not defined in the chart<commit_after>package helm

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/keel-hq/keel/approvals"
	"github.com/keel-hq/keel/internal/policy"
	"github.com/keel-hq/keel/types"
	"github.com/keel-hq/keel/util/image"

	hapi_chart "k8s.io/helm/pkg/proto/hapi/chart"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/keel-hq/keel/extension/notification"

	"github.com/ghodss/yaml"
	log "github.com/sirupsen/logrus"
	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/helm"
	"k8s.io/helm/pkg/strvals"
)

var helmVersionedUpdatesCounter = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "helm_versioned_updates_total",
		Help: "How many versioned helm charts were updated, partitioned by chart name.",
	},
	[]string{"chart"},
)

var helmUnversionedUpdatesCounter = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "helm_unversioned_updates_total",
		Help: "How many unversioned helm charts were updated, partitioned by chart name.",
	},
	[]string{"chart"},
)

func init() {
	prometheus.MustRegister(helmVersionedUpdatesCounter)
	prometheus.MustRegister(helmUnversionedUpdatesCounter)
}

// ErrPolicyNotSpecified helm related errors
var (
	ErrPolicyNotSpecified = errors.New("policy not specified")
)

// Manager - high level interface into helm provider related data used by
// triggers
type Manager interface {
	Images() ([]*image.Reference, error)
}

// ProviderName - helm provider name
const ProviderName = "helm"

// DefaultUpdateTimeout - update timeout in seconds
const DefaultUpdateTimeout = 300

// UpdatePlan - release update plan
type UpdatePlan struct {
	Namespace string
	Name      string

	Config *KeelChartConfig

	// chart
	Chart *hapi_chart.Chart

	// values to update path=value
	Values map[string]string

	// Current (last seen cluster version)
	CurrentVersion string
	// New version that's already in the deployment
	NewVersion string

	// ReleaseNotes is a slice of combined release notes.
	ReleaseNotes []string
}

// keel:
//   # keel policy (all/major/minor/patch/force)
//   policy: all
//   # trigger type, defaults to events such as pubsub, webhooks
//   trigger: poll
//   pollSchedule: "@every 2m"
//   # images to track and update
//   images:
//     - repository: image.repository
//       tag: image.tag

// Root - root element of the values yaml
type Root struct {
	Keel KeelChartConfig `json:"keel"`
}

// KeelChartConfig - keel related configuration taken from values.yaml
type KeelChartConfig struct {
	Policy               string            `json:"policy"`
	MatchTag             bool              `json:"matchTag"`
	Trigger              types.TriggerType `json:"trigger"`
	PollSchedule         string            `json:"pollSchedule"`
	Approvals            int               `json:"approvals"`        // Minimum required approvals
	ApprovalDeadline     int               `json:"approvalDeadline"` // Deadline in hours
	Images               []ImageDetails    `json:"images"`
	NotificationChannels []string          `json:"notificationChannels"` // optional notification channels

	Plc policy.Policy `json:"-"`
}

// ImageDetails - image details
type ImageDetails struct {
	RepositoryPath string `json:"repository"`
	TagPath        string `json:"tag"`
	DigestPath     string `json:"digest"`
	ReleaseNotes   string `json:"releaseNotes"`
}

// Provider - helm provider, responsible for managing release updates
type Provider struct {
	implementer Implementer

	sender notification.Sender

	approvalManager approvals.Manager

	events chan *types.Event
	stop   chan struct{}
}

// NewProvider - create new Helm provider
func NewProvider(implementer Implementer, sender notification.Sender, approvalManager approvals.Manager) *Provider {
	return &Provider{
		implementer:     implementer,
		approvalManager: approvalManager,
		sender:          sender,
		events:          make(chan *types.Event, 100),
		stop:            make(chan struct{}),
	}
}

// GetName - get provider name
func (p *Provider) GetName() string {
	return ProviderName
}

// Submit - submit event to provider
func (p *Provider) Submit(event types.Event) error {
	p.events <- &event
	return nil
}

// Start - starts kubernetes provider, waits for events
func (p *Provider) Start() error {
	return p.startInternal()
}

// Stop - stops kubernetes provider
func (p *Provider) Stop() {
	close(p.stop)
}

// TrackedImages - returns tracked images from all releases that have keel configuration
func (p *Provider) TrackedImages() ([]*types.TrackedImage, error) {
	var trackedImages []*types.TrackedImage

	releaseList, err := p.implementer.ListReleases()
	if err != nil {
		return nil, err
	}

	for _, release := range releaseList.Releases {
		// getting configuration
		vals, err := values(release.Chart, release.Config)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"release":   release.Name,
				"namespace": release.Namespace,
			}).Error("provider.helm: failed to get values.yaml for release")
			continue
		}

		cfg, err := getKeelConfig(vals)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"release":   release.Name,
				"namespace": release.Namespace,
			}).Debug("provider.helm: failed to get config for release")
			continue
		}

		if cfg.PollSchedule == "" {
			cfg.PollSchedule = types.KeelPollDefaultSchedule
		}
		// used to check pod secrets
		selector := fmt.Sprintf("app=%s,release=%s", release.Chart.Metadata.Name, release.Name)

		releaseImages, err := getImages(vals)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"release":   release.Name,
				"namespace": release.Namespace,
			}).Error("provider.helm: failed to get images for release")
			continue
		}

		for _, img := range releaseImages {
			img.Meta = map[string]string{
				"selector": selector,
			}
			img.Namespace = release.Namespace
			img.Provider = ProviderName
			trackedImages = append(trackedImages, img)
		}

	}

	return trackedImages, nil
}

func (p *Provider) startInternal() error {
	for {
		select {
		case event := <-p.events:
			log.WithFields(log.Fields{
				"repository": event.Repository.Name,
				"tag":        event.Repository.Tag,
				"registry":   event.Repository.Host,
			}).Info("provider.helm: processing event")
			err := p.processEvent(event)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err,
					"image": event.Repository.Name,
					"tag":   event.Repository.Tag,
				}).Error("provider.helm: failed to process event")
			}
		case <-p.stop:
			log.Info("provider.helm: got shutdown signal, stopping...")
			return nil
		}
	}
}

func (p *Provider) processEvent(event *types.Event) (err error) {
	plans, err := p.createUpdatePlans(event)
	if err != nil {
		return err
	}

	approved := p.checkForApprovals(event, plans)

	return p.applyPlans(approved)
}

func (p *Provider) createUpdatePlans(event *types.Event) ([]*UpdatePlan, error) {
	var plans []*UpdatePlan

	releaseList, err := p.implementer.ListReleases()
	if err != nil {
		return nil, err
	}

	for _, release := range releaseList.Releases {

		// plan, update, err := checkRelease(newVersion, &event.Repository, release.Namespace, release.Name, release.Chart, release.Config)
		plan, update, err := checkRelease(&event.Repository, release.Namespace, release.Name, release.Chart, release.Config)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"name":      release.Name,
				"namespace": release.Namespace,
			}).Error("provider.helm: failed to process versioned release")
			continue
		}
		if update {
			helmVersionedUpdatesCounter.With(prometheus.Labels{"chart": fmt.Sprintf("%s/%s", release.Namespace, release.Name)}).Inc()
			plans = append(plans, plan)
		}
	}

	return plans, nil
}

func (p *Provider) applyPlans(plans []*UpdatePlan) error {
	for _, plan := range plans {

		p.sender.Send(types.EventNotification{
			Name:      "update release",
			Message:   fmt.Sprintf("Preparing to update release %s/%s %s->%s (%s)", plan.Namespace, plan.Name, plan.CurrentVersion, plan.NewVersion, strings.Join(mapToSlice(plan.Values), ", ")),
			CreatedAt: time.Now(),
			Type:      types.NotificationPreReleaseUpdate,
			Level:     types.LevelDebug,
			Channels:  plan.Config.NotificationChannels,
		})

		err := updateHelmRelease(p.implementer, plan.Name, plan.Chart, plan.Values)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"name":      plan.Name,
				"namespace": plan.Namespace,
			}).Error("provider.helm: failed to apply plan")

			p.sender.Send(types.EventNotification{
				Name:      "update release",
				Message:   fmt.Sprintf("Release update failed %s/%s %s->%s (%s), error: %s", plan.Namespace, plan.Name, plan.CurrentVersion, plan.NewVersion, strings.Join(mapToSlice(plan.Values), ", "), err),
				CreatedAt: time.Now(),
				Type:      types.NotificationReleaseUpdate,
				Level:     types.LevelError,
				Channels:  plan.Config.NotificationChannels,
			})
			continue
		}

		err = p.updateComplete(plan)
		if err != nil {
			log.WithFields(log.Fields{
				"error":     err,
				"name":      plan.Name,
				"namespace": plan.Namespace,
			}).Warn("provider.helm: got error while resetting approvals counter after successful update")
		}

		var msg string
		if len(plan.ReleaseNotes) == 0 {
			msg = fmt.Sprintf("Successfully updated release %s/%s %s->%s (%s)", plan.Namespace, plan.Name, plan.CurrentVersion, plan.NewVersion, strings.Join(mapToSlice(plan.Values), ", "))
		} else {
			msg = fmt.Sprintf("Successfully updated release %s/%s %s->%s (%s). Release notes: %s", plan.Namespace, plan.Name, plan.CurrentVersion, plan.NewVersion, strings.Join(mapToSlice(plan.Values), ", "), strings.Join(plan.ReleaseNotes, ", "))
		}

		p.sender.Send(types.EventNotification{
			Name:      "update release",
			Message:   msg,
			CreatedAt: time.Now(),
			Type:      types.NotificationReleaseUpdate,
			Level:     types.LevelSuccess,
			Channels:  plan.Config.NotificationChannels,
		})

	}

	return nil
}

func updateHelmRelease(implementer Implementer, releaseName string, chart *hapi_chart.Chart, overrideValues map[string]string) error {

	overrideBts, err := convertToYaml(mapToSlice(overrideValues))
	if err != nil {
		return err
	}

	resp, err := implementer.UpdateReleaseFromChart(releaseName, chart,
		helm.UpdateValueOverrides(overrideBts),
		helm.UpgradeDryRun(false),
		helm.UpgradeRecreate(false),
		helm.UpgradeForce(true),
		helm.UpgradeDisableHooks(false),
		helm.UpgradeTimeout(DefaultUpdateTimeout),
		helm.ResetValues(false),
		helm.ReuseValues(true),
		helm.UpgradeWait(true))

	if err != nil {
		return err
	}

	log.WithFields(log.Fields{
		"version": resp.Release.Version,
		"release": releaseName,
	}).Info("provider.helm: release updated")
	return nil
}

func mapToSlice(values map[string]string) []string {
	converted := []string{}
	for k, v := range values {
		concat := k + "=" + v
		converted = append(converted, concat)
	}
	return converted
}

// parse
func convertToYaml(values []string) ([]byte, error) {
	base := map[string]interface{}{}
	for _, value := range values {
		if err := strvals.ParseInto(value, base); err != nil {
			return []byte{}, fmt.Errorf("failed parsing --set data: %s", err)
		}
	}

	return yaml.Marshal(base)
}

func getValueAsString(vals chartutil.Values, path string) (string, error) {
	valinterface, err := vals.PathValue(path)
	if err != nil {
		return "", err
	}
	valString, ok := valinterface.(string)
	if !ok {
		return "", fmt.Errorf("failed to convert value  to string")
	}

	return valString, nil
}

func values(chart *hapi_chart.Chart, config *hapi_chart.Config) (chartutil.Values, error) {
	return chartutil.CoalesceValues(chart, config)
}

func getKeelConfig(vals chartutil.Values) (*KeelChartConfig, error) {
	yamlFull, err := vals.YAML()
	if err != nil {
		return nil, fmt.Errorf("failed to get vals config, error: %s", err)
	}

	var r Root
	err = yaml.Unmarshal([]byte(yamlFull), &r)
	if err != nil {
		return nil, fmt.Errorf("failed to parse keel config: %s", err)
	}

	if r.Keel.Policy == "" {
		return nil, ErrPolicyNotSpecified
	}

	cfg := r.Keel

	cfg.Plc = policy.GetPolicy(cfg.Policy, &policy.Options{MatchTag: cfg.MatchTag})

	return &cfg, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Openprovider Authors. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package providers

import (
	"encoding/json"
	"errors"
	"net/http"
	"time"

	"github.com/openprovider/rates"
	"golang.org/x/text/currency"
)

// APILayer represents multi currency provider
type APILayer struct {
	currencies []currency.Unit
	lastURL    string
	historyURL string
}

const (
	apiLayerLastURL    = "http://apilayer.net/api/live"
	apiLayerHistoryURL = "http://apilayer.net/api/historical"
)

// APILayerCurrencies are valid types of currencies for that provider
var APILayerCurrencies = []string{
	AED, AFN, ALL, AMD, ANG, AOA, ARS, AUD, AWG, AZN, BAM, BBD, BDT, BGN,
	BHD, BIF, BMD, BND, BOB, BRL, BSD, BTC, BTN, BWP, BYR, BZD, CAD, CDF,
	CHF, CLF, CLP, CNY, COP, CRC, CUP, CVE, CZK, DJF, DKK, DOP, DZD, EEK,
	EGP, ERN, ETB, EUR, FJD, FKP, GBP, GEL, GGP, GHS, GIP, GMD, GNF, GTQ,
	GYD, HKD, HNL, HRK, HTG, HUF, IDR, ILS, IMP, INR, IQD, IRR, ISK, JEP,
	JMD, JOD, JPY, KES, KGS, KHR, KMF, KPW, KRW, KWD, KYD, KZT, LAK, LBP,
	LKR, LRD, LSL, LTL, LVL, LYD, MAD, MDL, MGA, MKD, MMK, MNT, MOP, MRO,
	MUR, MVR, MWK, MXN, MYR, MZN, NAD, NGN, NIO, NOK, NPR, NZD, OMR, PAB,
	PEN, PGK, PHP, PKR, PLN, PYG, QAR, RON, RSD, RUB, RWF, SAR, SBD, SCR,
	SDG, SEK, SGD, SHP, SLL, SOS, SRD, STD, SVC, SYP, SZL, THB, TJS, TMT,
	TND, TOP, TRY, TTD, TWD, TZS, UAH, UGX, USD, UYU, UZS, VEF, VND, VUV,
	WST, XAF, XAG, XAU, XCD, XDR, XOF, XPF, YER, ZAR, ZMK, ZMW, ZWL,
}

// NewAPILayerProvider inits APILayer provider record
func NewAPILayerProvider(options *rates.Options) *APILayer {
	apiLayer := new(APILayer)
	// init all units
	if len(options.Currencies) == 0 {
		options.Currencies = append(options.Currencies, APILayerCurrencies...)
	}
	for _, unit := range options.Currencies {
		if c, err := currency.ParseISO(unit); err == nil {
			apiLayer.currencies = append(apiLayer.currencies, c)
		}
	}
	if options.Token != "" {
		apiLayer.lastURL = apiLayerLastURL + "?access_key=" + options.Token
		if v, ok := options.Settings["date"]; ok {
			if date, ok := v.(string); ok {
				apiLayer.historyURL = apiLayerHistoryURL + "?access_key=" + options.Token + "&date=" + date
			}
		} else {
			apiLayer.historyURL = apiLayer.lastURL
		}
	}

	return apiLayer
}

type apiLayerEnvelope struct {
	Success   bool
	Terms     string
	Privacy   string
	TimeStamp int64
	Source    string
	Quotes    map[string]interface{}
	Error     struct {
		Code uint16
		Type string
		Info string
	}
}

// FetchLast gets exchange rates for the last day
func (al *APILayer) FetchLast() ([]rates.Rate, []error) {
	return al.fetch(al.lastURL)
}

// FetchHistory gets exchange rates for all existing days
func (al *APILayer) FetchHistory() ([]rates.Rate, []error) {
	return al.fetch(al.historyURL)
}

// FetchLast gets exchange rates for the last day
func (al *APILayer) fetch(url string) (alRates []rates.Rate, alErrors []error) {
	response, err := http.Get(url)
	if err != nil {
		alErrors = append(alErrors, err)
		return
	}
	defer response.Body.Close()

	var raw apiLayerEnvelope

	if err := json.NewDecoder(response.Body).Decode(&raw); err != nil {
		alErrors = append(alErrors, err)
		return
	}

	if !raw.Success {
		alErrors = append(alErrors, errors.New(raw.Error.Info))
		return
	}

	if raw.TimeStamp == 0 {
		alErrors = append(alErrors, errors.New("Could not determine date"))
		return
	}
	timestamp := time.Unix(raw.TimeStamp, 0)
	date := timestamp.Format(stdDateTime)

	for _, unit := range al.currencies {
		for item, value := range raw.Quotes {
			if item == USD+unit.String() {
				alRates = append(alRates, rates.Rate{
					Time:     timestamp,
					Date:     date,
					Base:     currency.USD,
					Unit:     unit,
					Currency: currency.USD.String() + "/" + unit.String(),
					Value:    value,
				})
			}
		}
	}
	return
}
<commit_msg>currencies filter for APILayer<commit_after>// Copyright 2016 Openprovider Authors. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package providers

import (
	"encoding/json"
	"errors"
	"net/http"
	"strings"
	"time"

	"github.com/openprovider/rates"
	"golang.org/x/text/currency"
)

// APILayer represents multi currency provider
type APILayer struct {
	currencies []currency.Unit
	lastURL    string
	historyURL string
}

const (
	apiLayerLastURL    = "http://apilayer.net/api/live"
	apiLayerHistoryURL = "http://apilayer.net/api/historical"
)

// APILayerCurrencies are valid types of currencies for that provider
var APILayerCurrencies = []string{
	AED, AFN, ALL, AMD, ANG, AOA, ARS, AUD, AWG, AZN, BAM, BBD, BDT, BGN,
	BHD, BIF, BMD, BND, BOB, BRL, BSD, BTC, BTN, BWP, BYR, BZD, CAD, CDF,
	CHF, CLF, CLP, CNY, COP, CRC, CUP, CVE, CZK, DJF, DKK, DOP, DZD, EEK,
	EGP, ERN, ETB, EUR, FJD, FKP, GBP, GEL, GGP, GHS, GIP, GMD, GNF, GTQ,
	GYD, HKD, HNL, HRK, HTG, HUF, IDR, ILS, IMP, INR, IQD, IRR, ISK, JEP,
	JMD, JOD, JPY, KES, KGS, KHR, KMF, KPW, KRW, KWD, KYD, KZT, LAK, LBP,
	LKR, LRD, LSL, LTL, LVL, LYD, MAD, MDL, MGA, MKD, MMK, MNT, MOP, MRO,
	MUR, MVR, MWK, MXN, MYR, MZN, NAD, NGN, NIO, NOK, NPR, NZD, OMR, PAB,
	PEN, PGK, PHP, PKR, PLN, PYG, QAR, RON, RSD, RUB, RWF, SAR, SBD, SCR,
	SDG, SEK, SGD, SHP, SLL, SOS, SRD, STD, SVC, SYP, SZL, THB, TJS, TMT,
	TND, TOP, TRY, TTD, TWD, TZS, UAH, UGX, USD, UYU, UZS, VEF, VND, VUV,
	WST, XAF, XAG, XAU, XCD, XDR, XOF, XPF, YER, ZAR, ZMK, ZMW, ZWL,
}

// NewAPILayerProvider inits APILayer provider record
func NewAPILayerProvider(options *rates.Options) *APILayer {
	apiLayer := new(APILayer)
	// init all units
	if len(options.Currencies) == 0 {
		options.Currencies = append(options.Currencies, APILayerCurrencies...)
	}
	for _, unit := range options.Currencies {
		if c, err := currency.ParseISO(unit); err == nil {
			apiLayer.currencies = append(apiLayer.currencies, c)
		}
	}
	if options.Token != "" {
		apiLayer.lastURL = apiLayerLastURL + "?access_key=" + options.Token
		if v, ok := options.Settings["date"]; ok {
			if date, ok := v.(string); ok {
				apiLayer.historyURL = apiLayerHistoryURL + "?access_key=" + options.Token + "&date=" + date
			}
		} else {
			apiLayer.historyURL = apiLayer.lastURL
		}
		if len(options.Currencies) < 10 {
			currencies := strings.Join(options.Currencies, ",")
			apiLayer.lastURL += "&currencies=" + currencies
			apiLayer.historyURL += "&currencies=" + currencies
		}
	}

	return apiLayer
}

type apiLayerEnvelope struct {
	Success   bool
	Terms     string
	Privacy   string
	TimeStamp int64
	Source    string
	Quotes    map[string]interface{}
	Error     struct {
		Code uint16
		Type string
		Info string
	}
}

// FetchLast gets exchange rates for the last day
func (al *APILayer) FetchLast() ([]rates.Rate, []error) {
	return al.fetch(al.lastURL)
}

// FetchHistory gets exchange rates for all existing days
func (al *APILayer) FetchHistory() ([]rates.Rate, []error) {
	return al.fetch(al.historyURL)
}

// FetchLast gets exchange rates for the last day
func (al *APILayer) fetch(url string) (alRates []rates.Rate, alErrors []error) {
	response, err := http.Get(url)
	if err != nil {
		alErrors = append(alErrors, err)
		return
	}
	defer response.Body.Close()

	var raw apiLayerEnvelope

	if err := json.NewDecoder(response.Body).Decode(&raw); err != nil {
		alErrors = append(alErrors, err)
		return
	}

	if !raw.Success {
		alErrors = append(alErrors, errors.New(raw.Error.Info))
		return
	}

	if raw.TimeStamp == 0 {
		alErrors = append(alErrors, errors.New("Could not determine date"))
		return
	}
	timestamp := time.Unix(raw.TimeStamp, 0)
	date := timestamp.Format(stdDateTime)

	for _, unit := range al.currencies {
		for item, value := range raw.Quotes {
			if item == USD+unit.String() {
				alRates = append(alRates, rates.Rate{
					Time:     timestamp,
					Date:     date,
					Base:     currency.USD,
					Unit:     unit,
					Currency: currency.USD.String() + "/" + unit.String(),
					Value:    value,
				})
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package grpcproxy

import (
	"github.com/coreos/etcd/clientv3"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/proxy/grpcproxy/cache"

	"golang.org/x/net/context"
)

type kvProxy struct {
	kv    clientv3.KV
	cache cache.Cache
}

func NewKvProxy(c *clientv3.Client) pb.KVServer {
	return &kvProxy{
		kv:    c.KV,
		cache: cache.NewCache(cache.DefaultMaxEntries),
	}
}

func (p *kvProxy) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) {
	if r.Serializable {
		resp, err := p.cache.Get(r)
		switch err {
		case nil:
			cacheHits.Inc()
			return resp, nil
		case cache.ErrCompacted:
			cacheHits.Inc()
			return nil, err
		}
	}
	cachedMisses.Inc()

	resp, err := p.kv.Do(ctx, RangeRequestToOp(r))
	if err != nil {
		return nil, err
	}

	// cache linearizable as serializable
	req := *r
	req.Serializable = true
	gresp := (*pb.RangeResponse)(resp.Get())
	p.cache.Add(&req, gresp)

	return gresp, nil
}

func (p *kvProxy) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) {
	p.cache.Invalidate(r.Key, nil)

	resp, err := p.kv.Do(ctx, PutRequestToOp(r))
	return (*pb.PutResponse)(resp.Put()), err
}

func (p *kvProxy) DeleteRange(ctx context.Context, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) {
	p.cache.Invalidate(r.Key, r.RangeEnd)

	resp, err := p.kv.Do(ctx, DelRequestToOp(r))
	return (*pb.DeleteRangeResponse)(resp.Del()), err
}

func (p *kvProxy) txnToCache(reqs []*pb.RequestOp, resps []*pb.ResponseOp) {
	for i := range resps {
		switch tv := resps[i].Response.(type) {
		case *pb.ResponseOp_ResponsePut:
			p.cache.Invalidate(reqs[i].GetRequestPut().Key, nil)
		case *pb.ResponseOp_ResponseDeleteRange:
			rdr := reqs[i].GetRequestDeleteRange()
			p.cache.Invalidate(rdr.Key, rdr.RangeEnd)
		case *pb.ResponseOp_ResponseRange:
			req := *(reqs[i].GetRequestRange())
			req.Serializable = true
			p.cache.Add(&req, tv.ResponseRange)
		}
	}
}

func (p *kvProxy) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) {
	txn := p.kv.Txn(ctx)
	cmps := make([]clientv3.Cmp, len(r.Compare))
	thenops := make([]clientv3.Op, len(r.Success))
	elseops := make([]clientv3.Op, len(r.Failure))

	for i := range r.Compare {
		cmps[i] = (clientv3.Cmp)(*r.Compare[i])
	}

	for i := range r.Success {
		thenops[i] = requestOpToOp(r.Success[i])
	}

	for i := range r.Failure {
		elseops[i] = requestOpToOp(r.Failure[i])
	}

	resp, err := txn.If(cmps...).Then(thenops...).Else(elseops...).Commit()

	if err != nil {
		return nil, err
	}
	// txn may claim an outdated key is updated; be safe and invalidate
	for _, cmp := range r.Compare {
		p.cache.Invalidate(cmp.Key, nil)
	}
	// update any fetched keys
	if resp.Succeeded {
		p.txnToCache(r.Success, resp.Responses)
	} else {
		p.txnToCache(r.Failure, resp.Responses)
	}

	return (*pb.TxnResponse)(resp), nil
}

func (p *kvProxy) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) {
	var opts []clientv3.CompactOption
	if r.Physical {
		opts = append(opts, clientv3.WithCompactPhysical())
	}

	resp, err := p.kv.Compact(ctx, r.Revision, opts...)
	if err == nil {
		p.cache.Compact(r.Revision)
	}

	return (*pb.CompactionResponse)(resp), err
}

func requestOpToOp(union *pb.RequestOp) clientv3.Op {
	switch tv := union.Request.(type) {
	case *pb.RequestOp_RequestRange:
		if tv.RequestRange != nil {
			return RangeRequestToOp(tv.RequestRange)
		}
	case *pb.RequestOp_RequestPut:
		if tv.RequestPut != nil {
			return PutRequestToOp(tv.RequestPut)
		}
	case *pb.RequestOp_RequestDeleteRange:
		if tv.RequestDeleteRange != nil {
			return DelRequestToOp(tv.RequestDeleteRange)
		}
	}
	panic("unknown request")
}

func RangeRequestToOp(r *pb.RangeRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	if len(r.RangeEnd) != 0 {
		opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
	}
	opts = append(opts, clientv3.WithRev(r.Revision))
	opts = append(opts, clientv3.WithLimit(r.Limit))
	opts = append(opts, clientv3.WithSort(
		clientv3.SortTarget(r.SortTarget),
		clientv3.SortOrder(r.SortOrder)),
	)
	opts = append(opts, clientv3.WithMaxCreateRev(r.MaxCreateRevision))
	opts = append(opts, clientv3.WithMinCreateRev(r.MinCreateRevision))
	opts = append(opts, clientv3.WithMaxModRev(r.MaxModRevision))
	opts = append(opts, clientv3.WithMinModRev(r.MinModRevision))

	if r.Serializable {
		opts = append(opts, clientv3.WithSerializable())
	}

	return clientv3.OpGet(string(r.Key), opts...)
}

func PutRequestToOp(r *pb.PutRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	opts = append(opts, clientv3.WithLease(clientv3.LeaseID(r.Lease)))

	return clientv3.OpPut(string(r.Key), string(r.Value), opts...)
}

func DelRequestToOp(r *pb.DeleteRangeRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	if len(r.RangeEnd) != 0 {
		opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
	}
	if r.PrevKv {
		opts = append(opts, clientv3.WithPrevKV())
	}
	return clientv3.OpDelete(string(r.Key), opts...)
}
<commit_msg>grpcproxy: handle 'IgnoreValue' field in PutRequest<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package grpcproxy

import (
	"github.com/coreos/etcd/clientv3"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/proxy/grpcproxy/cache"

	"golang.org/x/net/context"
)

type kvProxy struct {
	kv    clientv3.KV
	cache cache.Cache
}

func NewKvProxy(c *clientv3.Client) pb.KVServer {
	return &kvProxy{
		kv:    c.KV,
		cache: cache.NewCache(cache.DefaultMaxEntries),
	}
}

func (p *kvProxy) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) {
	if r.Serializable {
		resp, err := p.cache.Get(r)
		switch err {
		case nil:
			cacheHits.Inc()
			return resp, nil
		case cache.ErrCompacted:
			cacheHits.Inc()
			return nil, err
		}
	}
	cachedMisses.Inc()

	resp, err := p.kv.Do(ctx, RangeRequestToOp(r))
	if err != nil {
		return nil, err
	}

	// cache linearizable as serializable
	req := *r
	req.Serializable = true
	gresp := (*pb.RangeResponse)(resp.Get())
	p.cache.Add(&req, gresp)

	return gresp, nil
}

func (p *kvProxy) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) {
	p.cache.Invalidate(r.Key, nil)

	resp, err := p.kv.Do(ctx, PutRequestToOp(r))
	return (*pb.PutResponse)(resp.Put()), err
}

func (p *kvProxy) DeleteRange(ctx context.Context, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) {
	p.cache.Invalidate(r.Key, r.RangeEnd)

	resp, err := p.kv.Do(ctx, DelRequestToOp(r))
	return (*pb.DeleteRangeResponse)(resp.Del()), err
}

func (p *kvProxy) txnToCache(reqs []*pb.RequestOp, resps []*pb.ResponseOp) {
	for i := range resps {
		switch tv := resps[i].Response.(type) {
		case *pb.ResponseOp_ResponsePut:
			p.cache.Invalidate(reqs[i].GetRequestPut().Key, nil)
		case *pb.ResponseOp_ResponseDeleteRange:
			rdr := reqs[i].GetRequestDeleteRange()
			p.cache.Invalidate(rdr.Key, rdr.RangeEnd)
		case *pb.ResponseOp_ResponseRange:
			req := *(reqs[i].GetRequestRange())
			req.Serializable = true
			p.cache.Add(&req, tv.ResponseRange)
		}
	}
}

func (p *kvProxy) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) {
	txn := p.kv.Txn(ctx)
	cmps := make([]clientv3.Cmp, len(r.Compare))
	thenops := make([]clientv3.Op, len(r.Success))
	elseops := make([]clientv3.Op, len(r.Failure))

	for i := range r.Compare {
		cmps[i] = (clientv3.Cmp)(*r.Compare[i])
	}

	for i := range r.Success {
		thenops[i] = requestOpToOp(r.Success[i])
	}

	for i := range r.Failure {
		elseops[i] = requestOpToOp(r.Failure[i])
	}

	resp, err := txn.If(cmps...).Then(thenops...).Else(elseops...).Commit()

	if err != nil {
		return nil, err
	}
	// txn may claim an outdated key is updated; be safe and invalidate
	for _, cmp := range r.Compare {
		p.cache.Invalidate(cmp.Key, nil)
	}
	// update any fetched keys
	if resp.Succeeded {
		p.txnToCache(r.Success, resp.Responses)
	} else {
		p.txnToCache(r.Failure, resp.Responses)
	}

	return (*pb.TxnResponse)(resp), nil
}

func (p *kvProxy) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) {
	var opts []clientv3.CompactOption
	if r.Physical {
		opts = append(opts, clientv3.WithCompactPhysical())
	}

	resp, err := p.kv.Compact(ctx, r.Revision, opts...)
	if err == nil {
		p.cache.Compact(r.Revision)
	}

	return (*pb.CompactionResponse)(resp), err
}

func requestOpToOp(union *pb.RequestOp) clientv3.Op {
	switch tv := union.Request.(type) {
	case *pb.RequestOp_RequestRange:
		if tv.RequestRange != nil {
			return RangeRequestToOp(tv.RequestRange)
		}
	case *pb.RequestOp_RequestPut:
		if tv.RequestPut != nil {
			return PutRequestToOp(tv.RequestPut)
		}
	case *pb.RequestOp_RequestDeleteRange:
		if tv.RequestDeleteRange != nil {
			return DelRequestToOp(tv.RequestDeleteRange)
		}
	}
	panic("unknown request")
}

func RangeRequestToOp(r *pb.RangeRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	if len(r.RangeEnd) != 0 {
		opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
	}
	opts = append(opts, clientv3.WithRev(r.Revision))
	opts = append(opts, clientv3.WithLimit(r.Limit))
	opts = append(opts, clientv3.WithSort(
		clientv3.SortTarget(r.SortTarget),
		clientv3.SortOrder(r.SortOrder)),
	)
	opts = append(opts, clientv3.WithMaxCreateRev(r.MaxCreateRevision))
	opts = append(opts, clientv3.WithMinCreateRev(r.MinCreateRevision))
	opts = append(opts, clientv3.WithMaxModRev(r.MaxModRevision))
	opts = append(opts, clientv3.WithMinModRev(r.MinModRevision))

	if r.Serializable {
		opts = append(opts, clientv3.WithSerializable())
	}

	return clientv3.OpGet(string(r.Key), opts...)
}

func PutRequestToOp(r *pb.PutRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	opts = append(opts, clientv3.WithLease(clientv3.LeaseID(r.Lease)))
	if r.IgnoreValue {
		opts = append(opts, clientv3.WithIgnoreValue())
	}
	return clientv3.OpPut(string(r.Key), string(r.Value), opts...)
}

func DelRequestToOp(r *pb.DeleteRangeRequest) clientv3.Op {
	opts := []clientv3.OpOption{}
	if len(r.RangeEnd) != 0 {
		opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
	}
	if r.PrevKv {
		opts = append(opts, clientv3.WithPrevKV())
	}
	return clientv3.OpDelete(string(r.Key), opts...)
}
<|endoftext|>