<commit_before>/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"strings"

	authorization "k8s.io/api/authorization/v1"
	capi "k8s.io/api/certificates/v1"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/cloud-provider-gcp/pkg/csrmetrics"
	"k8s.io/klog/v2"
	certutil "k8s.io/kubernetes/pkg/apis/certificates/v1"
	"k8s.io/kubernetes/pkg/controller/certificates"
)

const (
	kubeletReadonlyCSRSignerName         = "gke.io/kubelet-readonly-client"
	kubeletReadonlyCSRRequestMetrics     = "kubelet_readonly_csr_request"
	podNameKey                           = "authentication.kubernetes.io/pod-name"
	kubeletAPILimitedReaderAnnotationKey = "autopilot.gke.io/kubelet-api-limited-reader"
	kubeletReadonlyCRCommonName          = "kubelet-ro-client:"
)

var (
	kubeletReadonlyUsageAllowList = []capi.KeyUsage{
		capi.UsageKeyEncipherment,
		capi.UsageDigitalSignature,
		capi.UsageClientAuth,
	}
)

type kubeletReadonlyCSRRequest struct {
	context           context.Context
	controllerContext *controllerContext
	csr               *capi.CertificateSigningRequest
	x509cr            *x509.CertificateRequest
	autopilotEnabled  bool
}

type kubeletReadonlyCSRResponse struct {
	result  bool
	err     error
	message string
}

// readonlyApprover handles approval/denial of CSRs access to read only port
// There is a list of approvers, requst must get all approval
type kubeletReadonlyCSRApprover struct {
	ctx        *controllerContext
	validators []kubeletReadonlyCsrValidator
}

func newKubeletReadonlyCSRApprover(ctx *controllerContext) kubeletReadonlyCSRApprover {
	return kubeletReadonlyCSRApprover{
		ctx:        ctx,
		validators: newKubeletReadonlyCsrValidator(),
	}
}

func (approver *kubeletReadonlyCSRApprover) handle(ctx context.Context, csr *capi.CertificateSigningRequest) error {
	if csr == nil || csr.Spec.SignerName != kubeletReadonlyCSRSignerName {
		return nil
	}
	klog.Infof("approver got CSR %q", csr.Name)
	recordMetric := csrmetrics.ApprovalStartRecorder(kubeletReadonlyCSRRequestMetrics)
	if len(csr.Status.Certificate) != 0 {
		return nil
	}
	if approved, denied := certificates.GetCertApprovalCondition(&csr.Status); approved || denied {
		return nil
	}

	x509cr, err := certutil.ParseCSR(csr.Spec.Request)
	if err != nil {
		recordMetric(csrmetrics.ApprovalStatusParseError)
		klog.Errorf("unable to parse csr %q: %v", csr.Name, err)
		return fmt.Errorf("unable to parse csr %q: %v", csr.Name, err)
	}

	for _, validator := range approver.validators {
		recordValidatorMetric := csrmetrics.ApprovalStartRecorder(validator.authFlowLabel)
		request := kubeletReadonlyCSRRequest{
			context:           ctx,
			controllerContext: approver.ctx,
			csr:               csr,
			x509cr:            x509cr,
			autopilotEnabled:  *autopilotEnabled,
		}
		response := validator.validate(request)

		switch {
		case response.err != nil:
			klog.Errorf("validating CSR %q failed: validator %q has error %v", csr.Name, validator.name, response.err)
			return fmt.Errorf("validating CSR %q failed: validator %q has error %v", csr.Name, validator.name, response.err)
		case response.result == false:
			klog.Infof("validator %q: denied CSR %q, with message: %s", validator.name, csr.Name, response.message)
			recordValidatorMetric(csrmetrics.ApprovalStatusDeny)
			return approver.updateCSR(csr, false, response.message)
		case response.result == true:
			klog.Infof("validator %q: approved CSR %q, with message: %s", validator.name, csr.Name, response.message)
			recordValidatorMetric(csrmetrics.ApprovalStatusApprove)
		}
	}

	klog.Infof("all validators approved CSR %q", csr.Name)
	recordMetric(csrmetrics.ApprovalStatusApprove)
	return approver.updateCSR(csr, true, "approved by kubelet readonly approver")
}

func (approver *kubeletReadonlyCSRApprover) updateCSR(csr *capi.CertificateSigningRequest, approved bool, msg string) error {
	if approved {
		csr.Status.Conditions = append(csr.Status.Conditions, capi.CertificateSigningRequestCondition{
			Type:    capi.CertificateApproved,
			Reason:  "AutoApproved",
			Message: msg,
			Status:  v1.ConditionTrue,
		})
	} else {
		csr.Status.Conditions = append(csr.Status.Conditions, capi.CertificateSigningRequestCondition{
			Type:    capi.CertificateDenied,
			Reason:  "AutoDenied",
			Message: msg,
			Status:  v1.ConditionTrue,
		})
	}
	updateRecordMetric := csrmetrics.OutboundRPCStartRecorder("k8s.CertificateSigningRequests.updateApproval")
	_, err := approver.ctx.client.CertificatesV1().CertificateSigningRequests().UpdateApproval(context.TODO(), csr.Name, csr, metav1.UpdateOptions{})
	if err != nil {
		klog.Errorf("error updating approval status for csr: %v", err)
		updateRecordMetric(csrmetrics.OutboundRPCStatusError)
		return fmt.Errorf("error updating approval status for csr: %v", err)
	}
	updateRecordMetric(csrmetrics.OutboundRPCStatusOK)
	return nil
}

type kubeletReadonlyCsrValidator struct {
	name          string
	authFlowLabel string

	// validate will validate the request and return approved or denied.
	// (false, err): will base on err type retry or not. If it is a non-retryable error then deny the request.
	// (true, nil): will continue to next validater. If it is the last one, will approve the request.
	// (false, nil): reject the request.
	validate   func(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse
	permission authorization.ResourceAttributes
}

func newKubeletReadonlyCsrValidator() []kubeletReadonlyCsrValidator {
	return []kubeletReadonlyCsrValidator{
		{
			name:          "csr usage validator",
			authFlowLabel: "kubelet_readonly_csr_usage_match",
			validate:      validateCSRUsage,
		},
		{
			name:          "pod annotation validator",
			authFlowLabel: "kubelet_readonly_pod_annotation_valid",
			validate:      validatePodAnnotation,
		},
		{
			name:          "rbac validator",
			authFlowLabel: "kubelet_readonly_rbac_exist",
			validate:      validateRbac,
		},
		{
			name:          "common name validator",
			authFlowLabel: "kubelet_readonly_common_name_match",
			validate:      validateCommonName,
		},
	}
}

func validateCSRUsage(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse {
	csr := request.csr
	if csr == nil || len(csr.Spec.Usages) == 0 {
		klog.Errorf("csr usage is empty")
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: "csr usage is empty",
		}
	}

	usageMap := map[capi.KeyUsage]struct{}{}
	for _, u := range kubeletReadonlyUsageAllowList {
		usageMap[u] = struct{}{}
	}

	for _, usage := range csr.Spec.Usages {
		if _, ok := usageMap[usage]; !ok {
			allowListString, err := json.Marshal(kubeletReadonlyUsageAllowList)
			if err != nil {
				klog.Errorf("error marshaling allowlist, error: %v", err)
				return kubeletReadonlyCSRResponse{
					result:  false,
					err:     err,
					message: fmt.Sprintf("error marshaling allowlist, error: %v", err),
				}
			}
			klog.Errorf("csr usage %v is not allowed, allowed usages: %v", usage, string(allowListString))
			return kubeletReadonlyCSRResponse{
				result:  false,
				err:     nil,
				message: fmt.Sprintf("csr usage %v is not allowed, allowed usages: %v", usage, string(allowListString)),
			}
		}
	}
	return kubeletReadonlyCSRResponse{
		result:  true,
		err:     nil,
		message: "csr usage is valid",
	}
}

func validatePodAnnotation(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse {
	if !request.autopilotEnabled {
		return kubeletReadonlyCSRResponse{
			result:  true,
			err:     nil,
			message: fmt.Sprintf("Bypassing annotation validation because cluster is not a GKE Autopilot cluster."),
		}
	}
	if request.csr == nil || len(request.csr.Spec.Extra) == 0 {
		klog.Errorf("csr Extra is empty")
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: "csr Extra is empty",
		}
	}

	podNames, ok := request.csr.Spec.Extra[podNameKey]
	if !ok {
		klog.Errorf("csr does not have an attached pod name")
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     fmt.Errorf("csr does not have an attached pod name"),
			message: "csr does not have pod name attached",
		}
	}

	for _, podName := range podNames {
		pod, err := request.controllerContext.client.CoreV1().Pods("").Get(request.context, podName, metav1.GetOptions{})
		if err != nil {
			klog.Errorf("error when get pod %s, error: %v", podName, err)
			return kubeletReadonlyCSRResponse{
				result:  false,
				err:     fmt.Errorf("get pod %s failed: %v", podName, err),
				message: fmt.Sprintf("get pod %s failed: %v", podName, err),
			}
		}

		if pod.Annotations[kubeletAPILimitedReaderAnnotationKey] != "true" {
			klog.Errorf("pod %s does not have annotation with key %s or the value is not \"true\"", podName, kubeletAPILimitedReaderAnnotationKey)
			return kubeletReadonlyCSRResponse{
				result:  false,
				err:     nil,
				message: fmt.Sprintf("pod %s does not have annotation with key %s or the value is not \"true\"", podName, kubeletAPILimitedReaderAnnotationKey),
			}
		}
		klog.Infof("pod %s does not have annotation with key %s or the value is not \"true\"", podName, kubeletAPILimitedReaderAnnotationKey)
	}

	return kubeletReadonlyCSRResponse{
		result:  true,
		err:     nil,
		message: fmt.Sprintf("Annotation validation passed."),
	}
}

// validateRbac include SAR validation
func validateRbac(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse {
	rattrs := authorization.ResourceAttributes{
		Group:    "certificates.k8s.io",
		Resource: "certificatesigningrequests/kubeletclient",
		Verb:     "create",
	}

	csr := request.csr
	extra := make(map[string]authorization.ExtraValue)
	for k, v := range csr.Spec.Extra {
		extra[k] = authorization.ExtraValue(v)
	}

	sar := &authorization.SubjectAccessReview{
		Spec: authorization.SubjectAccessReviewSpec{
			User:               csr.Spec.Username,
			UID:                csr.Spec.UID,
			Groups:             csr.Spec.Groups,
			Extra:              extra,
			ResourceAttributes: &rattrs,
		},
	}
	sar, err := request.controllerContext.client.AuthorizationV1().SubjectAccessReviews().Create(request.context, sar, metav1.CreateOptions{})
	if err != nil {
		klog.Errorf("subject access review request failed: %v", err)
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     err,
			message: fmt.Sprintf("subject access review request failed: %v", err),
		}
	}

	if !sar.Status.Allowed {
		klog.Errorf("user %s is not allowed to create a CSR", csr.Spec.Username)
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: fmt.Sprintf("user %s is not allowed to create a CSR", csr.Spec.Username),
		}
	}

	return kubeletReadonlyCSRResponse{
		result:  true,
		err:     nil,
		message: fmt.Sprintf("user %s is allowed to create a CSR", csr.Spec.Username),
	}
}

// validateCommonName will make sure the common name for x509 has kubelet-ro-client
func validateCommonName(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse {
	x509cr := request.x509cr
	if x509cr == nil || len(x509cr.Subject.CommonName) == 0 {
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: fmt.Sprintf("x509cr or CommonName is empty"),
		}
	}

	if !strings.HasPrefix(x509cr.Subject.CommonName, kubeletReadonlyCRCommonName) {
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: fmt.Sprintf("x509 common name should start with %s", kubeletReadonlyCRCommonName),
		}
	}
	return kubeletReadonlyCSRResponse{
		result:  true,
		err:     nil,
		message: fmt.Sprintf("x509 common name starts with %s", kubeletReadonlyCRCommonName),
	}
}
<commit_msg>Update cmd/gcp-controller-manager/kubelet_readonly_csr_approver.go<commit_after>/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"strings"

	authorization "k8s.io/api/authorization/v1"
	capi "k8s.io/api/certificates/v1"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/cloud-provider-gcp/pkg/csrmetrics"
	"k8s.io/klog/v2"
	certutil "k8s.io/kubernetes/pkg/apis/certificates/v1"
	"k8s.io/kubernetes/pkg/controller/certificates"
)

const (
	kubeletReadonlyCSRSignerName         = "gke.io/kubelet-readonly-client"
	kubeletReadonlyCSRRequestMetrics     = "kubelet_readonly_csr_request"
	podNameKey                           = "authentication.kubernetes.io/pod-name"
	kubeletAPILimitedReaderAnnotationKey = "autopilot.gke.io/kubelet-api-limited-reader"
	kubeletReadonlyCRCommonName          = "kubelet-ro-client:"
)

var (
	kubeletReadonlyUsageAllowList = []capi.KeyUsage{
		capi.UsageKeyEncipherment,
		capi.UsageDigitalSignature,
		capi.UsageClientAuth,
	}
)

type kubeletReadonlyCSRRequest struct {
	context           context.Context
	controllerContext *controllerContext
	csr               *capi.CertificateSigningRequest
	x509cr            *x509.CertificateRequest
	autopilotEnabled  bool
}

type kubeletReadonlyCSRResponse struct {
	result  bool
	err     error
	message string
}

// readonlyApprover handles approval/denial of CSRs access to read only port
// There is a list of approvers, requst must get all approval
type kubeletReadonlyCSRApprover struct {
	ctx        *controllerContext
	validators []kubeletReadonlyCsrValidator
}

func newKubeletReadonlyCSRApprover(ctx *controllerContext) kubeletReadonlyCSRApprover {
	return kubeletReadonlyCSRApprover{
		ctx:        ctx,
		validators: newKubeletReadonlyCsrValidator(),
	}
}

func (approver *kubeletReadonlyCSRApprover) handle(ctx context.Context, csr *capi.CertificateSigningRequest) error {
	if csr == nil || csr.Spec.SignerName != kubeletReadonlyCSRSignerName {
		return nil
	}
	klog.Infof("approver got CSR %q", csr.Name)
	recordMetric := csrmetrics.ApprovalStartRecorder(kubeletReadonlyCSRRequestMetrics)
	if len(csr.Status.Certificate) != 0 {
		return nil
	}
	if approved, denied := certificates.GetCertApprovalCondition(&csr.Status); approved || denied {
		return nil
	}

	x509cr, err := certutil.ParseCSR(csr.Spec.Request)
	if err != nil {
		recordMetric(csrmetrics.ApprovalStatusParseError)
		klog.Errorf("unable to parse csr %q: %v", csr.Name, err)
		return fmt.Errorf("unable to parse csr %q: %v", csr.Name, err)
	}

	for _, validator := range approver.validators {
		recordValidatorMetric := csrmetrics.ApprovalStartRecorder(validator.authFlowLabel)
		request := kubeletReadonlyCSRRequest{
			context:           ctx,
			controllerContext: approver.ctx,
			csr:               csr,
			x509cr:            x509cr,
			autopilotEnabled:  *autopilotEnabled,
		}
		response := validator.validate(request)

		switch {
		case response.err != nil:
			klog.Errorf("validating CSR %q failed: validator %q has error %v", csr.Name, validator.name, response.err)
			return fmt.Errorf("validating CSR %q failed: validator %q has error %v", csr.Name, validator.name, response.err)
		case response.result == false:
			klog.Infof("validator %q: denied CSR %q with message: %s", validator.name, csr.Name, response.message)
			recordValidatorMetric(csrmetrics.ApprovalStatusDeny)
			return approver.updateCSR(csr, false, response.message)
		case response.result == true:
			klog.Infof("validator %q: approved CSR %q, with message: %s", validator.name, csr.Name, response.message)
			recordValidatorMetric(csrmetrics.ApprovalStatusApprove)
		}
	}

	klog.Infof("all validators approved CSR %q", csr.Name)
	recordMetric(csrmetrics.ApprovalStatusApprove)
	return approver.updateCSR(csr, true, "approved by kubelet readonly approver")
}

func (approver *kubeletReadonlyCSRApprover) updateCSR(csr *capi.CertificateSigningRequest, approved bool, msg string) error {
	if approved {
		csr.Status.Conditions = append(csr.Status.Conditions, capi.CertificateSigningRequestCondition{
			Type:    capi.CertificateApproved,
			Reason:  "AutoApproved",
			Message: msg,
			Status:  v1.ConditionTrue,
		})
	} else {
		csr.Status.Conditions = append(csr.Status.Conditions, capi.CertificateSigningRequestCondition{
			Type:    capi.CertificateDenied,
			Reason:  "AutoDenied",
			Message: msg,
			Status:  v1.ConditionTrue,
		})
	}
	updateRecordMetric := csrmetrics.OutboundRPCStartRecorder("k8s.CertificateSigningRequests.updateApproval")
	_, err := approver.ctx.client.CertificatesV1().CertificateSigningRequests().UpdateApproval(context.TODO(), csr.Name, csr, metav1.UpdateOptions{})
	if err != nil {
		klog.Errorf("error updating approval status for csr: %v", err)
		updateRecordMetric(csrmetrics.OutboundRPCStatusError)
		return fmt.Errorf("error updating approval status for csr: %v", err)
	}
	updateRecordMetric(csrmetrics.OutboundRPCStatusOK)
	return nil
}

type kubeletReadonlyCsrValidator struct {
	name          string
	authFlowLabel string

	// validate will validate the request and return approved or denied.
	// (false, err): will base on err type retry or not. If it is a non-retryable error then deny the request.
	// (true, nil): will continue to next validater. If it is the last one, will approve the request.
	// (false, nil): reject the request.
	validate   func(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse
	permission authorization.ResourceAttributes
}

func newKubeletReadonlyCsrValidator() []kubeletReadonlyCsrValidator {
	return []kubeletReadonlyCsrValidator{
		{
			name:          "csr usage validator",
			authFlowLabel: "kubelet_readonly_csr_usage_match",
			validate:      validateCSRUsage,
		},
		{
			name:          "pod annotation validator",
			authFlowLabel: "kubelet_readonly_pod_annotation_valid",
			validate:      validatePodAnnotation,
		},
		{
			name:          "rbac validator",
			authFlowLabel: "kubelet_readonly_rbac_exist",
			validate:      validateRbac,
		},
		{
			name:          "common name validator",
			authFlowLabel: "kubelet_readonly_common_name_match",
			validate:      validateCommonName,
		},
	}
}

func validateCSRUsage(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse {
	csr := request.csr
	if csr == nil || len(csr.Spec.Usages) == 0 {
		klog.Errorf("csr usage is empty")
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: "csr usage is empty",
		}
	}

	usageMap := map[capi.KeyUsage]struct{}{}
	for _, u := range kubeletReadonlyUsageAllowList {
		usageMap[u] = struct{}{}
	}

	for _, usage := range csr.Spec.Usages {
		if _, ok := usageMap[usage]; !ok {
			allowListString, err := json.Marshal(kubeletReadonlyUsageAllowList)
			if err != nil {
				klog.Errorf("error marshaling allowlist, error: %v", err)
				return kubeletReadonlyCSRResponse{
					result:  false,
					err:     err,
					message: fmt.Sprintf("error marshaling allowlist, error: %v", err),
				}
			}
			klog.Errorf("csr usage %v is not allowed, allowed usages: %v", usage, string(allowListString))
			return kubeletReadonlyCSRResponse{
				result:  false,
				err:     nil,
				message: fmt.Sprintf("csr usage %v is not allowed, allowed usages: %v", usage, string(allowListString)),
			}
		}
	}
	return kubeletReadonlyCSRResponse{
		result:  true,
		err:     nil,
		message: "csr usage is valid",
	}
}

func validatePodAnnotation(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse {
	if !request.autopilotEnabled {
		return kubeletReadonlyCSRResponse{
			result:  true,
			err:     nil,
			message: fmt.Sprintf("Bypassing annotation validation because cluster is not a GKE Autopilot cluster."),
		}
	}
	if request.csr == nil || len(request.csr.Spec.Extra) == 0 {
		klog.Errorf("csr Extra is empty")
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: "csr Extra is empty",
		}
	}

	podNames, ok := request.csr.Spec.Extra[podNameKey]
	if !ok {
		klog.Errorf("csr does not have an attached pod name")
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     fmt.Errorf("csr does not have an attached pod name"),
			message: "csr does not have pod name attached",
		}
	}

	for _, podName := range podNames {
		pod, err := request.controllerContext.client.CoreV1().Pods("").Get(request.context, podName, metav1.GetOptions{})
		if err != nil {
			klog.Errorf("error when get pod %s, error: %v", podName, err)
			return kubeletReadonlyCSRResponse{
				result:  false,
				err:     fmt.Errorf("get pod %s failed: %v", podName, err),
				message: fmt.Sprintf("get pod %s failed: %v", podName, err),
			}
		}

		if pod.Annotations[kubeletAPILimitedReaderAnnotationKey] != "true" {
			klog.Errorf("pod %s does not have annotation with key %s or the value is not \"true\"", podName, kubeletAPILimitedReaderAnnotationKey)
			return kubeletReadonlyCSRResponse{
				result:  false,
				err:     nil,
				message: fmt.Sprintf("pod %s does not have annotation with key %s or the value is not \"true\"", podName, kubeletAPILimitedReaderAnnotationKey),
			}
		}
		klog.Infof("pod %s does not have annotation with key %s or the value is not \"true\"", podName, kubeletAPILimitedReaderAnnotationKey)
	}

	return kubeletReadonlyCSRResponse{
		result:  true,
		err:     nil,
		message: fmt.Sprintf("Annotation validation passed."),
	}
}

// validateRbac include SAR validation
func validateRbac(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse {
	rattrs := authorization.ResourceAttributes{
		Group:    "certificates.k8s.io",
		Resource: "certificatesigningrequests/kubeletclient",
		Verb:     "create",
	}

	csr := request.csr
	extra := make(map[string]authorization.ExtraValue)
	for k, v := range csr.Spec.Extra {
		extra[k] = authorization.ExtraValue(v)
	}

	sar := &authorization.SubjectAccessReview{
		Spec: authorization.SubjectAccessReviewSpec{
			User:               csr.Spec.Username,
			UID:                csr.Spec.UID,
			Groups:             csr.Spec.Groups,
			Extra:              extra,
			ResourceAttributes: &rattrs,
		},
	}
	sar, err := request.controllerContext.client.AuthorizationV1().SubjectAccessReviews().Create(request.context, sar, metav1.CreateOptions{})
	if err != nil {
		klog.Errorf("subject access review request failed: %v", err)
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     err,
			message: fmt.Sprintf("subject access review request failed: %v", err),
		}
	}

	if !sar.Status.Allowed {
		klog.Errorf("user %s is not allowed to create a CSR", csr.Spec.Username)
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: fmt.Sprintf("user %s is not allowed to create a CSR", csr.Spec.Username),
		}
	}

	return kubeletReadonlyCSRResponse{
		result:  true,
		err:     nil,
		message: fmt.Sprintf("user %s is allowed to create a CSR", csr.Spec.Username),
	}
}

// validateCommonName will make sure the common name for x509 has kubelet-ro-client
func validateCommonName(request kubeletReadonlyCSRRequest) kubeletReadonlyCSRResponse {
	x509cr := request.x509cr
	if x509cr == nil || len(x509cr.Subject.CommonName) == 0 {
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: fmt.Sprintf("x509cr or CommonName is empty"),
		}
	}

	if !strings.HasPrefix(x509cr.Subject.CommonName, kubeletReadonlyCRCommonName) {
		return kubeletReadonlyCSRResponse{
			result:  false,
			err:     nil,
			message: fmt.Sprintf("x509 common name should start with %s", kubeletReadonlyCRCommonName),
		}
	}
	return kubeletReadonlyCSRResponse{
		result:  true,
		err:     nil,
		message: fmt.Sprintf("x509 common name starts with %s", kubeletReadonlyCRCommonName),
	}
}
<|endoftext|>


<commit_before>package jsonpatch

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

const (
	eRaw = 0
	eDoc = 1
	eAry = 2
)

type lazyNode struct {
	raw   *json.RawMessage
	doc   partialDoc
	ary   partialArray
	which int
}

func newLazyNode(raw *json.RawMessage) *lazyNode {
	return &lazyNode{raw: raw, doc: nil, ary: nil, which: eRaw}
}

func (n *lazyNode) MarshalJSON() ([]byte, error) {
	switch n.which {
	case eRaw:
		return *n.raw, nil
	case eDoc:
		return json.Marshal(n.doc)
	case eAry:
		return json.Marshal(n.ary)
	default:
		return nil, fmt.Errorf("Unknown type")
	}
}

func (n *lazyNode) UnmarshalJSON(data []byte) error {
	dest := make(json.RawMessage, len(data))
	copy(dest, data)
	n.raw = &dest
	n.which = eRaw
	return nil
}

func (n *lazyNode) IntoDoc() (*partialDoc, error) {
	if n.which == eDoc {
		return &n.doc, nil
	}

	err := json.Unmarshal(*n.raw, &n.doc)

	if err != nil {
		return nil, err
	}

	n.which = eDoc
	return &n.doc, nil
}

func (n *lazyNode) IntoAry() (*partialArray, error) {
	if n.which == eAry {
		return &n.ary, nil
	}

	err := json.Unmarshal(*n.raw, &n.ary)

	if err != nil {
		return nil, err
	}

	n.which = eAry
	return &n.ary, nil
}

func (n *lazyNode) compact() []byte {
	buf := new(bytes.Buffer)

	err := json.Compact(buf, *n.raw)

	if err != nil {
		return *n.raw
	}

	return buf.Bytes()
}

func (n *lazyNode) tryDoc() bool {
	err := json.Unmarshal(*n.raw, &n.doc)

	if err != nil {
		return false
	}

	n.which = eDoc
	return true
}

func (n *lazyNode) tryAry() bool {
	err := json.Unmarshal(*n.raw, &n.ary)

	if err != nil {
		return false
	}

	n.which = eAry
	return true
}

func (n *lazyNode) equal(o *lazyNode) bool {
	if n.which == eRaw {
		if !n.tryDoc() && !n.tryAry() {
			if o.which != eRaw {
				return false
			}

			return bytes.Equal(n.compact(), o.compact())
		}
	}

	if n.which == eDoc {
		if o.which == eRaw {
			if !o.tryDoc() {
				return false
			}
		}

		if o.which != eDoc {
			return false
		}

		for k, v := range n.doc {
			ov, ok := o.doc[k]

			if !ok {
				return false
			}

			if !v.equal(ov) {
				return false
			}
		}

		return true
	}

	if o.which != eAry && !o.tryAry() {
		return false
	}

	if len(n.ary) != len(o.ary) {
		return false
	}

	for idx, val := range n.ary {
		if !val.equal(o.ary[idx]) {
			return false
		}
	}

	return true
}

// Indicate if 2 JSON documents have the same structural equality
func Equal(a, b []byte) bool {
	ra := make(json.RawMessage, len(a))
	copy(ra, a)
	la := newLazyNode(&ra)

	rb := make(json.RawMessage, len(b))
	copy(rb, b)
	lb := newLazyNode(&rb)

	return la.equal(lb)
}

type Operation map[string]*json.RawMessage
type Patch []Operation

type partialDoc map[string]*lazyNode
type partialArray []*lazyNode

type Container interface {
	Get(key string) (*lazyNode, error)
	Set(key string, val *lazyNode) error
	Remove(key string) error
}

func DecodePatch(buf []byte) (Patch, error) {
	var p Patch

	err := json.Unmarshal(buf, &p)

	if err != nil {
		return nil, err
	}

	return p, nil
}

func (p Patch) Operation(i int) Operation {
	if i < 0 || i >= len(p) {
		return nil
	}

	return p[i]
}

func (o Operation) Kind() string {
	if obj, ok := o["op"]; ok {
		var op string

		err := json.Unmarshal(*obj, &op)

		if err != nil {
			return "unknown"
		}

		return op
	}

	return "unknown"
}

func (o Operation) Path() string {
	if obj, ok := o["path"]; ok {
		var op string

		err := json.Unmarshal(*obj, &op)

		if err != nil {
			return "unknown"
		}

		return op
	}

	return "unknown"
}

func (o Operation) From() string {
	if obj, ok := o["from"]; ok {
		var op string

		err := json.Unmarshal(*obj, &op)

		if err != nil {
			return "unknown"
		}

		return op
	}

	return "unknown"
}

func (o Operation) Value() *lazyNode {
	if obj, ok := o["value"]; ok {
		return newLazyNode(obj)
	}

	return nil
}

func isArray(buf []byte) bool {
	for _, c := range buf {
		switch c {
		case ' ':
		case '\n':
		case '\t':
			continue
		case '[':
			return true
		default:
			break
		}
	}

	return false
}

func findObject(doc *partialDoc, path string) (Container, string) {
	split := strings.Split(path, "/")

	parts := split[1 : len(split)-1]

	key := split[len(split)-1]

	var err error

	for idx, part := range parts {
		next, ok := (*doc)[part]
		if !ok {
			return nil, ""
		}

		if isArray(*next.raw) {
			if idx == len(parts)-1 {
				ary, err := next.IntoAry()

				if err != nil {
					return nil, ""
				}

				return ary, key
			} else {
				return nil, ""
			}
		} else {
			doc, err = next.IntoDoc()

			if err != nil {
				return nil, ""
			}
		}
	}

	return doc, key
}

func (d *partialDoc) Set(key string, val *lazyNode) error {
	(*d)[key] = val
	return nil
}

func (d *partialDoc) Get(key string) (*lazyNode, error) {
	return (*d)[key], nil
}

func (d *partialDoc) Remove(key string) error {
	delete(*d, key)
	return nil
}

func (d *partialArray) Set(key string, val *lazyNode) error {
	if key == "-" {
		*d = append(*d, val)
		return nil
	}

	idx, err := strconv.Atoi(key)

	if err != nil {
		return err
	}

	ary := make([]*lazyNode, len(*d)+1)

	cur := *d

	copy(ary[0:idx], cur[0:idx])
	ary[idx] = val
	copy(ary[idx+1:], cur[idx:])

	*d = ary
	return nil
}

func (d *partialArray) Get(key string) (*lazyNode, error) {
	idx, err := strconv.Atoi(key)

	if err != nil {
		return nil, err
	}

	return (*d)[idx], nil
}

func (d *partialArray) Remove(key string) error {
	idx, err := strconv.Atoi(key)

	if err != nil {
		return err
	}

	cur := *d

	ary := make([]*lazyNode, len(cur)-1)

	copy(ary[0:idx], cur[0:idx])
	copy(ary[idx:], cur[idx+1:])

	*d = ary
	return nil

}

func (p Patch) add(doc *partialDoc, op Operation) error {
	path := op.Path()

	con, key := findObject(doc, path)

	if con == nil {
		return fmt.Errorf("Missing container: %s", path)
	}

	con.Set(key, op.Value())

	return nil
}

func (p Patch) remove(doc *partialDoc, op Operation) error {
	path := op.Path()

	con, key := findObject(doc, path)

	return con.Remove(key)
}

func (p Patch) replace(doc *partialDoc, op Operation) error {
	path := op.Path()

	con, key := findObject(doc, path)

	con.Set(key, op.Value())

	return nil
}

func (p Patch) move(doc *partialDoc, op Operation) error {
	from := op.From()

	con, key := findObject(doc, from)

	val, err := con.Get(key)

	if err != nil {
		return err
	}

	con.Remove(key)

	path := op.Path()

	con, key = findObject(doc, path)

	con.Set(key, val)

	return nil
}

var eTestFailed = fmt.Errorf("Testing value failed")

func (p Patch) test(doc *partialDoc, op Operation) error {
	path := op.Path()

	con, key := findObject(doc, path)

	val, err := con.Get(key)

	if err != nil {
		return err
	}

	if val.equal(op.Value()) {
		return nil
	}

	return eTestFailed
}

func (p Patch) Apply(doc []byte) ([]byte, error) {
	pd := new(partialDoc)

	err := json.Unmarshal(doc, pd)

	if err != nil {
		return nil, err
	}

	err = nil

	for _, op := range p {
		switch op.Kind() {
		case "add":
			err = p.add(pd, op)
		case "remove":
			err = p.remove(pd, op)
		case "replace":
			err = p.replace(pd, op)
		case "move":
			err = p.move(pd, op)
		case "test":
			err = p.test(pd, op)
		default:
			err = fmt.Errorf("Unexpected kind: %s", op.Kind())
		}

		if err != nil {
			return nil, err
		}
	}

	return json.Marshal(pd)
}
<commit_msg>Use iota to create enum values<commit_after>package jsonpatch

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

const (
	eRaw = iota
	eDoc = iota
	eAry = iota
)

type lazyNode struct {
	raw   *json.RawMessage
	doc   partialDoc
	ary   partialArray
	which int
}

func newLazyNode(raw *json.RawMessage) *lazyNode {
	return &lazyNode{raw: raw, doc: nil, ary: nil, which: eRaw}
}

func (n *lazyNode) MarshalJSON() ([]byte, error) {
	switch n.which {
	case eRaw:
		return *n.raw, nil
	case eDoc:
		return json.Marshal(n.doc)
	case eAry:
		return json.Marshal(n.ary)
	default:
		return nil, fmt.Errorf("Unknown type")
	}
}

func (n *lazyNode) UnmarshalJSON(data []byte) error {
	dest := make(json.RawMessage, len(data))
	copy(dest, data)
	n.raw = &dest
	n.which = eRaw
	return nil
}

func (n *lazyNode) IntoDoc() (*partialDoc, error) {
	if n.which == eDoc {
		return &n.doc, nil
	}

	err := json.Unmarshal(*n.raw, &n.doc)

	if err != nil {
		return nil, err
	}

	n.which = eDoc
	return &n.doc, nil
}

func (n *lazyNode) IntoAry() (*partialArray, error) {
	if n.which == eAry {
		return &n.ary, nil
	}

	err := json.Unmarshal(*n.raw, &n.ary)

	if err != nil {
		return nil, err
	}

	n.which = eAry
	return &n.ary, nil
}

func (n *lazyNode) compact() []byte {
	buf := new(bytes.Buffer)

	err := json.Compact(buf, *n.raw)

	if err != nil {
		return *n.raw
	}

	return buf.Bytes()
}

func (n *lazyNode) tryDoc() bool {
	err := json.Unmarshal(*n.raw, &n.doc)

	if err != nil {
		return false
	}

	n.which = eDoc
	return true
}

func (n *lazyNode) tryAry() bool {
	err := json.Unmarshal(*n.raw, &n.ary)

	if err != nil {
		return false
	}

	n.which = eAry
	return true
}

func (n *lazyNode) equal(o *lazyNode) bool {
	if n.which == eRaw {
		if !n.tryDoc() && !n.tryAry() {
			if o.which != eRaw {
				return false
			}

			return bytes.Equal(n.compact(), o.compact())
		}
	}

	if n.which == eDoc {
		if o.which == eRaw {
			if !o.tryDoc() {
				return false
			}
		}

		if o.which != eDoc {
			return false
		}

		for k, v := range n.doc {
			ov, ok := o.doc[k]

			if !ok {
				return false
			}

			if !v.equal(ov) {
				return false
			}
		}

		return true
	}

	if o.which != eAry && !o.tryAry() {
		return false
	}

	if len(n.ary) != len(o.ary) {
		return false
	}

	for idx, val := range n.ary {
		if !val.equal(o.ary[idx]) {
			return false
		}
	}

	return true
}

// Indicate if 2 JSON documents have the same structural equality
func Equal(a, b []byte) bool {
	ra := make(json.RawMessage, len(a))
	copy(ra, a)
	la := newLazyNode(&ra)

	rb := make(json.RawMessage, len(b))
	copy(rb, b)
	lb := newLazyNode(&rb)

	return la.equal(lb)
}

type Operation map[string]*json.RawMessage
type Patch []Operation

type partialDoc map[string]*lazyNode
type partialArray []*lazyNode

type Container interface {
	Get(key string) (*lazyNode, error)
	Set(key string, val *lazyNode) error
	Remove(key string) error
}

func DecodePatch(buf []byte) (Patch, error) {
	var p Patch

	err := json.Unmarshal(buf, &p)

	if err != nil {
		return nil, err
	}

	return p, nil
}

func (p Patch) Operation(i int) Operation {
	if i < 0 || i >= len(p) {
		return nil
	}

	return p[i]
}

func (o Operation) Kind() string {
	if obj, ok := o["op"]; ok {
		var op string

		err := json.Unmarshal(*obj, &op)

		if err != nil {
			return "unknown"
		}

		return op
	}

	return "unknown"
}

func (o Operation) Path() string {
	if obj, ok := o["path"]; ok {
		var op string

		err := json.Unmarshal(*obj, &op)

		if err != nil {
			return "unknown"
		}

		return op
	}

	return "unknown"
}

func (o Operation) From() string {
	if obj, ok := o["from"]; ok {
		var op string

		err := json.Unmarshal(*obj, &op)

		if err != nil {
			return "unknown"
		}

		return op
	}

	return "unknown"
}

func (o Operation) Value() *lazyNode {
	if obj, ok := o["value"]; ok {
		return newLazyNode(obj)
	}

	return nil
}

func isArray(buf []byte) bool {
	for _, c := range buf {
		switch c {
		case ' ':
		case '\n':
		case '\t':
			continue
		case '[':
			return true
		default:
			break
		}
	}

	return false
}

func findObject(doc *partialDoc, path string) (Container, string) {
	split := strings.Split(path, "/")

	parts := split[1 : len(split)-1]

	key := split[len(split)-1]

	var err error

	for idx, part := range parts {
		next, ok := (*doc)[part]
		if !ok {
			return nil, ""
		}

		if isArray(*next.raw) {
			if idx == len(parts)-1 {
				ary, err := next.IntoAry()

				if err != nil {
					return nil, ""
				}

				return ary, key
			} else {
				return nil, ""
			}
		} else {
			doc, err = next.IntoDoc()

			if err != nil {
				return nil, ""
			}
		}
	}

	return doc, key
}

func (d *partialDoc) Set(key string, val *lazyNode) error {
	(*d)[key] = val
	return nil
}

func (d *partialDoc) Get(key string) (*lazyNode, error) {
	return (*d)[key], nil
}

func (d *partialDoc) Remove(key string) error {
	delete(*d, key)
	return nil
}

func (d *partialArray) Set(key string, val *lazyNode) error {
	if key == "-" {
		*d = append(*d, val)
		return nil
	}

	idx, err := strconv.Atoi(key)

	if err != nil {
		return err
	}

	ary := make([]*lazyNode, len(*d)+1)

	cur := *d

	copy(ary[0:idx], cur[0:idx])
	ary[idx] = val
	copy(ary[idx+1:], cur[idx:])

	*d = ary
	return nil
}

func (d *partialArray) Get(key string) (*lazyNode, error) {
	idx, err := strconv.Atoi(key)

	if err != nil {
		return nil, err
	}

	return (*d)[idx], nil
}

func (d *partialArray) Remove(key string) error {
	idx, err := strconv.Atoi(key)

	if err != nil {
		return err
	}

	cur := *d

	ary := make([]*lazyNode, len(cur)-1)

	copy(ary[0:idx], cur[0:idx])
	copy(ary[idx:], cur[idx+1:])

	*d = ary
	return nil

}

func (p Patch) add(doc *partialDoc, op Operation) error {
	path := op.Path()

	con, key := findObject(doc, path)

	if con == nil {
		return fmt.Errorf("Missing container: %s", path)
	}

	con.Set(key, op.Value())

	return nil
}

func (p Patch) remove(doc *partialDoc, op Operation) error {
	path := op.Path()

	con, key := findObject(doc, path)

	return con.Remove(key)
}

func (p Patch) replace(doc *partialDoc, op Operation) error {
	path := op.Path()

	con, key := findObject(doc, path)

	con.Set(key, op.Value())

	return nil
}

func (p Patch) move(doc *partialDoc, op Operation) error {
	from := op.From()

	con, key := findObject(doc, from)

	val, err := con.Get(key)

	if err != nil {
		return err
	}

	con.Remove(key)

	path := op.Path()

	con, key = findObject(doc, path)

	con.Set(key, val)

	return nil
}

var eTestFailed = fmt.Errorf("Testing value failed")

func (p Patch) test(doc *partialDoc, op Operation) error {
	path := op.Path()

	con, key := findObject(doc, path)

	val, err := con.Get(key)

	if err != nil {
		return err
	}

	if val.equal(op.Value()) {
		return nil
	}

	return eTestFailed
}

func (p Patch) Apply(doc []byte) ([]byte, error) {
	pd := new(partialDoc)

	err := json.Unmarshal(doc, pd)

	if err != nil {
		return nil, err
	}

	err = nil

	for _, op := range p {
		switch op.Kind() {
		case "add":
			err = p.add(pd, op)
		case "remove":
			err = p.remove(pd, op)
		case "replace":
			err = p.replace(pd, op)
		case "move":
			err = p.move(pd, op)
		case "test":
			err = p.test(pd, op)
		default:
			err = fmt.Errorf("Unexpected kind: %s", op.Kind())
		}

		if err != nil {
			return nil, err
		}
	}

	return json.Marshal(pd)
}
<|endoftext|>


<commit_before>package cassandra

import (
	"flag"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/raintank/met"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/idx/memory"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS %s.metric_def_idx (
    id text PRIMARY KEY,
    def blob,
) WITH compaction = {'class': 'SizeTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`

var (
	idxCasOk             met.Count
	idxCasFail           met.Count
	idxCasAddDuration    met.Timer
	idxCasDeleteDuration met.Timer

	Enabled        bool
	keyspace       string
	hosts          string
	consistency    string
	timeout        time.Duration
	numConns       int
	writeQueueSize int
	protoVer       int
	maxStale       time.Duration
	pruneInterval  time.Duration
)

func ConfigSetup() {
	casIdx := flag.NewFlagSet("cassandra-idx", flag.ExitOnError)
	casIdx.BoolVar(&Enabled, "enabled", false, "")
	casIdx.StringVar(&keyspace, "keyspace", "metric", "Cassandra keyspace to store metricDefinitions in.")
	casIdx.StringVar(&hosts, "hosts", "localhost:9042", "comma separated list of cassandra addresses in host:port form")
	casIdx.StringVar(&consistency, "consistency", "one", "write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one")
	casIdx.DurationVar(&timeout, "timeout", time.Second, "cassandra request timeout")
	casIdx.IntVar(&numConns, "num-conns", 10, "number of concurrent connections to cassandra")
	casIdx.IntVar(&writeQueueSize, "write-queue-size", 100000, "Max number of metricDefs allowed to be unwritten to cassandra")
	casIdx.IntVar(&protoVer, "protocol-version", 4, "cql protocol version to use")
	casIdx.DurationVar(&maxStale, "max-stale", 0, "clear series from the index if they have not been seen for this much time.")
	casIdx.DurationVar(&pruneInterval, "prune-interval", time.Hour*3, "Interval at which the index should be checked for stale series.")
	globalconf.Register("cassandra-idx", casIdx)
}

type writeReq struct {
	def      *schema.MetricDefinition
	recvTime time.Time
}

// Implements the the "MetricIndex" interface
type CasIdx struct {
	memory.MemoryIdx
	cluster    *gocql.ClusterConfig
	session    *gocql.Session
	writeQueue chan writeReq
	shutdown   chan struct{}
	wg         sync.WaitGroup
}

func New() *CasIdx {
	cluster := gocql.NewCluster(strings.Split(hosts, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = timeout
	cluster.NumConns = numConns
	cluster.ProtoVersion = protoVer

	return &CasIdx{
		MemoryIdx:  *memory.New(),
		cluster:    cluster,
		writeQueue: make(chan writeReq, writeQueueSize),
		shutdown:   make(chan struct{}),
	}
}

func (c *CasIdx) Init(stats met.Backend) error {
	log.Info("initializing cassandra-idx. Hosts=%s", hosts)
	if err := c.MemoryIdx.Init(stats); err != nil {
		return err
	}

	var err error
	tmpSession, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	// ensure the keyspace and table exist.
	err = tmpSession.Query(fmt.Sprintf(keyspace_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra keyspace. %s", err)
		return err
	}
	err = tmpSession.Query(fmt.Sprintf(table_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra table. %s", err)
		return err
	}
	tmpSession.Close()
	c.cluster.Keyspace = keyspace
	session, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	c.session = session

	idxCasOk = stats.NewCount("idx.cassandra.ok")
	idxCasFail = stats.NewCount("idx.cassandra.fail")
	idxCasAddDuration = stats.NewTimer("idx.cassandra.add_duration", 0)
	idxCasDeleteDuration = stats.NewTimer("idx.cassandra.delete_duration", 0)

	for i := 0; i < numConns; i++ {
		c.wg.Add(1)
		go c.processWriteQueue()
	}
	//Rebuild the in-memory index.

	c.rebuildIndex()
	if maxStale > 0 {
		go c.prune()
	}
	return nil
}

func (c *CasIdx) Stop() {
	log.Info("cassandra-idx stopping")
	c.MemoryIdx.Stop()
	close(c.writeQueue)
	c.wg.Wait()
	c.session.Close()
}

func (c *CasIdx) Add(data *schema.MetricData) {
	existing, err := c.MemoryIdx.Get(data.Id)
	inMemory := true
	if err != nil {
		if err == idx.DefNotFound {
			inMemory = false
		} else {
			log.Error(3, "cassandra-idx Failed to query Memory Index for %s. %s", data.Id, err)
			return
		}
	}
	if inMemory {
		log.Debug("cassandra-idx def already seen before. Just updating memory Index")
		existing.LastUpdate = data.Time
		c.MemoryIdx.AddDef(&existing)
		return
	}
	def := schema.MetricDefinitionFromMetricData(data)
	c.MemoryIdx.AddDef(def)
	c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
}

func (c *CasIdx) rebuildIndex() {
	log.Info("cassandra-idx Rebuilding Memory Index from metricDefinitions in Cassandra")
	pre := time.Now()
	defs := make([]schema.MetricDefinition, 0)
	iter := c.session.Query("SELECT def from metric_def_idx").Iter()

	var data []byte
	mdef := schema.MetricDefinition{}
	for iter.Scan(&data) {
		_, err := mdef.UnmarshalMsg(data)
		if err != nil {
			log.Error(3, "cassandra-idx Bad definition in index. %s - %s", data, err)
			continue
		}
		defs = append(defs, mdef)
	}
	c.MemoryIdx.Load(defs)
	log.Info("Rebuilding Memory Index Complete. Took %s", time.Since(pre).String())
}

func (c *CasIdx) processWriteQueue() {
	log.Info("cassandra-idx writeQueue handler started.")
	data := make([]byte, 0)
	var success bool
	var attempts int
	var err error
	var req writeReq
	for req = range c.writeQueue {
		data = data[:0]
		data, err = req.def.MarshalMsg(data)
		if err != nil {
			log.Error(3, "Failed to marshal metricDef. %s", err)
			continue
		}
		success = false
		attempts = 0
		for !success {
			if err := c.session.Query(`INSERT INTO metric_def_idx (id, def) VALUES (?, ?)`, req.def.Id, data).Exec(); err != nil {
				idxCasFail.Inc(1)
				if (attempts % 20) == 0 {
					log.Warn("cassandra-idx Failed to write def to cassandra. it will be retried. %s", err)
				}
				sleepTime := 100 * attempts
				if sleepTime > 2000 {
					sleepTime = 2000
				}
				time.Sleep(time.Duration(sleepTime) * time.Millisecond)
				attempts++
			} else {
				success = true
				idxCasAddDuration.Value(time.Since(req.recvTime))
				idxCasOk.Inc(1)
				log.Debug("cassandra-idx metricDef saved to cassandra. %s", req.def.Id)
			}
		}
	}
	log.Info("cassandra-idx writeQueue handler ended.")
	c.wg.Done()
}

func (c *CasIdx) Delete(orgId int, pattern string) error {
	ids, err := c.MemoryIdx.DeleteWithReport(orgId, pattern)
	if err != nil {
		return err
	}
	for _, id := range ids {
		err := c.session.Query("DELETE FROM metric_def_idx where id=?", id).Exec()
		if err != nil {
			log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", id, err)
			return err
		}
	}
	return nil
}

func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]schema.MetricDefinition, error) {
	pruned, err := c.MemoryIdx.Prune(orgId, oldest)
	// if an error was encountered then pruned is probably a partial list of metricDefs
	// deleted, so lets still try and delete these from Cassandra.
	for _, def := range pruned {
		err := c.session.Query("DELETE FROM metric_def_idx where id=?", def.Id).Exec()
		if err != nil {
			log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", def.Id, err)
			continue
		}
	}
	return pruned, err
}

func (c *CasIdx) prune() {
	ticker := time.NewTicker(pruneInterval)
	for range ticker.C {
		staleTs := time.Now().Add(maxStale * -1)
		_, err := c.Prune(-1, staleTs)
		if err != nil {
			log.Error(3, "cassandra-idx: prune error. %s", err)
		}
	}
}
<commit_msg>error out during init if pruneInterval is 0<commit_after>package cassandra

import (
	"flag"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/raintank/met"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/idx/memory"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS %s.metric_def_idx (
    id text PRIMARY KEY,
    def blob,
) WITH compaction = {'class': 'SizeTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`

var (
	idxCasOk             met.Count
	idxCasFail           met.Count
	idxCasAddDuration    met.Timer
	idxCasDeleteDuration met.Timer

	Enabled        bool
	keyspace       string
	hosts          string
	consistency    string
	timeout        time.Duration
	numConns       int
	writeQueueSize int
	protoVer       int
	maxStale       time.Duration
	pruneInterval  time.Duration
)

func ConfigSetup() {
	casIdx := flag.NewFlagSet("cassandra-idx", flag.ExitOnError)
	casIdx.BoolVar(&Enabled, "enabled", false, "")
	casIdx.StringVar(&keyspace, "keyspace", "metric", "Cassandra keyspace to store metricDefinitions in.")
	casIdx.StringVar(&hosts, "hosts", "localhost:9042", "comma separated list of cassandra addresses in host:port form")
	casIdx.StringVar(&consistency, "consistency", "one", "write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one")
	casIdx.DurationVar(&timeout, "timeout", time.Second, "cassandra request timeout")
	casIdx.IntVar(&numConns, "num-conns", 10, "number of concurrent connections to cassandra")
	casIdx.IntVar(&writeQueueSize, "write-queue-size", 100000, "Max number of metricDefs allowed to be unwritten to cassandra")
	casIdx.IntVar(&protoVer, "protocol-version", 4, "cql protocol version to use")
	casIdx.DurationVar(&maxStale, "max-stale", 0, "clear series from the index if they have not been seen for this much time.")
	casIdx.DurationVar(&pruneInterval, "prune-interval", time.Hour*3, "Interval at which the index should be checked for stale series.")
	globalconf.Register("cassandra-idx", casIdx)
}

type writeReq struct {
	def      *schema.MetricDefinition
	recvTime time.Time
}

// Implements the the "MetricIndex" interface
type CasIdx struct {
	memory.MemoryIdx
	cluster    *gocql.ClusterConfig
	session    *gocql.Session
	writeQueue chan writeReq
	shutdown   chan struct{}
	wg         sync.WaitGroup
}

func New() *CasIdx {
	cluster := gocql.NewCluster(strings.Split(hosts, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = timeout
	cluster.NumConns = numConns
	cluster.ProtoVersion = protoVer

	return &CasIdx{
		MemoryIdx:  *memory.New(),
		cluster:    cluster,
		writeQueue: make(chan writeReq, writeQueueSize),
		shutdown:   make(chan struct{}),
	}
}

func (c *CasIdx) Init(stats met.Backend) error {
	log.Info("initializing cassandra-idx. Hosts=%s", hosts)
	if err := c.MemoryIdx.Init(stats); err != nil {
		return err
	}

	var err error
	tmpSession, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	// ensure the keyspace and table exist.
	err = tmpSession.Query(fmt.Sprintf(keyspace_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra keyspace. %s", err)
		return err
	}
	err = tmpSession.Query(fmt.Sprintf(table_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra table. %s", err)
		return err
	}
	tmpSession.Close()
	c.cluster.Keyspace = keyspace
	session, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	c.session = session

	idxCasOk = stats.NewCount("idx.cassandra.ok")
	idxCasFail = stats.NewCount("idx.cassandra.fail")
	idxCasAddDuration = stats.NewTimer("idx.cassandra.add_duration", 0)
	idxCasDeleteDuration = stats.NewTimer("idx.cassandra.delete_duration", 0)

	for i := 0; i < numConns; i++ {
		c.wg.Add(1)
		go c.processWriteQueue()
	}
	//Rebuild the in-memory index.

	c.rebuildIndex()
	if maxStale > 0 {
		if pruneInterval == 0 {
			return fmt.Errorf("pruneInterval must be greater then 0")
		}
		go c.prune()
	}
	return nil
}

func (c *CasIdx) Stop() {
	log.Info("cassandra-idx stopping")
	c.MemoryIdx.Stop()
	close(c.writeQueue)
	c.wg.Wait()
	c.session.Close()
}

func (c *CasIdx) Add(data *schema.MetricData) {
	existing, err := c.MemoryIdx.Get(data.Id)
	inMemory := true
	if err != nil {
		if err == idx.DefNotFound {
			inMemory = false
		} else {
			log.Error(3, "cassandra-idx Failed to query Memory Index for %s. %s", data.Id, err)
			return
		}
	}
	if inMemory {
		log.Debug("cassandra-idx def already seen before. Just updating memory Index")
		existing.LastUpdate = data.Time
		c.MemoryIdx.AddDef(&existing)
		return
	}
	def := schema.MetricDefinitionFromMetricData(data)
	c.MemoryIdx.AddDef(def)
	c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
}

func (c *CasIdx) rebuildIndex() {
	log.Info("cassandra-idx Rebuilding Memory Index from metricDefinitions in Cassandra")
	pre := time.Now()
	defs := make([]schema.MetricDefinition, 0)
	iter := c.session.Query("SELECT def from metric_def_idx").Iter()

	var data []byte
	mdef := schema.MetricDefinition{}
	for iter.Scan(&data) {
		_, err := mdef.UnmarshalMsg(data)
		if err != nil {
			log.Error(3, "cassandra-idx Bad definition in index. %s - %s", data, err)
			continue
		}
		defs = append(defs, mdef)
	}
	c.MemoryIdx.Load(defs)
	log.Info("Rebuilding Memory Index Complete. Took %s", time.Since(pre).String())
}

func (c *CasIdx) processWriteQueue() {
	log.Info("cassandra-idx writeQueue handler started.")
	data := make([]byte, 0)
	var success bool
	var attempts int
	var err error
	var req writeReq
	for req = range c.writeQueue {
		data = data[:0]
		data, err = req.def.MarshalMsg(data)
		if err != nil {
			log.Error(3, "Failed to marshal metricDef. %s", err)
			continue
		}
		success = false
		attempts = 0
		for !success {
			if err := c.session.Query(`INSERT INTO metric_def_idx (id, def) VALUES (?, ?)`, req.def.Id, data).Exec(); err != nil {
				idxCasFail.Inc(1)
				if (attempts % 20) == 0 {
					log.Warn("cassandra-idx Failed to write def to cassandra. it will be retried. %s", err)
				}
				sleepTime := 100 * attempts
				if sleepTime > 2000 {
					sleepTime = 2000
				}
				time.Sleep(time.Duration(sleepTime) * time.Millisecond)
				attempts++
			} else {
				success = true
				idxCasAddDuration.Value(time.Since(req.recvTime))
				idxCasOk.Inc(1)
				log.Debug("cassandra-idx metricDef saved to cassandra. %s", req.def.Id)
			}
		}
	}
	log.Info("cassandra-idx writeQueue handler ended.")
	c.wg.Done()
}

func (c *CasIdx) Delete(orgId int, pattern string) error {
	ids, err := c.MemoryIdx.DeleteWithReport(orgId, pattern)
	if err != nil {
		return err
	}
	for _, id := range ids {
		err := c.session.Query("DELETE FROM metric_def_idx where id=?", id).Exec()
		if err != nil {
			log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", id, err)
			return err
		}
	}
	return nil
}

func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]schema.MetricDefinition, error) {
	pruned, err := c.MemoryIdx.Prune(orgId, oldest)
	// if an error was encountered then pruned is probably a partial list of metricDefs
	// deleted, so lets still try and delete these from Cassandra.
	for _, def := range pruned {
		err := c.session.Query("DELETE FROM metric_def_idx where id=?", def.Id).Exec()
		if err != nil {
			log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", def.Id, err)
			continue
		}
	}
	return pruned, err
}

func (c *CasIdx) prune() {
	ticker := time.NewTicker(pruneInterval)
	for range ticker.C {
		staleTs := time.Now().Add(maxStale * -1)
		_, err := c.Prune(-1, staleTs)
		if err != nil {
			log.Error(3, "cassandra-idx: prune error. %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package openapi

import (
	"errors"
	"strings"
)

var ErrTypeAssertion = errors.New("type assertion error")

func resolve(root *Document, ref string) (interface{}, error) {
	switch {
	case strings.HasPrefix(ref, "#/"):
		path := strings.Split(ref, "/")
		if len(path) < 2 {
			return nil, errors.New("ref string length invalid")
		}
		return root.resolve(cdr(path))
	default:
		return nil, errors.New("cannot resolve relative document")
	}
}

func (doc *Document) resolve(path []string) (interface{}, error) {
	switch s := car(path); s {
	case "components":
		return doc.Components.resolve(cdr(path))
	default:
		return nil, errors.New("unknown reference path: " + s)
	}
}

func (components *Components) resolve(path []string) (interface{}, error) {
	if len(path) != 2 {
		return nil, errors.New("cannot resolve")
	}
	var ret interface{}
	var ok bool
	switch s := car(path); s {
	case "schemas":
		ret, ok = components.Schemas[car(cdr(path))]
	case "responses":
		ret, ok = components.Responses[car(cdr(path))]
	case "parameters":
		ret, ok = components.Parameters[car(cdr(path))]
	case "examples":
		ret, ok = components.Examples[car(cdr(path))]
	case "requestBodies":
		ret, ok = components.RequestBodies[car(cdr(path))]
	case "headers":
		ret, ok = components.Headers[car(cdr(path))]
	case "securitySchemes":
		ret, ok = components.SecuritySchemes[car(cdr(path))]
	case "links":
		ret, ok = components.Links[car(cdr(path))]
	case "callbacks":
		ret, ok = components.Callbacks[car(cdr(path))]
	default:
		return nil, errors.New("unknown reference path: " + s)
	}
	if !ok {
		return nil, errors.New("not found: " + car(cdr(path)))
	}
	return ret, nil
}

// ResolveSchema resolves a schema reference string.
func ResolveSchema(root *Document, ref string) (*Schema, error) {
	si, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if s, ok := si.(*Schema); ok {
		return s, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveResponse resolves a response reference string.
func ResolveResponse(root *Document, ref string) (*Response, error) {
	ri, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if r, ok := ri.(*Response); ok {
		return r, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveParameter resolves a response reference string.
func ResolveParameter(root *Document, ref string) (*Parameter, error) {
	pi, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if p, ok := pi.(*Parameter); ok {
		return p, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveExample resolves an example reference string.
func ResolveExample(root *Document, ref string) (*Example, error) {
	ei, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if e, ok := ei.(*Example); ok {
		return e, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveRequestBody resolves a requestBody reference string.
func ResolveRequestBody(root *Document, ref string) (*RequestBody, error) {
	ri, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if r, ok := ri.(*RequestBody); ok {
		return r, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveHeader resolves a header reference string.
func ResolveHeader(root *Document, ref string) (*Header, error) {
	hi, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if h, ok := hi.(*Header); ok {
		return h, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveSecurityScheme resolves a securityScheme reference string.
func ResolveSecurityScheme(root *Document, ref string) (*SecurityScheme, error) {
	si, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if s, ok := si.(*SecurityScheme); ok {
		return s, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveLink resolves a link reference string.
func ResolveLink(root *Document, ref string) (*Link, error) {
	li, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if l, ok := li.(*Link); ok {
		return l, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveCallback resolves a callback reference string.
func ResolveCallback(root *Document, ref string) (*Callback, error) {
	ci, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if c, ok := ci.(*Callback); ok {
		return c, nil
	}
	return nil, ErrTypeAssertion
}
<commit_msg>refactor same function calling<commit_after>package openapi

import (
	"errors"
	"strings"
)

var ErrTypeAssertion = errors.New("type assertion error")

func resolve(root *Document, ref string) (interface{}, error) {
	switch {
	case strings.HasPrefix(ref, "#/"):
		path := strings.Split(ref, "/")
		if len(path) < 2 {
			return nil, errors.New("ref string length invalid")
		}
		return root.resolve(cdr(path))
	default:
		return nil, errors.New("cannot resolve relative document")
	}
}

func (doc *Document) resolve(path []string) (interface{}, error) {
	switch s := car(path); s {
	case "components":
		return doc.Components.resolve(cdr(path))
	default:
		return nil, errors.New("unknown reference path: " + s)
	}
}

func (components *Components) resolve(path []string) (interface{}, error) {
	if len(path) != 2 {
		return nil, errors.New("cannot resolve")
	}
	var ret interface{}
	var ok bool
	next := car(cdr(path))
	switch s := car(path); s {
	case "schemas":
		ret, ok = components.Schemas[next]
	case "responses":
		ret, ok = components.Responses[next]
	case "parameters":
		ret, ok = components.Parameters[next]
	case "examples":
		ret, ok = components.Examples[next]
	case "requestBodies":
		ret, ok = components.RequestBodies[next]
	case "headers":
		ret, ok = components.Headers[next]
	case "securitySchemes":
		ret, ok = components.SecuritySchemes[next]
	case "links":
		ret, ok = components.Links[next]
	case "callbacks":
		ret, ok = components.Callbacks[next]
	default:
		return nil, errors.New("unknown reference path: " + s)
	}
	if !ok {
		return nil, errors.New("not found: " + next)
	}
	return ret, nil
}

// ResolveSchema resolves a schema reference string.
func ResolveSchema(root *Document, ref string) (*Schema, error) {
	si, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if s, ok := si.(*Schema); ok {
		return s, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveResponse resolves a response reference string.
func ResolveResponse(root *Document, ref string) (*Response, error) {
	ri, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if r, ok := ri.(*Response); ok {
		return r, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveParameter resolves a response reference string.
func ResolveParameter(root *Document, ref string) (*Parameter, error) {
	pi, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if p, ok := pi.(*Parameter); ok {
		return p, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveExample resolves an example reference string.
func ResolveExample(root *Document, ref string) (*Example, error) {
	ei, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if e, ok := ei.(*Example); ok {
		return e, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveRequestBody resolves a requestBody reference string.
func ResolveRequestBody(root *Document, ref string) (*RequestBody, error) {
	ri, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if r, ok := ri.(*RequestBody); ok {
		return r, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveHeader resolves a header reference string.
func ResolveHeader(root *Document, ref string) (*Header, error) {
	hi, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if h, ok := hi.(*Header); ok {
		return h, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveSecurityScheme resolves a securityScheme reference string.
func ResolveSecurityScheme(root *Document, ref string) (*SecurityScheme, error) {
	si, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if s, ok := si.(*SecurityScheme); ok {
		return s, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveLink resolves a link reference string.
func ResolveLink(root *Document, ref string) (*Link, error) {
	li, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if l, ok := li.(*Link); ok {
		return l, nil
	}
	return nil, ErrTypeAssertion
}

// ResolveCallback resolves a callback reference string.
func ResolveCallback(root *Document, ref string) (*Callback, error) {
	ci, err := resolve(root, ref)
	if err != nil {
		return nil, err
	}
	if c, ok := ci.(*Callback); ok {
		return c, nil
	}
	return nil, ErrTypeAssertion
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import "fmt"

const MAJOR uint = 2
const MINOR uint = 15
const PATCH uint = 0

var COMMIT string = ""
var IDENTIFIER string = ""
var METADATA string = ""

func Version() string {
	var suffix string = ""
	if len(IDENTIFIER) > 0 {
		suffix = fmt.Sprintf("-%s", IDENTIFIER)
	}

	if len(COMMIT) > 0 || len(METADATA) > 0 {
		suffix = suffix + "+"
	}

	if len(COMMIT) > 0 {
		suffix = fmt.Sprintf("%s"+"commit.%s", suffix, COMMIT)

	}

	if len(METADATA) > 0 {
		if len(COMMIT) > 0 {
			suffix = suffix + "."
		}
		suffix = suffix + METADATA
	}

	return fmt.Sprintf("%d.%d.%d%s", MAJOR, MINOR, PATCH, suffix)
}
<commit_msg>v2.16.0<commit_after>// Copyright (C) 2018 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import "fmt"

const MAJOR uint = 2
const MINOR uint = 16
const PATCH uint = 0

var COMMIT string = ""
var IDENTIFIER string = ""
var METADATA string = ""

func Version() string {
	var suffix string = ""
	if len(IDENTIFIER) > 0 {
		suffix = fmt.Sprintf("-%s", IDENTIFIER)
	}

	if len(COMMIT) > 0 || len(METADATA) > 0 {
		suffix = suffix + "+"
	}

	if len(COMMIT) > 0 {
		suffix = fmt.Sprintf("%s"+"commit.%s", suffix, COMMIT)

	}

	if len(METADATA) > 0 {
		if len(COMMIT) > 0 {
			suffix = suffix + "."
		}
		suffix = suffix + METADATA
	}

	return fmt.Sprintf("%d.%d.%d%s", MAJOR, MINOR, PATCH, suffix)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package frontend provides a web server that serves documentation and meta
// tags to instruct "go get" where to find the Upspin source repository.
package frontend // import "upspin.io/serverutil/frontend"

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/russross/blackfriday"

	"upspin.io/config"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/serverutil/web"
	"upspin.io/upspin"

	_ "upspin.io/pack/eeintegrity"
	_ "upspin.io/transports"
)

func Main() {
	docPath := flag.String("docpath", defaultDocPath(), "location of folder containing documentation")
	local := flag.Bool("local", false, "run local testing instance")
	flags.Parse(flags.Server)

	if *local {
		// Hack to serve locally. If the flag has not been set explicitly (its default
		// value is ":80"), overwrite it to the local port.
		if flags.HTTPAddr == ":80" {
			flags.HTTPAddr = "localhost:8080"
		}
		flags.InsecureHTTP = true
		http.Handle("/", &reloadHandler{
			dir: *docPath,
			load: func() (http.Handler, error) {
				return newServer(nil, *docPath)
			},
		})
	} else {
		cfg, err := config.FromFile(flags.Config)
		if err != nil {
			log.Fatal(err)
		}
		s, err := newServer(cfg, *docPath)
		if err != nil {
			log.Fatal(err)
		}
		http.Handle("/", s)
	}
}

const (
	extMarkdown  = ".md"
	docHostname  = "upspin.io"      // redirect doc requests to this host
	testHostname = "test.upspin.io" // don't redirect requests to this host
	augieUser    = "augie@upspin.io"
)

// sourceRepo is a map from each custom domain to their repo base URLs.
var sourceRepo = map[string]string{
	"upspin.io": "https://upspin.googlesource.com/upspin",

	"android.upspin.io":      "https://upspin.googlesource.com/android",
	"augie.upspin.io":        "https://upspin.googlesource.com/augie",
	"aws.upspin.io":          "https://upspin.googlesource.com/aws",
	"b2.upspin.io":           "https://upspin.googlesource.com/b2",
	"digitalocean.upspin.io": "https://upspin.googlesource.com/digitalocean",
	"drive.upspin.io":        "https://upspin.googlesource.com/drive",
	"dropbox.upspin.io":      "https://upspin.googlesource.com/dropbox",
	"exp.upspin.io":          "https://upspin.googlesource.com/exp",
	"gcp.upspin.io":          "https://upspin.googlesource.com/gcp",
	"openstack.upspin.io":    "https://upspin.googlesource.com/openstack",
}

func defaultDocPath() string {
	p, err := build.Import("upspin.io/doc", "", build.FindOnly)
	if err != nil {
		return ""
	}
	return p.Dir
}

type server struct {
	handlers http.Handler // stack of wrapped http.Handlers
	docHTML  map[string][]byte
	docTitle map[string]string
	tmpl     struct {
		doc, download *template.Template
	}
}

// newServer initializes and returns a new HTTP server serving documentation
// from the given directory. A nil Config prevents the server from building and
// serving binaries for download.
func newServer(cfg upspin.Config, docs string) (http.Handler, error) {
	s := &server{}

	if err := s.parseTemplates(filepath.Join(docs, "templates")); err != nil {
		return nil, fmt.Errorf("parsing templates: %v", err)
	}
	if err := s.parseDocs(docs); err != nil {
		return nil, fmt.Errorf("parsing docs: %v", err)
	}

	mux := http.NewServeMux()
	mux.Handle("/", http.HandlerFunc(s.handleDoc))
	mux.Handle("/images/", http.FileServer(http.Dir(docs)))
	mux.Handle("/issue/", redirectHandler("/issue/", "https://github.com/upspin/upspin/issues/"))
	mux.Handle("/cl/", redirectHandler("/cl/", "https://upspin-review.googlesource.com/c/"))
	if cfg != nil {
		mux.Handle(downloadPath, newDownloadHandler(cfg, s.tmpl.download))
		mux.Handle("/"+releaseUser+"/", web.New(cfg, isWriter(releaseUser)))
		mux.Handle("/"+augieUser+"/", web.New(cfg, isWriter(augieUser)))
	}
	s.handlers = goGetHandler{gziphandler.GzipHandler(canonicalHostHandler{mux})}

	return s, nil
}

// isWriter is a serverutil/web.IsWriter implementation.
type isWriter upspin.UserName

// IsWriter reports whether the given user matches isWriter.
func (w isWriter) IsWriter(u upspin.UserName) bool {
	return upspin.UserName(w) == u
}

type pageData struct {
	Title    string
	Content  interface{}
	FileName string
}

func (s *server) handleDoc(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	switch r.URL.Path {
	case "/":
		s.renderDoc(w, "index.md")
	case "/doc/":
		s.renderDoc(w, "doc.md")
	case "/doc":
		http.Redirect(w, r, "/doc/", http.StatusFound)
	default:
		if !strings.HasPrefix(r.URL.Path, "/doc/") {
			http.NotFound(w, r)
			return
		}
		base := filepath.Base(r.URL.Path)
		switch base {
		case "index.md":
			http.Redirect(w, r, "/", http.StatusFound)
		case "doc.md":
			http.Redirect(w, r, "/doc/", http.StatusFound)
		default:
			s.renderDoc(w, base)
		}
	}
}

func (s *server) renderDoc(w http.ResponseWriter, fn string) {
	b, ok := s.docHTML[fn]
	if !ok {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}
	if err := s.tmpl.doc.Execute(w, pageData{
		Title:    s.docTitle[fn] + " · Upspin",
		Content:  template.HTML(b),
		FileName: fn,
	}); err != nil {
		log.Error.Printf("Error executing doc content template: %s", err)
		return
	}
}

// ServeHTTP satisfies the http.Handler interface for a server.
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.TLS != nil {
		w.Header().Set("Strict-Transport-Security", "max-age=86400; includeSubDomains")
	}
	s.handlers.ServeHTTP(w, r)
}

func (s *server) parseTemplates(dir string) (err error) {
	s.tmpl.doc, err = template.ParseFiles(filepath.Join(dir, "base.tmpl"), filepath.Join(dir, "doc.tmpl"))
	if err != nil {
		return err
	}
	s.tmpl.download, err = template.ParseFiles(filepath.Join(dir, "base.tmpl"), filepath.Join(dir, "download.tmpl"))
	return err
}

func (s *server) parseDocs(dir string) error {
	fis, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	var (
		html  = map[string][]byte{}
		title = map[string]string{}
	)
	for _, fi := range fis {
		fn := fi.Name()
		if filepath.Ext(fn) != extMarkdown {
			continue
		}
		b, err := ioutil.ReadFile(filepath.Join(dir, fn))
		if err != nil {
			return err
		}
		html[fn] = blackfriday.MarkdownCommon(b)
		title[fn] = docTitle(b)
	}
	s.docHTML = html
	s.docTitle = title
	return nil
}

// docTitle extracts the first Markdown header in the given document body.
// It expects the first line to be of the form
// 	# Title string
// If not, it will return "Untitled".
func docTitle(b []byte) string {
	if len(b) > 2 && b[0] == '#' {
		if i := bytes.IndexByte(b, '\n'); i != -1 {
			// On Windows we need to strip out the '\r' as well
			if b[i-1] == '\r' {
				i--
			}
			return string(b[2:i])
		}
	}
	return "Untitled"
}

type goGetHandler struct {
	http.Handler
}

func (h goGetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Query().Get("go-get") == "1" {
		base := r.Host
		repo, ok := sourceRepo[base]
		if !ok {
			http.NotFound(w, r)
			return
		}
		fmt.Fprintf(w, `<meta name="go-import" content="%v git %v">`, base, repo)
		return
	}
	h.Handler.ServeHTTP(w, r)
}

type canonicalHostHandler struct {
	http.Handler
}

func (h canonicalHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Redirect requests to foo.upspin.io to upspin.io.
	if r.Host != docHostname && r.Host != testHostname && strings.HasSuffix(r.Host, "."+docHostname) {
		u := *r.URL
		u.Host = docHostname
		http.Redirect(w, r, u.String(), http.StatusFound)
		return
	}
	h.Handler.ServeHTTP(w, r)
}

// reloadHandler is a http.Handler wrapper that watches the contents of dir for
// modifications and reloads the underlying http.Handler when changes are
// made. It is intended to be used in local mode to facilitate rapid editing.
type reloadHandler struct {
	dir  string
	load func() (http.Handler, error)

	mu           sync.Mutex
	handler      http.Handler
	lastScan     time.Time
	lastModified time.Time
}

func (h *reloadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// scanInterval is time to wait between scanning for new changes.
	// It should be set long enough to avoid scanning on every request,
	// but not so long as to make editing tedious.
	const scanInterval = 5 * time.Second

	h.mu.Lock()
	if h.handler != nil && time.Since(h.lastScan) > scanInterval {
		err := filepath.Walk(h.dir, func(_ string, fi os.FileInfo, _ error) error {
			if t := fi.ModTime(); t.After(h.lastModified) {
				h.lastModified = t
				h.handler = nil
			}
			return nil
		})
		if err != nil {
			h.mu.Unlock()
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	if h.handler == nil {
		var err error
		h.handler, err = h.load()
		if err != nil {
			h.mu.Unlock()
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	handler := h.handler
	h.mu.Unlock()

	handler.ServeHTTP(w, r)
}
<commit_msg>serverutil/frontend: make the GitHub repos the canonical source<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package frontend provides a web server that serves documentation and meta
// tags to instruct "go get" where to find the Upspin source repository.
package frontend // import "upspin.io/serverutil/frontend"

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/russross/blackfriday"

	"upspin.io/config"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/serverutil/web"
	"upspin.io/upspin"

	_ "upspin.io/pack/eeintegrity"
	_ "upspin.io/transports"
)

func Main() {
	docPath := flag.String("docpath", defaultDocPath(), "location of folder containing documentation")
	local := flag.Bool("local", false, "run local testing instance")
	flags.Parse(flags.Server)

	if *local {
		// Hack to serve locally. If the flag has not been set explicitly (its default
		// value is ":80"), overwrite it to the local port.
		if flags.HTTPAddr == ":80" {
			flags.HTTPAddr = "localhost:8080"
		}
		flags.InsecureHTTP = true
		http.Handle("/", &reloadHandler{
			dir: *docPath,
			load: func() (http.Handler, error) {
				return newServer(nil, *docPath)
			},
		})
	} else {
		cfg, err := config.FromFile(flags.Config)
		if err != nil {
			log.Fatal(err)
		}
		s, err := newServer(cfg, *docPath)
		if err != nil {
			log.Fatal(err)
		}
		http.Handle("/", s)
	}
}

const (
	extMarkdown  = ".md"
	docHostname  = "upspin.io"      // redirect doc requests to this host
	testHostname = "test.upspin.io" // don't redirect requests to this host
	augieUser    = "augie@upspin.io"
)

// sourceRepo is a map from each custom domain to their repo base URLs.
var sourceRepo = map[string]string{
	"upspin.io": "https://github.com/upspin/upspin",

	"android.upspin.io":      "https://github.com/upspin/android",
	"augie.upspin.io":        "https://github.com/upspin/augie",
	"aws.upspin.io":          "https://github.com/upspin/aws",
	"b2.upspin.io":           "https://github.com/upspin/b2",
	"digitalocean.upspin.io": "https://github.com/upspin/digitalocean",
	"drive.upspin.io":        "https://github.com/upspin/drive",
	"dropbox.upspin.io":      "https://github.com/upspin/dropbox",
	"exp.upspin.io":          "https://github.com/upspin/exp",
	"gcp.upspin.io":          "https://github.com/upspin/gcp",
	"openstack.upspin.io":    "https://github.com/upspin/openstack",
}

func defaultDocPath() string {
	p, err := build.Import("upspin.io/doc", "", build.FindOnly)
	if err != nil {
		return ""
	}
	return p.Dir
}

type server struct {
	handlers http.Handler // stack of wrapped http.Handlers
	docHTML  map[string][]byte
	docTitle map[string]string
	tmpl     struct {
		doc, download *template.Template
	}
}

// newServer initializes and returns a new HTTP server serving documentation
// from the given directory. A nil Config prevents the server from building and
// serving binaries for download.
func newServer(cfg upspin.Config, docs string) (http.Handler, error) {
	s := &server{}

	if err := s.parseTemplates(filepath.Join(docs, "templates")); err != nil {
		return nil, fmt.Errorf("parsing templates: %v", err)
	}
	if err := s.parseDocs(docs); err != nil {
		return nil, fmt.Errorf("parsing docs: %v", err)
	}

	mux := http.NewServeMux()
	mux.Handle("/", http.HandlerFunc(s.handleDoc))
	mux.Handle("/images/", http.FileServer(http.Dir(docs)))
	mux.Handle("/issue/", redirectHandler("/issue/", "https://github.com/upspin/upspin/issues/"))
	mux.Handle("/cl/", redirectHandler("/cl/", "https://upspin-review.googlesource.com/c/"))
	if cfg != nil {
		mux.Handle(downloadPath, newDownloadHandler(cfg, s.tmpl.download))
		mux.Handle("/"+releaseUser+"/", web.New(cfg, isWriter(releaseUser)))
		mux.Handle("/"+augieUser+"/", web.New(cfg, isWriter(augieUser)))
	}
	s.handlers = goGetHandler{gziphandler.GzipHandler(canonicalHostHandler{mux})}

	return s, nil
}

// isWriter is a serverutil/web.IsWriter implementation.
type isWriter upspin.UserName

// IsWriter reports whether the given user matches isWriter.
func (w isWriter) IsWriter(u upspin.UserName) bool {
	return upspin.UserName(w) == u
}

type pageData struct {
	Title    string
	Content  interface{}
	FileName string
}

func (s *server) handleDoc(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	switch r.URL.Path {
	case "/":
		s.renderDoc(w, "index.md")
	case "/doc/":
		s.renderDoc(w, "doc.md")
	case "/doc":
		http.Redirect(w, r, "/doc/", http.StatusFound)
	default:
		if !strings.HasPrefix(r.URL.Path, "/doc/") {
			http.NotFound(w, r)
			return
		}
		base := filepath.Base(r.URL.Path)
		switch base {
		case "index.md":
			http.Redirect(w, r, "/", http.StatusFound)
		case "doc.md":
			http.Redirect(w, r, "/doc/", http.StatusFound)
		default:
			s.renderDoc(w, base)
		}
	}
}

func (s *server) renderDoc(w http.ResponseWriter, fn string) {
	b, ok := s.docHTML[fn]
	if !ok {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}
	if err := s.tmpl.doc.Execute(w, pageData{
		Title:    s.docTitle[fn] + " · Upspin",
		Content:  template.HTML(b),
		FileName: fn,
	}); err != nil {
		log.Error.Printf("Error executing doc content template: %s", err)
		return
	}
}

// ServeHTTP satisfies the http.Handler interface for a server.
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.TLS != nil {
		w.Header().Set("Strict-Transport-Security", "max-age=86400; includeSubDomains")
	}
	s.handlers.ServeHTTP(w, r)
}

func (s *server) parseTemplates(dir string) (err error) {
	s.tmpl.doc, err = template.ParseFiles(filepath.Join(dir, "base.tmpl"), filepath.Join(dir, "doc.tmpl"))
	if err != nil {
		return err
	}
	s.tmpl.download, err = template.ParseFiles(filepath.Join(dir, "base.tmpl"), filepath.Join(dir, "download.tmpl"))
	return err
}

func (s *server) parseDocs(dir string) error {
	fis, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	var (
		html  = map[string][]byte{}
		title = map[string]string{}
	)
	for _, fi := range fis {
		fn := fi.Name()
		if filepath.Ext(fn) != extMarkdown {
			continue
		}
		b, err := ioutil.ReadFile(filepath.Join(dir, fn))
		if err != nil {
			return err
		}
		html[fn] = blackfriday.MarkdownCommon(b)
		title[fn] = docTitle(b)
	}
	s.docHTML = html
	s.docTitle = title
	return nil
}

// docTitle extracts the first Markdown header in the given document body.
// It expects the first line to be of the form
//
//	# Title string
//
// If not, it will return "Untitled".
func docTitle(b []byte) string {
	if len(b) > 2 && b[0] == '#' {
		if i := bytes.IndexByte(b, '\n'); i != -1 {
			// On Windows we need to strip out the '\r' as well
			if b[i-1] == '\r' {
				i--
			}
			return string(b[2:i])
		}
	}
	return "Untitled"
}

type goGetHandler struct {
	http.Handler
}

func (h goGetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Query().Get("go-get") == "1" {
		base := r.Host
		repo, ok := sourceRepo[base]
		if !ok {
			http.NotFound(w, r)
			return
		}
		fmt.Fprintf(w, `<meta name="go-import" content="%v git %v">`, base, repo)
		return
	}
	h.Handler.ServeHTTP(w, r)
}

type canonicalHostHandler struct {
	http.Handler
}

func (h canonicalHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Redirect requests to foo.upspin.io to upspin.io.
	if r.Host != docHostname && r.Host != testHostname && strings.HasSuffix(r.Host, "."+docHostname) {
		u := *r.URL
		u.Host = docHostname
		http.Redirect(w, r, u.String(), http.StatusFound)
		return
	}
	h.Handler.ServeHTTP(w, r)
}

// reloadHandler is a http.Handler wrapper that watches the contents of dir for
// modifications and reloads the underlying http.Handler when changes are
// made. It is intended to be used in local mode to facilitate rapid editing.
type reloadHandler struct {
	dir  string
	load func() (http.Handler, error)

	mu           sync.Mutex
	handler      http.Handler
	lastScan     time.Time
	lastModified time.Time
}

func (h *reloadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// scanInterval is time to wait between scanning for new changes.
	// It should be set long enough to avoid scanning on every request,
	// but not so long as to make editing tedious.
	const scanInterval = 5 * time.Second

	h.mu.Lock()
	if h.handler != nil && time.Since(h.lastScan) > scanInterval {
		err := filepath.Walk(h.dir, func(_ string, fi os.FileInfo, _ error) error {
			if t := fi.ModTime(); t.After(h.lastModified) {
				h.lastModified = t
				h.handler = nil
			}
			return nil
		})
		if err != nil {
			h.mu.Unlock()
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	if h.handler == nil {
		var err error
		h.handler, err = h.load()
		if err != nil {
			h.mu.Unlock()
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	handler := h.handler
	h.mu.Unlock()

	handler.ServeHTTP(w, r)
}
<|endoftext|>


<commit_before>package slack

import (
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/hirakiuc/gonta/internal/bot"

	simpleJson "github.com/bitly/go-simplejson"
)

type RtmStartApi struct {
	Token string
}

const BASE_API = "https://slack.com/api/rtm.start"

func (api *RtmStartApi) requestUrl() string {
	return fmt.Sprintf(BASE_API+"?token=%s", api.Token)
}

func (api *RtmStartApi) WssUrl() (url string, err error) {
	res, err := http.Get(api.requestUrl())
	if err != nil {
		log.Error("Failed to get rtm.start method: %v", err)
		return "", err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Error("Failed to receive message: %v", err)
		return "", err
	}

	js, err := simpleJson.NewJson(body)
	if err != nil {
		log.Error("Failed to parse json: %v", err)
		return "", err
	}

	bot := bot.Initialize(js)
	log.Info(bot.String())

	wss, err := js.Get("url").String()
	if err != nil {
		log.Error("Failed to find wssurl: %v", err)
		return
	}

	return wss, nil
}
<commit_msg>fix return interface<commit_after>package slack

import (
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/hirakiuc/gonta/internal/bot"

	simpleJson "github.com/bitly/go-simplejson"
)

type RtmStartApi struct {
	Token string
}

const BASE_API = "https://slack.com/api/rtm.start"

func (api *RtmStartApi) requestUrl() string {
	return fmt.Sprintf(BASE_API+"?token=%s", api.Token)
}

func (api *RtmStartApi) WssUrl() (url string, err error) {
	res, err := http.Get(api.requestUrl())
	if err != nil {
		log.Error("Failed to get rtm.start method: %v", err)
		return "", err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Error("Failed to receive message: %v", err)
		return "", err
	}

	js, err := simpleJson.NewJson(body)
	if err != nil {
		log.Error("Failed to parse json: %v", err)
		return "", err
	}

	bot := bot.Initialize(js)
	log.Info(bot.String())

	wss, err := js.Get("url").String()
	if err != nil {
		log.Error("Failed to find wssurl: %v", err)
		return "", err
	}

	return wss, nil
}
<|endoftext|>