<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"reflect"
	"strconv"

	corev1 "k8s.io/api/core/v1"
	extv1beta1 "k8s.io/api/extensions/v1beta1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/klog"

	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack/cert-manager/pkg/util"
)

const (
	// editInPlaceAnnotation is used to toggle the use of ingressClass instead
	// of ingress on the created Certificate resource
	editInPlaceAnnotation = "certmanager.k8s.io/acme-http01-edit-in-place"
	// issuerNameAnnotation can be used to override the issuer specified on the
	// created Certificate resource.
	issuerNameAnnotation = "certmanager.k8s.io/issuer"
	// clusterIssuerNameAnnotation can be used to override the issuer specified on the
	// created Certificate resource. The Certificate will reference the
	// specified *ClusterIssuer* instead of normal issuer.
	clusterIssuerNameAnnotation = "certmanager.k8s.io/cluster-issuer"
	// acmeIssuerChallengeTypeAnnotation can be used to override the default ACME challenge
	// type to be used when the specified issuer is an ACME issuer
	acmeIssuerChallengeTypeAnnotation = "certmanager.k8s.io/acme-challenge-type"
	// acmeIssuerDNS01ProviderNameAnnotation can be used to override the default dns01 provider
	// configured on the issuer if the challenge type is set to dns01
	acmeIssuerDNS01ProviderNameAnnotation = "certmanager.k8s.io/acme-dns01-provider"
	// acmeIssuerHTTP01IngressClassAnnotation can be used to override the http01 ingressClass
	// if the challenge type is set to http01
	acmeIssuerHTTP01IngressClassAnnotation = "certmanager.k8s.io/acme-http01-ingress-class"

	ingressClassAnnotation = util.IngressKey
)

var ingressGVK = extv1beta1.SchemeGroupVersion.WithKind("Ingress")

func (c *Controller) Sync(ctx context.Context, ing *extv1beta1.Ingress) error {
	if !shouldSync(ing, c.defaults.autoCertificateAnnotations) {
		klog.Infof("Not syncing ingress %s/%s as it does not contain necessary annotations", ing.Namespace, ing.Name)
		return nil
	}

	issuerName, issuerKind := c.issuerForIngress(ing)
	if issuerName == "" {
		c.Recorder.Eventf(ing, corev1.EventTypeWarning, "BadConfig", "Issuer name annotation is not set and a default issuer has not been configured")
		return nil
	}

	issuer, err := c.helper.GetGenericIssuer(v1alpha1.ObjectReference{
		Name: issuerName,
		Kind: issuerKind,
	}, ing.Namespace)
	if apierrors.IsNotFound(err) {
		c.Recorder.Eventf(ing, corev1.EventTypeWarning, "BadConfig", "%s resource %q not found", issuerKind, issuerName)
		return nil
	}
	if err != nil {
		return err
	}

	errs := c.validateIngress(ing)
	if len(errs) > 0 {
		errMsg := errs[0].Error()
		if len(errs) > 1 {
			errMsg = utilerrors.NewAggregate(errs).Error()
		}
		c.Recorder.Eventf(ing, corev1.EventTypeWarning, "BadConfig", errMsg)
		return nil
	}

	newCrts, updateCrts, err := c.buildCertificates(ing, issuer, issuerKind)
	if err != nil {
		return err
	}

	for _, crt := range newCrts {
		_, err := c.CMClient.CertmanagerV1alpha1().Certificates(crt.Namespace).Create(crt)
		if err != nil {
			return err
		}
		c.Recorder.Eventf(ing, corev1.EventTypeNormal, "CreateCertificate", "Successfully created Certificate %q", crt.Name)
	}

	for _, crt := range updateCrts {
		_, err := c.CMClient.CertmanagerV1alpha1().Certificates(crt.Namespace).Update(crt)
		if err != nil {
			return err
		}
		c.Recorder.Eventf(ing, corev1.EventTypeNormal, "UpdateCertificate", "Successfully updated Certificate %q", crt.Name)
	}

	return nil
}

func (c *Controller) validateIngress(ing *extv1beta1.Ingress) []error {
	var errs []error
	if ing.Annotations != nil {
		challengeType := ing.Annotations[acmeIssuerChallengeTypeAnnotation]
		switch challengeType {
		case "", "http01":
		case "dns01":
			providerName := ing.Annotations[acmeIssuerDNS01ProviderNameAnnotation]
			if providerName == "" {
				errs = append(errs, fmt.Errorf("No acme dns01 challenge provider specified"))
			}
		default:
			errs = append(errs, fmt.Errorf("Invalid acme challenge type specified %q", challengeType))
		}
	}
	for i, tls := range ing.Spec.TLS {
		// validate the ingress TLS block
		if len(tls.Hosts) == 0 {
			errs = append(errs, fmt.Errorf("Secret %q for ingress TLS has no hosts specified", tls.SecretName))
		}
		if tls.SecretName == "" {
			errs = append(errs, fmt.Errorf("TLS entry %d for hosts %v must specify a secretName", i, tls.Hosts))
		}
	}
	return errs
}

func (c *Controller) buildCertificates(ing *extv1beta1.Ingress, issuer v1alpha1.GenericIssuer, issuerKind string) (new, update []*v1alpha1.Certificate, _ error) {
	var newCrts []*v1alpha1.Certificate
	var updateCrts []*v1alpha1.Certificate
	for _, tls := range ing.Spec.TLS {
		existingCrt, err := c.certificateLister.Certificates(ing.Namespace).Get(tls.SecretName)
		if !apierrors.IsNotFound(err) && err != nil {
			return nil, nil, err
		}

		crt := &v1alpha1.Certificate{
			ObjectMeta: metav1.ObjectMeta{
				Name:            tls.SecretName,
				Namespace:       ing.Namespace,
				Labels:          ing.Labels,
				OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ing, ingressGVK)},
			},
			Spec: v1alpha1.CertificateSpec{
				DNSNames:   tls.Hosts,
				SecretName: tls.SecretName,
				IssuerRef: v1alpha1.ObjectReference{
					Name: issuer.GetObjectMeta().Name,
					Kind: issuerKind,
				},
			},
		}

		err = c.setIssuerSpecificConfig(crt, issuer, ing, tls)
		if err != nil {
			return nil, nil, err
		}

		// check if a Certificate for this TLS entry already exists, and if it
		// does then skip this entry
		if existingCrt != nil {
			klog.Infof("Certificate %q for ingress %q already exists", tls.SecretName, ing.Name)

			if !certNeedsUpdate(existingCrt, crt) {
				klog.Infof("Certificate %q for ingress %q is up to date", tls.SecretName, ing.Name)
				continue
			}

			updateCrt := existingCrt.DeepCopy()

			updateCrt.Spec.DNSNames = tls.Hosts
			updateCrt.Spec.SecretName = tls.SecretName
			updateCrt.Spec.IssuerRef.Name = issuer.GetObjectMeta().Name
			updateCrt.Spec.IssuerRef.Kind = issuerKind
			updateCrt.Labels = ing.Labels
			err = c.setIssuerSpecificConfig(updateCrt, issuer, ing, tls)
			if err != nil {
				return nil, nil, err
			}
			updateCrts = append(updateCrts, updateCrt)
		} else {
			newCrts = append(newCrts, crt)
		}
	}
	return newCrts, updateCrts, nil
}

// certNeedsUpdate checks and returns true if two Certificates differ
func certNeedsUpdate(a, b *v1alpha1.Certificate) bool {
	if a.Name != b.Name {
		return true
	}

	// TODO: we may need to allow users to edit the managed Certificate resources
	// to add their own labels directly.
	// Right now, we'll reset/remove the label values back automatically.
	// Let's hope no other controllers do this automatically, else we'll start fighting...
	if !reflect.DeepEqual(a.Labels, b.Labels) {
		return true
	}

	if len(a.Spec.DNSNames) != len(b.Spec.DNSNames) {
		return true
	}

	for i := range a.Spec.DNSNames {
		if a.Spec.DNSNames[i] != b.Spec.DNSNames[i] {
			return true
		}
	}

	if a.Spec.SecretName != b.Spec.SecretName {
		return true
	}

	if a.Spec.IssuerRef.Name != b.Spec.IssuerRef.Name {
		return true
	}

	if a.Spec.IssuerRef.Kind != b.Spec.IssuerRef.Kind {
		return true
	}

	var configA, configB []v1alpha1.DomainSolverConfig

	if a.Spec.ACME != nil {
		configA = a.Spec.ACME.Config
	}

	if b.Spec.ACME != nil {
		configB = b.Spec.ACME.Config
	}

	if !reflect.DeepEqual(configA, configB) {
		return true
	}

	return false
}

func (c *Controller) setIssuerSpecificConfig(crt *v1alpha1.Certificate, issuer v1alpha1.GenericIssuer, ing *extv1beta1.Ingress, tls extv1beta1.IngressTLS) error {
	ingAnnotations := ing.Annotations
	if ingAnnotations == nil {
		ingAnnotations = map[string]string{}
	}
	// for ACME issuers
	if issuer.GetSpec().ACME != nil {
		challengeType, ok := ingAnnotations[acmeIssuerChallengeTypeAnnotation]
		if !ok {
			challengeType = c.defaults.acmeIssuerChallengeType
		}
		domainCfg := v1alpha1.DomainSolverConfig{
			Domains: tls.Hosts,
		}
		switch challengeType {
		case "http01":
			editInPlaceVal, ok := ingAnnotations[editInPlaceAnnotation]
			editInPlace := editInPlaceVal == "true"
			// If the HTTP01 issuer is not enabled, skip setting the ACME field
			// on the Certificate resource.
			if issuer.GetSpec().ACME.HTTP01 == nil {
				if editInPlace {
					c.Recorder.Eventf(ing, corev1.EventTypeWarning, "Unsupported", "%s annotation cannot be enabled when using new format solver type. "+
						"Re-enable the old format HTTP01 solver, or otherwise create a specific HTTP01 solver for this Ingress.", editInPlaceAnnotation)
				}
				crt.Spec.ACME = nil
				return nil
			}
			domainCfg.HTTP01 = &v1alpha1.HTTP01SolverConfig{}
			// If annotation isn't present, or it's set to true, edit the existing ingress
			if ok && editInPlace {
				domainCfg.HTTP01.Ingress = ing.Name
			} else {
				ingressClass, ok := ingAnnotations[acmeIssuerHTTP01IngressClassAnnotation]
				if ok {
					domainCfg.HTTP01.IngressClass = &ingressClass
				} else {
					ingressClass, ok := ingAnnotations[ingressClassAnnotation]
					if ok {
						domainCfg.HTTP01.IngressClass = &ingressClass
					}
				}
			}
		case "dns01":
			// If the DNS01 issuer is not enabled, skip setting the ACME field
			// on the Certificate resource.
			if issuer.GetSpec().ACME.DNS01 == nil {
				crt.Spec.ACME = nil
				return nil
			}
			dnsProvider, ok := ingAnnotations[acmeIssuerDNS01ProviderNameAnnotation]
			if !ok {
				dnsProvider = c.defaults.acmeIssuerDNS01ProviderName
			}
			if dnsProvider == "" {
				return fmt.Errorf("no acme issuer dns01 challenge provider specified")
			}
			domainCfg.DNS01 = &v1alpha1.DNS01SolverConfig{Provider: dnsProvider}
		// If no challenge type is specified, don't set the ACME field at all
		// and instead rely on the 'new API format' to provide solver config.
		case "":
			crt.Spec.ACME = nil
			return nil
		default:
			return fmt.Errorf("invalid acme issuer challenge type specified %q", challengeType)
		}
		crt.Spec.ACME = &v1alpha1.ACMECertificateConfig{Config: []v1alpha1.DomainSolverConfig{domainCfg}}
	}
	return nil
}

// shouldSync returns true if this ingress should have a Certificate resource
// created for it
func shouldSync(ing *extv1beta1.Ingress, autoCertificateAnnotations []string) bool {
	annotations := ing.Annotations
	if annotations == nil {
		annotations = map[string]string{}
	}
	if _, ok := annotations[issuerNameAnnotation]; ok {
		return true
	}
	if _, ok := annotations[clusterIssuerNameAnnotation]; ok {
		return true
	}
	for _, x := range autoCertificateAnnotations {
		if s, ok := annotations[x]; ok {
			if b, _ := strconv.ParseBool(s); b {
				return true
			}
		}
	}
	if _, ok := annotations[acmeIssuerChallengeTypeAnnotation]; ok {
		return true
	}
	if _, ok := annotations[acmeIssuerDNS01ProviderNameAnnotation]; ok {
		return true
	}
	return false
}

// issuerForIngress will determine the issuer that should be specified on a
// Certificate created for the given Ingress resource. If one is not set, the
// default issuer given to the controller will be used.
func (c *Controller) issuerForIngress(ing *extv1beta1.Ingress) (name string, kind string) {
	name = c.defaults.issuerName
	kind = c.defaults.issuerKind
	annotations := ing.Annotations
	if annotations == nil {
		annotations = map[string]string{}
	}
	if issuerName, ok := annotations[issuerNameAnnotation]; ok {
		name = issuerName
		kind = v1alpha1.IssuerKind
	}
	if issuerName, ok := annotations[clusterIssuerNameAnnotation]; ok {
		name = issuerName
		kind = v1alpha1.ClusterIssuerKind
	}
	return name, kind
}
<commit_msg>Ensure only the sole owner of a certificate can modify it<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"reflect"
	"strconv"

	corev1 "k8s.io/api/core/v1"
	extv1beta1 "k8s.io/api/extensions/v1beta1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/klog"

	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack/cert-manager/pkg/util"
)

const (
	// editInPlaceAnnotation is used to toggle the use of ingressClass instead
	// of ingress on the created Certificate resource
	editInPlaceAnnotation = "certmanager.k8s.io/acme-http01-edit-in-place"
	// issuerNameAnnotation can be used to override the issuer specified on the
	// created Certificate resource.
	issuerNameAnnotation = "certmanager.k8s.io/issuer"
	// clusterIssuerNameAnnotation can be used to override the issuer specified on the
	// created Certificate resource. The Certificate will reference the
	// specified *ClusterIssuer* instead of normal issuer.
	clusterIssuerNameAnnotation = "certmanager.k8s.io/cluster-issuer"
	// acmeIssuerChallengeTypeAnnotation can be used to override the default ACME challenge
	// type to be used when the specified issuer is an ACME issuer
	acmeIssuerChallengeTypeAnnotation = "certmanager.k8s.io/acme-challenge-type"
	// acmeIssuerDNS01ProviderNameAnnotation can be used to override the default dns01 provider
	// configured on the issuer if the challenge type is set to dns01
	acmeIssuerDNS01ProviderNameAnnotation = "certmanager.k8s.io/acme-dns01-provider"
	// acmeIssuerHTTP01IngressClassAnnotation can be used to override the http01 ingressClass
	// if the challenge type is set to http01
	acmeIssuerHTTP01IngressClassAnnotation = "certmanager.k8s.io/acme-http01-ingress-class"

	ingressClassAnnotation = util.IngressKey
)

var ingressGVK = extv1beta1.SchemeGroupVersion.WithKind("Ingress")

func (c *Controller) Sync(ctx context.Context, ing *extv1beta1.Ingress) error {
	if !shouldSync(ing, c.defaults.autoCertificateAnnotations) {
		klog.Infof("Not syncing ingress %s/%s as it does not contain necessary annotations", ing.Namespace, ing.Name)
		return nil
	}

	issuerName, issuerKind := c.issuerForIngress(ing)
	if issuerName == "" {
		c.Recorder.Eventf(ing, corev1.EventTypeWarning, "BadConfig", "Issuer name annotation is not set and a default issuer has not been configured")
		return nil
	}

	issuer, err := c.helper.GetGenericIssuer(v1alpha1.ObjectReference{
		Name: issuerName,
		Kind: issuerKind,
	}, ing.Namespace)
	if apierrors.IsNotFound(err) {
		c.Recorder.Eventf(ing, corev1.EventTypeWarning, "BadConfig", "%s resource %q not found", issuerKind, issuerName)
		return nil
	}
	if err != nil {
		return err
	}

	errs := c.validateIngress(ing)
	if len(errs) > 0 {
		errMsg := errs[0].Error()
		if len(errs) > 1 {
			errMsg = utilerrors.NewAggregate(errs).Error()
		}
		c.Recorder.Eventf(ing, corev1.EventTypeWarning, "BadConfig", errMsg)
		return nil
	}

	newCrts, updateCrts, err := c.buildCertificates(ing, issuer, issuerKind)
	if err != nil {
		return err
	}

	for _, crt := range newCrts {
		_, err := c.CMClient.CertmanagerV1alpha1().Certificates(crt.Namespace).Create(crt)
		if err != nil {
			return err
		}
		c.Recorder.Eventf(ing, corev1.EventTypeNormal, "CreateCertificate", "Successfully created Certificate %q", crt.Name)
	}

	for _, crt := range updateCrts {
		_, err := c.CMClient.CertmanagerV1alpha1().Certificates(crt.Namespace).Update(crt)
		if err != nil {
			return err
		}
		c.Recorder.Eventf(ing, corev1.EventTypeNormal, "UpdateCertificate", "Successfully updated Certificate %q", crt.Name)
	}

	return nil
}

func (c *Controller) validateIngress(ing *extv1beta1.Ingress) []error {
	var errs []error
	if ing.Annotations != nil {
		challengeType := ing.Annotations[acmeIssuerChallengeTypeAnnotation]
		switch challengeType {
		case "", "http01":
		case "dns01":
			providerName := ing.Annotations[acmeIssuerDNS01ProviderNameAnnotation]
			if providerName == "" {
				errs = append(errs, fmt.Errorf("No acme dns01 challenge provider specified"))
			}
		default:
			errs = append(errs, fmt.Errorf("Invalid acme challenge type specified %q", challengeType))
		}
	}
	for i, tls := range ing.Spec.TLS {
		// validate the ingress TLS block
		if len(tls.Hosts) == 0 {
			errs = append(errs, fmt.Errorf("Secret %q for ingress TLS has no hosts specified", tls.SecretName))
		}
		if tls.SecretName == "" {
			errs = append(errs, fmt.Errorf("TLS entry %d for hosts %v must specify a secretName", i, tls.Hosts))
		}
	}
	return errs
}

func (c *Controller) buildCertificates(ing *extv1beta1.Ingress, issuer v1alpha1.GenericIssuer, issuerKind string) (new, update []*v1alpha1.Certificate, _ error) {
	var newCrts []*v1alpha1.Certificate
	var updateCrts []*v1alpha1.Certificate
	for _, tls := range ing.Spec.TLS {
		existingCrt, err := c.certificateLister.Certificates(ing.Namespace).Get(tls.SecretName)
		if !apierrors.IsNotFound(err) && err != nil {
			return nil, nil, err
		}

		ingOwnerRef := *metav1.NewControllerRef(ing, ingressGVK)
		crt := &v1alpha1.Certificate{
			ObjectMeta: metav1.ObjectMeta{
				Name:            tls.SecretName,
				Namespace:       ing.Namespace,
				Labels:          ing.Labels,
				OwnerReferences: []metav1.OwnerReference{ingOwnerRef},
			},
			Spec: v1alpha1.CertificateSpec{
				DNSNames:   tls.Hosts,
				SecretName: tls.SecretName,
				IssuerRef: v1alpha1.ObjectReference{
					Name: issuer.GetObjectMeta().Name,
					Kind: issuerKind,
				},
			},
		}

		err = c.setIssuerSpecificConfig(crt, issuer, ing, tls)
		if err != nil {
			return nil, nil, err
		}

		// check if a Certificate for this TLS entry already exists, and if it
		// does then skip this entry
		if existingCrt != nil {
			klog.Infof("Certificate %q for ingress %q already exists", tls.SecretName, ing.Name)

			ownerRefs := existingCrt.GetOwnerReferences()
			if len(ownerRefs) == 0 {
				klog.Infof("Certificate %q has no owners and cannot be updated for ingress %q", tls.SecretName, ing.Name)
				continue
			}

			notOwned := false
			for _, ownerRef := range ownerRefs {
				notOwned = notOwned || !reflect.DeepEqual(ingOwnerRef, ownerRef)
			}

			if notOwned {
				klog.Infof("Certificate %q is not (solely) owned by ingress %q", tls.SecretName, ing.Name)
				continue
			}

			if !certNeedsUpdate(existingCrt, crt) {
				klog.Infof("Certificate %q for ingress %q is up to date", tls.SecretName, ing.Name)
				continue
			}

			updateCrt := existingCrt.DeepCopy()

			updateCrt.Spec.DNSNames = tls.Hosts
			updateCrt.Spec.SecretName = tls.SecretName
			updateCrt.Spec.IssuerRef.Name = issuer.GetObjectMeta().Name
			updateCrt.Spec.IssuerRef.Kind = issuerKind
			updateCrt.Labels = ing.Labels
			err = c.setIssuerSpecificConfig(updateCrt, issuer, ing, tls)
			if err != nil {
				return nil, nil, err
			}
			updateCrts = append(updateCrts, updateCrt)
		} else {
			newCrts = append(newCrts, crt)
		}
	}
	return newCrts, updateCrts, nil
}

// certNeedsUpdate checks and returns true if two Certificates differ
func certNeedsUpdate(a, b *v1alpha1.Certificate) bool {
	if a.Name != b.Name {
		return true
	}

	// TODO: we may need to allow users to edit the managed Certificate resources
	// to add their own labels directly.
	// Right now, we'll reset/remove the label values back automatically.
	// Let's hope no other controllers do this automatically, else we'll start fighting...
	if !reflect.DeepEqual(a.Labels, b.Labels) {
		return true
	}

	if len(a.Spec.DNSNames) != len(b.Spec.DNSNames) {
		return true
	}

	for i := range a.Spec.DNSNames {
		if a.Spec.DNSNames[i] != b.Spec.DNSNames[i] {
			return true
		}
	}

	if a.Spec.SecretName != b.Spec.SecretName {
		return true
	}

	if a.Spec.IssuerRef.Name != b.Spec.IssuerRef.Name {
		return true
	}

	if a.Spec.IssuerRef.Kind != b.Spec.IssuerRef.Kind {
		return true
	}

	var configA, configB []v1alpha1.DomainSolverConfig

	if a.Spec.ACME != nil {
		configA = a.Spec.ACME.Config
	}

	if b.Spec.ACME != nil {
		configB = b.Spec.ACME.Config
	}

	if !reflect.DeepEqual(configA, configB) {
		return true
	}

	return false
}

func (c *Controller) setIssuerSpecificConfig(crt *v1alpha1.Certificate, issuer v1alpha1.GenericIssuer, ing *extv1beta1.Ingress, tls extv1beta1.IngressTLS) error {
	ingAnnotations := ing.Annotations
	if ingAnnotations == nil {
		ingAnnotations = map[string]string{}
	}
	// for ACME issuers
	if issuer.GetSpec().ACME != nil {
		challengeType, ok := ingAnnotations[acmeIssuerChallengeTypeAnnotation]
		if !ok {
			challengeType = c.defaults.acmeIssuerChallengeType
		}
		domainCfg := v1alpha1.DomainSolverConfig{
			Domains: tls.Hosts,
		}
		switch challengeType {
		case "http01":
			editInPlaceVal, ok := ingAnnotations[editInPlaceAnnotation]
			editInPlace := editInPlaceVal == "true"
			// If the HTTP01 issuer is not enabled, skip setting the ACME field
			// on the Certificate resource.
			if issuer.GetSpec().ACME.HTTP01 == nil {
				if editInPlace {
					c.Recorder.Eventf(ing, corev1.EventTypeWarning, "Unsupported", "%s annotation cannot be enabled when using new format solver type. "+
						"Re-enable the old format HTTP01 solver, or otherwise create a specific HTTP01 solver for this Ingress.", editInPlaceAnnotation)
				}
				crt.Spec.ACME = nil
				return nil
			}
			domainCfg.HTTP01 = &v1alpha1.HTTP01SolverConfig{}
			// If annotation isn't present, or it's set to true, edit the existing ingress
			if ok && editInPlace {
				domainCfg.HTTP01.Ingress = ing.Name
			} else {
				ingressClass, ok := ingAnnotations[acmeIssuerHTTP01IngressClassAnnotation]
				if ok {
					domainCfg.HTTP01.IngressClass = &ingressClass
				} else {
					ingressClass, ok := ingAnnotations[ingressClassAnnotation]
					if ok {
						domainCfg.HTTP01.IngressClass = &ingressClass
					}
				}
			}
		case "dns01":
			// If the DNS01 issuer is not enabled, skip setting the ACME field
			// on the Certificate resource.
			if issuer.GetSpec().ACME.DNS01 == nil {
				crt.Spec.ACME = nil
				return nil
			}
			dnsProvider, ok := ingAnnotations[acmeIssuerDNS01ProviderNameAnnotation]
			if !ok {
				dnsProvider = c.defaults.acmeIssuerDNS01ProviderName
			}
			if dnsProvider == "" {
				return fmt.Errorf("no acme issuer dns01 challenge provider specified")
			}
			domainCfg.DNS01 = &v1alpha1.DNS01SolverConfig{Provider: dnsProvider}
		// If no challenge type is specified, don't set the ACME field at all
		// and instead rely on the 'new API format' to provide solver config.
		case "":
			crt.Spec.ACME = nil
			return nil
		default:
			return fmt.Errorf("invalid acme issuer challenge type specified %q", challengeType)
		}
		crt.Spec.ACME = &v1alpha1.ACMECertificateConfig{Config: []v1alpha1.DomainSolverConfig{domainCfg}}
	}
	return nil
}

// shouldSync returns true if this ingress should have a Certificate resource
// created for it
func shouldSync(ing *extv1beta1.Ingress, autoCertificateAnnotations []string) bool {
	annotations := ing.Annotations
	if annotations == nil {
		annotations = map[string]string{}
	}
	if _, ok := annotations[issuerNameAnnotation]; ok {
		return true
	}
	if _, ok := annotations[clusterIssuerNameAnnotation]; ok {
		return true
	}
	for _, x := range autoCertificateAnnotations {
		if s, ok := annotations[x]; ok {
			if b, _ := strconv.ParseBool(s); b {
				return true
			}
		}
	}
	if _, ok := annotations[acmeIssuerChallengeTypeAnnotation]; ok {
		return true
	}
	if _, ok := annotations[acmeIssuerDNS01ProviderNameAnnotation]; ok {
		return true
	}
	return false
}

// issuerForIngress will determine the issuer that should be specified on a
// Certificate created for the given Ingress resource. If one is not set, the
// default issuer given to the controller will be used.
func (c *Controller) issuerForIngress(ing *extv1beta1.Ingress) (name string, kind string) {
	name = c.defaults.issuerName
	kind = c.defaults.issuerKind
	annotations := ing.Annotations
	if annotations == nil {
		annotations = map[string]string{}
	}
	if issuerName, ok := annotations[issuerNameAnnotation]; ok {
		name = issuerName
		kind = v1alpha1.IssuerKind
	}
	if issuerName, ok := annotations[clusterIssuerNameAnnotation]; ok {
		name = issuerName
		kind = v1alpha1.ClusterIssuerKind
	}
	return name, kind
}
<|endoftext|>


<commit_before>package bitcoind

type MiningInfo struct {
	Block            uint64  `json:"block"`
	CurrentBlocksize uint64  `json:"currentblocksize"`
	CurrentBlockTx   uint64  `json:"currentblocktx"`
	Difficulty       float64 `json:"difficulty"`
	Errors           string  `json:"errors"`
	GenProcLimit     int32   `json:"genproclimit"`
	NetworkHashps    uint64  `json:"networkhashps"`
	PooledtTx        uint64  `json:"pooledtx"`
	Testnet          bool    `json:"testnet"`
	Generate         bool    `json:"generate"`
	HashesPersec     uint64  `json:"hashespersec"`
}
<commit_msg>add doc<commit_after>package bitcoind

type MiningInfo struct {
	// The current block
	Blocks uint64 `json:"block"`
	// The last block size
	CurrentBlocksize uint64  `json:"currentblocksize"`
	CurrentBlockTx   uint64  `json:"currentblocktx"` // The last block transaction
	Difficulty       float64 `json:"difficulty"`
	Errors           string  `json:"errors"`
	GenProcLimit     int32   `json:"genproclimit"`
	NetworkHashps    uint64  `json:"networkhashps"`
	PooledtTx        uint64  `json:"pooledtx"`
	Testnet          bool    `json:"testnet"`
	Generate         bool    `json:"generate"`
	HashesPersec     uint64  `json:"hashespersec"`
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build js,wasm

package http

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"syscall/js"
)

// jsFetchMode is a Request.Header map key that, if present,
// signals that the map entry is actually an option to the Fetch API mode setting.
// Valid values are: "cors", "no-cors", "same-origin", "navigate"
// The default is "same-origin".
//
// Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters
const jsFetchMode = "js.fetch:mode"

// jsFetchCreds is a Request.Header map key that, if present,
// signals that the map entry is actually an option to the Fetch API credentials setting.
// Valid values are: "omit", "same-origin", "include"
// The default is "same-origin".
//
// Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters
const jsFetchCreds = "js.fetch:credentials"

// jsFetchRedirect is a Request.Header map key that, if present,
// signals that the map entry is actually an option to the Fetch API redirect setting.
// Valid values are: "follow", "error", "manual"
// The default is "follow".
//
// Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters
const jsFetchRedirect = "js.fetch:redirect"

// RoundTrip implements the RoundTripper interface using the WHATWG Fetch API.
func (t *Transport) RoundTrip(req *Request) (*Response, error) {
	if useFakeNetwork() {
		return t.roundTrip(req)
	}

	ac := js.Global().Get("AbortController")
	if ac != js.Undefined() {
		// Some browsers that support WASM don't necessarily support
		// the AbortController. See
		// https://developer.mozilla.org/en-US/docs/Web/API/AbortController#Browser_compatibility.
		ac = ac.New()
	}

	opt := js.Global().Get("Object").New()
	// See https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
	// for options available.
	opt.Set("method", req.Method)
	opt.Set("credentials", "same-origin")
	if h := req.Header.Get(jsFetchCreds); h != "" {
		opt.Set("credentials", h)
		req.Header.Del(jsFetchCreds)
	}
	if h := req.Header.Get(jsFetchMode); h != "" {
		opt.Set("mode", h)
		req.Header.Del(jsFetchMode)
	}
	if h := req.Header.Get(jsFetchRedirect); h != "" {
		opt.Set("redirect", h)
		req.Header.Del(jsFetchRedirect)
	}
	if ac != js.Undefined() {
		opt.Set("signal", ac.Get("signal"))
	}
	headers := js.Global().Get("Headers").New()
	for key, values := range req.Header {
		for _, value := range values {
			headers.Call("append", key, value)
		}
	}
	opt.Set("headers", headers)

	if req.Body != nil {
		// TODO(johanbrandhorst): Stream request body when possible.
		// See https://bugs.chromium.org/p/chromium/issues/detail?id=688906 for Blink issue.
		// See https://bugzilla.mozilla.org/show_bug.cgi?id=1387483 for Firefox issue.
		// See https://github.com/web-platform-tests/wpt/issues/7693 for WHATWG tests issue.
		// See https://developer.mozilla.org/en-US/docs/Web/API/Streams_API for more details on the Streams API
		// and browser support.
		body, err := ioutil.ReadAll(req.Body)
		if err != nil {
			req.Body.Close() // RoundTrip must always close the body, including on errors.
			return nil, err
		}
		req.Body.Close()
		a := js.TypedArrayOf(body)
		defer a.Release()
		opt.Set("body", a)
	}
	respPromise := js.Global().Call("fetch", req.URL.String(), opt)
	var (
		respCh = make(chan *Response, 1)
		errCh  = make(chan error, 1)
	)
	success := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		result := args[0]
		header := Header{}
		// https://developer.mozilla.org/en-US/docs/Web/API/Headers/entries
		headersIt := result.Get("headers").Call("entries")
		for {
			n := headersIt.Call("next")
			if n.Get("done").Bool() {
				break
			}
			pair := n.Get("value")
			key, value := pair.Index(0).String(), pair.Index(1).String()
			ck := CanonicalHeaderKey(key)
			header[ck] = append(header[ck], value)
		}

		contentLength := int64(0)
		if cl, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64); err == nil {
			contentLength = cl
		}

		b := result.Get("body")
		var body io.ReadCloser
		// The body is undefined when the browser does not support streaming response bodies (Firefox),
		// and null in certain error cases, i.e. when the request is blocked because of CORS settings.
		if b != js.Undefined() && b != js.Null() {
			body = &streamReader{stream: b.Call("getReader")}
		} else {
			// Fall back to using ArrayBuffer
			// https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer
			body = &arrayReader{arrayPromise: result.Call("arrayBuffer")}
		}

		select {
		case respCh <- &Response{
			Status:        result.Get("status").String() + " " + StatusText(result.Get("status").Int()),
			StatusCode:    result.Get("status").Int(),
			Header:        header,
			ContentLength: contentLength,
			Body:          body,
			Request:       req,
		}:
		case <-req.Context().Done():
		}

		return nil
	})
	defer success.Release()
	failure := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		err := fmt.Errorf("net/http: fetch() failed: %s", args[0].String())
		select {
		case errCh <- err:
		case <-req.Context().Done():
		}
		return nil
	})
	defer failure.Release()
	respPromise.Call("then", success, failure)
	select {
	case <-req.Context().Done():
		if ac != js.Undefined() {
			// Abort the Fetch request
			ac.Call("abort")
		}
		return nil, req.Context().Err()
	case resp := <-respCh:
		return resp, nil
	case err := <-errCh:
		return nil, err
	}
}

var errClosed = errors.New("net/http: reader is closed")

// useFakeNetwork is used to determine whether the request is made
// by a test and should be made to use the fake in-memory network.
func useFakeNetwork() bool {
	return len(os.Args) > 0 && strings.HasSuffix(os.Args[0], ".test")
}

// streamReader implements an io.ReadCloser wrapper for ReadableStream.
// See https://fetch.spec.whatwg.org/#readablestream for more information.
type streamReader struct {
	pending []byte
	stream  js.Value
	err     error // sticky read error
}

func (r *streamReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	if len(r.pending) == 0 {
		var (
			bCh   = make(chan []byte, 1)
			errCh = make(chan error, 1)
		)
		success := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
			result := args[0]
			if result.Get("done").Bool() {
				errCh <- io.EOF
				return nil
			}
			value := make([]byte, result.Get("value").Get("byteLength").Int())
			a := js.TypedArrayOf(value)
			a.Call("set", result.Get("value"))
			a.Release()
			bCh <- value
			return nil
		})
		defer success.Release()
		failure := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
			// Assumes it's a TypeError. See
			// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
			// for more information on this type. See
			// https://streams.spec.whatwg.org/#byob-reader-read for the spec on
			// the read method.
			errCh <- errors.New(args[0].Get("message").String())
			return nil
		})
		defer failure.Release()
		r.stream.Call("read").Call("then", success, failure)
		select {
		case b := <-bCh:
			r.pending = b
		case err := <-errCh:
			r.err = err
			return 0, err
		}
	}
	n = copy(p, r.pending)
	r.pending = r.pending[n:]
	return n, nil
}

func (r *streamReader) Close() error {
	// This ignores any error returned from cancel method. So far, I did not encounter any concrete
	// situation where reporting the error is meaningful. Most users ignore error from resp.Body.Close().
	// If there's a need to report error here, it can be implemented and tested when that need comes up.
	r.stream.Call("cancel")
	if r.err == nil {
		r.err = errClosed
	}
	return nil
}

// arrayReader implements an io.ReadCloser wrapper for ArrayBuffer.
// https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer.
type arrayReader struct {
	arrayPromise js.Value
	pending      []byte
	read         bool
	err          error // sticky read error
}

func (r *arrayReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	if !r.read {
		r.read = true
		var (
			bCh   = make(chan []byte, 1)
			errCh = make(chan error, 1)
		)
		success := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
			// Wrap the input ArrayBuffer with a Uint8Array
			uint8arrayWrapper := js.Global().Get("Uint8Array").New(args[0])
			value := make([]byte, uint8arrayWrapper.Get("byteLength").Int())
			a := js.TypedArrayOf(value)
			a.Call("set", uint8arrayWrapper)
			a.Release()
			bCh <- value
			return nil
		})
		defer success.Release()
		failure := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
			// Assumes it's a TypeError. See
			// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
			// for more information on this type.
			// See https://fetch.spec.whatwg.org/#concept-body-consume-body for reasons this might error.
			errCh <- errors.New(args[0].Get("message").String())
			return nil
		})
		defer failure.Release()
		r.arrayPromise.Call("then", success, failure)
		select {
		case b := <-bCh:
			r.pending = b
		case err := <-errCh:
			return 0, err
		}
	}
	if len(r.pending) == 0 {
		return 0, io.EOF
	}
	n = copy(p, r.pending)
	r.pending = r.pending[n:]
	return n, nil
}

func (r *arrayReader) Close() error {
	if r.err == nil {
		r.err = errClosed
	}
	return nil
}
<commit_msg>net/http: remove "number:" from Response.Status string<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build js,wasm

package http

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"syscall/js"
)

// jsFetchMode is a Request.Header map key that, if present,
// signals that the map entry is actually an option to the Fetch API mode setting.
// Valid values are: "cors", "no-cors", "same-origin", "navigate"
// The default is "same-origin".
//
// Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters
const jsFetchMode = "js.fetch:mode"

// jsFetchCreds is a Request.Header map key that, if present,
// signals that the map entry is actually an option to the Fetch API credentials setting.
// Valid values are: "omit", "same-origin", "include"
// The default is "same-origin".
//
// Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters
const jsFetchCreds = "js.fetch:credentials"

// jsFetchRedirect is a Request.Header map key that, if present,
// signals that the map entry is actually an option to the Fetch API redirect setting.
// Valid values are: "follow", "error", "manual"
// The default is "follow".
//
// Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters
const jsFetchRedirect = "js.fetch:redirect"

// RoundTrip implements the RoundTripper interface using the WHATWG Fetch API.
func (t *Transport) RoundTrip(req *Request) (*Response, error) {
	if useFakeNetwork() {
		return t.roundTrip(req)
	}

	ac := js.Global().Get("AbortController")
	if ac != js.Undefined() {
		// Some browsers that support WASM don't necessarily support
		// the AbortController. See
		// https://developer.mozilla.org/en-US/docs/Web/API/AbortController#Browser_compatibility.
		ac = ac.New()
	}

	opt := js.Global().Get("Object").New()
	// See https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
	// for options available.
	opt.Set("method", req.Method)
	opt.Set("credentials", "same-origin")
	if h := req.Header.Get(jsFetchCreds); h != "" {
		opt.Set("credentials", h)
		req.Header.Del(jsFetchCreds)
	}
	if h := req.Header.Get(jsFetchMode); h != "" {
		opt.Set("mode", h)
		req.Header.Del(jsFetchMode)
	}
	if h := req.Header.Get(jsFetchRedirect); h != "" {
		opt.Set("redirect", h)
		req.Header.Del(jsFetchRedirect)
	}
	if ac != js.Undefined() {
		opt.Set("signal", ac.Get("signal"))
	}
	headers := js.Global().Get("Headers").New()
	for key, values := range req.Header {
		for _, value := range values {
			headers.Call("append", key, value)
		}
	}
	opt.Set("headers", headers)

	if req.Body != nil {
		// TODO(johanbrandhorst): Stream request body when possible.
		// See https://bugs.chromium.org/p/chromium/issues/detail?id=688906 for Blink issue.
		// See https://bugzilla.mozilla.org/show_bug.cgi?id=1387483 for Firefox issue.
		// See https://github.com/web-platform-tests/wpt/issues/7693 for WHATWG tests issue.
		// See https://developer.mozilla.org/en-US/docs/Web/API/Streams_API for more details on the Streams API
		// and browser support.
		body, err := ioutil.ReadAll(req.Body)
		if err != nil {
			req.Body.Close() // RoundTrip must always close the body, including on errors.
			return nil, err
		}
		req.Body.Close()
		a := js.TypedArrayOf(body)
		defer a.Release()
		opt.Set("body", a)
	}
	respPromise := js.Global().Call("fetch", req.URL.String(), opt)
	var (
		respCh = make(chan *Response, 1)
		errCh  = make(chan error, 1)
	)
	success := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		result := args[0]
		header := Header{}
		// https://developer.mozilla.org/en-US/docs/Web/API/Headers/entries
		headersIt := result.Get("headers").Call("entries")
		for {
			n := headersIt.Call("next")
			if n.Get("done").Bool() {
				break
			}
			pair := n.Get("value")
			key, value := pair.Index(0).String(), pair.Index(1).String()
			ck := CanonicalHeaderKey(key)
			header[ck] = append(header[ck], value)
		}

		contentLength := int64(0)
		if cl, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64); err == nil {
			contentLength = cl
		}

		b := result.Get("body")
		var body io.ReadCloser
		// The body is undefined when the browser does not support streaming response bodies (Firefox),
		// and null in certain error cases, i.e. when the request is blocked because of CORS settings.
		if b != js.Undefined() && b != js.Null() {
			body = &streamReader{stream: b.Call("getReader")}
		} else {
			// Fall back to using ArrayBuffer
			// https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer
			body = &arrayReader{arrayPromise: result.Call("arrayBuffer")}
		}

		code := result.Get("status").Int()
		select {
		case respCh <- &Response{
			Status:        fmt.Sprintf("%d %s", code, StatusText(code)),
			StatusCode:    code,
			Header:        header,
			ContentLength: contentLength,
			Body:          body,
			Request:       req,
		}:
		case <-req.Context().Done():
		}

		return nil
	})
	defer success.Release()
	failure := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		err := fmt.Errorf("net/http: fetch() failed: %s", args[0].String())
		select {
		case errCh <- err:
		case <-req.Context().Done():
		}
		return nil
	})
	defer failure.Release()
	respPromise.Call("then", success, failure)
	select {
	case <-req.Context().Done():
		if ac != js.Undefined() {
			// Abort the Fetch request
			ac.Call("abort")
		}
		return nil, req.Context().Err()
	case resp := <-respCh:
		return resp, nil
	case err := <-errCh:
		return nil, err
	}
}

var errClosed = errors.New("net/http: reader is closed")

// useFakeNetwork is used to determine whether the request is made
// by a test and should be made to use the fake in-memory network.
func useFakeNetwork() bool {
	return len(os.Args) > 0 && strings.HasSuffix(os.Args[0], ".test")
}

// streamReader implements an io.ReadCloser wrapper for ReadableStream.
// See https://fetch.spec.whatwg.org/#readablestream for more information.
type streamReader struct {
	pending []byte
	stream  js.Value
	err     error // sticky read error
}

func (r *streamReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	if len(r.pending) == 0 {
		var (
			bCh   = make(chan []byte, 1)
			errCh = make(chan error, 1)
		)
		success := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
			result := args[0]
			if result.Get("done").Bool() {
				errCh <- io.EOF
				return nil
			}
			value := make([]byte, result.Get("value").Get("byteLength").Int())
			a := js.TypedArrayOf(value)
			a.Call("set", result.Get("value"))
			a.Release()
			bCh <- value
			return nil
		})
		defer success.Release()
		failure := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
			// Assumes it's a TypeError. See
			// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
			// for more information on this type. See
			// https://streams.spec.whatwg.org/#byob-reader-read for the spec on
			// the read method.
			errCh <- errors.New(args[0].Get("message").String())
			return nil
		})
		defer failure.Release()
		r.stream.Call("read").Call("then", success, failure)
		select {
		case b := <-bCh:
			r.pending = b
		case err := <-errCh:
			r.err = err
			return 0, err
		}
	}
	n = copy(p, r.pending)
	r.pending = r.pending[n:]
	return n, nil
}

func (r *streamReader) Close() error {
	// This ignores any error returned from cancel method. So far, I did not encounter any concrete
	// situation where reporting the error is meaningful. Most users ignore error from resp.Body.Close().
	// If there's a need to report error here, it can be implemented and tested when that need comes up.
	r.stream.Call("cancel")
	if r.err == nil {
		r.err = errClosed
	}
	return nil
}

// arrayReader implements an io.ReadCloser wrapper for ArrayBuffer.
// https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer.
type arrayReader struct {
	arrayPromise js.Value
	pending      []byte
	read         bool
	err          error // sticky read error
}

func (r *arrayReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	if !r.read {
		r.read = true
		var (
			bCh   = make(chan []byte, 1)
			errCh = make(chan error, 1)
		)
		success := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
			// Wrap the input ArrayBuffer with a Uint8Array
			uint8arrayWrapper := js.Global().Get("Uint8Array").New(args[0])
			value := make([]byte, uint8arrayWrapper.Get("byteLength").Int())
			a := js.TypedArrayOf(value)
			a.Call("set", uint8arrayWrapper)
			a.Release()
			bCh <- value
			return nil
		})
		defer success.Release()
		failure := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
			// Assumes it's a TypeError. See
			// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
			// for more information on this type.
			// See https://fetch.spec.whatwg.org/#concept-body-consume-body for reasons this might error.
			errCh <- errors.New(args[0].Get("message").String())
			return nil
		})
		defer failure.Release()
		r.arrayPromise.Call("then", success, failure)
		select {
		case b := <-bCh:
			r.pending = b
		case err := <-errCh:
			return 0, err
		}
	}
	if len(r.pending) == 0 {
		return 0, io.EOF
	}
	n = copy(p, r.pending)
	r.pending = r.pending[n:]
	return n, nil
}

func (r *arrayReader) Close() error {
	if r.err == nil {
		r.err = errClosed
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 ibelie, Chen Jie, Joungtao. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package uuid

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"github.com/ibelie/ruid"
	"github.com/ibelie/tygo"
)

type ID UUID

var ZERO = ID{}

func New() ID {
	return ID(NewV1())
}

func (u ID) Hash() ruid.ID {
	return ID(md5.Sum(u[:]))
}

func (u ID) Lt(o ruid.ID) bool {
	i := o.(ID)
	return bytes.Compare(u[:], i[:]) < 0
}

func (u ID) Ge(o ruid.ID) bool {
	i := o.(ID)
	return bytes.Compare(u[:], i[:]) >= 0
}

func (u ID) Nonzero() bool {
	return u != ZERO
}

func (u ID) String() string {
	return base64.RawURLEncoding.EncodeToString(u[:])
}

func (u ID) ByteSize() (size int) {
	return 16
}

func (u ID) Serialize(output *tygo.ProtoBuf) {
	output.Write(u[:])
}

func (u *ID) Deserialize(input *tygo.ProtoBuf) (err error) {
	_, err = input.Read(u[:])
	return
}

type UUIdentity int

var UUIdent UUIdentity = 0

func (_ UUIdentity) New() ruid.ID {
	return ID(NewV1())
}

func (_ UUIdentity) Zero() ruid.ID {
	return ZERO
}

func (_ UUIdentity) Deserialize(input *tygo.ProtoBuf) (ruid.ID, error) {
	u := ID{}
	_, err := input.Read(u[:])
	return u, err
}

func (_ UUIdentity) GetIDs(bytes []byte) (ids []ruid.ID) {
	for i, n := 0, len(bytes)-16; i <= n; i += 16 {
		u := ID{}
		copy(u[:], bytes[i:])
		ids = append(ids, u)
	}
	return
}
<commit_msg>use ruid base64 encode<commit_after>// Copyright 2017 ibelie, Chen Jie, Joungtao. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package uuid

import (
	"bytes"
	"crypto/md5"
	"github.com/ibelie/ruid"
	"github.com/ibelie/tygo"
)

type ID UUID

var ZERO = ID{}

func New() ID {
	return ID(NewV1())
}

func (u ID) Hash() ruid.ID {
	return ID(md5.Sum(u[:]))
}

func (u ID) Lt(o ruid.ID) bool {
	i := o.(ID)
	return bytes.Compare(u[:], i[:]) < 0
}

func (u ID) Ge(o ruid.ID) bool {
	i := o.(ID)
	return bytes.Compare(u[:], i[:]) >= 0
}

func (u ID) Nonzero() bool {
	return u != ZERO
}

func (u ID) String() string {
	return ruid.RUIDEncoding.EncodeToString(u[:])
}

func (u ID) ByteSize() (size int) {
	return 16
}

func (u ID) Serialize(output *tygo.ProtoBuf) {
	output.Write(u[:])
}

func (u *ID) Deserialize(input *tygo.ProtoBuf) (err error) {
	_, err = input.Read(u[:])
	return
}

type UUIdentity int

var UUIdent UUIdentity = 0

func (_ UUIdentity) New() ruid.ID {
	return ID(NewV1())
}

func (_ UUIdentity) Zero() ruid.ID {
	return ZERO
}

func (_ UUIdentity) Deserialize(input *tygo.ProtoBuf) (ruid.ID, error) {
	u := ID{}
	_, err := input.Read(u[:])
	return u, err
}

func (_ UUIdentity) GetIDs(bytes []byte) (ids []ruid.ID) {
	for i, n := 0, len(bytes)-16; i <= n; i += 16 {
		u := ID{}
		copy(u[:], bytes[i:])
		ids = append(ids, u)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"strconv"
	"testing"
	"time"

	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/pkg/testutil"
)

// TestMetricDbSizeBoot checks that the db size metric is set on boot.
func TestMetricDbSizeBoot(t *testing.T) {
	defer testutil.AfterTest(t)
	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	v, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
	if err != nil {
		t.Fatal(err)
	}

	if v == "0" {
		t.Fatalf("expected non-zero, got %q", v)
	}
}

// TestMetricDbSizeDefrag checks that the db size metric is set after defrag.
func TestMetricDbSizeDefrag(t *testing.T) {
	defer testutil.AfterTest(t)
	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	kvc := toGRPC(clus.Client(0)).KV
	mc := toGRPC(clus.Client(0)).Maintenance

	// expand the db size
	numPuts := 25 // large enough to write more than 1 page
	putreq := &pb.PutRequest{Key: []byte("k"), Value: make([]byte, 4096)}
	for i := 0; i < numPuts; i++ {
		time.Sleep(10 * time.Millisecond) // to execute multiple backend txn
		if _, err := kvc.Put(context.TODO(), putreq); err != nil {
			t.Fatal(err)
		}
	}

	// wait for backend txn sync
	time.Sleep(500 * time.Millisecond)

	expected := numPuts * len(putreq.Value)
	beforeDefrag, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	bv, err := strconv.Atoi(beforeDefrag)
	if err != nil {
		t.Fatal(err)
	}
	if bv < expected {
		t.Fatalf("expected db size greater than %d, got %d", expected, bv)
	}
	beforeDefragInUse, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_use_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	biu, err := strconv.Atoi(beforeDefragInUse)
	if err != nil {
		t.Fatal(err)
	}
	if biu < expected {
		t.Fatalf("expected db size in use is greater than %d, got %d", expected, biu)
	}

	// clear out historical keys, in use bytes should free pages
	creq := &pb.CompactionRequest{Revision: int64(numPuts), Physical: true}
	if _, kerr := kvc.Compact(context.TODO(), creq); kerr != nil {
		t.Fatal(kerr)
	}

	// Put to move PendingPages to FreePages
	if _, err = kvc.Put(context.TODO(), putreq); err != nil {
		t.Fatal(err)
	}
	time.Sleep(500 * time.Millisecond)

	afterCompactionInUse, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_use_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	aciu, err := strconv.Atoi(afterCompactionInUse)
	if err != nil {
		t.Fatal(err)
	}
	if biu <= aciu {
		t.Fatalf("expected less than %d, got %d after compaction", biu, aciu)
	}

	// defrag should give freed space back to fs
	mc.Defragment(context.TODO(), &pb.DefragmentRequest{})

	afterDefrag, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	av, err := strconv.Atoi(afterDefrag)
	if err != nil {
		t.Fatal(err)
	}
	if bv <= av {
		t.Fatalf("expected less than %d, got %d after defrag", bv, av)
	}

	afterDefragInUse, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_use_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	adiu, err := strconv.Atoi(afterDefragInUse)
	if err != nil {
		t.Fatal(err)
	}
	if adiu > av {
		t.Fatalf("db size in use (%d) is expected less than db size (%d) after defrag", adiu, av)
	}
}
<commit_msg>integration: add "TestMetricQuotaBackendBytes"<commit_after>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"strconv"
	"testing"
	"time"

	"github.com/coreos/etcd/etcdserver"

	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/pkg/testutil"
)

// TestMetricDbSizeBoot checks that the db size metric is set on boot.
func TestMetricDbSizeBoot(t *testing.T) {
	defer testutil.AfterTest(t)
	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	v, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
	if err != nil {
		t.Fatal(err)
	}

	if v == "0" {
		t.Fatalf("expected non-zero, got %q", v)
	}
}

// TestMetricDbSizeDefrag checks that the db size metric is set after defrag.
func TestMetricDbSizeDefrag(t *testing.T) {
	defer testutil.AfterTest(t)
	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	kvc := toGRPC(clus.Client(0)).KV
	mc := toGRPC(clus.Client(0)).Maintenance

	// expand the db size
	numPuts := 25 // large enough to write more than 1 page
	putreq := &pb.PutRequest{Key: []byte("k"), Value: make([]byte, 4096)}
	for i := 0; i < numPuts; i++ {
		time.Sleep(10 * time.Millisecond) // to execute multiple backend txn
		if _, err := kvc.Put(context.TODO(), putreq); err != nil {
			t.Fatal(err)
		}
	}

	// wait for backend txn sync
	time.Sleep(500 * time.Millisecond)

	expected := numPuts * len(putreq.Value)
	beforeDefrag, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	bv, err := strconv.Atoi(beforeDefrag)
	if err != nil {
		t.Fatal(err)
	}
	if bv < expected {
		t.Fatalf("expected db size greater than %d, got %d", expected, bv)
	}
	beforeDefragInUse, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_use_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	biu, err := strconv.Atoi(beforeDefragInUse)
	if err != nil {
		t.Fatal(err)
	}
	if biu < expected {
		t.Fatalf("expected db size in use is greater than %d, got %d", expected, biu)
	}

	// clear out historical keys, in use bytes should free pages
	creq := &pb.CompactionRequest{Revision: int64(numPuts), Physical: true}
	if _, kerr := kvc.Compact(context.TODO(), creq); kerr != nil {
		t.Fatal(kerr)
	}

	// Put to move PendingPages to FreePages
	if _, err = kvc.Put(context.TODO(), putreq); err != nil {
		t.Fatal(err)
	}
	time.Sleep(500 * time.Millisecond)

	afterCompactionInUse, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_use_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	aciu, err := strconv.Atoi(afterCompactionInUse)
	if err != nil {
		t.Fatal(err)
	}
	if biu <= aciu {
		t.Fatalf("expected less than %d, got %d after compaction", biu, aciu)
	}

	// defrag should give freed space back to fs
	mc.Defragment(context.TODO(), &pb.DefragmentRequest{})

	afterDefrag, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	av, err := strconv.Atoi(afterDefrag)
	if err != nil {
		t.Fatal(err)
	}
	if bv <= av {
		t.Fatalf("expected less than %d, got %d after defrag", bv, av)
	}

	afterDefragInUse, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_use_in_bytes")
	if err != nil {
		t.Fatal(err)
	}
	adiu, err := strconv.Atoi(afterDefragInUse)
	if err != nil {
		t.Fatal(err)
	}
	if adiu > av {
		t.Fatalf("db size in use (%d) is expected less than db size (%d) after defrag", adiu, av)
	}
}

func TestMetricQuotaBackendBytes(t *testing.T) {
	defer testutil.AfterTest(t)
	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	qs, err := clus.Members[0].Metric("etcd_server_quota_backend_bytes")
	if err != nil {
		t.Fatal(err)
	}
	qv, err := strconv.ParseFloat(qs, 64)
	if err != nil {
		t.Fatal(err)
	}
	if int64(qv) != etcdserver.DefaultQuotaBytes {
		t.Fatalf("expected %d, got %f", etcdserver.DefaultQuotaBytes, qv)
	}
}
<|endoftext|>


<commit_before>package gitignore

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"os/user"
	"strings"

	"github.com/go-git/go-billy/v5"
	"github.com/go-git/go-git/v5/plumbing/format/config"
	gioutil "github.com/go-git/go-git/v5/utils/ioutil"
)

const (
	commentPrefix = "#"
	coreSection   = "core"
	excludesfile  = "excludesfile"
	gitDir        = ".git"
	gitignoreFile = ".gitignore"
	gitconfigFile = ".gitconfig"
	systemFile    = "/etc/gitconfig"
)

// readIgnoreFile reads a specific git ignore file.
func readIgnoreFile(fs billy.Filesystem, path []string, ignoreFile string) (ps []Pattern, err error) {
	f, err := fs.Open(fs.Join(append(path, ignoreFile)...))
	if err == nil {
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			s := scanner.Text()
			if !strings.HasPrefix(s, commentPrefix) && len(strings.TrimSpace(s)) > 0 {
				ps = append(ps, ParsePattern(s, path))
			}
		}
	} else if !os.IsNotExist(err) {
		return nil, err
	}

	return
}

// ReadPatterns reads gitignore patterns recursively traversing through the directory
// structure. The result is in the ascending order of priority (last higher).
func ReadPatterns(fs billy.Filesystem, path []string) (ps []Pattern, err error) {
	ps, _ = readIgnoreFile(fs, path, gitignoreFile)

	var fis []os.FileInfo
	fis, err = fs.ReadDir(fs.Join(path...))
	if err != nil {
		return
	}

	for _, fi := range fis {
		if fi.IsDir() && fi.Name() != gitDir {
			var subps []Pattern
			subps, err = ReadPatterns(fs, append(path, fi.Name()))
			if err != nil {
				return
			}

			if len(subps) > 0 {
				ps = append(ps, subps...)
			}
		}
	}

	return
}

func loadPatterns(fs billy.Filesystem, path string) (ps []Pattern, err error) {
	f, err := fs.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}

	defer gioutil.CheckClose(f, &err)

	b, err := ioutil.ReadAll(f)
	if err != nil {
		return
	}

	d := config.NewDecoder(bytes.NewBuffer(b))

	raw := config.New()
	if err = d.Decode(raw); err != nil {
		return
	}

	s := raw.Section(coreSection)
	efo := s.Options.Get(excludesfile)
	if efo == "" {
		return nil, nil
	}

	ps, err = readIgnoreFile(fs, nil, efo)
	if os.IsNotExist(err) {
		return nil, nil
	}

	return
}

// LoadGlobalPatterns loads gitignore patterns from from the gitignore file
// declared in a user's ~/.gitconfig file.  If the ~/.gitconfig file does not
// exist the function will return nil.  If the core.excludesfile property
// is not declared, the function will return nil.  If the file pointed to by
// the core.excludesfile property does not exist, the function will return nil.
//
// The function assumes fs is rooted at the root filesystem.
func LoadGlobalPatterns(fs billy.Filesystem) (ps []Pattern, err error) {
	usr, err := user.Current()
	if err != nil {
		return
	}

	return loadPatterns(fs, fs.Join(usr.HomeDir, gitconfigFile))
}

// LoadSystemPatterns loads gitignore patterns from from the gitignore file
// declared in a system's /etc/gitconfig file.  If the ~/.gitconfig file does
// not exist the function will return nil.  If the core.excludesfile property
// is not declared, the function will return nil.  If the file pointed to by
// the core.excludesfile property does not exist, the function will return nil.
//
// The function assumes fs is rooted at the root filesystem.
func LoadSystemPatterns(fs billy.Filesystem) (ps []Pattern, err error) {
	return loadPatterns(fs, systemFile)
}
<commit_msg>plumbing: gitignore, Fix gitconfig path in LoadSystemPatterns doc (#256)<commit_after>package gitignore

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"os/user"
	"strings"

	"github.com/go-git/go-billy/v5"
	"github.com/go-git/go-git/v5/plumbing/format/config"
	gioutil "github.com/go-git/go-git/v5/utils/ioutil"
)

const (
	commentPrefix = "#"
	coreSection   = "core"
	excludesfile  = "excludesfile"
	gitDir        = ".git"
	gitignoreFile = ".gitignore"
	gitconfigFile = ".gitconfig"
	systemFile    = "/etc/gitconfig"
)

// readIgnoreFile reads a specific git ignore file.
func readIgnoreFile(fs billy.Filesystem, path []string, ignoreFile string) (ps []Pattern, err error) {
	f, err := fs.Open(fs.Join(append(path, ignoreFile)...))
	if err == nil {
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			s := scanner.Text()
			if !strings.HasPrefix(s, commentPrefix) && len(strings.TrimSpace(s)) > 0 {
				ps = append(ps, ParsePattern(s, path))
			}
		}
	} else if !os.IsNotExist(err) {
		return nil, err
	}

	return
}

// ReadPatterns reads gitignore patterns recursively traversing through the directory
// structure. The result is in the ascending order of priority (last higher).
func ReadPatterns(fs billy.Filesystem, path []string) (ps []Pattern, err error) {
	ps, _ = readIgnoreFile(fs, path, gitignoreFile)

	var fis []os.FileInfo
	fis, err = fs.ReadDir(fs.Join(path...))
	if err != nil {
		return
	}

	for _, fi := range fis {
		if fi.IsDir() && fi.Name() != gitDir {
			var subps []Pattern
			subps, err = ReadPatterns(fs, append(path, fi.Name()))
			if err != nil {
				return
			}

			if len(subps) > 0 {
				ps = append(ps, subps...)
			}
		}
	}

	return
}

func loadPatterns(fs billy.Filesystem, path string) (ps []Pattern, err error) {
	f, err := fs.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}

	defer gioutil.CheckClose(f, &err)

	b, err := ioutil.ReadAll(f)
	if err != nil {
		return
	}

	d := config.NewDecoder(bytes.NewBuffer(b))

	raw := config.New()
	if err = d.Decode(raw); err != nil {
		return
	}

	s := raw.Section(coreSection)
	efo := s.Options.Get(excludesfile)
	if efo == "" {
		return nil, nil
	}

	ps, err = readIgnoreFile(fs, nil, efo)
	if os.IsNotExist(err) {
		return nil, nil
	}

	return
}

// LoadGlobalPatterns loads gitignore patterns from from the gitignore file
// declared in a user's ~/.gitconfig file.  If the ~/.gitconfig file does not
// exist the function will return nil.  If the core.excludesfile property
// is not declared, the function will return nil.  If the file pointed to by
// the core.excludesfile property does not exist, the function will return nil.
//
// The function assumes fs is rooted at the root filesystem.
func LoadGlobalPatterns(fs billy.Filesystem) (ps []Pattern, err error) {
	usr, err := user.Current()
	if err != nil {
		return
	}

	return loadPatterns(fs, fs.Join(usr.HomeDir, gitconfigFile))
}

// LoadSystemPatterns loads gitignore patterns from from the gitignore file
// declared in a system's /etc/gitconfig file.  If the /etc/gitconfig file does
// not exist the function will return nil.  If the core.excludesfile property
// is not declared, the function will return nil.  If the file pointed to by
// the core.excludesfile property does not exist, the function will return nil.
//
// The function assumes fs is rooted at the root filesystem.
func LoadSystemPatterns(fs billy.Filesystem) (ps []Pattern, err error) {
	return loadPatterns(fs, systemFile)
}
<|endoftext|>


<commit_before>package effio

// methods for loading & wrangling fio's JSON output
// License: Apache 2.0

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
)

type FioJsonHistogram map[float64]float64

// three kinds of latency, slat, clat, lat all with the same fields
type FioJsonLatency struct {
	Min        float64          `json:"min"`
	Max        float64          `json:"max"`
	Mean       float64          `json:"mean"`
	Stdev      float64          `json:"stdev"`
	Percentile FioJsonHistogram `json:"percentile"`
}

// same data for Mixed/Read/Write/Trim, which are present depends on
// how fio was run
type FioJsonJobStats struct {
	IoBytes   int             `json:"io_bytes"`
	Bandwidth float64         `json:"bw"`
	BwMin     float64         `json:"bw_min"`
	BwMax     float64         `json:"bw_max"`
	BwAgg     float64         `json:"bw_agg"`
	BwMean    float64         `json:"bw_mean"`
	BwStdev   float64         `json:"bw_dev"`
	Iops      int             `json:"iops"`
	Runtime   int             `json:"runtime"`
	Slat      *FioJsonLatency `json:"slat"`
	Clat      *FioJsonLatency `json:"clat"`
	Lat       *FioJsonLatency `json:"lat"`
}

// each fio session can have multiple jobs, each job is reported
// in an array called client_stats

type FioJsonJob struct {
	Name              string           `json:"jobname"`
	Description       string           `json:"desc"`
	Groupid           int              `json:"groupid"`
	Error             int              `json:"error"`
	Mixed             *FioJsonJobStats `json:"mixed"` // fio config dependent
	Read              *FioJsonJobStats `json:"read"`  // fio config dependent
	Write             *FioJsonJobStats `json:"write"` // fio config dependent
	Trim              *FioJsonJobStats `json:"trim"`  // fio config dependent
	UsrCpu            float64          `json:"usr_cpu"`
	SysCpu            float64          `json:"sys_cpu"`
	ContextSwitches   int              `json:"ctx"`
	MajorFaults       int              `json:"majf"`
	MinorFaults       int              `json:"minf"`
	IODepthLevel      FioJsonHistogram `json:"iodepth_level"`
	LatencyUsec       FioJsonHistogram `json:"latency_us"`
	LatencyMsec       FioJsonHistogram `json:"latency_ms"`
	LatencyDepth      int              `json:"latency_depth"`
	LatencyTarget     int              `json:"latency_target"`
	LatencyPercentile float64          `json:"latency_percentile"`
	LatencyWindow     int              `json:"latency_window"`
	Hostname          string           `json:"hostname"`
	Port              int              `json:"port"`
}

type FioJsonDiskUtil struct {
	Name        string  `json:"name"`
	ReadIos     int     `json:"read_ios"`
	WriteIos    int     `json:"write_ios"`
	ReadMerges  int     `json:"read_merges"`
	WriteMerges int     `json:"write_merges"`
	ReadTicks   int     `json:"read_ticks"`
	WriteTicks  int     `json:"write_ticks"`
	InQueue     int     `json:"in_queue"`
	Util        float64 `json:"util"`
}

type FioJsonData struct {
	Filename      string            `json:"filename"`
	FioVersion    string            `json:"fio version"`
	HeaderGarbage string            `json:"garbage"`
	Jobs          []FioJsonJob      `json:"jobs"`
	DiskUtil      []FioJsonDiskUtil `json:"disk_util"`
}

func LoadFioJsonData(filename string) (fdata FioJsonData) {
	dataBytes, err := ioutil.ReadFile(filename)

	if os.IsNotExist(err) {
		log.Fatalf("Could not read file %s: %s", filename, err)
	}

	// data loaded OK
	fdata.Filename = filename

	// fio writes a bunch of crap out to the output file before the JSON so for
	// now do the easy thing and find the first { after a \n and call it good
	offset := bytes.Index(dataBytes, []byte("\n{"))
	// bytes.Index will return -1 for not found, in which case we assume that it
	// been trimmed from the input file and start at index 0
	if offset == -1 {
		offset = 0
	}

	err = json.Unmarshal(dataBytes[offset:], &fdata)
	if err != nil {
		log.Fatalf("Could not parse JSON: %s", err)
	}

	fdata.HeaderGarbage = string(dataBytes[0:offset])

	return
}

// some of the bucket keys are in the form ">=50.00" which of course
// cannot be unmarshaled into a number, so clean that up before trying
func (hst *FioJsonHistogram) UnmarshalJSON(data []byte) error {
	hststr := make(map[string]float64, len(data) / 8)

	err := json.Unmarshal(data, &hststr)
	if err != nil {
		return err
	}

	out := make(FioJsonHistogram, len(hststr))

	for k, v := range hststr {
		// remove the ">=" fio puts in some of the keys
		cleaned := strings.TrimPrefix(k, ">=")
		fkey, _ := strconv.ParseFloat(cleaned, 64)
		out[fkey] = v
	}

	hst = &out

	return nil
}

// JSON doesn't officially support anything but strings as keys
// so the floats have to be converted with this handler.
func (hst FioJsonHistogram) MarshalJSON() ([]byte, error) {
	started := false
	buf := make([]byte, 8192) // lazy
	sep := ""
	buf[0] = '{'
	bufidx := 1
	for key, val := range hst {
		if started {
			sep = ","
		} else {
			started = true
		}
		out := []byte(fmt.Sprintf("\"%g\":%g%s", key, val, sep))
		bufidx += copy(buf[bufidx:bufidx+len(out)], out)
	}
	buf[bufidx] = '}'

	return buf[0 : bufidx+1], nil
}
<commit_msg>go fmt<commit_after>package effio

// methods for loading & wrangling fio's JSON output
// License: Apache 2.0

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
)

type FioJsonHistogram map[float64]float64

// three kinds of latency, slat, clat, lat all with the same fields
type FioJsonLatency struct {
	Min        float64          `json:"min"`
	Max        float64          `json:"max"`
	Mean       float64          `json:"mean"`
	Stdev      float64          `json:"stdev"`
	Percentile FioJsonHistogram `json:"percentile"`
}

// same data for Mixed/Read/Write/Trim, which are present depends on
// how fio was run
type FioJsonJobStats struct {
	IoBytes   int             `json:"io_bytes"`
	Bandwidth float64         `json:"bw"`
	BwMin     float64         `json:"bw_min"`
	BwMax     float64         `json:"bw_max"`
	BwAgg     float64         `json:"bw_agg"`
	BwMean    float64         `json:"bw_mean"`
	BwStdev   float64         `json:"bw_dev"`
	Iops      int             `json:"iops"`
	Runtime   int             `json:"runtime"`
	Slat      *FioJsonLatency `json:"slat"`
	Clat      *FioJsonLatency `json:"clat"`
	Lat       *FioJsonLatency `json:"lat"`
}

// each fio session can have multiple jobs, each job is reported
// in an array called client_stats

type FioJsonJob struct {
	Name              string           `json:"jobname"`
	Description       string           `json:"desc"`
	Groupid           int              `json:"groupid"`
	Error             int              `json:"error"`
	Mixed             *FioJsonJobStats `json:"mixed"` // fio config dependent
	Read              *FioJsonJobStats `json:"read"`  // fio config dependent
	Write             *FioJsonJobStats `json:"write"` // fio config dependent
	Trim              *FioJsonJobStats `json:"trim"`  // fio config dependent
	UsrCpu            float64          `json:"usr_cpu"`
	SysCpu            float64          `json:"sys_cpu"`
	ContextSwitches   int              `json:"ctx"`
	MajorFaults       int              `json:"majf"`
	MinorFaults       int              `json:"minf"`
	IODepthLevel      FioJsonHistogram `json:"iodepth_level"`
	LatencyUsec       FioJsonHistogram `json:"latency_us"`
	LatencyMsec       FioJsonHistogram `json:"latency_ms"`
	LatencyDepth      int              `json:"latency_depth"`
	LatencyTarget     int              `json:"latency_target"`
	LatencyPercentile float64          `json:"latency_percentile"`
	LatencyWindow     int              `json:"latency_window"`
	Hostname          string           `json:"hostname"`
	Port              int              `json:"port"`
}

type FioJsonDiskUtil struct {
	Name        string  `json:"name"`
	ReadIos     int     `json:"read_ios"`
	WriteIos    int     `json:"write_ios"`
	ReadMerges  int     `json:"read_merges"`
	WriteMerges int     `json:"write_merges"`
	ReadTicks   int     `json:"read_ticks"`
	WriteTicks  int     `json:"write_ticks"`
	InQueue     int     `json:"in_queue"`
	Util        float64 `json:"util"`
}

type FioJsonData struct {
	Filename      string            `json:"filename"`
	FioVersion    string            `json:"fio version"`
	HeaderGarbage string            `json:"garbage"`
	Jobs          []FioJsonJob      `json:"jobs"`
	DiskUtil      []FioJsonDiskUtil `json:"disk_util"`
}

func LoadFioJsonData(filename string) (fdata FioJsonData) {
	dataBytes, err := ioutil.ReadFile(filename)

	if os.IsNotExist(err) {
		log.Fatalf("Could not read file %s: %s", filename, err)
	}

	// data loaded OK
	fdata.Filename = filename

	// fio writes a bunch of crap out to the output file before the JSON so for
	// now do the easy thing and find the first { after a \n and call it good
	offset := bytes.Index(dataBytes, []byte("\n{"))
	// bytes.Index will return -1 for not found, in which case we assume that it
	// been trimmed from the input file and start at index 0
	if offset == -1 {
		offset = 0
	}

	err = json.Unmarshal(dataBytes[offset:], &fdata)
	if err != nil {
		log.Fatalf("Could not parse JSON: %s", err)
	}

	fdata.HeaderGarbage = string(dataBytes[0:offset])

	return
}

// some of the bucket keys are in the form ">=50.00" which of course
// cannot be unmarshaled into a number, so clean that up before trying
func (hst *FioJsonHistogram) UnmarshalJSON(data []byte) error {
	hststr := make(map[string]float64, len(data)/8)

	err := json.Unmarshal(data, &hststr)
	if err != nil {
		return err
	}

	out := make(FioJsonHistogram, len(hststr))

	for k, v := range hststr {
		// remove the ">=" fio puts in some of the keys
		cleaned := strings.TrimPrefix(k, ">=")
		fkey, _ := strconv.ParseFloat(cleaned, 64)
		out[fkey] = v
	}

	hst = &out

	return nil
}

// JSON doesn't officially support anything but strings as keys
// so the floats have to be converted with this handler.
func (hst FioJsonHistogram) MarshalJSON() ([]byte, error) {
	started := false
	buf := make([]byte, 8192) // lazy
	sep := ""
	buf[0] = '{'
	bufidx := 1
	for key, val := range hst {
		if started {
			sep = ","
		} else {
			started = true
		}
		out := []byte(fmt.Sprintf("\"%g\":%g%s", key, val, sep))
		bufidx += copy(buf[bufidx:bufidx+len(out)], out)
	}
	buf[bufidx] = '}'

	return buf[0 : bufidx+1], nil
}
<|endoftext|>


<commit_before>package conntrack

import (
	"encoding/binary"
	"fmt"
	"net"
	"syscall"
	"unsafe"
)

const (
	sizeofGenmsg = uint32(unsafe.Sizeof(nfgenmsg{}))
)

type ConntrackListReq struct {
	Header syscall.NlMsghdr
	Body   nfgenmsg
}

func (c *ConntrackListReq) toWireFormat() []byte {
	// adapted from syscall/NetlinkRouteRequest.toWireFormat
	b := make([]byte, c.Header.Len)
	*(*uint32)(unsafe.Pointer(&b[0:4][0])) = c.Header.Len
	*(*uint16)(unsafe.Pointer(&b[4:6][0])) = c.Header.Type
	*(*uint16)(unsafe.Pointer(&b[6:8][0])) = c.Header.Flags
	*(*uint32)(unsafe.Pointer(&b[8:12][0])) = c.Header.Seq
	*(*uint32)(unsafe.Pointer(&b[12:16][0])) = c.Header.Pid
	b[16] = byte(c.Body.Family)
	b[17] = byte(c.Body.Version)
	*(*uint16)(unsafe.Pointer(&b[18:20][0])) = c.Body.ResID
	return b
}

func connectNetfilter(bufferSize int, groups uint32) (int, *syscall.SockaddrNetlink, error) {
	s, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_NETFILTER)
	if err != nil {
		return 0, nil, err
	}
	lsa := &syscall.SockaddrNetlink{
		Family: syscall.AF_NETLINK,
		Groups: groups,
	}
	if err := syscall.Bind(s, lsa); err != nil {
		return 0, nil, err
	}
	if bufferSize > 0 {
		if err := syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, bufferSize); err != nil {
			if err := syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_RCVBUF, bufferSize); err != nil {
				return 0, nil, err
			}
		}
	}
	return s, lsa, nil
}

// Established lists all established TCP connections.
func Established(bufferSize int) ([]Flow, error) {
	s, lsa, err := connectNetfilter(bufferSize, 0)
	if err != nil {
		return nil, err
	}
	defer syscall.Close(s)

	var flows []Flow
	msg := ConntrackListReq{
		Header: syscall.NlMsghdr{
			Len:   syscall.NLMSG_HDRLEN + sizeofGenmsg,
			Type:  (NFNL_SUBSYS_CTNETLINK << 8) | uint16(IpctnlMsgCtGet),
			Flags: syscall.NLM_F_REQUEST | syscall.NLM_F_DUMP,
			Pid:   0,
			Seq:   0,
		},
		Body: nfgenmsg{
			Family:  syscall.AF_INET,
			Version: NFNETLINK_V0,
			ResID:   0,
		},
	}
	wb := msg.toWireFormat()
	if err := syscall.Sendto(s, wb, 0, lsa); err != nil {
		return nil, err
	}

	readMsgs(s, func(c Flow) {
		if c.MsgType != NfctMsgUpdate {
			return
		}
		flows = append(flows, c)
	})
	return flows, nil
}

// Follow gives a channel with all changes.
func Follow(bufferSize int) (<-chan Flow, func(), error) {
	s, _, err := connectNetfilter(bufferSize, NF_NETLINK_CONNTRACK_NEW|NF_NETLINK_CONNTRACK_UPDATE|NF_NETLINK_CONNTRACK_DESTROY)
	stop := func() {
		syscall.Close(s)
	}
	if err != nil {
		return nil, stop, err
	}

	res := make(chan Flow, 1)
	go func() {
		defer syscall.Close(s)
		if err := readMsgs(s, func(c Flow) {
			res <- c
		}); err != nil {
			close(res)
			return
		}
	}()
	return res, stop, nil
}

func readMsgs(s int, cb func(Flow)) error {
	for {
		// TODO(jpg): Re-use the receive buffer.
		// Will require copying any byte slices we will take pointers to
		rb := make([]byte, syscall.Getpagesize())
		nr, _, err := syscall.Recvfrom(s, rb, 0)
		if err != nil {
			return err
		}

		msgs, err := syscall.ParseNetlinkMessage(rb[:nr])
		if err != nil {
			return err
		}
		for _, msg := range msgs {
			if err := nfnlIsError(msg.Header); err != nil {
			}
			if nflnSubsysID(msg.Header.Type) != NFNL_SUBSYS_CTNETLINK {
				return fmt.Errorf(
					"unexpected subsys_id: %d\n",
					nflnSubsysID(msg.Header.Type),
				)
			}
			flow, err := parsePayload(msg.Data[sizeofGenmsg:])
			if err != nil {
				return err
			}

			// Disregard non-TCP flows for now
			if flow.Original.Proto != syscall.IPPROTO_TCP {
				continue
			}

			// Taken from conntrack/parse.c:__parse_message_type
			switch CntlMsgTypes(nflnMsgType(msg.Header.Type)) {
			case IpctnlMsgCtNew:
				flow.MsgType = NfctMsgUpdate
				if msg.Header.Flags&(syscall.NLM_F_CREATE|syscall.NLM_F_EXCL) > 0 {
					flow.MsgType = NfctMsgNew
				}
			case IpctnlMsgCtDelete:
				flow.MsgType = NfctMsgDestroy
			}

			cb(*flow)
		}
	}
}

type Flow struct {
	MsgType  NfConntrackMsg
	Original Meta
	Reply    Meta
	State    TCPState
	ID       uint32
}

type Layer3 struct {
	SrcIP net.IP
	DstIP net.IP
}

type Layer4 struct {
	SrcPort uint16
	DstPort uint16
	Proto   uint8
}

type Meta struct {
	Layer3
	Layer4
}

type nfgenmsg struct {
	Family  uint8  /* AF_xxx */
	Version uint8  /* nfnetlink version */
	ResID   uint16 /* resource id */
}

func parsePayload(b []byte) (*Flow, error) {
	// Adapted from libnetfilter_conntrack/src/conntrack/parse_mnl.c
	flow := &Flow{}
	attrs, err := parseAttrs(b)
	if err != nil {
		return flow, err
	}
	for _, attr := range attrs {
		switch CtattrType(attr.Typ) {
		case CtaTupleOrig:
			parseTuple(attr.Msg, &flow.Original)
		case CtaTupleReply:
			parseTuple(attr.Msg, &flow.Reply)
		case CtaProtoinfo:
			parseProtoinfo(attr.Msg, flow)
		case CtaId:
			flow.ID = binary.BigEndian.Uint32(attr.Msg)
		}
	}
	return flow, nil
}

func parseTuple(b []byte, meta *Meta) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		switch CtattrTuple(attr.Typ) {
		case CtaTupleUnspec:
		case CtaTupleIp:
			if err := parseIP(attr.Msg, meta); err != nil {
				return err
			}
		case CtaTupleProto:
			parseProto(attr.Msg, meta)
		}
	}
	return nil
}

func parseIP(b []byte, meta *Meta) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		// TODO(jpg) IPv6 support
		switch CtattrIp(attr.Typ) {
		case CtaIpV4Src:
			meta.SrcIP = net.IP(attr.Msg) // TODO: copy so we can reuse the buffer?
		case CtaIpV4Dst:
			meta.DstIP = net.IP(attr.Msg) // TODO: copy so we can reuse the buffer?
		}
	}
	return nil
}

func parseProto(b []byte, meta *Meta) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		switch CtattrL4proto(attr.Typ) {
		case CtaProtoNum:
			meta.Proto = uint8(attr.Msg[0])
		case CtaProtoSrcPort:
			meta.SrcPort = binary.BigEndian.Uint16(attr.Msg)
		case CtaProtoDstPort:
			meta.DstPort = binary.BigEndian.Uint16(attr.Msg)
		}
	}
	return nil
}

func parseProtoinfo(b []byte, flow *Flow) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		switch CtattrProtoinfo(attr.Typ) {
		case CtaProtoinfoTcp:
			if err := parseProtoinfoTCP(attr.Msg, flow); err != nil {
				return err
			}
		default:
			// we're not interested in other protocols
		}
	}
	return nil
}

func parseProtoinfoTCP(b []byte, flow *Flow) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		switch CtattrProtoinfoTcp(attr.Typ) {
		case CtaProtoinfoTcpState:
			flow.State = TCPState(attr.Msg[0])
		default:
			// we're not interested in other protocols
		}
	}
	return nil
}
<commit_msg>Use Nfgenmsg from unix package instead of declaring locally<commit_after>package conntrack

import (
	"encoding/binary"
	"fmt"
	"net"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

const (
	sizeofGenmsg = uint32(unsafe.Sizeof(unix.Nfgenmsg{})) // TODO
)

type ConntrackListReq struct {
	Header syscall.NlMsghdr
	Body   unix.Nfgenmsg
}

func (c *ConntrackListReq) toWireFormat() []byte {
	// adapted from syscall/NetlinkRouteRequest.toWireFormat
	b := make([]byte, c.Header.Len)
	*(*uint32)(unsafe.Pointer(&b[0:4][0])) = c.Header.Len
	*(*uint16)(unsafe.Pointer(&b[4:6][0])) = c.Header.Type
	*(*uint16)(unsafe.Pointer(&b[6:8][0])) = c.Header.Flags
	*(*uint32)(unsafe.Pointer(&b[8:12][0])) = c.Header.Seq
	*(*uint32)(unsafe.Pointer(&b[12:16][0])) = c.Header.Pid
	b[16] = byte(c.Body.Nfgen_family)
	b[17] = byte(c.Body.Version)
	*(*uint16)(unsafe.Pointer(&b[18:20][0])) = c.Body.Res_id
	return b
}

func connectNetfilter(bufferSize int, groups uint32) (int, *syscall.SockaddrNetlink, error) {
	s, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_NETFILTER)
	if err != nil {
		return 0, nil, err
	}
	lsa := &syscall.SockaddrNetlink{
		Family: syscall.AF_NETLINK,
		Groups: groups,
	}
	if err := syscall.Bind(s, lsa); err != nil {
		return 0, nil, err
	}
	if bufferSize > 0 {
		if err := syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, bufferSize); err != nil {
			if err := syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_RCVBUF, bufferSize); err != nil {
				return 0, nil, err
			}
		}
	}
	return s, lsa, nil
}

// Established lists all established TCP connections.
func Established(bufferSize int) ([]Flow, error) {
	s, lsa, err := connectNetfilter(bufferSize, 0)
	if err != nil {
		return nil, err
	}
	defer syscall.Close(s)

	var flows []Flow
	msg := ConntrackListReq{
		Header: syscall.NlMsghdr{
			Len:   syscall.NLMSG_HDRLEN + sizeofGenmsg,
			Type:  (NFNL_SUBSYS_CTNETLINK << 8) | uint16(IpctnlMsgCtGet),
			Flags: syscall.NLM_F_REQUEST | syscall.NLM_F_DUMP,
			Pid:   0,
			Seq:   0,
		},
		Body: unix.Nfgenmsg{
			Nfgen_family: syscall.AF_INET,
			Version:      NFNETLINK_V0,
			Res_id:       0,
		},
	}
	wb := msg.toWireFormat()
	if err := syscall.Sendto(s, wb, 0, lsa); err != nil {
		return nil, err
	}

	readMsgs(s, func(c Flow) {
		if c.MsgType != NfctMsgUpdate {
			return
		}
		flows = append(flows, c)
	})
	return flows, nil
}

// Follow gives a channel with all changes.
func Follow(bufferSize int) (<-chan Flow, func(), error) {
	s, _, err := connectNetfilter(bufferSize, NF_NETLINK_CONNTRACK_NEW|NF_NETLINK_CONNTRACK_UPDATE|NF_NETLINK_CONNTRACK_DESTROY)
	stop := func() {
		syscall.Close(s)
	}
	if err != nil {
		return nil, stop, err
	}

	res := make(chan Flow, 1)
	go func() {
		defer syscall.Close(s)
		if err := readMsgs(s, func(c Flow) {
			res <- c
		}); err != nil {
			close(res)
			return
		}
	}()
	return res, stop, nil
}

func readMsgs(s int, cb func(Flow)) error {
	for {
		// TODO(jpg): Re-use the receive buffer.
		// Will require copying any byte slices we will take pointers to
		rb := make([]byte, syscall.Getpagesize())
		nr, _, err := syscall.Recvfrom(s, rb, 0)
		if err != nil {
			return err
		}

		msgs, err := syscall.ParseNetlinkMessage(rb[:nr])
		if err != nil {
			return err
		}
		for _, msg := range msgs {
			if err := nfnlIsError(msg.Header); err != nil {
			}
			if nflnSubsysID(msg.Header.Type) != NFNL_SUBSYS_CTNETLINK {
				return fmt.Errorf(
					"unexpected subsys_id: %d\n",
					nflnSubsysID(msg.Header.Type),
				)
			}
			flow, err := parsePayload(msg.Data[sizeofGenmsg:])
			if err != nil {
				return err
			}

			// Disregard non-TCP flows for now
			if flow.Original.Proto != syscall.IPPROTO_TCP {
				continue
			}

			// Taken from conntrack/parse.c:__parse_message_type
			switch CntlMsgTypes(nflnMsgType(msg.Header.Type)) {
			case IpctnlMsgCtNew:
				flow.MsgType = NfctMsgUpdate
				if msg.Header.Flags&(syscall.NLM_F_CREATE|syscall.NLM_F_EXCL) > 0 {
					flow.MsgType = NfctMsgNew
				}
			case IpctnlMsgCtDelete:
				flow.MsgType = NfctMsgDestroy
			}

			cb(*flow)
		}
	}
}

type Flow struct {
	MsgType  NfConntrackMsg
	Original Meta
	Reply    Meta
	State    TCPState
	ID       uint32
}

type Layer3 struct {
	SrcIP net.IP
	DstIP net.IP
}

type Layer4 struct {
	SrcPort uint16
	DstPort uint16
	Proto   uint8
}

type Meta struct {
	Layer3
	Layer4
}

func parsePayload(b []byte) (*Flow, error) {
	// Adapted from libnetfilter_conntrack/src/conntrack/parse_mnl.c
	flow := &Flow{}
	attrs, err := parseAttrs(b)
	if err != nil {
		return flow, err
	}
	for _, attr := range attrs {
		switch CtattrType(attr.Typ) {
		case CtaTupleOrig:
			parseTuple(attr.Msg, &flow.Original)
		case CtaTupleReply:
			parseTuple(attr.Msg, &flow.Reply)
		case CtaProtoinfo:
			parseProtoinfo(attr.Msg, flow)
		case CtaId:
			flow.ID = binary.BigEndian.Uint32(attr.Msg)
		}
	}
	return flow, nil
}

func parseTuple(b []byte, meta *Meta) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		switch CtattrTuple(attr.Typ) {
		case CtaTupleUnspec:
		case CtaTupleIp:
			if err := parseIP(attr.Msg, meta); err != nil {
				return err
			}
		case CtaTupleProto:
			parseProto(attr.Msg, meta)
		}
	}
	return nil
}

func parseIP(b []byte, meta *Meta) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		// TODO(jpg) IPv6 support
		switch CtattrIp(attr.Typ) {
		case CtaIpV4Src:
			meta.SrcIP = net.IP(attr.Msg) // TODO: copy so we can reuse the buffer?
		case CtaIpV4Dst:
			meta.DstIP = net.IP(attr.Msg) // TODO: copy so we can reuse the buffer?
		}
	}
	return nil
}

func parseProto(b []byte, meta *Meta) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		switch CtattrL4proto(attr.Typ) {
		case CtaProtoNum:
			meta.Proto = uint8(attr.Msg[0])
		case CtaProtoSrcPort:
			meta.SrcPort = binary.BigEndian.Uint16(attr.Msg)
		case CtaProtoDstPort:
			meta.DstPort = binary.BigEndian.Uint16(attr.Msg)
		}
	}
	return nil
}

func parseProtoinfo(b []byte, flow *Flow) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		switch CtattrProtoinfo(attr.Typ) {
		case CtaProtoinfoTcp:
			if err := parseProtoinfoTCP(attr.Msg, flow); err != nil {
				return err
			}
		default:
			// we're not interested in other protocols
		}
	}
	return nil
}

func parseProtoinfoTCP(b []byte, flow *Flow) error {
	attrs, err := parseAttrs(b)
	if err != nil {
		return fmt.Errorf("invalid tuple attr: %s", err)
	}
	for _, attr := range attrs {
		switch CtattrProtoinfoTcp(attr.Typ) {
		case CtaProtoinfoTcpState:
			flow.State = TCPState(attr.Msg[0])
		default:
			// we're not interested in other protocols
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"fmt"
	"encoding/json"

	"testing"
	"net/http/httptest"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Handlers", func() {

	var (
		w *httptest.ResponseRecorder
	)

	BeforeEach(func() {
		w = httptest.NewRecorder()
	})

	Describe("Get todos", func() {
		Context("Plain", func() {
			It("should return 2 todos", func() {
				r, err := http.NewRequest("GET", "/", nil)
				Expect(err).NotTo(HaveOccurred())

				TodoIndex(w, r)

				var ts Todos
				err = json.NewDecoder(w.Body).Decode(&ts)

				fmt.Printf("the returned body is {}", ts)
			})
		})
	})
})

func TestHandlers(t *testing.T){
	RegisterFailHandler(Fail)
	RunSpecs(t, "Handler suite")
}

//func TestTodoIndex(t *testing.T){
//	w := httptest.NewRecorder()
//	r, err := http.NewRequest("GET", "/", nil)
//
//	if err != nil {
//		t.Fatal("request creation error")
//	}
//
//	TodoIndex(w, r)
//
//	if w.Code != 200 {
//		t.Fatalf("wrong status code: {}", w.Code)
//	}
//
//	var ts Todos
//	err = json.NewDecoder(w.Body).Decode(&ts)
//
//
//
//	fmt.Printf("the returned body is {}", ts)
//}

//func TestCreateTodo(t *testing.T){
//	TodoCreate()
//}
<commit_msg>handlers test improved<commit_after>package main

import (
	"net/http"
	"fmt"
	"encoding/json"

	"testing"
	"net/http/httptest"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Handlers", func() {

	var (
		w *httptest.ResponseRecorder
	)

	BeforeEach(func() {
		w = httptest.NewRecorder()
	})

	Describe("Get todos", func() {
		Context("Plain", func() {
			It("should return 2 todos", func() {
				r, err := http.NewRequest("GET", "/", nil)
				Ω(err).NotTo(HaveOccurred())

				TodoIndex(w, r)

				var ts Todos
				err = json.NewDecoder(w.Body).Decode(&ts)

				Ω(len(ts)).To(BeEquivalentTo(2))
				fmt.Printf("the returned body is {}", ts)
			})
		})
	})
})

func TestHandlers(t *testing.T){
	RegisterFailHandler(Fail)
	RunSpecs(t, "Handler suite")
}

//func TestTodoIndex(t *testing.T){
//	w := httptest.NewRecorder()
//	r, err := http.NewRequest("GET", "/", nil)
//
//	if err != nil {
//		t.Fatal("request creation error")
//	}
//
//	TodoIndex(w, r)
//
//	if w.Code != 200 {
//		t.Fatalf("wrong status code: {}", w.Code)
//	}
//
//	var ts Todos
//	err = json.NewDecoder(w.Body).Decode(&ts)
//
//
//
//	fmt.Printf("the returned body is {}", ts)
//}

//func TestCreateTodo(t *testing.T){
//	TodoCreate()
//}
<|endoftext|>