<commit_before>/*
Copyright 2017 Heptio Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package backup

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"strings"
	"time"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	kuberrs "k8s.io/apimachinery/pkg/util/errors"

	api "github.com/heptio/ark/pkg/apis/ark/v1"
	"github.com/heptio/ark/pkg/client"
	"github.com/heptio/ark/pkg/discovery"
	"github.com/heptio/ark/pkg/util/collections"
)

// Backupper performs backups.
type Backupper interface {
	// Backup takes a backup using the specification in the api.Backup and writes backup data to the
	// given writers.
	Backup(backup *api.Backup, data, log io.Writer) error
}

// kubernetesBackupper implements Backupper.
type kubernetesBackupper struct {
	dynamicFactory  client.DynamicFactory
	discoveryHelper discovery.Helper
	actions         map[schema.GroupResource]Action
	itemBackupper   itemBackupper
}

var _ Backupper = &kubernetesBackupper{}

// ActionContext contains contextual information for actions.
type ActionContext struct {
	logger *logger
}

func (ac ActionContext) log(msg string, args ...interface{}) {
	ac.logger.log(msg, args...)
}

// Action is an actor that performs an operation on an individual item being backed up.
type Action interface {
	// Execute is invoked on an item being backed up. If an error is returned, the Backup is marked as
	// failed.
	Execute(ctx ActionContext, item map[string]interface{}, backup *api.Backup) error
}

// NewKubernetesBackupper creates a new kubernetesBackupper.
func NewKubernetesBackupper(
	discoveryHelper discovery.Helper,
	dynamicFactory client.DynamicFactory,
	actions map[string]Action,
) (Backupper, error) {
	resolvedActions, err := resolveActions(discoveryHelper.Mapper(), actions)
	if err != nil {
		return nil, err
	}

	return &kubernetesBackupper{
		discoveryHelper: discoveryHelper,
		dynamicFactory:  dynamicFactory,
		actions:         resolvedActions,
		itemBackupper:   &realItemBackupper{},
	}, nil
}

// resolveActions resolves the string-based map of group-resources to actions and returns a map of
// schema.GroupResources to actions.
func resolveActions(mapper meta.RESTMapper, actions map[string]Action) (map[schema.GroupResource]Action, error) {
	ret := make(map[schema.GroupResource]Action)

	for resource, action := range actions {
		gr, err := resolveGroupResource(mapper, resource)
		if err != nil {
			return nil, err
		}
		ret[gr] = action
	}

	return ret, nil
}

// resolveResources uses the RESTMapper to resolve resources to their fully-qualified group-resource
// names. fn is invoked for each resolved resource. resolveResources returns a list of any resources that failed to resolve.
func (ctx *backupContext) resolveResources(mapper meta.RESTMapper, resources []string, allowAll bool, fn func(string)) {
	for _, resource := range resources {
		if allowAll && resource == "*" {
			fn("*")
			return
		}
		gr, err := resolveGroupResource(mapper, resource)
		if err != nil {
			ctx.log("Unable to resolve resource %q: %v", resource, err)
			continue
		}
		fn(gr.String())
	}
}

// getResourceIncludesExcludes takes the lists of resources to include and exclude, uses the
// RESTMapper to resolve them to fully-qualified group-resource names, and returns an
// IncludesExcludes list.
func (ctx *backupContext) getResourceIncludesExcludes(mapper meta.RESTMapper, includes, excludes []string) *collections.IncludesExcludes {
	resources := collections.NewIncludesExcludes()

	ctx.resolveResources(mapper, includes, true, func(s string) { resources.Includes(s) })
	ctx.resolveResources(mapper, excludes, false, func(s string) { resources.Excludes(s) })

	ctx.log("Including resources: %v", strings.Join(resources.GetIncludes(), ", "))
	ctx.log("Excluding resources: %v", strings.Join(resources.GetExcludes(), ", "))

	return resources
}

// resolveGroupResource uses the RESTMapper to resolve resource to a fully-qualified
// schema.GroupResource. If the RESTMapper is unable to do so, an error is returned instead.
func resolveGroupResource(mapper meta.RESTMapper, resource string) (schema.GroupResource, error) {
	gvr, err := mapper.ResourceFor(schema.ParseGroupResource(resource).WithVersion(""))
	if err != nil {
		return schema.GroupResource{}, err
	}
	return gvr.GroupResource(), nil
}

// getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to
// include and exclude from the backup.
func getNamespaceIncludesExcludes(backup *api.Backup) *collections.IncludesExcludes {
	return collections.NewIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...)
}

type logger struct {
	w io.Writer
}

func (l *logger) log(msg string, args ...interface{}) {
	// TODO use a real logger that supports writing to files
	now := time.Now().Format(time.RFC3339)
	fmt.Fprintf(l.w, now+" "+msg+"\n", args...)
}

type backupContext struct {
	backup                    *api.Backup
	w                         tarWriter
	logger                    *logger
	namespaceIncludesExcludes *collections.IncludesExcludes
	resourceIncludesExcludes  *collections.IncludesExcludes
	// deploymentsBackedUp marks whether we've seen and are backing up the deployments resource, from
	// either the apps or extensions api groups. We only want to back them up once, from whichever api
	// group we see first.
	deploymentsBackedUp bool
	// networkPoliciesBackedUp marks whether we've seen and are backing up the networkpolicies
	// resource, from either the networking.k8s.io or extensions api groups. We only want to back them
	// up once, from whichever api group we see first.
	networkPoliciesBackedUp bool
}

func (ctx *backupContext) log(msg string, args ...interface{}) {
	ctx.logger.log(msg, args...)
}

// Backup backs up the items specified in the Backup, placing them in a gzip-compressed tar file
// written to data. The finalized api.Backup is written to metadata.
func (kb *kubernetesBackupper) Backup(backup *api.Backup, data, log io.Writer) error {
	gzippedData := gzip.NewWriter(data)
	defer gzippedData.Close()

	tw := tar.NewWriter(gzippedData)
	defer tw.Close()

	gzippedLog := gzip.NewWriter(log)
	defer gzippedLog.Close()

	var errs []error

	ctx := &backupContext{
		backup: backup,
		w:      tw,
		logger: &logger{w: gzippedLog},
		namespaceIncludesExcludes: getNamespaceIncludesExcludes(backup),
	}

	ctx.log("Starting backup")

	ctx.resourceIncludesExcludes = ctx.getResourceIncludesExcludes(kb.discoveryHelper.Mapper(), backup.Spec.IncludedResources, backup.Spec.ExcludedResources)

	for _, group := range kb.discoveryHelper.Resources() {
		ctx.log("Processing group %s", group.GroupVersion)
		if err := kb.backupGroup(ctx, group); err != nil {
			errs = append(errs, err)
		}
	}

	err := kuberrs.NewAggregate(errs)
	if err == nil {
		ctx.log("Backup completed successfully")
	} else {
		ctx.log("Backup completed with errors: %v", err)
	}

	return err
}

type tarWriter interface {
	io.Closer
	Write([]byte) (int, error)
	WriteHeader(*tar.Header) error
}

// backupGroup backs up a single API group.
func (kb *kubernetesBackupper) backupGroup(ctx *backupContext, group *metav1.APIResourceList) error {
	var errs []error

	for _, resource := range group.APIResources {
		ctx.log("Processing resource %s/%s", group.GroupVersion, resource.Name)
		if err := kb.backupResource(ctx, group, resource); err != nil {
			errs = append(errs, err)
		}
	}

	return kuberrs.NewAggregate(errs)
}

const (
	appsDeploymentsResource           = "deployments.apps"
	extensionsDeploymentsResource     = "deployments.extensions"
	networkingNetworkPoliciesResource = "networkpolicies.networking.k8s.io"
	extensionsNetworkPoliciesResource = "networkpolicies.extensions"
)

// backupResource backs up all the objects for a given group-version-resource.
func (kb *kubernetesBackupper) backupResource(
	ctx *backupContext,
	group *metav1.APIResourceList,
	resource metav1.APIResource,
) error {
	var errs []error

	gv, err := schema.ParseGroupVersion(group.GroupVersion)
	if err != nil {
		return err
	}
	gvr := schema.GroupVersionResource{Group: gv.Group, Version: gv.Version}
	gr := schema.GroupResource{Group: gv.Group, Resource: resource.Name}
	grString := gr.String()

	if !ctx.resourceIncludesExcludes.ShouldInclude(grString) {
		ctx.log("Resource %s is excluded", grString)
		return nil
	}

	if grString == appsDeploymentsResource || grString == extensionsDeploymentsResource {
		if ctx.deploymentsBackedUp {
			var other string
			if grString == appsDeploymentsResource {
				other = extensionsDeploymentsResource
			} else {
				other = appsDeploymentsResource
			}
			ctx.log("Skipping resource %q because it's a duplicate of %q", grString, other)
			return nil
		}

		ctx.deploymentsBackedUp = true
	}

	if grString == networkingNetworkPoliciesResource || grString == extensionsNetworkPoliciesResource {
		if ctx.networkPoliciesBackedUp {
			var other string
			if grString == networkingNetworkPoliciesResource {
				other = extensionsNetworkPoliciesResource
			} else {
				other = networkingNetworkPoliciesResource
			}
			ctx.log("Skipping resource %q because it's a duplicate of %q", grString, other)
			return nil
		}

		ctx.networkPoliciesBackedUp = true
	}

	var namespacesToList []string
	if resource.Namespaced {
		namespacesToList = getNamespacesToList(ctx.namespaceIncludesExcludes)
	} else {
		namespacesToList = []string{""}
	}
	for _, namespace := range namespacesToList {
		resourceClient, err := kb.dynamicFactory.ClientForGroupVersionResource(gvr, resource, namespace)
		if err != nil {
			return err
		}

		labelSelector := ""
		if ctx.backup.Spec.LabelSelector != nil {
			labelSelector = metav1.FormatLabelSelector(ctx.backup.Spec.LabelSelector)
		}
		unstructuredList, err := resourceClient.List(metav1.ListOptions{LabelSelector: labelSelector})
		if err != nil {
			return err
		}

		// do the backup
		items, err := meta.ExtractList(unstructuredList)
		if err != nil {
			return err
		}

		action := kb.actions[gr]

		for _, item := range items {
			unstructured, ok := item.(runtime.Unstructured)
			if !ok {
				errs = append(errs, fmt.Errorf("unexpected type %T", item))
				continue
			}

			obj := unstructured.UnstructuredContent()

			if err := kb.itemBackupper.backupItem(ctx, obj, grString, action); err != nil {
				errs = append(errs, err)
			}
		}
	}

	return kuberrs.NewAggregate(errs)
}

// getNamespacesToList examines ie and resolves the includes and excludes to a full list of
// namespaces to list. If ie is nil or it includes *, the result is just "" (list across all
// namespaces). Otherwise, the result is a list of every included namespace minus all excluded ones.
func getNamespacesToList(ie *collections.IncludesExcludes) []string {
	if ie == nil {
		return []string{""}
	}

	if ie.ShouldInclude("*") {
		// "" means all namespaces
		return []string{""}
	}

	var list []string
	for _, i := range ie.GetIncludes() {
		if ie.ShouldInclude(i) {
			list = append(list, i)
		}
	}

	return list
}

type itemBackupper interface {
	backupItem(ctx *backupContext, item map[string]interface{}, groupResource string, action Action) error
}

type realItemBackupper struct{}

// backupItem backs up an individual item to tarWriter. The item may be excluded based on the
// namespaces IncludesExcludes list.
func (*realItemBackupper) backupItem(ctx *backupContext, item map[string]interface{}, groupResource string, action Action) error {
	// Never save status
	delete(item, "status")

	metadata, err := collections.GetMap(item, "metadata")
	if err != nil {
		return err
	}

	name, err := collections.GetString(metadata, "name")
	if err != nil {
		return err
	}

	namespace, err := collections.GetString(metadata, "namespace")
	if err == nil {
		if !ctx.namespaceIncludesExcludes.ShouldInclude(namespace) {
			ctx.log("Excluding item %s because namespace %s is excluded", name, namespace)
			return nil
		}
	}

	if action != nil {
		ctx.log("Executing action on %s, ns=%s, name=%s", groupResource, namespace, name)

		actionCtx := ActionContext{logger: ctx.logger}
		if err := action.Execute(actionCtx, item, ctx.backup); err != nil {
			return err
		}
	}

	ctx.log("Backing up resource=%s, ns=%s, name=%s", groupResource, namespace, name)

	var filePath string
	if namespace != "" {
		filePath = strings.Join([]string{api.NamespaceScopedDir, namespace, groupResource, name + ".json"}, "/")
	} else {
		filePath = strings.Join([]string{api.ClusterScopedDir, groupResource, name + ".json"}, "/")
	}

	itemBytes, err := json.Marshal(item)
	if err != nil {
		return err
	}

	hdr := &tar.Header{
		Name:     filePath,
		Size:     int64(len(itemBytes)),
		Typeflag: tar.TypeReg,
		Mode:     0755,
		ModTime:  time.Now(),
	}

	if err := ctx.w.WriteHeader(hdr); err != nil {
		return err
	}

	if _, err := ctx.w.Write(itemBytes); err != nil {
		return err
	}

	return nil
}
<commit_msg>Clarify Backup() param names<commit_after>/*
Copyright 2017 Heptio Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package backup

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"strings"
	"time"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	kuberrs "k8s.io/apimachinery/pkg/util/errors"

	api "github.com/heptio/ark/pkg/apis/ark/v1"
	"github.com/heptio/ark/pkg/client"
	"github.com/heptio/ark/pkg/discovery"
	"github.com/heptio/ark/pkg/util/collections"
)

// Backupper performs backups.
type Backupper interface {
	// Backup takes a backup using the specification in the api.Backup and writes backup and log data
	// to the given writers.
	Backup(backup *api.Backup, backupFile, logFile io.Writer) error
}

// kubernetesBackupper implements Backupper.
type kubernetesBackupper struct {
	dynamicFactory  client.DynamicFactory
	discoveryHelper discovery.Helper
	actions         map[schema.GroupResource]Action
	itemBackupper   itemBackupper
}

var _ Backupper = &kubernetesBackupper{}

// ActionContext contains contextual information for actions.
type ActionContext struct {
	logger *logger
}

func (ac ActionContext) log(msg string, args ...interface{}) {
	ac.logger.log(msg, args...)
}

// Action is an actor that performs an operation on an individual item being backed up.
type Action interface {
	// Execute is invoked on an item being backed up. If an error is returned, the Backup is marked as
	// failed.
	Execute(ctx ActionContext, item map[string]interface{}, backup *api.Backup) error
}

// NewKubernetesBackupper creates a new kubernetesBackupper.
func NewKubernetesBackupper(
	discoveryHelper discovery.Helper,
	dynamicFactory client.DynamicFactory,
	actions map[string]Action,
) (Backupper, error) {
	resolvedActions, err := resolveActions(discoveryHelper.Mapper(), actions)
	if err != nil {
		return nil, err
	}

	return &kubernetesBackupper{
		discoveryHelper: discoveryHelper,
		dynamicFactory:  dynamicFactory,
		actions:         resolvedActions,
		itemBackupper:   &realItemBackupper{},
	}, nil
}

// resolveActions resolves the string-based map of group-resources to actions and returns a map of
// schema.GroupResources to actions.
func resolveActions(mapper meta.RESTMapper, actions map[string]Action) (map[schema.GroupResource]Action, error) {
	ret := make(map[schema.GroupResource]Action)

	for resource, action := range actions {
		gr, err := resolveGroupResource(mapper, resource)
		if err != nil {
			return nil, err
		}
		ret[gr] = action
	}

	return ret, nil
}

// resolveResources uses the RESTMapper to resolve resources to their fully-qualified group-resource
// names. fn is invoked for each resolved resource. resolveResources returns a list of any resources that failed to resolve.
func (ctx *backupContext) resolveResources(mapper meta.RESTMapper, resources []string, allowAll bool, fn func(string)) {
	for _, resource := range resources {
		if allowAll && resource == "*" {
			fn("*")
			return
		}
		gr, err := resolveGroupResource(mapper, resource)
		if err != nil {
			ctx.log("Unable to resolve resource %q: %v", resource, err)
			continue
		}
		fn(gr.String())
	}
}

// getResourceIncludesExcludes takes the lists of resources to include and exclude, uses the
// RESTMapper to resolve them to fully-qualified group-resource names, and returns an
// IncludesExcludes list.
func (ctx *backupContext) getResourceIncludesExcludes(mapper meta.RESTMapper, includes, excludes []string) *collections.IncludesExcludes {
	resources := collections.NewIncludesExcludes()

	ctx.resolveResources(mapper, includes, true, func(s string) { resources.Includes(s) })
	ctx.resolveResources(mapper, excludes, false, func(s string) { resources.Excludes(s) })

	ctx.log("Including resources: %v", strings.Join(resources.GetIncludes(), ", "))
	ctx.log("Excluding resources: %v", strings.Join(resources.GetExcludes(), ", "))

	return resources
}

// resolveGroupResource uses the RESTMapper to resolve resource to a fully-qualified
// schema.GroupResource. If the RESTMapper is unable to do so, an error is returned instead.
func resolveGroupResource(mapper meta.RESTMapper, resource string) (schema.GroupResource, error) {
	gvr, err := mapper.ResourceFor(schema.ParseGroupResource(resource).WithVersion(""))
	if err != nil {
		return schema.GroupResource{}, err
	}
	return gvr.GroupResource(), nil
}

// getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to
// include and exclude from the backup.
func getNamespaceIncludesExcludes(backup *api.Backup) *collections.IncludesExcludes {
	return collections.NewIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...)
}

type logger struct {
	w io.Writer
}

func (l *logger) log(msg string, args ...interface{}) {
	// TODO use a real logger that supports writing to files
	now := time.Now().Format(time.RFC3339)
	fmt.Fprintf(l.w, now+" "+msg+"\n", args...)
}

type backupContext struct {
	backup                    *api.Backup
	w                         tarWriter
	logger                    *logger
	namespaceIncludesExcludes *collections.IncludesExcludes
	resourceIncludesExcludes  *collections.IncludesExcludes
	// deploymentsBackedUp marks whether we've seen and are backing up the deployments resource, from
	// either the apps or extensions api groups. We only want to back them up once, from whichever api
	// group we see first.
	deploymentsBackedUp bool
	// networkPoliciesBackedUp marks whether we've seen and are backing up the networkpolicies
	// resource, from either the networking.k8s.io or extensions api groups. We only want to back them
	// up once, from whichever api group we see first.
	networkPoliciesBackedUp bool
}

func (ctx *backupContext) log(msg string, args ...interface{}) {
	ctx.logger.log(msg, args...)
}

// Backup backs up the items specified in the Backup, placing them in a gzip-compressed tar file
// written to backupFile. The finalized api.Backup is written to metadata.
func (kb *kubernetesBackupper) Backup(backup *api.Backup, backupFile, logFile io.Writer) error {
	gzippedData := gzip.NewWriter(backupFile)
	defer gzippedData.Close()

	tw := tar.NewWriter(gzippedData)
	defer tw.Close()

	gzippedLog := gzip.NewWriter(logFile)
	defer gzippedLog.Close()

	var errs []error

	ctx := &backupContext{
		backup: backup,
		w:      tw,
		logger: &logger{w: gzippedLog},
		namespaceIncludesExcludes: getNamespaceIncludesExcludes(backup),
	}

	ctx.log("Starting backup")

	ctx.resourceIncludesExcludes = ctx.getResourceIncludesExcludes(kb.discoveryHelper.Mapper(), backup.Spec.IncludedResources, backup.Spec.ExcludedResources)

	for _, group := range kb.discoveryHelper.Resources() {
		ctx.log("Processing group %s", group.GroupVersion)
		if err := kb.backupGroup(ctx, group); err != nil {
			errs = append(errs, err)
		}
	}

	err := kuberrs.NewAggregate(errs)
	if err == nil {
		ctx.log("Backup completed successfully")
	} else {
		ctx.log("Backup completed with errors: %v", err)
	}

	return err
}

type tarWriter interface {
	io.Closer
	Write([]byte) (int, error)
	WriteHeader(*tar.Header) error
}

// backupGroup backs up a single API group.
func (kb *kubernetesBackupper) backupGroup(ctx *backupContext, group *metav1.APIResourceList) error {
	var errs []error

	for _, resource := range group.APIResources {
		ctx.log("Processing resource %s/%s", group.GroupVersion, resource.Name)
		if err := kb.backupResource(ctx, group, resource); err != nil {
			errs = append(errs, err)
		}
	}

	return kuberrs.NewAggregate(errs)
}

const (
	appsDeploymentsResource           = "deployments.apps"
	extensionsDeploymentsResource     = "deployments.extensions"
	networkingNetworkPoliciesResource = "networkpolicies.networking.k8s.io"
	extensionsNetworkPoliciesResource = "networkpolicies.extensions"
)

// backupResource backs up all the objects for a given group-version-resource.
func (kb *kubernetesBackupper) backupResource(
	ctx *backupContext,
	group *metav1.APIResourceList,
	resource metav1.APIResource,
) error {
	var errs []error

	gv, err := schema.ParseGroupVersion(group.GroupVersion)
	if err != nil {
		return err
	}
	gvr := schema.GroupVersionResource{Group: gv.Group, Version: gv.Version}
	gr := schema.GroupResource{Group: gv.Group, Resource: resource.Name}
	grString := gr.String()

	if !ctx.resourceIncludesExcludes.ShouldInclude(grString) {
		ctx.log("Resource %s is excluded", grString)
		return nil
	}

	if grString == appsDeploymentsResource || grString == extensionsDeploymentsResource {
		if ctx.deploymentsBackedUp {
			var other string
			if grString == appsDeploymentsResource {
				other = extensionsDeploymentsResource
			} else {
				other = appsDeploymentsResource
			}
			ctx.log("Skipping resource %q because it's a duplicate of %q", grString, other)
			return nil
		}

		ctx.deploymentsBackedUp = true
	}

	if grString == networkingNetworkPoliciesResource || grString == extensionsNetworkPoliciesResource {
		if ctx.networkPoliciesBackedUp {
			var other string
			if grString == networkingNetworkPoliciesResource {
				other = extensionsNetworkPoliciesResource
			} else {
				other = networkingNetworkPoliciesResource
			}
			ctx.log("Skipping resource %q because it's a duplicate of %q", grString, other)
			return nil
		}

		ctx.networkPoliciesBackedUp = true
	}

	var namespacesToList []string
	if resource.Namespaced {
		namespacesToList = getNamespacesToList(ctx.namespaceIncludesExcludes)
	} else {
		namespacesToList = []string{""}
	}
	for _, namespace := range namespacesToList {
		resourceClient, err := kb.dynamicFactory.ClientForGroupVersionResource(gvr, resource, namespace)
		if err != nil {
			return err
		}

		labelSelector := ""
		if ctx.backup.Spec.LabelSelector != nil {
			labelSelector = metav1.FormatLabelSelector(ctx.backup.Spec.LabelSelector)
		}
		unstructuredList, err := resourceClient.List(metav1.ListOptions{LabelSelector: labelSelector})
		if err != nil {
			return err
		}

		// do the backup
		items, err := meta.ExtractList(unstructuredList)
		if err != nil {
			return err
		}

		action := kb.actions[gr]

		for _, item := range items {
			unstructured, ok := item.(runtime.Unstructured)
			if !ok {
				errs = append(errs, fmt.Errorf("unexpected type %T", item))
				continue
			}

			obj := unstructured.UnstructuredContent()

			if err := kb.itemBackupper.backupItem(ctx, obj, grString, action); err != nil {
				errs = append(errs, err)
			}
		}
	}

	return kuberrs.NewAggregate(errs)
}

// getNamespacesToList examines ie and resolves the includes and excludes to a full list of
// namespaces to list. If ie is nil or it includes *, the result is just "" (list across all
// namespaces). Otherwise, the result is a list of every included namespace minus all excluded ones.
func getNamespacesToList(ie *collections.IncludesExcludes) []string {
	if ie == nil {
		return []string{""}
	}

	if ie.ShouldInclude("*") {
		// "" means all namespaces
		return []string{""}
	}

	var list []string
	for _, i := range ie.GetIncludes() {
		if ie.ShouldInclude(i) {
			list = append(list, i)
		}
	}

	return list
}

type itemBackupper interface {
	backupItem(ctx *backupContext, item map[string]interface{}, groupResource string, action Action) error
}

type realItemBackupper struct{}

// backupItem backs up an individual item to tarWriter. The item may be excluded based on the
// namespaces IncludesExcludes list.
func (*realItemBackupper) backupItem(ctx *backupContext, item map[string]interface{}, groupResource string, action Action) error {
	// Never save status
	delete(item, "status")

	metadata, err := collections.GetMap(item, "metadata")
	if err != nil {
		return err
	}

	name, err := collections.GetString(metadata, "name")
	if err != nil {
		return err
	}

	namespace, err := collections.GetString(metadata, "namespace")
	if err == nil {
		if !ctx.namespaceIncludesExcludes.ShouldInclude(namespace) {
			ctx.log("Excluding item %s because namespace %s is excluded", name, namespace)
			return nil
		}
	}

	if action != nil {
		ctx.log("Executing action on %s, ns=%s, name=%s", groupResource, namespace, name)

		actionCtx := ActionContext{logger: ctx.logger}
		if err := action.Execute(actionCtx, item, ctx.backup); err != nil {
			return err
		}
	}

	ctx.log("Backing up resource=%s, ns=%s, name=%s", groupResource, namespace, name)

	var filePath string
	if namespace != "" {
		filePath = strings.Join([]string{api.NamespaceScopedDir, namespace, groupResource, name + ".json"}, "/")
	} else {
		filePath = strings.Join([]string{api.ClusterScopedDir, groupResource, name + ".json"}, "/")
	}

	itemBytes, err := json.Marshal(item)
	if err != nil {
		return err
	}

	hdr := &tar.Header{
		Name:     filePath,
		Size:     int64(len(itemBytes)),
		Typeflag: tar.TypeReg,
		Mode:     0755,
		ModTime:  time.Now(),
	}

	if err := ctx.w.WriteHeader(hdr); err != nil {
		return err
	}

	if _, err := ctx.w.Write(itemBytes); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package couchdb

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"

	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/labstack/echo/v4"
)

// Proxy generate a httputil.ReverseProxy which forwards the request to the
// correct route.
func Proxy(db Database, doctype, path string) *httputil.ReverseProxy {
	couchURL := config.CouchURL()
	couchAuth := config.GetConfig().CouchDB.Auth

	director := func(req *http.Request) {
		req.URL.Scheme = couchURL.Scheme
		req.URL.Host = couchURL.Host
		req.Header.Del(echo.HeaderAuthorization) // drop stack auth
		req.Header.Del(echo.HeaderCookie)
		req.URL.RawPath = "/" + makeDBName(db, doctype) + "/" + path
		req.URL.Path, _ = url.PathUnescape(req.URL.RawPath)
		if couchAuth != nil {
			if p, ok := couchAuth.Password(); ok {
				req.SetBasicAuth(couchAuth.Username(), p)
			}
		}
	}

	var transport http.RoundTripper
	if client := config.GetConfig().CouchDB.Client; client != nil {
		transport = client.Transport
	} else {
		transport = http.DefaultTransport
	}

	return &httputil.ReverseProxy{
		Director:  director,
		Transport: transport,
	}
}

// ProxyBulkDocs generates a httputil.ReverseProxy to forward the couchdb
// request on the _bulk_docs endpoint. This endpoint is specific since it will
// mutate many document in database, the stack has to read the response from
// couch to emit the correct realtime events.
func ProxyBulkDocs(db Database, doctype string, req *http.Request) (*httputil.ReverseProxy, *http.Request, error) {
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, nil, err
	}

	var reqValue struct {
		Docs     []JSONDoc `json:"docs"`
		NewEdits *bool     `json:"new_edits"`
	}

	if err = json.Unmarshal(body, &reqValue); err != nil {
		return nil, nil, echo.NewHTTPError(http.StatusBadRequest,
			"request body is not valid JSON")
	}

	docs := make(map[string]JSONDoc)
	for _, d := range reqValue.Docs {
		docs[d.ID()] = d
	}

	// reset body to proxy
	req.Body = ioutil.NopCloser(bytes.NewReader(body))

	var transport http.RoundTripper
	if client := config.GetConfig().CouchDB.Client; client != nil {
		transport = client.Transport
	} else {
		transport = http.DefaultTransport
	}

	p := Proxy(db, doctype, "/_bulk_docs")
	p.Transport = &bulkTransport{
		RoundTripper: transport,
		OnResponseRead: func(data []byte) {
			type respValue struct {
				ID    string `json:"id"`
				Rev   string `json:"rev"`
				OK    bool   `json:"ok"`
				Error string `json:"error"`
			}

			// When using the 'new_edits' flag (like pouchdb), the couchdb response
			// does not contain any value. We only rely on the request data and
			// expect no error.
			if reqValue.NewEdits != nil && !*reqValue.NewEdits {
				for _, doc := range reqValue.Docs {
					rev := doc.Rev()
					var event string
					if strings.HasPrefix(rev, "1-") {
						event = realtime.EventCreate
					} else {
						event = realtime.EventUpdate
					}
					RTEvent(db, event, doc, nil)
				}
			} else {
				var respValues []*respValue
				if err = json.Unmarshal(data, &respValues); err != nil {
					return
				}

				docs := make(map[string]JSONDoc)
				for _, doc := range reqValue.Docs {
					docs[doc.ID()] = doc
				}

				for _, r := range respValues {
					if r.Error != "" || !r.OK {
						continue
					}
					doc, ok := docs[r.ID]
					if !ok {
						continue
					}
					var event string
					if doc.Rev() == "" {
						event = realtime.EventCreate
					} else if doc.Get("_deleted") == true {
						event = realtime.EventDelete
					} else {
						event = realtime.EventUpdate
					}
					doc.SetRev(r.Rev)
					RTEvent(db, event, doc, nil)
				}
			}
		},
	}

	return p, req, nil
}

type bulkTransport struct {
	http.RoundTripper
	OnResponseRead func([]byte)
}

func (t *bulkTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	resp, err = t.RoundTripper.RoundTrip(req)
	if err != nil {
		return nil, newConnectionError(err)
	}
	defer func() {
		if errc := resp.Body.Close(); err == nil && errc != nil {
			err = errc
		}
	}()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == http.StatusCreated {
		go t.OnResponseRead(b)
	}
	resp.Body = ioutil.NopCloser(bytes.NewReader(b))
	return resp, nil
}
<commit_msg>Fix real-time and triggers on replication (#2004)<commit_after>package couchdb

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"

	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/labstack/echo/v4"
)

// Proxy generate a httputil.ReverseProxy which forwards the request to the
// correct route.
func Proxy(db Database, doctype, path string) *httputil.ReverseProxy {
	couchURL := config.CouchURL()
	couchAuth := config.GetConfig().CouchDB.Auth

	director := func(req *http.Request) {
		req.URL.Scheme = couchURL.Scheme
		req.URL.Host = couchURL.Host
		req.Header.Del(echo.HeaderAuthorization) // drop stack auth
		req.Header.Del(echo.HeaderCookie)
		req.URL.RawPath = "/" + makeDBName(db, doctype) + "/" + path
		req.URL.Path, _ = url.PathUnescape(req.URL.RawPath)
		if couchAuth != nil {
			if p, ok := couchAuth.Password(); ok {
				req.SetBasicAuth(couchAuth.Username(), p)
			}
		}
	}

	var transport http.RoundTripper
	if client := config.GetConfig().CouchDB.Client; client != nil {
		transport = client.Transport
	} else {
		transport = http.DefaultTransport
	}

	return &httputil.ReverseProxy{
		Director:  director,
		Transport: transport,
	}
}

// ProxyBulkDocs generates a httputil.ReverseProxy to forward the couchdb
// request on the _bulk_docs endpoint. This endpoint is specific since it will
// mutate many document in database, the stack has to read the response from
// couch to emit the correct realtime events.
func ProxyBulkDocs(db Database, doctype string, req *http.Request) (*httputil.ReverseProxy, *http.Request, error) {
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, nil, err
	}

	var reqValue struct {
		Docs     []JSONDoc `json:"docs"`
		NewEdits *bool     `json:"new_edits"`
	}

	if err = json.Unmarshal(body, &reqValue); err != nil {
		return nil, nil, echo.NewHTTPError(http.StatusBadRequest,
			"request body is not valid JSON")
	}

	docs := make(map[string]JSONDoc)
	for _, d := range reqValue.Docs {
		docs[d.ID()] = d
	}

	// reset body to proxy
	req.Body = ioutil.NopCloser(bytes.NewReader(body))

	var transport http.RoundTripper
	if client := config.GetConfig().CouchDB.Client; client != nil {
		transport = client.Transport
	} else {
		transport = http.DefaultTransport
	}

	p := Proxy(db, doctype, "/_bulk_docs")
	p.Transport = &bulkTransport{
		RoundTripper: transport,
		OnResponseRead: func(data []byte) {
			type respValue struct {
				ID    string `json:"id"`
				Rev   string `json:"rev"`
				OK    bool   `json:"ok"`
				Error string `json:"error"`
			}

			// When using the 'new_edits' flag (like pouchdb), the couchdb response
			// does not contain any value. We only rely on the request data and
			// expect no error.
			if reqValue.NewEdits != nil && !*reqValue.NewEdits {
				for _, doc := range reqValue.Docs {
					doc.Type = doctype
					rev := doc.Rev()
					var event string
					if strings.HasPrefix(rev, "1-") {
						event = realtime.EventCreate
					} else {
						event = realtime.EventUpdate
					}
					RTEvent(db, event, doc, nil)
				}
			} else {
				var respValues []*respValue
				if err = json.Unmarshal(data, &respValues); err != nil {
					return
				}

				docs := make(map[string]JSONDoc)
				for _, doc := range reqValue.Docs {
					docs[doc.ID()] = doc
				}

				for _, r := range respValues {
					if r.Error != "" || !r.OK {
						continue
					}
					doc, ok := docs[r.ID]
					if !ok {
						continue
					}
					var event string
					if doc.Rev() == "" {
						event = realtime.EventCreate
					} else if doc.Get("_deleted") == true {
						event = realtime.EventDelete
					} else {
						event = realtime.EventUpdate
					}
					doc.SetRev(r.Rev)
					RTEvent(db, event, doc, nil)
				}
			}
		},
	}

	return p, req, nil
}

type bulkTransport struct {
	http.RoundTripper
	OnResponseRead func([]byte)
}

func (t *bulkTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	resp, err = t.RoundTripper.RoundTrip(req)
	if err != nil {
		return nil, newConnectionError(err)
	}
	defer func() {
		if errc := resp.Body.Close(); err == nil && errc != nil {
			err = errc
		}
	}()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == http.StatusCreated {
		go t.OnResponseRead(b)
	}
	resp.Body = ioutil.NopCloser(bytes.NewReader(b))
	return resp, nil
}
<|endoftext|>


<commit_before>package ami

// Agents lists agents and their status.
func Agents(client Client, actionID string) ([]Response, error) {
	return requestList(client, "Agents", actionID, "AgentsEntry", "AgentsComplete")
}

// AgentLogoff sets an agent as no longer logged in.
func AgentLogoff(client Client, actionID, agent string, soft bool) (Response, error) {
	return send(client, "AgentLogoff", actionID, map[string]interface{}{
		"Agent": agent,
		"Soft":  soft,
	})
}
<commit_msg>Change event type name for Agents<commit_after>package ami

// Agents lists agents and their status.
func Agents(client Client, actionID string) ([]Response, error) {
	return requestList(client, "Agents", actionID, "Agents", "AgentsComplete")
}

// AgentLogoff sets an agent as no longer logged in.
func AgentLogoff(client Client, actionID, agent string, soft bool) (Response, error) {
	return send(client, "AgentLogoff", actionID, map[string]interface{}{
		"Agent": agent,
		"Soft":  soft,
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubectl

import (
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/runtime"
	api "k8s.io/kubernetes/pkg/apis/core"
	"k8s.io/kubernetes/pkg/kubectl/resource"
)

// GetOriginalConfiguration retrieves the original configuration of the object
// from the annotation, or nil if no annotation was found.
func GetOriginalConfiguration(mapping *meta.RESTMapping, obj runtime.Object) ([]byte, error) {
	annots, err := mapping.MetadataAccessor.Annotations(obj)
	if err != nil {
		return nil, err
	}

	if annots == nil {
		return nil, nil
	}

	original, ok := annots[api.LastAppliedConfigAnnotation]
	if !ok {
		return nil, nil
	}

	return []byte(original), nil
}

// SetOriginalConfiguration sets the original configuration of the object
// as the annotation on the object for later use in computing a three way patch.
func SetOriginalConfiguration(info *resource.Info, original []byte) error {
	if len(original) < 1 {
		return nil
	}

	accessor := info.Mapping.MetadataAccessor
	annots, err := accessor.Annotations(info.Object)
	if err != nil {
		return err
	}

	if annots == nil {
		annots = map[string]string{}
	}

	annots[api.LastAppliedConfigAnnotation] = string(original)
	return info.Mapping.MetadataAccessor.SetAnnotations(info.Object, annots)
}

// GetModifiedConfiguration retrieves the modified configuration of the object.
// If annotate is true, it embeds the result as an annotation in the modified
// configuration. If an object was read from the command input, it will use that
// version of the object. Otherwise, it will use the version from the server.
func GetModifiedConfiguration(info *resource.Info, annotate bool, codec runtime.Encoder) ([]byte, error) {
	// First serialize the object without the annotation to prevent recursion,
	// then add that serialization to it as the annotation and serialize it again.
	var modified []byte

	// Otherwise, use the server side version of the object.
	accessor := info.Mapping.MetadataAccessor
	// Get the current annotations from the object.
	annots, err := accessor.Annotations(info.Object)
	if err != nil {
		return nil, err
	}

	if annots == nil {
		annots = map[string]string{}
	}

	original := annots[api.LastAppliedConfigAnnotation]
	delete(annots, api.LastAppliedConfigAnnotation)
	if err := accessor.SetAnnotations(info.Object, annots); err != nil {
		return nil, err
	}

	modified, err = runtime.Encode(codec, info.Object)
	if err != nil {
		return nil, err
	}

	if annotate {
		annots[api.LastAppliedConfigAnnotation] = string(modified)
		if err := info.Mapping.MetadataAccessor.SetAnnotations(info.Object, annots); err != nil {
			return nil, err
		}

		modified, err = runtime.Encode(codec, info.Object)
		if err != nil {
			return nil, err
		}
	}

	// Restore the object to its original condition.
	annots[api.LastAppliedConfigAnnotation] = original
	if err := info.Mapping.MetadataAccessor.SetAnnotations(info.Object, annots); err != nil {
		return nil, err
	}

	return modified, nil
}

// UpdateApplyAnnotation calls CreateApplyAnnotation if the last applied
// configuration annotation is already present. Otherwise, it does nothing.
func UpdateApplyAnnotation(info *resource.Info, codec runtime.Encoder) error {
	if original, err := GetOriginalConfiguration(info.Mapping, info.Object); err != nil || len(original) <= 0 {
		return err
	}
	return CreateApplyAnnotation(info, codec)
}

// CreateApplyAnnotation gets the modified configuration of the object,
// without embedding it again, and then sets it on the object as the annotation.
func CreateApplyAnnotation(info *resource.Info, codec runtime.Encoder) error {
	modified, err := GetModifiedConfiguration(info, false, codec)
	if err != nil {
		return err
	}
	return SetOriginalConfiguration(info, modified)
}

// Create the annotation used by kubectl apply only when createAnnotation is true
// Otherwise, only update the annotation when it already exists
func CreateOrUpdateAnnotation(createAnnotation bool, info *resource.Info, codec runtime.Encoder) error {
	if createAnnotation {
		return CreateApplyAnnotation(info, codec)
	}
	return UpdateApplyAnnotation(info, codec)
}
<commit_msg>remove internal version api from apply<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubectl

import (
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/kubernetes/pkg/kubectl/resource"
)

// GetOriginalConfiguration retrieves the original configuration of the object
// from the annotation, or nil if no annotation was found.
func GetOriginalConfiguration(mapping *meta.RESTMapping, obj runtime.Object) ([]byte, error) {
	annots, err := mapping.MetadataAccessor.Annotations(obj)
	if err != nil {
		return nil, err
	}

	if annots == nil {
		return nil, nil
	}

	original, ok := annots[v1.LastAppliedConfigAnnotation]
	if !ok {
		return nil, nil
	}

	return []byte(original), nil
}

// SetOriginalConfiguration sets the original configuration of the object
// as the annotation on the object for later use in computing a three way patch.
func SetOriginalConfiguration(info *resource.Info, original []byte) error {
	if len(original) < 1 {
		return nil
	}

	accessor := info.Mapping.MetadataAccessor
	annots, err := accessor.Annotations(info.Object)
	if err != nil {
		return err
	}

	if annots == nil {
		annots = map[string]string{}
	}

	annots[v1.LastAppliedConfigAnnotation] = string(original)
	return info.Mapping.MetadataAccessor.SetAnnotations(info.Object, annots)
}

// GetModifiedConfiguration retrieves the modified configuration of the object.
// If annotate is true, it embeds the result as an annotation in the modified
// configuration. If an object was read from the command input, it will use that
// version of the object. Otherwise, it will use the version from the server.
func GetModifiedConfiguration(info *resource.Info, annotate bool, codec runtime.Encoder) ([]byte, error) {
	// First serialize the object without the annotation to prevent recursion,
	// then add that serialization to it as the annotation and serialize it again.
	var modified []byte

	// Otherwise, use the server side version of the object.
	accessor := info.Mapping.MetadataAccessor
	// Get the current annotations from the object.
	annots, err := accessor.Annotations(info.Object)
	if err != nil {
		return nil, err
	}

	if annots == nil {
		annots = map[string]string{}
	}

	original := annots[v1.LastAppliedConfigAnnotation]
	delete(annots, v1.LastAppliedConfigAnnotation)
	if err := accessor.SetAnnotations(info.Object, annots); err != nil {
		return nil, err
	}

	modified, err = runtime.Encode(codec, info.Object)
	if err != nil {
		return nil, err
	}

	if annotate {
		annots[v1.LastAppliedConfigAnnotation] = string(modified)
		if err := info.Mapping.MetadataAccessor.SetAnnotations(info.Object, annots); err != nil {
			return nil, err
		}

		modified, err = runtime.Encode(codec, info.Object)
		if err != nil {
			return nil, err
		}
	}

	// Restore the object to its original condition.
	annots[v1.LastAppliedConfigAnnotation] = original
	if err := info.Mapping.MetadataAccessor.SetAnnotations(info.Object, annots); err != nil {
		return nil, err
	}

	return modified, nil
}

// UpdateApplyAnnotation calls CreateApplyAnnotation if the last applied
// configuration annotation is already present. Otherwise, it does nothing.
func UpdateApplyAnnotation(info *resource.Info, codec runtime.Encoder) error {
	if original, err := GetOriginalConfiguration(info.Mapping, info.Object); err != nil || len(original) <= 0 {
		return err
	}
	return CreateApplyAnnotation(info, codec)
}

// CreateApplyAnnotation gets the modified configuration of the object,
// without embedding it again, and then sets it on the object as the annotation.
func CreateApplyAnnotation(info *resource.Info, codec runtime.Encoder) error {
	modified, err := GetModifiedConfiguration(info, false, codec)
	if err != nil {
		return err
	}
	return SetOriginalConfiguration(info, modified)
}

// Create the annotation used by kubectl apply only when createAnnotation is true
// Otherwise, only update the annotation when it already exists
func CreateOrUpdateAnnotation(createAnnotation bool, info *resource.Info, codec runtime.Encoder) error {
	if createAnnotation {
		return CreateApplyAnnotation(info, codec)
	}
	return UpdateApplyAnnotation(info, codec)
}
<|endoftext|>


<commit_before>// Copyright 2016 David Lavieri.  All rights reserved.
// Use of this source code is governed by a MIT License
// License that can be found in the LICENSE file.

package goradix

import "testing"

// ----------------------- Benchmarks ------------------------ //

func BenchmarkLookUpStringSingle(b *testing.B) {
	rx := New()
	insertData(rx, sampleData2)

	toLookUp := randomString(sampleData2())

	for i := 0; i < b.N; i++ {
		rx.LookUp(toLookUp)
	}
}

func BenchmarkLookUpStringRandom(b *testing.B) {
	rx := New()
	insertData(rx, sampleData2)
	sd2 := sampleData2()

	for i := 0; i < b.N; i++ {
		if i%20 == 0 {
			rx = New()
		}

		rx.LookUp(randomString(sd2))
	}
}

func BenchmarkLookUpBytesSingle(b *testing.B) {
	rx := New()
	insertDataBytes(rx, sampleData3)

	toLookUp := randomBytes(sampleData3())

	for i := 0; i < b.N; i++ {
		rx.LookUpBytes(toLookUp)
	}
}

func BenchmarkLookUpBytesRandom(b *testing.B) {
	rx := New()
	insertDataBytes(rx, sampleData3)
	sd3 := sampleData3()

	for i := 0; i < b.N; i++ {
		if i%20 == 0 {
			rx = New()
		}

		rx.LookUpBytes(randomBytes(sd3))
	}
}
<commit_msg>fix lookup benchmarks<commit_after>// Copyright 2016 David Lavieri.  All rights reserved.
// Use of this source code is governed by a MIT License
// License that can be found in the LICENSE file.

package goradix

import "testing"

// ----------------------- Benchmarks ------------------------ //

func BenchmarkLookUpStringSingle(b *testing.B) {
	rx := New()
	insertData(rx, sampleData2)

	toLookUp := randomString(sampleData2())

	for i := 0; i < b.N; i++ {
		rx.LookUp(toLookUp)
	}
}

func BenchmarkLookUpStringRandom(b *testing.B) {
	rx := New()

	sd2 := sampleData2()

	for i := 0; i < b.N; i++ {
		if i%20 == 0 {
			b.StopTimer()
			rx = New()
			insertData(rx, sampleData2)
			b.StartTimer()
		}

		rx.LookUp(randomString(sd2))
	}
}

func BenchmarkLookUpBytesSingle(b *testing.B) {
	rx := New()
	insertDataBytes(rx, sampleData3)

	toLookUp := randomBytes(sampleData3())

	for i := 0; i < b.N; i++ {
		rx.LookUpBytes(toLookUp)
	}
}

func BenchmarkLookUpBytesRandom(b *testing.B) {
	rx := New()
	insertDataBytes(rx, sampleData3)
	sd3 := sampleData3()

	for i := 0; i < b.N; i++ {
		if i%20 == 0 {
			b.StopTimer()
			rx = New()
			insertDataBytes(rx, sampleData3)
			b.StartTimer()
		}

		rx.LookUpBytes(randomBytes(sd3))
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package server

import (
	"context"
	"os"
	"time"

	"github.com/neovim/go-client/nvim"
	"github.com/pkg/errors"
	"go.uber.org/zap"

	"github.com/zchee/nvim-go/pkg/logger"
)

type Server struct {
	*nvim.Nvim
}

func NewServer(pctx context.Context) (*Server, error) {
	log := logger.FromContext(pctx).Named("server")

	const envNvimListenAddress = "NVIM_LISTEN_ADDRESS"
	addr := os.Getenv(envNvimListenAddress)
	if addr == "" {
		return nil, errors.Errorf("%s not set", envNvimListenAddress)
	}

	zapLogf := func(format string, a ...interface{}) {
		log.Info("", zap.Any(format, a))
	}

	ctx, cancel := context.WithTimeout(pctx, 1*time.Second)
	defer cancel()

	var n *nvim.Nvim
	var tempDelay time.Duration
	for {
		var err error
		n, err = nvim.Dial(addr, nvim.DialContext(ctx), nvim.DialServe(false), nvim.DialLogf(zapLogf))
		if err != nil {
			if tempDelay == 0 {
				tempDelay = 5 * time.Millisecond
			} else {
				tempDelay *= 2
			}
			if max := 1 * time.Second; tempDelay > max {
				tempDelay = max
			}
			log.Info("Dial error", zap.Error(err), zap.Duration("retrying in", tempDelay))
			timer := time.NewTimer(tempDelay)
			select {
			case <-timer.C:
			}
			continue
		}
		tempDelay = 0

		return &Server{Nvim: n}, nil
	}
}

func Dial(pctx context.Context) (*nvim.Nvim, error) {
	const envNvimListenAddress = "NVIM_LISTEN_ADDRESS"
	addr := os.Getenv(envNvimListenAddress) // NVIM_LISTEN_ADDRESS env can get if launched nvim process
	if addr == "" {
		return nil, errors.Errorf("failed get %s", envNvimListenAddress)
	}

	ctx, cancel := context.WithTimeout(pctx, 1*time.Second)
	defer cancel()

	var n *nvim.Nvim
	dialOpts := []nvim.DialOption{
		nvim.DialContext(ctx),
		nvim.DialServe(false),
		nvim.DialLogf(func(format string, a ...interface{}) {
			logger.FromContext(ctx).Info("", zap.Any(format, a))
		}),
	}

	var tempDelay time.Duration // how long to sleep on accept failure
	var err error
	for {
		n, err = nvim.Dial(addr, dialOpts...)
		if err != nil {
			if tempDelay == 0 {
				tempDelay = 5 * time.Millisecond
			} else {
				tempDelay *= 2
			}
			if max := 1 * time.Second; tempDelay > max {
				tempDelay = max
			}
			logger.FromContext(ctx).Error("Dial error", zap.Error(err), zap.Duration("retrying in", tempDelay))
			timer := time.NewTimer(tempDelay)
			select {
			case <-timer.C:
			}
			continue
		}
		tempDelay = 0

		return n, nil
	}
}

func (s *Server) Serve() error {
	return s.Nvim.Serve()
}

func (s *Server) Close() error {
	return s.Nvim.Close()
}
<commit_msg>pkg/server: change Dial logic and Server struct<commit_after>// Copyright 2017 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package server

import (
	"context"
	"fmt"
	"net"
	"os"
	"time"

	"github.com/neovim/go-client/nvim"
	"github.com/pkg/errors"
	"go.uber.org/zap"

	"github.com/zchee/nvim-go/pkg/logger"
	"github.com/zchee/nvim-go/pkg/nctx"
)

type Server struct {
	Nvim *nvim.Nvim
}

func NewServer(ctx context.Context) (*Server, error) {
	n, err := Dial(ctx)
	if err != nil {
		return nil, err
	}

	return &Server{
		Nvim: n,
	}, nil
}

var DefaultDialer = func(ctx context.Context, network, address string) (net.Conn, error) {
	return (&net.Dialer{
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
		DualStack: true,
	}).DialContext(ctx, network, address)
}

var DefaultLogFunc = func(ctx context.Context) func(string, ...interface{}) {
	return func(format string, a ...interface{}) {
		logger.FromContext(ctx).Info(fmt.Sprintf(format, a))
	}
}

func Dial(ctx context.Context) (n *nvim.Nvim, err error) {
	log := logger.FromContext(ctx).Named("server")
	ctx = logger.NewContext(ctx, log)

	addr := os.Getenv(nctx.ListenAddress) // NVIM_LISTEN_ADDRESS environment can get if already launch the nvim process
	if addr == "" {
		return nil, errors.Errorf("failed get %s", nctx.ListenAddress)
	}

	dialOpts := []nvim.DialOption{
		nvim.DialContext(ctx),
		nvim.DialServe(false),
		nvim.DialNetDial(DefaultDialer),
		nvim.DialLogf(DefaultLogFunc(ctx)),
	}

	var tmpDelay time.Duration // how long to sleep on accept failure
	for {
		n, err = nvim.Dial(addr, dialOpts...)
		if err != nil {
			log.Error("Dial error", zap.Error(err), zap.Duration("retrying", tmpDelay))

			if tmpDelay == 0 {
				tmpDelay = 5 * time.Millisecond
			} else {
				tmpDelay *= 2
			}
			if max := 1 * time.Second; tmpDelay > max {
				tmpDelay = max
			}
			timer := time.NewTimer(tmpDelay)
			select {
			case <-timer.C:
			}
			continue
		}
		tmpDelay = 0

		return n, nil
	}
}

func (s *Server) Serve() error {
	return s.Nvim.Serve()
}

func (s *Server) Close() error {
	return s.Nvim.Close()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package net

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/golang/glog"
	"golang.org/x/net/http2"
)

// JoinPreservingTrailingSlash does a path.Join of the specified elements,
// preserving any trailing slash on the last non-empty segment
func JoinPreservingTrailingSlash(elem ...string) string {
	// do the basic path join
	result := path.Join(elem...)

	// find the last non-empty segment
	for i := len(elem) - 1; i >= 0; i-- {
		if len(elem[i]) > 0 {
			// if the last segment ended in a slash, ensure our result does as well
			if strings.HasSuffix(elem[i], "/") && !strings.HasSuffix(result, "/") {
				result += "/"
			}
			break
		}
	}

	return result
}

// IsProbableEOF returns true if the given error resembles a connection termination
// scenario that would justify assuming that the watch is empty.
// These errors are what the Go http stack returns back to us which are general
// connection closure errors (strongly correlated) and callers that need to
// differentiate probable errors in connection behavior between normal "this is
// disconnected" should use the method.
func IsProbableEOF(err error) bool {
	if uerr, ok := err.(*url.Error); ok {
		err = uerr.Err
	}
	switch {
	case err == io.EOF:
		return true
	case err.Error() == "http: can't write HTTP request on broken connection":
		return true
	case strings.Contains(err.Error(), "connection reset by peer"):
		return true
	case strings.Contains(strings.ToLower(err.Error()), "use of closed network connection"):
		return true
	}
	return false
}

var defaultTransport = http.DefaultTransport.(*http.Transport)

// SetOldTransportDefaults applies the defaults from http.DefaultTransport
// for the Proxy, Dial, and TLSHandshakeTimeout fields if unset
func SetOldTransportDefaults(t *http.Transport) *http.Transport {
	if t.Proxy == nil || isDefault(t.Proxy) {
		// http.ProxyFromEnvironment doesn't respect CIDRs and that makes it impossible to exclude things like pod and service IPs from proxy settings
		// ProxierWithNoProxyCIDR allows CIDR rules in NO_PROXY
		t.Proxy = NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment)
	}
	if t.Dial == nil {
		t.Dial = defaultTransport.Dial
	}
	if t.TLSHandshakeTimeout == 0 {
		t.TLSHandshakeTimeout = defaultTransport.TLSHandshakeTimeout
	}
	return t
}

// SetTransportDefaults applies the defaults from http.DefaultTransport
// for the Proxy, Dial, and TLSHandshakeTimeout fields if unset
func SetTransportDefaults(t *http.Transport) *http.Transport {
	t = SetOldTransportDefaults(t)
	// Allow clients to disable http2 if needed.
	if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
		glog.Infof("HTTP2 has been explicitly disabled")
	} else {
		if err := http2.ConfigureTransport(t); err != nil {
			glog.Warningf("Transport failed http2 configuration: %v", err)
		}
	}
	return t
}

type RoundTripperWrapper interface {
	http.RoundTripper
	WrappedRoundTripper() http.RoundTripper
}

type DialFunc func(net, addr string) (net.Conn, error)

func DialerFor(transport http.RoundTripper) (DialFunc, error) {
	if transport == nil {
		return nil, nil
	}

	switch transport := transport.(type) {
	case *http.Transport:
		return transport.Dial, nil
	case RoundTripperWrapper:
		return DialerFor(transport.WrappedRoundTripper())
	default:
		return nil, fmt.Errorf("unknown transport type: %T", transport)
	}
}

type TLSClientConfigHolder interface {
	TLSClientConfig() *tls.Config
}

func TLSClientConfig(transport http.RoundTripper) (*tls.Config, error) {
	if transport == nil {
		return nil, nil
	}

	switch transport := transport.(type) {
	case *http.Transport:
		return transport.TLSClientConfig, nil
	case TLSClientConfigHolder:
		return transport.TLSClientConfig(), nil
	case RoundTripperWrapper:
		return TLSClientConfig(transport.WrappedRoundTripper())
	default:
		return nil, fmt.Errorf("unknown transport type: %T", transport)
	}
}

func FormatURL(scheme string, host string, port int, path string) *url.URL {
	return &url.URL{
		Scheme: scheme,
		Host:   net.JoinHostPort(host, strconv.Itoa(port)),
		Path:   path,
	}
}

func GetHTTPClient(req *http.Request) string {
	if userAgent, ok := req.Header["User-Agent"]; ok {
		if len(userAgent) > 0 {
			return userAgent[0]
		}
	}
	return "unknown"
}

// SourceIPs splits the comma separated X-Forwarded-For header or returns the X-Real-Ip header or req.RemoteAddr,
// in that order, ignoring invalid IPs. It returns nil if all of these are empty or invalid.
func SourceIPs(req *http.Request) []net.IP {
	hdr := req.Header
	// First check the X-Forwarded-For header for requests via proxy.
	hdrForwardedFor := hdr.Get("X-Forwarded-For")
	forwardedForIPs := []net.IP{}
	if hdrForwardedFor != "" {
		// X-Forwarded-For can be a csv of IPs in case of multiple proxies.
		// Use the first valid one.
		parts := strings.Split(hdrForwardedFor, ",")
		for _, part := range parts {
			ip := net.ParseIP(strings.TrimSpace(part))
			if ip != nil {
				forwardedForIPs = append(forwardedForIPs, ip)
			}
		}
	}
	if len(forwardedForIPs) > 0 {
		return forwardedForIPs
	}

	// Try the X-Real-Ip header.
	hdrRealIp := hdr.Get("X-Real-Ip")
	if hdrRealIp != "" {
		ip := net.ParseIP(hdrRealIp)
		if ip != nil {
			return []net.IP{ip}
		}
	}

	// Fallback to Remote Address in request, which will give the correct client IP when there is no proxy.
	// Remote Address in Go's HTTP server is in the form host:port so we need to split that first.
	host, _, err := net.SplitHostPort(req.RemoteAddr)
	if err == nil {
		if remoteIP := net.ParseIP(host); remoteIP != nil {
			return []net.IP{remoteIP}
		}
	}

	// Fallback if Remote Address was just IP.
	if remoteIP := net.ParseIP(req.RemoteAddr); remoteIP != nil {
		return []net.IP{remoteIP}
	}

	return nil
}

// Extracts and returns the clients IP from the given request.
// Looks at X-Forwarded-For header, X-Real-Ip header and request.RemoteAddr in that order.
// Returns nil if none of them are set or is set to an invalid value.
func GetClientIP(req *http.Request) net.IP {
	ips := SourceIPs(req)
	if len(ips) == 0 {
		return nil
	}
	return ips[0]
}

// Prepares the X-Forwarded-For header for another forwarding hop by appending the previous sender's
// IP address to the X-Forwarded-For chain.
func AppendForwardedForHeader(req *http.Request) {
	// Copied from net/http/httputil/reverseproxy.go:
	if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
		// If we aren't the first proxy retain prior
		// X-Forwarded-For information as a comma+space
		// separated list and fold multiple headers into one.
		if prior, ok := req.Header["X-Forwarded-For"]; ok {
			clientIP = strings.Join(prior, ", ") + ", " + clientIP
		}
		req.Header.Set("X-Forwarded-For", clientIP)
	}
}

var defaultProxyFuncPointer = fmt.Sprintf("%p", http.ProxyFromEnvironment)

// isDefault checks to see if the transportProxierFunc is pointing to the default one
func isDefault(transportProxier func(*http.Request) (*url.URL, error)) bool {
	transportProxierPointer := fmt.Sprintf("%p", transportProxier)
	return transportProxierPointer == defaultProxyFuncPointer
}

// NewProxierWithNoProxyCIDR constructs a Proxier function that respects CIDRs in NO_PROXY and delegates if
// no matching CIDRs are found
func NewProxierWithNoProxyCIDR(delegate func(req *http.Request) (*url.URL, error)) func(req *http.Request) (*url.URL, error) {
	// we wrap the default method, so we only need to perform our check if the NO_PROXY (or no_proxy) envvar has a CIDR in it
	noProxyEnv := os.Getenv("NO_PROXY")
	if noProxyEnv == "" {
		noProxyEnv = os.Getenv("no_proxy")
	}
	noProxyRules := strings.Split(noProxyEnv, ",")

	cidrs := []*net.IPNet{}
	for _, noProxyRule := range noProxyRules {
		_, cidr, _ := net.ParseCIDR(noProxyRule)
		if cidr != nil {
			cidrs = append(cidrs, cidr)
		}
	}

	if len(cidrs) == 0 {
		return delegate
	}

	return func(req *http.Request) (*url.URL, error) {
		ip := net.ParseIP(req.URL.Hostname())
		if ip == nil {
			return delegate(req)
		}

		for _, cidr := range cidrs {
			if cidr.Contains(ip) {
				return nil, nil
			}
		}

		return delegate(req)
	}
}

// DialerFunc implements Dialer for the provided function.
type DialerFunc func(req *http.Request) (net.Conn, error)

func (fn DialerFunc) Dial(req *http.Request) (net.Conn, error) {
	return fn(req)
}

// Dialer dials a host and writes a request to it.
type Dialer interface {
	// Dial connects to the host specified by req's URL, writes the request to the connection, and
	// returns the opened net.Conn.
	Dial(req *http.Request) (net.Conn, error)
}

// ConnectWithRedirects uses dialer to send req, following up to 10 redirects (relative to
// originalLocation). It returns the opened net.Conn and the raw response bytes.
func ConnectWithRedirects(originalMethod string, originalLocation *url.URL, header http.Header, originalBody io.Reader, dialer Dialer) (net.Conn, []byte, error) {
	const (
		maxRedirects    = 10
		maxResponseSize = 16384 // play it safe to allow the potential for lots of / large headers
	)

	var (
		location         = originalLocation
		method           = originalMethod
		intermediateConn net.Conn
		rawResponse      = bytes.NewBuffer(make([]byte, 0, 256))
		body             = originalBody
	)

	defer func() {
		if intermediateConn != nil {
			intermediateConn.Close()
		}
	}()

redirectLoop:
	for redirects := 0; ; redirects++ {
		if redirects > maxRedirects {
			return nil, nil, fmt.Errorf("too many redirects (%d)", redirects)
		}

		req, err := http.NewRequest(method, location.String(), body)
		if err != nil {
			return nil, nil, err
		}

		req.Header = header

		intermediateConn, err = dialer.Dial(req)
		if err != nil {
			return nil, nil, err
		}

		// Peek at the backend response.
		rawResponse.Reset()
		respReader := bufio.NewReader(io.TeeReader(
			io.LimitReader(intermediateConn, maxResponseSize), // Don't read more than maxResponseSize bytes.
			rawResponse)) // Save the raw response.
		resp, err := http.ReadResponse(respReader, nil)
		if err != nil {
			// Unable to read the backend response; let the client handle it.
			glog.Warningf("Error reading backend response: %v", err)
			break redirectLoop
		}

		switch resp.StatusCode {
		case http.StatusFound:
			// Redirect, continue.
		default:
			// Don't redirect.
			break redirectLoop
		}

		// Redirected requests switch to "GET" according to the HTTP spec:
		// https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3
		method = "GET"
		// don't send a body when following redirects
		body = nil

		resp.Body.Close() // not used

		// Reset the connection.
		intermediateConn.Close()
		intermediateConn = nil

		// Prepare to follow the redirect.
		redirectStr := resp.Header.Get("Location")
		if redirectStr == "" {
			return nil, nil, fmt.Errorf("%d response missing Location header", resp.StatusCode)
		}
		// We have to parse relative to the current location, NOT originalLocation. For example,
		// if we request http://foo.com/a and get back "http://bar.com/b", the result should be
		// http://bar.com/b. If we then make that request and get back a redirect to "/c", the result
		// should be http://bar.com/c, not http://foo.com/c.
		location, err = location.Parse(redirectStr)
		if err != nil {
			return nil, nil, fmt.Errorf("malformed Location header: %v", err)
		}
	}

	connToReturn := intermediateConn
	intermediateConn = nil // Don't close the connection when we return it.
	return connToReturn, rawResponse.Bytes(), nil
}

// CloneRequest creates a shallow copy of the request along with a deep copy of the Headers.
func CloneRequest(req *http.Request) *http.Request {
	r := new(http.Request)

	// shallow clone
	*r = *req

	// deep copy headers
	r.Header = CloneHeader(req.Header)

	return r
}

// CloneHeader creates a deep copy of an http.Header.
func CloneHeader(in http.Header) http.Header {
	out := make(http.Header, len(in))
	for key, values := range in {
		newValues := make([]string, len(values))
		copy(newValues, values)
		out[key] = newValues
	}
	return out
}
<commit_msg>Check nil error in IsProbableEOF()<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package net

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/golang/glog"
	"golang.org/x/net/http2"
)

// JoinPreservingTrailingSlash does a path.Join of the specified elements,
// preserving any trailing slash on the last non-empty segment
func JoinPreservingTrailingSlash(elem ...string) string {
	// do the basic path join
	result := path.Join(elem...)

	// find the last non-empty segment
	for i := len(elem) - 1; i >= 0; i-- {
		if len(elem[i]) > 0 {
			// if the last segment ended in a slash, ensure our result does as well
			if strings.HasSuffix(elem[i], "/") && !strings.HasSuffix(result, "/") {
				result += "/"
			}
			break
		}
	}

	return result
}

// IsProbableEOF returns true if the given error resembles a connection termination
// scenario that would justify assuming that the watch is empty.
// These errors are what the Go http stack returns back to us which are general
// connection closure errors (strongly correlated) and callers that need to
// differentiate probable errors in connection behavior between normal "this is
// disconnected" should use the method.
func IsProbableEOF(err error) bool {
	if err == nil {
		return false
	}
	if uerr, ok := err.(*url.Error); ok {
		err = uerr.Err
	}
	switch {
	case err == io.EOF:
		return true
	case err.Error() == "http: can't write HTTP request on broken connection":
		return true
	case strings.Contains(err.Error(), "connection reset by peer"):
		return true
	case strings.Contains(strings.ToLower(err.Error()), "use of closed network connection"):
		return true
	}
	return false
}

var defaultTransport = http.DefaultTransport.(*http.Transport)

// SetOldTransportDefaults applies the defaults from http.DefaultTransport
// for the Proxy, Dial, and TLSHandshakeTimeout fields if unset
func SetOldTransportDefaults(t *http.Transport) *http.Transport {
	if t.Proxy == nil || isDefault(t.Proxy) {
		// http.ProxyFromEnvironment doesn't respect CIDRs and that makes it impossible to exclude things like pod and service IPs from proxy settings
		// ProxierWithNoProxyCIDR allows CIDR rules in NO_PROXY
		t.Proxy = NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment)
	}
	if t.Dial == nil {
		t.Dial = defaultTransport.Dial
	}
	if t.TLSHandshakeTimeout == 0 {
		t.TLSHandshakeTimeout = defaultTransport.TLSHandshakeTimeout
	}
	return t
}

// SetTransportDefaults applies the defaults from http.DefaultTransport
// for the Proxy, Dial, and TLSHandshakeTimeout fields if unset
func SetTransportDefaults(t *http.Transport) *http.Transport {
	t = SetOldTransportDefaults(t)
	// Allow clients to disable http2 if needed.
	if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
		glog.Infof("HTTP2 has been explicitly disabled")
	} else {
		if err := http2.ConfigureTransport(t); err != nil {
			glog.Warningf("Transport failed http2 configuration: %v", err)
		}
	}
	return t
}

type RoundTripperWrapper interface {
	http.RoundTripper
	WrappedRoundTripper() http.RoundTripper
}

type DialFunc func(net, addr string) (net.Conn, error)

func DialerFor(transport http.RoundTripper) (DialFunc, error) {
	if transport == nil {
		return nil, nil
	}

	switch transport := transport.(type) {
	case *http.Transport:
		return transport.Dial, nil
	case RoundTripperWrapper:
		return DialerFor(transport.WrappedRoundTripper())
	default:
		return nil, fmt.Errorf("unknown transport type: %T", transport)
	}
}

type TLSClientConfigHolder interface {
	TLSClientConfig() *tls.Config
}

func TLSClientConfig(transport http.RoundTripper) (*tls.Config, error) {
	if transport == nil {
		return nil, nil
	}

	switch transport := transport.(type) {
	case *http.Transport:
		return transport.TLSClientConfig, nil
	case TLSClientConfigHolder:
		return transport.TLSClientConfig(), nil
	case RoundTripperWrapper:
		return TLSClientConfig(transport.WrappedRoundTripper())
	default:
		return nil, fmt.Errorf("unknown transport type: %T", transport)
	}
}

func FormatURL(scheme string, host string, port int, path string) *url.URL {
	return &url.URL{
		Scheme: scheme,
		Host:   net.JoinHostPort(host, strconv.Itoa(port)),
		Path:   path,
	}
}

func GetHTTPClient(req *http.Request) string {
	if userAgent, ok := req.Header["User-Agent"]; ok {
		if len(userAgent) > 0 {
			return userAgent[0]
		}
	}
	return "unknown"
}

// SourceIPs splits the comma separated X-Forwarded-For header or returns the X-Real-Ip header or req.RemoteAddr,
// in that order, ignoring invalid IPs. It returns nil if all of these are empty or invalid.
func SourceIPs(req *http.Request) []net.IP {
	hdr := req.Header
	// First check the X-Forwarded-For header for requests via proxy.
	hdrForwardedFor := hdr.Get("X-Forwarded-For")
	forwardedForIPs := []net.IP{}
	if hdrForwardedFor != "" {
		// X-Forwarded-For can be a csv of IPs in case of multiple proxies.
		// Use the first valid one.
		parts := strings.Split(hdrForwardedFor, ",")
		for _, part := range parts {
			ip := net.ParseIP(strings.TrimSpace(part))
			if ip != nil {
				forwardedForIPs = append(forwardedForIPs, ip)
			}
		}
	}
	if len(forwardedForIPs) > 0 {
		return forwardedForIPs
	}

	// Try the X-Real-Ip header.
	hdrRealIp := hdr.Get("X-Real-Ip")
	if hdrRealIp != "" {
		ip := net.ParseIP(hdrRealIp)
		if ip != nil {
			return []net.IP{ip}
		}
	}

	// Fallback to Remote Address in request, which will give the correct client IP when there is no proxy.
	// Remote Address in Go's HTTP server is in the form host:port so we need to split that first.
	host, _, err := net.SplitHostPort(req.RemoteAddr)
	if err == nil {
		if remoteIP := net.ParseIP(host); remoteIP != nil {
			return []net.IP{remoteIP}
		}
	}

	// Fallback if Remote Address was just IP.
	if remoteIP := net.ParseIP(req.RemoteAddr); remoteIP != nil {
		return []net.IP{remoteIP}
	}

	return nil
}

// Extracts and returns the clients IP from the given request.
// Looks at X-Forwarded-For header, X-Real-Ip header and request.RemoteAddr in that order.
// Returns nil if none of them are set or is set to an invalid value.
func GetClientIP(req *http.Request) net.IP {
	ips := SourceIPs(req)
	if len(ips) == 0 {
		return nil
	}
	return ips[0]
}

// Prepares the X-Forwarded-For header for another forwarding hop by appending the previous sender's
// IP address to the X-Forwarded-For chain.
func AppendForwardedForHeader(req *http.Request) {
	// Copied from net/http/httputil/reverseproxy.go:
	if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
		// If we aren't the first proxy retain prior
		// X-Forwarded-For information as a comma+space
		// separated list and fold multiple headers into one.
		if prior, ok := req.Header["X-Forwarded-For"]; ok {
			clientIP = strings.Join(prior, ", ") + ", " + clientIP
		}
		req.Header.Set("X-Forwarded-For", clientIP)
	}
}

var defaultProxyFuncPointer = fmt.Sprintf("%p", http.ProxyFromEnvironment)

// isDefault checks to see if the transportProxierFunc is pointing to the default one
func isDefault(transportProxier func(*http.Request) (*url.URL, error)) bool {
	transportProxierPointer := fmt.Sprintf("%p", transportProxier)
	return transportProxierPointer == defaultProxyFuncPointer
}

// NewProxierWithNoProxyCIDR constructs a Proxier function that respects CIDRs in NO_PROXY and delegates if
// no matching CIDRs are found
func NewProxierWithNoProxyCIDR(delegate func(req *http.Request) (*url.URL, error)) func(req *http.Request) (*url.URL, error) {
	// we wrap the default method, so we only need to perform our check if the NO_PROXY (or no_proxy) envvar has a CIDR in it
	noProxyEnv := os.Getenv("NO_PROXY")
	if noProxyEnv == "" {
		noProxyEnv = os.Getenv("no_proxy")
	}
	noProxyRules := strings.Split(noProxyEnv, ",")

	cidrs := []*net.IPNet{}
	for _, noProxyRule := range noProxyRules {
		_, cidr, _ := net.ParseCIDR(noProxyRule)
		if cidr != nil {
			cidrs = append(cidrs, cidr)
		}
	}

	if len(cidrs) == 0 {
		return delegate
	}

	return func(req *http.Request) (*url.URL, error) {
		ip := net.ParseIP(req.URL.Hostname())
		if ip == nil {
			return delegate(req)
		}

		for _, cidr := range cidrs {
			if cidr.Contains(ip) {
				return nil, nil
			}
		}

		return delegate(req)
	}
}

// DialerFunc implements Dialer for the provided function.
type DialerFunc func(req *http.Request) (net.Conn, error)

func (fn DialerFunc) Dial(req *http.Request) (net.Conn, error) {
	return fn(req)
}

// Dialer dials a host and writes a request to it.
type Dialer interface {
	// Dial connects to the host specified by req's URL, writes the request to the connection, and
	// returns the opened net.Conn.
	Dial(req *http.Request) (net.Conn, error)
}

// ConnectWithRedirects uses dialer to send req, following up to 10 redirects (relative to
// originalLocation). It returns the opened net.Conn and the raw response bytes.
func ConnectWithRedirects(originalMethod string, originalLocation *url.URL, header http.Header, originalBody io.Reader, dialer Dialer) (net.Conn, []byte, error) {
	const (
		maxRedirects    = 10
		maxResponseSize = 16384 // play it safe to allow the potential for lots of / large headers
	)

	var (
		location         = originalLocation
		method           = originalMethod
		intermediateConn net.Conn
		rawResponse      = bytes.NewBuffer(make([]byte, 0, 256))
		body             = originalBody
	)

	defer func() {
		if intermediateConn != nil {
			intermediateConn.Close()
		}
	}()

redirectLoop:
	for redirects := 0; ; redirects++ {
		if redirects > maxRedirects {
			return nil, nil, fmt.Errorf("too many redirects (%d)", redirects)
		}

		req, err := http.NewRequest(method, location.String(), body)
		if err != nil {
			return nil, nil, err
		}

		req.Header = header

		intermediateConn, err = dialer.Dial(req)
		if err != nil {
			return nil, nil, err
		}

		// Peek at the backend response.
		rawResponse.Reset()
		respReader := bufio.NewReader(io.TeeReader(
			io.LimitReader(intermediateConn, maxResponseSize), // Don't read more than maxResponseSize bytes.
			rawResponse)) // Save the raw response.
		resp, err := http.ReadResponse(respReader, nil)
		if err != nil {
			// Unable to read the backend response; let the client handle it.
			glog.Warningf("Error reading backend response: %v", err)
			break redirectLoop
		}

		switch resp.StatusCode {
		case http.StatusFound:
			// Redirect, continue.
		default:
			// Don't redirect.
			break redirectLoop
		}

		// Redirected requests switch to "GET" according to the HTTP spec:
		// https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3
		method = "GET"
		// don't send a body when following redirects
		body = nil

		resp.Body.Close() // not used

		// Reset the connection.
		intermediateConn.Close()
		intermediateConn = nil

		// Prepare to follow the redirect.
		redirectStr := resp.Header.Get("Location")
		if redirectStr == "" {
			return nil, nil, fmt.Errorf("%d response missing Location header", resp.StatusCode)
		}
		// We have to parse relative to the current location, NOT originalLocation. For example,
		// if we request http://foo.com/a and get back "http://bar.com/b", the result should be
		// http://bar.com/b. If we then make that request and get back a redirect to "/c", the result
		// should be http://bar.com/c, not http://foo.com/c.
		location, err = location.Parse(redirectStr)
		if err != nil {
			return nil, nil, fmt.Errorf("malformed Location header: %v", err)
		}
	}

	connToReturn := intermediateConn
	intermediateConn = nil // Don't close the connection when we return it.
	return connToReturn, rawResponse.Bytes(), nil
}

// CloneRequest creates a shallow copy of the request along with a deep copy of the Headers.
func CloneRequest(req *http.Request) *http.Request {
	r := new(http.Request)

	// shallow clone
	*r = *req

	// deep copy headers
	r.Header = CloneHeader(req.Header)

	return r
}

// CloneHeader creates a deep copy of an http.Header.
func CloneHeader(in http.Header) http.Header {
	out := make(http.Header, len(in))
	for key, values := range in {
		newValues := make([]string, len(values))
		copy(newValues, values)
		out[key] = newValues
	}
	return out
}
<|endoftext|>


<commit_before>package analyze

import (
	"math"
	"math/rand"

	"github.com/phil-mansfield/shellfish/math/sort"
)

type Shell func(phi, theta float64) float64

func randomAngle() (phi, theta float64) {
	u, v := rand.Float64(), rand.Float64()
	return 2 * math.Pi * u, math.Acos(2*v - 1)
}

func cartesian(phi, theta, r float64) (x, y, z float64) {
	sinP, cosP := math.Sincos(phi)
	sinT, cosT := math.Sincos(theta)
	return r * sinT * cosP, r * sinT * sinP, r * cosT
}

func (s Shell) CartesianSampledVolume(samples int, rMax float64) float64 {
	inside := 0
	for i := 0; i < samples; i++ {
		x := rand.Float64()*(2*rMax) - rMax
		y := rand.Float64()*(2*rMax) - rMax
		z := rand.Float64()*(2*rMax) - rMax

		r := math.Sqrt(x*x + y*y + z*z)
		phi := math.Atan2(y, x)
		th := math.Acos(z / r)

		rs := s(phi, th)
		if r < rs {
			inside++
		}
	}

	return float64(inside) / float64(samples) * (rMax * rMax * rMax * 8)
}

func (s Shell) Volume(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		sum += r * r * r
	}
	r3 := sum / float64(samples)
	return r3 * 4 * (math.Pi / 3)
}

func (s Shell) MeanRadius(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, th := randomAngle()
		r := s(phi, th)
		sum += r
	}
	return sum / float64(samples)
}

func (s Shell) MedianRadius(samples int) float64 {
	rs := make([]float64, samples)
	for i := range rs {
		phi, th := randomAngle()
		rs[i] = s(phi, th)
	}
	return sort.Median(rs, rs)
}

func (s Shell) Moments(samples int) (Ix, Iy, Iz float64) {
	xSum, ySum, zSum, rSum := 0.0, 0.0, 0.0, 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		x, y, z := cartesian(phi, theta, r)
		xSum += (y*y + z*z) * r * r
		ySum += (x*x + z*z) * r * r
		zSum += (x*x + y*y) * r * r
		rSum += r * r
	}
	return xSum / rSum, ySum / rSum, zSum / rSum
}

func (s Shell) SurfaceArea(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		sum += r * r
	}
	return sum / float64(samples) * 4 * math.Pi
}

func (s1 Shell) DiffVolume(s2 Shell, samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r1, r2 := s1(phi, theta), s2(phi, theta)
		r := (r1 + r2) / 2
		dr := math.Abs(r1 - r2)
		sum += dr * r * r
	}
	return sum / float64(samples) * (4 * math.Pi) / 3
}

func (s1 Shell) MaxDiff(s2 Shell, samples int) float64 {
	max := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r1, r2 := s1(phi, theta), s2(phi, theta)
		dr := math.Abs(r1 - r2)
		if dr > max {
			max = dr
		}
	}
	return max
}

func (s Shell) RadialRange(samples int) (low, high float64) {
	phi, theta := randomAngle()
	low = s(phi, theta)
	high = low
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		if r > high {
			high = r
		}
		if r < low {
			low = r
		}
	}
	return low, high
}

func (s Shell) Contains(x, y, z float64) bool {
	r := math.Sqrt(x*x + y*y + z*z)
	phi := math.Atan2(y, x)
	theta := math.Acos(z / r)
	return s(phi, theta) > r
}

type Tracers struct {
	Vol, Sa, Ix, Iy, Iz float64
}

func CumulativeTracers(
	shells [][]Shell, samples int,
) (means, stds []Tracers) {
	rings := len(shells[0])
	sums, sqrs := make([]Tracers, rings), make([]Tracers, rings)

	for ir := range shells[0] {
		for ih := range shells {
			shell := shells[ih][ir]

			vol := shell.Volume(samples)
			sa := shell.SurfaceArea(samples)
			ix, iy, iz := shell.Moments(samples)

			sums[ir].Vol += vol
			sums[ir].Sa += sa
			sums[ir].Ix += ix
			sums[ir].Iy += iy
			sums[ir].Iz += iz

			sqrs[ir].Vol += vol * vol
			sqrs[ir].Sa += sa * sa
			sqrs[ir].Ix += ix * ix
			sqrs[ir].Iy += iy * iy
			sqrs[ir].Iz += iz * iz
		}
	}

	means, stds = make([]Tracers, rings), make([]Tracers, rings)
	n := float64(len(shells))
	for i := range means {
		means[i].Vol = sums[i].Vol / n
		means[i].Sa = sums[i].Sa / n
		means[i].Ix = sums[i].Ix / n
		means[i].Iy = sums[i].Iy / n
		means[i].Iz = sums[i].Iz / n

		stds[i].Vol = math.Sqrt(sqrs[i].Vol/n - means[i].Vol*means[i].Vol)
		stds[i].Sa = math.Sqrt(sqrs[i].Sa/n - means[i].Sa*means[i].Sa)
		stds[i].Ix = math.Sqrt(sqrs[i].Ix/n - means[i].Ix*means[i].Ix)
		stds[i].Iy = math.Sqrt(sqrs[i].Iy/n - means[i].Iy*means[i].Iy)
		stds[i].Iz = math.Sqrt(sqrs[i].Iz/n - means[i].Iz*means[i].Iz)
	}

	return means, stds
}
<commit_msg>Implemented RadiusHistogram()<commit_after>package analyze

import (
	"math"
	"math/rand"

	"github.com/phil-mansfield/shellfish/math/sort"
)

type Shell func(phi, theta float64) float64

func randomAngle() (phi, theta float64) {
	u, v := rand.Float64(), rand.Float64()
	return 2 * math.Pi * u, math.Acos(2*v - 1)
}

func cartesian(phi, theta, r float64) (x, y, z float64) {
	sinP, cosP := math.Sincos(phi)
	sinT, cosT := math.Sincos(theta)
	return r * sinT * cosP, r * sinT * sinP, r * cosT
}

func (s Shell) CartesianSampledVolume(samples int, rMax float64) float64 {
	inside := 0
	for i := 0; i < samples; i++ {
		x := rand.Float64()*(2*rMax) - rMax
		y := rand.Float64()*(2*rMax) - rMax
		z := rand.Float64()*(2*rMax) - rMax

		r := math.Sqrt(x*x + y*y + z*z)
		phi := math.Atan2(y, x)
		th := math.Acos(z / r)

		rs := s(phi, th)
		if r < rs {
			inside++
		}
	}

	return float64(inside) / float64(samples) * (rMax * rMax * rMax * 8)
}

func (s Shell) Volume(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		sum += r * r * r
	}
	r3 := sum / float64(samples)
	return r3 * 4 * (math.Pi / 3)
}

func (s Shell) MeanRadius(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, th := randomAngle()
		r := s(phi, th)
		sum += r
	}
	return sum / float64(samples)
}

func (s Shell) MedianRadius(samples int) float64 {
	rs := make([]float64, samples)
	for i := range rs {
		phi, th := randomAngle()
		rs[i] = s(phi, th)
	}
	return sort.Median(rs, rs)
}

func (s Shell) Moments(samples int) (Ix, Iy, Iz float64) {
	xSum, ySum, zSum, rSum := 0.0, 0.0, 0.0, 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		x, y, z := cartesian(phi, theta, r)
		xSum += (y*y + z*z) * r * r
		ySum += (x*x + z*z) * r * r
		zSum += (x*x + y*y) * r * r
		rSum += r * r
	}
	return xSum / rSum, ySum / rSum, zSum / rSum
}

func (s Shell) SurfaceArea(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		sum += r * r
	}
	return sum / float64(samples) * 4 * math.Pi
}

func (s1 Shell) DiffVolume(s2 Shell, samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r1, r2 := s1(phi, theta), s2(phi, theta)
		r := (r1 + r2) / 2
		dr := math.Abs(r1 - r2)
		sum += dr * r * r
	}
	return sum / float64(samples) * (4 * math.Pi) / 3
}

func (s1 Shell) MaxDiff(s2 Shell, samples int) float64 {
	max := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r1, r2 := s1(phi, theta), s2(phi, theta)
		dr := math.Abs(r1 - r2)
		if dr > max {
			max = dr
		}
	}
	return max
}

func (s Shell) RadialRange(samples int) (low, high float64) {
	phi, theta := randomAngle()
	low = s(phi, theta)
	high = low
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		if r > high {
			high = r
		}
		if r < low {
			low = r
		}
	}
	return low, high
}

func  (s Shell) RadiusHistogram(
	samples, bins int, rMin, rMax float64,
) (rs, ns []float64) {
	rs, ns = make([]float64, bins), make([]float64, bins)
	dr := (rMax - rMin) / float64(bins)
	for i := range rs {
		rs[i] = rMin + dr*(float64(i) + 0.5)
	}

	count := 0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		ri := (r - rMin) / dr
		if ri < 0 { continue }
		idx := int(ri)
		if idx >= bins { continue }
		ns[idx]++
		count++
	}

	for i := range ns {
		ns[i] /= float64(count) * dr
	}

	return rs, ns
}

func (s Shell) Contains(x, y, z float64) bool {
	r := math.Sqrt(x*x + y*y + z*z)
	phi := math.Atan2(y, x)
	theta := math.Acos(z / r)
	return s(phi, theta) > r
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2019 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ipam

import (
	"context"
	"fmt"
	"math"
	"os"
	"strings"

	"github.com/olekukonko/tablewriter"

	"github.com/projectcalico/calicoctl/calicoctl/commands/constants"
	"github.com/projectcalico/libcalico-go/lib/ipam"

	docopt "github.com/docopt/docopt-go"

	"github.com/projectcalico/calicoctl/calicoctl/commands/argutils"
	"github.com/projectcalico/calicoctl/calicoctl/commands/clientmgr"
)

// IPAM takes keyword with an IP address then calls the subcommands.
func Show(args []string) error {
	doc := constants.DatastoreIntro + `Usage:
  calicoctl ipam show [--ip=<IP> | --show-blocks] [--config=<CONFIG>]

Options:
  -h --help             Show this screen.
     --ip=<IP>          Report whether this specific IP address is in use.
     --show-blocks      Show detailed information for IP blocks as well as pools.
  -c --config=<CONFIG>  Path to the file containing connection configuration in
                        YAML or JSON format.
                        [default: ` + constants.DefaultConfigPath + `]

Description:
  The ipam show command prints information about a given IP address, or about
  overall IP usage.
`
	parsedArgs, err := docopt.Parse(doc, args, true, "", false, false)
	if err != nil {
		return fmt.Errorf("Invalid option: 'calicoctl %s'. Use flag '--help' to read about a specific subcommand.", strings.Join(args, " "))
	}
	if len(parsedArgs) == 0 {
		return nil
	}

	ctx := context.Background()

	// Create a new backend client from env vars.
	cf := parsedArgs["--config"].(string)
	client, err := clientmgr.NewClient(cf)
	if err != nil {
		fmt.Println(err)
	}

	showBlocks := parsedArgs["--show-blocks"].(bool)

	ipamClient := client.IPAM()
	if passedIP := parsedArgs["--ip"]; passedIP != nil {
		ip := argutils.ValidateIP(passedIP.(string))
		attr, err := ipamClient.GetAssignmentAttributes(ctx, ip)

		// IP address is not assigned, this prints message like `IP 192.168.71.1
		// is not assigned in block`. This is not exactly an error, so not
		// returning it to the caller.
		if err != nil {
			fmt.Println(err)
			return nil
		}

		// IP address is assigned.
		fmt.Printf("IP %s is in use\n", ip)
		if len(attr) != 0 {
			fmt.Printf("Attributes: %v\n", attr)
		} else {
			fmt.Println("No attributes defined")
		}
		return nil
	}

	usage, err := ipamClient.GetUtilization(ctx, ipam.GetUtilizationArgs{})
	if err != nil {
		return err
	}
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"GROUPING", "CIDR", "IPS TOTAL", "IPS IN USE", "IPS FREE"})
	genRow := func(kind, cidr string, available, capacity float64) []string {
		return []string{
			kind,
			cidr,
			fmt.Sprintf("%.5g", capacity),
			// Note: the '+capacity/2' bits here give us rounding to the nearest
			// integer, instead of rounding down, and so ensure that the two percentages
			// add up to 100.
			fmt.Sprintf("%.5g (%.f%%)", capacity-available, 100*(capacity-available)/capacity),
			fmt.Sprintf("%.5g (%.f%%)", available, 100*available/capacity),
		}
	}
	for _, poolUse := range usage {
		var blockRows [][]string
		var poolInUse float64
		for _, blockUse := range poolUse.Blocks {
			blockRows = append(blockRows, genRow("Block", blockUse.CIDR.String(), float64(blockUse.Available), float64(blockUse.Capacity)))
			poolInUse += float64(blockUse.Capacity - blockUse.Available)
		}
		ones, bits := poolUse.CIDR.Mask.Size()
		poolCapacity := math.Pow(2, float64(bits-ones))
		if ones > 0 {
			// Only show the IP Pool row for a real IP Pool and not for the orphaned
			// block case.
			table.Append(genRow("IP Pool", poolUse.CIDR.String(), poolCapacity-poolInUse, poolCapacity))
		}
		if showBlocks {
			table.AppendBulk(blockRows)
		}
	}
	table.Render()

	return nil
}
<commit_msg>Fix showing IPS IN USE from a large pool as 0<commit_after>// Copyright (c) 2016-2019 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ipam

import (
	"context"
	"fmt"
	"math"
	"os"
	"strings"

	"github.com/olekukonko/tablewriter"

	"github.com/projectcalico/calicoctl/calicoctl/commands/constants"
	"github.com/projectcalico/libcalico-go/lib/ipam"

	docopt "github.com/docopt/docopt-go"

	"github.com/projectcalico/calicoctl/calicoctl/commands/argutils"
	"github.com/projectcalico/calicoctl/calicoctl/commands/clientmgr"
)

// IPAM takes keyword with an IP address then calls the subcommands.
func Show(args []string) error {
	doc := constants.DatastoreIntro + `Usage:
  calicoctl ipam show [--ip=<IP> | --show-blocks] [--config=<CONFIG>]

Options:
  -h --help             Show this screen.
     --ip=<IP>          Report whether this specific IP address is in use.
     --show-blocks      Show detailed information for IP blocks as well as pools.
  -c --config=<CONFIG>  Path to the file containing connection configuration in
                        YAML or JSON format.
                        [default: ` + constants.DefaultConfigPath + `]

Description:
  The ipam show command prints information about a given IP address, or about
  overall IP usage.
`
	parsedArgs, err := docopt.Parse(doc, args, true, "", false, false)
	if err != nil {
		return fmt.Errorf("Invalid option: 'calicoctl %s'. Use flag '--help' to read about a specific subcommand.", strings.Join(args, " "))
	}
	if len(parsedArgs) == 0 {
		return nil
	}

	ctx := context.Background()

	// Create a new backend client from env vars.
	cf := parsedArgs["--config"].(string)
	client, err := clientmgr.NewClient(cf)
	if err != nil {
		fmt.Println(err)
	}

	showBlocks := parsedArgs["--show-blocks"].(bool)

	ipamClient := client.IPAM()
	if passedIP := parsedArgs["--ip"]; passedIP != nil {
		ip := argutils.ValidateIP(passedIP.(string))
		attr, err := ipamClient.GetAssignmentAttributes(ctx, ip)

		// IP address is not assigned, this prints message like `IP 192.168.71.1
		// is not assigned in block`. This is not exactly an error, so not
		// returning it to the caller.
		if err != nil {
			fmt.Println(err)
			return nil
		}

		// IP address is assigned.
		fmt.Printf("IP %s is in use\n", ip)
		if len(attr) != 0 {
			fmt.Printf("Attributes: %v\n", attr)
		} else {
			fmt.Println("No attributes defined")
		}
		return nil
	}

	usage, err := ipamClient.GetUtilization(ctx, ipam.GetUtilizationArgs{})
	if err != nil {
		return err
	}
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"GROUPING", "CIDR", "IPS TOTAL", "IPS IN USE", "IPS FREE"})
	genRow := func(kind, cidr string, inUse, capacity float64) []string {
		return []string{
			kind,
			cidr,
			fmt.Sprintf("%.5g", capacity),
			// Note: the '+capacity/2' bits here give us rounding to the nearest
			// integer, instead of rounding down, and so ensure that the two percentages
			// add up to 100.
			fmt.Sprintf("%.5g (%.f%%)", inUse, 100*inUse/capacity),
			fmt.Sprintf("%.5g (%.f%%)", capacity-inUse, 100*(capacity-inUse)/capacity),
		}
	}
	for _, poolUse := range usage {
		var blockRows [][]string
		var poolInUse float64
		for _, blockUse := range poolUse.Blocks {
			blockRows = append(blockRows, genRow("Block", blockUse.CIDR.String(), float64(blockUse.Capacity-blockUse.Available), float64(blockUse.Capacity)))
			poolInUse += float64(blockUse.Capacity - blockUse.Available)
		}
		ones, bits := poolUse.CIDR.Mask.Size()
		poolCapacity := math.Pow(2, float64(bits-ones))
		if ones > 0 {
			// Only show the IP Pool row for a real IP Pool and not for the orphaned
			// block case.
			table.Append(genRow("IP Pool", poolUse.CIDR.String(), poolInUse, poolCapacity))
		}
		if showBlocks {
			table.AppendBulk(blockRows)
		}
	}
	table.Render()

	return nil
}
<|endoftext|>