<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package ipcache

import (
	"context"
	"fmt"
	"net"
	"net/netip"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/ip"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/labels/cidr"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/source"
)

// AllocateCIDRs attempts to allocate identities for a list of CIDRs. If any
// allocation fails, all allocations are rolled back and the error is returned.
// When an identity is freshly allocated for a CIDR, it is added to the
// ipcache if 'newlyAllocatedIdentities' is 'nil', otherwise the newly allocated
// identities are placed in 'newlyAllocatedIdentities' and it is the caller's
// responsibility to upsert them into ipcache by calling UpsertGeneratedIdentities().
//
// Previously used numeric identities for the given prefixes may be passed in as the
// 'oldNIDs' parameter; nil slice must be passed if no previous numeric identities exist.
// Previously used NID is allocated if still available. Non-availability is not an error.
//
// Upon success, the caller must also arrange for the resulting identities to
// be released via a subsequent call to ReleaseCIDRIdentitiesByCIDR().
func (ipc *IPCache) AllocateCIDRs(
	prefixes []netip.Prefix, oldNIDs []identity.NumericIdentity, newlyAllocatedIdentities map[netip.Prefix]*identity.Identity,
) ([]*identity.Identity, error) {
	// maintain list of used identities to undo on error
	usedIdentities := make([]*identity.Identity, 0, len(prefixes))

	// Maintain list of newly allocated identities to update ipcache,
	// but upsert them to ipcache only if no map was given by the caller.
	upsert := false
	if newlyAllocatedIdentities == nil {
		upsert = true
		newlyAllocatedIdentities = map[netip.Prefix]*identity.Identity{}
	}

	allocateCtx, cancel := context.WithTimeout(context.Background(), option.Config.IPAllocationTimeout)
	defer cancel()

	ipc.metadata.RLock()
	ipc.Lock()
	allocatedIdentities := make(map[netip.Prefix]*identity.Identity, len(prefixes))
	for i, prefix := range prefixes {
		lbls := cidr.GetCIDRLabels(prefix)
		lbls.MergeLabels(ipc.metadata.getLocked(prefix).ToLabels())
		oldNID := identity.InvalidIdentity
		if oldNIDs != nil && len(oldNIDs) > i {
			oldNID = oldNIDs[i]
		}
		id, isNew, err := ipc.allocate(allocateCtx, prefix, lbls, oldNID)
		if err != nil {
			ipc.IdentityAllocator.ReleaseSlice(context.Background(), nil, usedIdentities)
			ipc.Unlock()
			return nil, err
		}

		usedIdentities = append(usedIdentities, id)
		allocatedIdentities[prefix] = id
		if isNew {
			newlyAllocatedIdentities[prefix] = id
		}
	}
	ipc.Unlock()
	ipc.metadata.RUnlock()

	// Only upsert into ipcache if identity wasn't allocated
	// before and the caller does not care doing this
	if upsert {
		ipc.UpsertGeneratedIdentities(newlyAllocatedIdentities, nil)
	}

	identities := make([]*identity.Identity, 0, len(allocatedIdentities))
	for _, id := range allocatedIdentities {
		identities = append(identities, id)
	}
	return identities, nil
}

// AllocateCIDRsForIPs performs the same action as AllocateCIDRs but for IP
// addresses instead of CIDRs.
//
// Upon success, the caller must also arrange for the resulting identities to
// be released via a subsequent call to ReleaseCIDRIdentitiesByID().
func (ipc *IPCache) AllocateCIDRsForIPs(
	prefixes []net.IP, newlyAllocatedIdentities map[netip.Prefix]*identity.Identity,
) ([]*identity.Identity, error) {
	return ipc.AllocateCIDRs(ip.IPsToNetPrefixes(prefixes), nil, newlyAllocatedIdentities)
}

func cidrLabelToPrefix(id *identity.Identity) (prefix netip.Prefix, ok bool) {
	var err error

	label := id.CIDRLabel.String()
	if !strings.HasPrefix(label, labels.LabelSourceCIDR) {
		log.WithFields(logrus.Fields{
			logfields.Identity: id.ID,
		}).Warning("BUG: Attempting to upsert non-CIDR identity")
		return
	}

	if prefix, err = netip.ParsePrefix(strings.TrimPrefix(label, labels.LabelSourceCIDR+":")); err != nil {
		log.WithFields(logrus.Fields{
			logfields.Identity: id.ID,
			logfields.Labels:   label,
		}).Warning("BUG: Attempting to upsert identity with bad CIDR label")
		return
	}
	return prefix, true
}

// UpsertGeneratedIdentities unconditionally upserts 'newlyAllocatedIdentities'
// into the ipcache, then also upserts any CIDR identities in 'usedIdentities'
// that were not already upserted. If any 'usedIdentities' are upserted, these
// are counted separately as they may provide an indication of another logic
// error elsewhere in the codebase that is causing premature ipcache deletions.
func (ipc *IPCache) UpsertGeneratedIdentities(newlyAllocatedIdentities map[netip.Prefix]*identity.Identity, usedIdentities []*identity.Identity) {
	for prefix, id := range newlyAllocatedIdentities {
		ipc.Upsert(prefix.String(), nil, 0, nil, Identity{
			ID:     id.ID,
			Source: source.Generated,
		})
	}
	if len(usedIdentities) == 0 {
		return
	}

	toUpsert := make(map[netip.Prefix]*identity.Identity)
	ipc.mutex.RLock()
	for _, id := range usedIdentities {
		prefix, ok := cidrLabelToPrefix(id)
		if !ok {
			continue
		}
		if _, ok := ipc.LookupByIPRLocked(prefix.String()); ok {
			// Already there; continue
			continue
		}
		toUpsert[prefix] = id
	}
	ipc.mutex.RUnlock()
	for prefix, id := range toUpsert {
		metrics.IPCacheErrorsTotal.WithLabelValues(
			metricTypeRecover, metricErrorUnexpected,
		).Inc()
		ipc.Upsert(prefix.String(), nil, 0, nil, Identity{
			ID:     id.ID,
			Source: source.Generated,
		})
	}
}

// allocate will allocate a new identity for the given prefix based on the
// given set of labels. This function performs both global and local (CIDR)
// identity allocation and the set of labels determine which identity
// allocation type is to occur.
//
// If the identity is a CIDR identity, then its corresponding Identity will
// have its CIDR labels set correctly.
//
// A possible previously used numeric identity for these labels can be passed
// in as the 'oldNID' parameter; identity.InvalidIdentity must be passed if no
// previous numeric identity exists.
//
// It is up to the caller to provide the full set of labels for identity
// allocation.
func (ipc *IPCache) allocate(ctx context.Context, prefix netip.Prefix, lbls labels.Labels, oldNID identity.NumericIdentity) (*identity.Identity, bool, error) {
	id, isNew, err := ipc.IdentityAllocator.AllocateIdentity(ctx, lbls, false, oldNID)
	if err != nil {
		return nil, isNew, fmt.Errorf("failed to allocate identity for cidr %s: %s", prefix, err)
	}

	if lbls.Has(labels.LabelWorld[labels.IDNameWorld]) {
		id.CIDRLabel = labels.NewLabelsFromModel([]string{labels.LabelSourceCIDR + ":" + prefix.String()})
	}

	return id, isNew, err
}

func (ipc *IPCache) releaseCIDRIdentities(ctx context.Context, prefixes []netip.Prefix) {
	// Create a critical section for identity release + removal from ipcache.
	// Otherwise, it's possible to trigger the following race condition:
	//
	// Goroutine 1                | Goroutine 2
	// releaseCIDRIdentities()    | AllocateCIDRs()
	// -> Release(..., id, ...)   |
	//                            | -> allocate(...)
	//                            | -> ipc.UpsertGeneratedIdentities(...)
	// -> ipc.deleteLocked(...)   |
	//
	// In this case, the expectation from Goroutine 2 is that an identity
	// is allocated and that identity is in the ipcache, but the result
	// is that the identity is allocated but the ipcache entry is missing.
	ipc.Lock()
	defer ipc.Unlock()

	toDelete := make([]netip.Prefix, 0, len(prefixes))
	for _, prefix := range prefixes {
		lbls := cidr.GetCIDRLabels(prefix)
		id := ipc.IdentityAllocator.LookupIdentity(ctx, lbls)
		if id == nil {
			log.Errorf("Unable to find identity of previously used CIDR %s", prefix.String())
			continue
		}
		released, err := ipc.IdentityAllocator.Release(ctx, id, false)
		if err != nil {
			log.WithFields(logrus.Fields{
				logfields.Identity: id,
				logfields.CIDR:     prefix,
			}).WithError(err).Warning("Unable to release CIDR identity. Ignoring error. Identity may be leaked")
		}
		if released {
			toDelete = append(toDelete, prefix)
		}
	}

	for _, prefix := range toDelete {
		ipc.deleteLocked(prefix.String(), source.Generated)
	}
}

// ReleaseCIDRIdentitiesByCIDR releases the identities of a list of CIDRs.
// When the last use of the identity is released, the ipcache entry is deleted.
func (ipc *IPCache) ReleaseCIDRIdentitiesByCIDR(prefixes []netip.Prefix) {
	ipc.deferredPrefixRelease.enqueue(prefixes, "cidr-prefix-release")
}

// ReleaseCIDRIdentitiesByID releases the specified identities.
// When the last use of the identity is released, the ipcache entry is deleted.
func (ipc *IPCache) ReleaseCIDRIdentitiesByID(ctx context.Context, identities []identity.NumericIdentity) {
	prefixes := make([]netip.Prefix, 0, len(identities))
	for _, nid := range identities {
		if id := ipc.IdentityAllocator.LookupIdentityByID(ctx, nid); id != nil {
			prefix, ok := cidrLabelToPrefix(id)
			if !ok {
				log.WithFields(logrus.Fields{
					logfields.Identity: nid,
					logfields.Labels:   id.Labels,
				}).Warn("Unexpected release of non-CIDR identity, will leak this identity. Please report this issue to the developers.")
				continue
			}
			prefixes = append(prefixes, prefix)
		} else {
			log.WithFields(logrus.Fields{
				logfields.Identity: nid,
			}).Warn("Unexpected release of numeric identity that is no longer allocated")
		}
	}

	ipc.deferredPrefixRelease.enqueue(prefixes, "selector-prefix-release")
}
<commit_msg>ipcache: Release metadata mutex in loop error condition<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package ipcache

import (
	"context"
	"fmt"
	"net"
	"net/netip"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/ip"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/labels/cidr"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/source"
)

// AllocateCIDRs attempts to allocate identities for a list of CIDRs. If any
// allocation fails, all allocations are rolled back and the error is returned.
// When an identity is freshly allocated for a CIDR, it is added to the
// ipcache if 'newlyAllocatedIdentities' is 'nil', otherwise the newly allocated
// identities are placed in 'newlyAllocatedIdentities' and it is the caller's
// responsibility to upsert them into ipcache by calling UpsertGeneratedIdentities().
//
// Previously used numeric identities for the given prefixes may be passed in as the
// 'oldNIDs' parameter; nil slice must be passed if no previous numeric identities exist.
// Previously used NID is allocated if still available. Non-availability is not an error.
//
// Upon success, the caller must also arrange for the resulting identities to
// be released via a subsequent call to ReleaseCIDRIdentitiesByCIDR().
func (ipc *IPCache) AllocateCIDRs(
	prefixes []netip.Prefix, oldNIDs []identity.NumericIdentity, newlyAllocatedIdentities map[netip.Prefix]*identity.Identity,
) ([]*identity.Identity, error) {
	// maintain list of used identities to undo on error
	usedIdentities := make([]*identity.Identity, 0, len(prefixes))

	// Maintain list of newly allocated identities to update ipcache,
	// but upsert them to ipcache only if no map was given by the caller.
	upsert := false
	if newlyAllocatedIdentities == nil {
		upsert = true
		newlyAllocatedIdentities = map[netip.Prefix]*identity.Identity{}
	}

	allocateCtx, cancel := context.WithTimeout(context.Background(), option.Config.IPAllocationTimeout)
	defer cancel()

	ipc.metadata.RLock()
	ipc.Lock()
	allocatedIdentities := make(map[netip.Prefix]*identity.Identity, len(prefixes))
	for i, prefix := range prefixes {
		lbls := cidr.GetCIDRLabels(prefix)
		lbls.MergeLabels(ipc.metadata.getLocked(prefix).ToLabels())
		oldNID := identity.InvalidIdentity
		if oldNIDs != nil && len(oldNIDs) > i {
			oldNID = oldNIDs[i]
		}
		id, isNew, err := ipc.allocate(allocateCtx, prefix, lbls, oldNID)
		if err != nil {
			ipc.IdentityAllocator.ReleaseSlice(context.Background(), nil, usedIdentities)
			ipc.Unlock()
			ipc.metadata.RUnlock()
			return nil, err
		}

		usedIdentities = append(usedIdentities, id)
		allocatedIdentities[prefix] = id
		if isNew {
			newlyAllocatedIdentities[prefix] = id
		}
	}
	ipc.Unlock()
	ipc.metadata.RUnlock()

	// Only upsert into ipcache if identity wasn't allocated
	// before and the caller does not care doing this
	if upsert {
		ipc.UpsertGeneratedIdentities(newlyAllocatedIdentities, nil)
	}

	identities := make([]*identity.Identity, 0, len(allocatedIdentities))
	for _, id := range allocatedIdentities {
		identities = append(identities, id)
	}
	return identities, nil
}

// AllocateCIDRsForIPs performs the same action as AllocateCIDRs but for IP
// addresses instead of CIDRs.
//
// Upon success, the caller must also arrange for the resulting identities to
// be released via a subsequent call to ReleaseCIDRIdentitiesByID().
func (ipc *IPCache) AllocateCIDRsForIPs(
	prefixes []net.IP, newlyAllocatedIdentities map[netip.Prefix]*identity.Identity,
) ([]*identity.Identity, error) {
	return ipc.AllocateCIDRs(ip.IPsToNetPrefixes(prefixes), nil, newlyAllocatedIdentities)
}

func cidrLabelToPrefix(id *identity.Identity) (prefix netip.Prefix, ok bool) {
	var err error

	label := id.CIDRLabel.String()
	if !strings.HasPrefix(label, labels.LabelSourceCIDR) {
		log.WithFields(logrus.Fields{
			logfields.Identity: id.ID,
		}).Warning("BUG: Attempting to upsert non-CIDR identity")
		return
	}

	if prefix, err = netip.ParsePrefix(strings.TrimPrefix(label, labels.LabelSourceCIDR+":")); err != nil {
		log.WithFields(logrus.Fields{
			logfields.Identity: id.ID,
			logfields.Labels:   label,
		}).Warning("BUG: Attempting to upsert identity with bad CIDR label")
		return
	}
	return prefix, true
}

// UpsertGeneratedIdentities unconditionally upserts 'newlyAllocatedIdentities'
// into the ipcache, then also upserts any CIDR identities in 'usedIdentities'
// that were not already upserted. If any 'usedIdentities' are upserted, these
// are counted separately as they may provide an indication of another logic
// error elsewhere in the codebase that is causing premature ipcache deletions.
func (ipc *IPCache) UpsertGeneratedIdentities(newlyAllocatedIdentities map[netip.Prefix]*identity.Identity, usedIdentities []*identity.Identity) {
	for prefix, id := range newlyAllocatedIdentities {
		ipc.Upsert(prefix.String(), nil, 0, nil, Identity{
			ID:     id.ID,
			Source: source.Generated,
		})
	}
	if len(usedIdentities) == 0 {
		return
	}

	toUpsert := make(map[netip.Prefix]*identity.Identity)
	ipc.mutex.RLock()
	for _, id := range usedIdentities {
		prefix, ok := cidrLabelToPrefix(id)
		if !ok {
			continue
		}
		if _, ok := ipc.LookupByIPRLocked(prefix.String()); ok {
			// Already there; continue
			continue
		}
		toUpsert[prefix] = id
	}
	ipc.mutex.RUnlock()
	for prefix, id := range toUpsert {
		metrics.IPCacheErrorsTotal.WithLabelValues(
			metricTypeRecover, metricErrorUnexpected,
		).Inc()
		ipc.Upsert(prefix.String(), nil, 0, nil, Identity{
			ID:     id.ID,
			Source: source.Generated,
		})
	}
}

// allocate will allocate a new identity for the given prefix based on the
// given set of labels. This function performs both global and local (CIDR)
// identity allocation and the set of labels determine which identity
// allocation type is to occur.
//
// If the identity is a CIDR identity, then its corresponding Identity will
// have its CIDR labels set correctly.
//
// A possible previously used numeric identity for these labels can be passed
// in as the 'oldNID' parameter; identity.InvalidIdentity must be passed if no
// previous numeric identity exists.
//
// It is up to the caller to provide the full set of labels for identity
// allocation.
func (ipc *IPCache) allocate(ctx context.Context, prefix netip.Prefix, lbls labels.Labels, oldNID identity.NumericIdentity) (*identity.Identity, bool, error) {
	id, isNew, err := ipc.IdentityAllocator.AllocateIdentity(ctx, lbls, false, oldNID)
	if err != nil {
		return nil, isNew, fmt.Errorf("failed to allocate identity for cidr %s: %s", prefix, err)
	}

	if lbls.Has(labels.LabelWorld[labels.IDNameWorld]) {
		id.CIDRLabel = labels.NewLabelsFromModel([]string{labels.LabelSourceCIDR + ":" + prefix.String()})
	}

	return id, isNew, err
}

func (ipc *IPCache) releaseCIDRIdentities(ctx context.Context, prefixes []netip.Prefix) {
	// Create a critical section for identity release + removal from ipcache.
	// Otherwise, it's possible to trigger the following race condition:
	//
	// Goroutine 1                | Goroutine 2
	// releaseCIDRIdentities()    | AllocateCIDRs()
	// -> Release(..., id, ...)   |
	//                            | -> allocate(...)
	//                            | -> ipc.UpsertGeneratedIdentities(...)
	// -> ipc.deleteLocked(...)   |
	//
	// In this case, the expectation from Goroutine 2 is that an identity
	// is allocated and that identity is in the ipcache, but the result
	// is that the identity is allocated but the ipcache entry is missing.
	ipc.Lock()
	defer ipc.Unlock()

	toDelete := make([]netip.Prefix, 0, len(prefixes))
	for _, prefix := range prefixes {
		lbls := cidr.GetCIDRLabels(prefix)
		id := ipc.IdentityAllocator.LookupIdentity(ctx, lbls)
		if id == nil {
			log.Errorf("Unable to find identity of previously used CIDR %s", prefix.String())
			continue
		}
		released, err := ipc.IdentityAllocator.Release(ctx, id, false)
		if err != nil {
			log.WithFields(logrus.Fields{
				logfields.Identity: id,
				logfields.CIDR:     prefix,
			}).WithError(err).Warning("Unable to release CIDR identity. Ignoring error. Identity may be leaked")
		}
		if released {
			toDelete = append(toDelete, prefix)
		}
	}

	for _, prefix := range toDelete {
		ipc.deleteLocked(prefix.String(), source.Generated)
	}
}

// ReleaseCIDRIdentitiesByCIDR releases the identities of a list of CIDRs.
// When the last use of the identity is released, the ipcache entry is deleted.
func (ipc *IPCache) ReleaseCIDRIdentitiesByCIDR(prefixes []netip.Prefix) {
	ipc.deferredPrefixRelease.enqueue(prefixes, "cidr-prefix-release")
}

// ReleaseCIDRIdentitiesByID releases the specified identities.
// When the last use of the identity is released, the ipcache entry is deleted.
func (ipc *IPCache) ReleaseCIDRIdentitiesByID(ctx context.Context, identities []identity.NumericIdentity) {
	prefixes := make([]netip.Prefix, 0, len(identities))
	for _, nid := range identities {
		if id := ipc.IdentityAllocator.LookupIdentityByID(ctx, nid); id != nil {
			prefix, ok := cidrLabelToPrefix(id)
			if !ok {
				log.WithFields(logrus.Fields{
					logfields.Identity: nid,
					logfields.Labels:   id.Labels,
				}).Warn("Unexpected release of non-CIDR identity, will leak this identity. Please report this issue to the developers.")
				continue
			}
			prefixes = append(prefixes, prefix)
		} else {
			log.WithFields(logrus.Fields{
				logfields.Identity: nid,
			}).Warn("Unexpected release of numeric identity that is no longer allocated")
		}
	}

	ipc.deferredPrefixRelease.enqueue(prefixes, "selector-prefix-release")
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags. It provides an approximation of the Kubernetes
// version for ad-hoc builds (e.g. `go build`) that cannot get the version
// information from git.
//
// If you are looking at these fields in the git tree, they look
// strange. They are modified on the fly by the build process. The
// in-tree values are dummy values used for "git archive", which also
// works for GitHub tar downloads.
//
// When releasing a new Kubernetes version, this file is updated by
// build/mark_new_version.sh to reflect the new version, and then a
// git annotated tag (using format vX.Y where X == Major version and Y
// == Minor version) is created to point to the commit that updates
// pkg/version/base.go
var (
	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion
	// instead. First step in deprecation, keep the fields but make
	// them irrelevant. (Next we'll take it out, which may muck with
	// scripts consuming the kubectl version output - but most of
	// these should be looking at gitVersion already anyways.)
	gitMajor string = "1"  // major version, always numeric
	gitMinor string = "4+" // minor version, numeric possibly followed by "+"

	// semantic version, derived by build scripts (see
	// https://github.com/kubernetes/kubernetes/blob/master/docs/design/versioning.md
	// for a detailed discussion of this field)
	//
	// TODO: This field is still called "gitVersion" for legacy
	// reasons. For prerelease versions, the build metadata on the
	// semantic version is a git hash, but the version itself is no
	// longer the direct output of "git describe", but a slight
	// translation to be semver compliant.
	gitVersion   string = "v1.4.0-beta.6+$Format:%h$"
	gitCommit    string = "$Format:%H$"    // sha1 from git, output of $(git rev-parse HEAD)
	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"

	buildDate string = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
)
<commit_msg>Kubernetes version v1.4.0-beta.7<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags. It provides an approximation of the Kubernetes
// version for ad-hoc builds (e.g. `go build`) that cannot get the version
// information from git.
//
// If you are looking at these fields in the git tree, they look
// strange. They are modified on the fly by the build process. The
// in-tree values are dummy values used for "git archive", which also
// works for GitHub tar downloads.
//
// When releasing a new Kubernetes version, this file is updated by
// build/mark_new_version.sh to reflect the new version, and then a
// git annotated tag (using format vX.Y where X == Major version and Y
// == Minor version) is created to point to the commit that updates
// pkg/version/base.go
var (
	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion
	// instead. First step in deprecation, keep the fields but make
	// them irrelevant. (Next we'll take it out, which may muck with
	// scripts consuming the kubectl version output - but most of
	// these should be looking at gitVersion already anyways.)
	gitMajor string = "1"  // major version, always numeric
	gitMinor string = "4+" // minor version, numeric possibly followed by "+"

	// semantic version, derived by build scripts (see
	// https://github.com/kubernetes/kubernetes/blob/master/docs/design/versioning.md
	// for a detailed discussion of this field)
	//
	// TODO: This field is still called "gitVersion" for legacy
	// reasons. For prerelease versions, the build metadata on the
	// semantic version is a git hash, but the version itself is no
	// longer the direct output of "git describe", but a slight
	// translation to be semver compliant.
	gitVersion   string = "v1.4.0-beta.7+$Format:%h$"
	gitCommit    string = "$Format:%H$"    // sha1 from git, output of $(git rev-parse HEAD)
	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"

	buildDate string = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
)
<|endoftext|>


<commit_before>package vfs

import (
	"bytes"
	"image"
	"io"
	"io/ioutil"
	"time"

	// Packages image/... are not used explicitly in the code below,
	// but are imported for its initialization side-effect
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	// Same for image/webp
	_ "golang.org/x/image/webp"

	"github.com/cozy/goexif2/exif"
	"github.com/dhowden/tag"
)

// MetadataExtractorVersion is the version number of the metadata extractor.
// It will be used later to know which files can be re-examined to get more
// metadata when the extractor is improved.
const MetadataExtractorVersion = 2

// Metadata is a list of metadata specific to each mimetype:
// id3 for music, exif for jpegs, etc.
type Metadata map[string]interface{}

// NewMetadata returns a new metadata object, with the version field set
func NewMetadata() Metadata {
	m := Metadata{}
	m["extractor_version"] = MetadataExtractorVersion
	return m
}

// MetaExtractor is an interface for extracting metadata from a file
type MetaExtractor interface {
	io.WriteCloser
	Abort(error)
	Result() Metadata
}

// NewMetaExtractor returns an extractor for metadata if the mime type has one,
// or null else
func NewMetaExtractor(doc *FileDoc) *MetaExtractor {
	var e MetaExtractor
	switch doc.Mime {
	case "image/jpeg":
		e = NewExifExtractor()
	case "image/png", "image/gif":
		e = NewImageExtractor()
	case "audio/mp3", "audio/mpeg", "audio/ogg", "audio/x-m4a", "audio/flac":
		e = NewAudioExtractor()
	}
	if e != nil {
		return &e
	}
	return nil
}

// ImageExtractor is used to extract width/height from images
type ImageExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	ch chan interface{}
}

// NewImageExtractor returns an extractor for images
func NewImageExtractor() *ImageExtractor {
	e := &ImageExtractor{}
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *ImageExtractor) Start() {
	cfg, _, err := image.DecodeConfig(e.r)
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- cfg
	}
}

// Write is called to push some bytes to the extractor
func (e *ImageExtractor) Write(p []byte) (n int, err error) {
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *ImageExtractor) Close() error {
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *ImageExtractor) Abort(err error) {
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *ImageExtractor) Result() Metadata {
	m := NewMetadata()
	m["datetime"] = time.Now()
	cfg := <-e.ch
	switch cfg := cfg.(type) {
	case image.Config:
		m["width"] = cfg.Width
		m["height"] = cfg.Height
	}
	return m
}

// ExifExtractor is used to extract EXIF metadata from jpegs
type ExifExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	im *ImageExtractor
	ch chan interface{}
}

// NewExifExtractor returns an extractor for EXIF metadata
func NewExifExtractor() *ExifExtractor {
	e := &ExifExtractor{}
	e.im = NewImageExtractor()
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *ExifExtractor) Start() {
	x, err := exif.Decode(e.r)
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- x
	}
}

// Write is called to push some bytes to the extractor
func (e *ExifExtractor) Write(p []byte) (n int, err error) {
	e.im.Write(p)
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *ExifExtractor) Close() error {
	e.im.Close()
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *ExifExtractor) Abort(err error) {
	e.im.Abort(err)
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *ExifExtractor) Result() Metadata {
	m := e.im.Result()
	x := <-e.ch
	switch x := x.(type) {
	case *exif.Exif:
		if dt, err := x.DateTime(); err == nil {
			m["datetime"] = dt
		}
		if flash, err := x.Flash(); err == nil {
			m["flash"] = flash
		}
		if lat, long, err := x.LatLong(); err == nil {
			m["gps"] = map[string]float64{
				"lat":  lat,
				"long": long,
			}
		}
		if o, err := x.Get("Orientation"); err == nil {
			if orientation, err := o.Int(0); err == nil {
				m["orientation"] = orientation
				if orientation > 4 {
					m["width"], m["height"] = m["height"], m["width"]
				}
			}
		}
	}
	return m
}

// AudioExtractor is used to extract album/artist/etc. from audio
type AudioExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	ch chan interface{}
}

// NewAudioExtractor returns an extractor for audio
func NewAudioExtractor() *AudioExtractor {
	e := &AudioExtractor{}
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *AudioExtractor) Start() {
	buf, err := ioutil.ReadAll(e.r)
	if err != nil {
		e.ch <- err
		return
	}
	tags, err := tag.ReadFrom(bytes.NewReader(buf))
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- tags
	}
}

// Write is called to push some bytes to the extractor
func (e *AudioExtractor) Write(p []byte) (n int, err error) {
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *AudioExtractor) Close() error {
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *AudioExtractor) Abort(err error) {
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *AudioExtractor) Result() Metadata {
	m := NewMetadata()
	tags := <-e.ch
	switch tags := tags.(type) {
	case tag.Metadata:
		if album := tags.Album(); album != "" {
			m["album"] = album
		}
		if artist := tags.Artist(); artist != "" {
			m["artist"] = artist
		}
		if composer := tags.Composer(); composer != "" {
			m["composer"] = composer
		}
		if genre := tags.Genre(); genre != "" {
			m["genre"] = genre
		}
		if title := tags.Title(); title != "" {
			m["title"] = title
		}
		if year := tags.Year(); year != 0 {
			m["year"] = year
		}
		if track, _ := tags.Track(); track != 0 {
			m["track"] = track
		}
	}
	return m
}
<commit_msg>Fix a leak of the metadata extractor Start goroutine<commit_after>package vfs

import (
	"bytes"
	"image"
	"io"
	"io/ioutil"
	"time"

	// Packages image/... are not used explicitly in the code below,
	// but are imported for its initialization side-effect
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	// Same for image/webp
	_ "golang.org/x/image/webp"

	"github.com/cozy/goexif2/exif"
	"github.com/dhowden/tag"
)

// MetadataExtractorVersion is the version number of the metadata extractor.
// It will be used later to know which files can be re-examined to get more
// metadata when the extractor is improved.
const MetadataExtractorVersion = 2

// Metadata is a list of metadata specific to each mimetype:
// id3 for music, exif for jpegs, etc.
type Metadata map[string]interface{}

// NewMetadata returns a new metadata object, with the version field set
func NewMetadata() Metadata {
	m := Metadata{}
	m["extractor_version"] = MetadataExtractorVersion
	return m
}

// MetaExtractor is an interface for extracting metadata from a file
type MetaExtractor interface {
	io.WriteCloser
	Abort(error)
	Result() Metadata
}

// NewMetaExtractor returns an extractor for metadata if the mime type has one,
// or null else
func NewMetaExtractor(doc *FileDoc) *MetaExtractor {
	var e MetaExtractor
	switch doc.Mime {
	case "image/jpeg":
		e = NewExifExtractor()
	case "image/png", "image/gif":
		e = NewImageExtractor()
	case "audio/mp3", "audio/mpeg", "audio/ogg", "audio/x-m4a", "audio/flac":
		e = NewAudioExtractor()
	}
	if e != nil {
		return &e
	}
	return nil
}

// ImageExtractor is used to extract width/height from images
type ImageExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	ch chan interface{}
}

// NewImageExtractor returns an extractor for images
func NewImageExtractor() *ImageExtractor {
	e := &ImageExtractor{}
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *ImageExtractor) Start() {
	cfg, _, err := image.DecodeConfig(e.r)
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- cfg
	}
}

// Write is called to push some bytes to the extractor
func (e *ImageExtractor) Write(p []byte) (n int, err error) {
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *ImageExtractor) Close() error {
	err := e.w.Close()
	if err != nil {
		<-e.ch
	}
	return err
}

// Abort is called when the extractor can be discarded
func (e *ImageExtractor) Abort(err error) {
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *ImageExtractor) Result() Metadata {
	m := NewMetadata()
	m["datetime"] = time.Now()
	cfg := <-e.ch
	switch cfg := cfg.(type) {
	case image.Config:
		m["width"] = cfg.Width
		m["height"] = cfg.Height
	}
	return m
}

// ExifExtractor is used to extract EXIF metadata from jpegs
type ExifExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	im *ImageExtractor
	ch chan interface{}
}

// NewExifExtractor returns an extractor for EXIF metadata
func NewExifExtractor() *ExifExtractor {
	e := &ExifExtractor{}
	e.im = NewImageExtractor()
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *ExifExtractor) Start() {
	x, err := exif.Decode(e.r)
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- x
	}
}

// Write is called to push some bytes to the extractor
func (e *ExifExtractor) Write(p []byte) (n int, err error) {
	e.im.Write(p)
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *ExifExtractor) Close() error {
	e.im.Close()
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *ExifExtractor) Abort(err error) {
	e.im.Abort(err)
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *ExifExtractor) Result() Metadata {
	m := e.im.Result()
	x := <-e.ch
	switch x := x.(type) {
	case *exif.Exif:
		if dt, err := x.DateTime(); err == nil {
			m["datetime"] = dt
		}
		if flash, err := x.Flash(); err == nil {
			m["flash"] = flash
		}
		if lat, long, err := x.LatLong(); err == nil {
			m["gps"] = map[string]float64{
				"lat":  lat,
				"long": long,
			}
		}
		if o, err := x.Get("Orientation"); err == nil {
			if orientation, err := o.Int(0); err == nil {
				m["orientation"] = orientation
				if orientation > 4 {
					m["width"], m["height"] = m["height"], m["width"]
				}
			}
		}
	}
	return m
}

// AudioExtractor is used to extract album/artist/etc. from audio
type AudioExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	ch chan interface{}
}

// NewAudioExtractor returns an extractor for audio
func NewAudioExtractor() *AudioExtractor {
	e := &AudioExtractor{}
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *AudioExtractor) Start() {
	buf, err := ioutil.ReadAll(e.r)
	if err != nil {
		e.ch <- err
		return
	}
	tags, err := tag.ReadFrom(bytes.NewReader(buf))
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- tags
	}
}

// Write is called to push some bytes to the extractor
func (e *AudioExtractor) Write(p []byte) (n int, err error) {
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *AudioExtractor) Close() error {
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *AudioExtractor) Abort(err error) {
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *AudioExtractor) Result() Metadata {
	m := NewMetadata()
	tags := <-e.ch
	switch tags := tags.(type) {
	case tag.Metadata:
		if album := tags.Album(); album != "" {
			m["album"] = album
		}
		if artist := tags.Artist(); artist != "" {
			m["artist"] = artist
		}
		if composer := tags.Composer(); composer != "" {
			m["composer"] = composer
		}
		if genre := tags.Genre(); genre != "" {
			m["genre"] = genre
		}
		if title := tags.Title(); title != "" {
			m["title"] = title
		}
		if year := tags.Year(); year != 0 {
			m["year"] = year
		}
		if track, _ := tags.Track(); track != 0 {
			m["track"] = track
		}
	}
	return m
}
<|endoftext|>


<commit_before>// Copyright 2019 OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package carbonexporter

import (
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"go.opentelemetry.io/collector/pdata/pcommon"
	"go.opentelemetry.io/collector/pdata/pmetric"
)

func TestSanitizeTagKey(t *testing.T) {
	tests := []struct {
		name string
		key  string
		want string
	}{
		{
			name: "no_changes",
			key:  "a valid tag key",
			want: "a valid tag key",
		},
		{
			name: "remove_tag_set",
			key:  "a" + tagKeyValueSeparator + "c",
			want: "a" + string(sanitizedRune) + "c",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := sanitizeTagKey(tt.key)
			assert.Equal(t, tt.want, got)
		})
	}
}

func TestSanitizeTagValue(t *testing.T) {
	tests := []struct {
		name  string
		value string
		want  string
	}{
		{
			name:  "no_changes",
			value: "a valid tag value",
			want:  "a valid tag value",
		},
		{
			name:  "replace_tilde",
			value: "a~c",
			want:  "a" + string(sanitizedRune) + "c",
		},
		{
			name:  "replace_semicol",
			value: "a;c",
			want:  "a" + string(sanitizedRune) + "c",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := sanitizeTagValue(tt.value)
			assert.Equal(t, tt.want, got)
		})
	}
}

func TestBuildPath(t *testing.T) {
	tests := []struct {
		name       string
		attributes pcommon.Map
		want       string
	}{
		{
			name: "happy_path",
			attributes: func() pcommon.Map {
				attr := pcommon.NewMap()
				attr.FromRaw(map[string]interface{}{"key0": "val0"})
				return attr
			}(),
			want: "happy_path;key0=val0",
		},
		{
			name: "empty_value",
			attributes: func() pcommon.Map {
				attr := pcommon.NewMap()
				attr.FromRaw(map[string]interface{}{"k0": "", "k1": "v1"})
				return attr
			}(),
			want: "empty_value;k0=" + tagValueEmptyPlaceholder + ";k1=v1",
		},
		{
			name: "int_value",
			attributes: func() pcommon.Map {
				attr := pcommon.NewMap()
				attr.FromRaw(map[string]interface{}{"k": 1})
				return attr
			}(),
			want: "int_value;k=1",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := buildPath(tt.name, tt.attributes)
			assert.Equal(t, tt.want, got)
		})
	}
}

func TestToPlaintext(t *testing.T) {
	expectedTagsStr := ";k0=v0;k1=v1"

	unixSecs := int64(1574092046)
	expectedUnixSecsStr := strconv.FormatInt(unixSecs, 10)
	unixNSecs := int64(11 * time.Millisecond)
	tsUnix := time.Unix(unixSecs, unixNSecs)

	doubleVal := 1234.5678
	expectedDobuleValStr := strconv.FormatFloat(doubleVal, 'g', -1, 64)
	int64Val := int64(123)
	expectedInt64ValStr := "123"

	distributionCount := uint64(16)
	distributionSum := float64(34.56)
	distributionBounds := []float64{1.5, 2, 4}
	distributionCounts := []uint64{4, 2, 3, 7}

	summaryCount := uint64(11)
	summarySum := float64(111)
	summaryQuantiles := []float64{90, 95, 99, 99.9}
	summaryQuantileValues := []float64{100, 6, 4, 1}
	tests := []struct {
		name                       string
		metricsDataFn              func() pmetric.Metrics
		wantLines                  []string
		wantNumConvertedTimeseries int
		wantNumDroppedTimeseries   int
	}{
		{
			name: "no_dims",
			metricsDataFn: func() pmetric.Metrics {
				md := pmetric.NewMetrics()
				ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()
				ms.AppendEmpty().SetName("gauge_double_no_dims")
				dps1 := ms.At(0).SetEmptyGauge().DataPoints()
				dps1.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps1.At(0).SetDoubleValue(doubleVal)
				ms.AppendEmpty().SetName("gauge_int_no_dims")
				dps2 := ms.At(1).SetEmptyGauge().DataPoints()
				dps2.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps2.At(0).SetIntValue(int64Val)

				ms.AppendEmpty().SetName("cumulative_double_no_dims")
				ms.At(2).SetEmptySum().SetIsMonotonic(true)
				dps3 := ms.At(2).Sum().DataPoints()
				dps3.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps3.At(0).SetDoubleValue(doubleVal)
				ms.AppendEmpty().SetName("cumulative_int_no_dims")
				ms.At(3).SetEmptySum().SetIsMonotonic(true)
				dps4 := ms.At(3).Sum().DataPoints()
				dps4.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps4.At(0).SetIntValue(int64Val)
				return md

			},
			wantLines: []string{
				"gauge_double_no_dims " + expectedDobuleValStr + " " + expectedUnixSecsStr,
				"gauge_int_no_dims " + expectedInt64ValStr + " " + expectedUnixSecsStr,
				"cumulative_double_no_dims " + expectedDobuleValStr + " " + expectedUnixSecsStr,
				"cumulative_int_no_dims " + expectedInt64ValStr + " " + expectedUnixSecsStr,
			},
			wantNumConvertedTimeseries: 4,
		},
		{
			name: "with_dims",
			metricsDataFn: func() pmetric.Metrics {
				md := pmetric.NewMetrics()
				ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()
				ms.AppendEmpty().SetName("gauge_double_with_dims")
				dps1 := ms.At(0).SetEmptyGauge().DataPoints()
				dps1.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps1.At(0).Attributes().FromRaw(map[string]interface{}{"k0": "v0", "k1": "v1"})
				dps1.At(0).Attributes().Sort() // ensures result order
				dps1.At(0).SetDoubleValue(doubleVal)
				ms.AppendEmpty().SetName("gauge_int_with_dims")
				dps2 := ms.At(1).SetEmptyGauge().DataPoints()
				dps2.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps2.At(0).Attributes().FromRaw(map[string]interface{}{"k0": "v0", "k1": "v1"})
				dps2.At(0).Attributes().Sort() // ensures result order
				dps2.At(0).SetIntValue(int64Val)

				ms.AppendEmpty().SetName("cumulative_double_with_dims")
				ms.At(2).SetEmptySum().SetIsMonotonic(true)
				dps3 := ms.At(2).Sum().DataPoints()
				dps3.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps3.At(0).Attributes().FromRaw(map[string]interface{}{"k0": "v0", "k1": "v1"})
				dps3.At(0).Attributes().Sort() // ensures result order
				dps3.At(0).SetDoubleValue(doubleVal)
				ms.AppendEmpty().SetName("cumulative_int_with_dims")
				ms.At(3).SetEmptySum().SetIsMonotonic(true)
				dps4 := ms.At(3).Sum().DataPoints()
				dps4.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps4.At(0).Attributes().FromRaw(map[string]interface{}{"k0": "v0", "k1": "v1"})
				dps4.At(0).Attributes().Sort() // ensures result order
				dps4.At(0).SetIntValue(int64Val)
				return md
			},
			wantLines: []string{
				"gauge_double_with_dims" + expectedTagsStr + " " + expectedDobuleValStr + " " + expectedUnixSecsStr,
				"gauge_int_with_dims" + expectedTagsStr + " " + expectedInt64ValStr + " " + expectedUnixSecsStr,
				"cumulative_double_with_dims" + expectedTagsStr + " " + expectedDobuleValStr + " " + expectedUnixSecsStr,
				"cumulative_int_with_dims" + expectedTagsStr + " " + expectedInt64ValStr + " " + expectedUnixSecsStr,
			},
			wantNumConvertedTimeseries: 4,
		},
		{
			name: "distributions",
			metricsDataFn: func() pmetric.Metrics {
				md := pmetric.NewMetrics()
				ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()
				ms.AppendEmpty().SetName("distrib")
				ms.At(0).SetEmptyHistogram().SetAggregationTemporality(pmetric.MetricAggregationTemporalityCumulative)
				dp := ms.At(0).SetEmptyHistogram().DataPoints().AppendEmpty()
				dp.SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dp.Attributes().FromRaw(map[string]interface{}{"k0": "v0", "k1": "v1"})
				dp.Attributes().Sort() // ensures result order
				dp.SetCount(distributionCount)
				dp.SetSum(distributionSum)
				dp.ExplicitBounds().FromRaw(distributionBounds)
				dp.BucketCounts().FromRaw(distributionCounts)
				return md
			},
			wantLines: expectedDistributionLines(
				"distrib", expectedTagsStr, expectedUnixSecsStr,
				distributionSum,
				distributionCount,
				distributionBounds,
				distributionCounts),
			wantNumConvertedTimeseries: 1,
		},
		{
			name: "summary",
			metricsDataFn: func() pmetric.Metrics {
				md := pmetric.NewMetrics()
				ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()
				ms.AppendEmpty().SetName("summary")
				dp := ms.At(0).SetEmptySummary().DataPoints().AppendEmpty()
				dp.SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dp.Attributes().FromRaw(map[string]interface{}{"k0": "v0", "k1": "v1"})
				dp.Attributes().Sort() // ensures result order
				dp.SetCount(summaryCount)
				dp.SetSum(summarySum)
				for i := range summaryQuantiles {
					qv := dp.QuantileValues().AppendEmpty()
					qv.SetQuantile(summaryQuantiles[i] / 100)
					qv.SetValue(summaryQuantileValues[i])
				}
				return md
			},
			wantLines: expectedSummaryLines(
				"summary", expectedTagsStr, expectedUnixSecsStr,
				summarySum,
				summaryCount,
				summaryQuantiles,
				summaryQuantileValues),
			wantNumConvertedTimeseries: 1,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gotLines := metricDataToPlaintext(tt.metricsDataFn())
			got := strings.Split(gotLines, "\n")
			got = got[:len(got)-1]
			assert.Equal(t, tt.wantLines, got)
		})
	}
}

func expectedDistributionLines(
	metricName, tags, timestampStr string,
	sum float64,
	count uint64,
	bounds []float64,
	counts []uint64,
) []string {
	lines := []string{
		metricName + ".count" + tags + " " + formatInt64(int64(count)) + " " + timestampStr,
		metricName + tags + " " + formatFloatForLabel(sum) + " " + timestampStr,
	}

	for i, bound := range bounds {
		lines = append(lines,
			metricName+".bucket"+tags+";upper_bound="+formatFloatForLabel(bound)+" "+formatInt64(int64(counts[i]))+" "+timestampStr)
	}
	lines = append(lines,
		metricName+".bucket"+tags+";upper_bound=inf "+formatInt64(int64(counts[len(bounds)]))+" "+timestampStr)

	return lines
}

func expectedSummaryLines(
	metricName, tags, timestampStr string,
	sum float64,
	count uint64,
	summaryQuantiles []float64,
	summaryQuantileValues []float64,
) []string {
	lines := []string{
		metricName + ".count" + tags + " " + formatInt64(int64(count)) + " " + timestampStr,
		metricName + tags + " " + formatFloatForValue(sum) + " " + timestampStr,
	}

	for i := range summaryQuantiles {
		lines = append(lines,
			metricName+".quantile"+tags+";quantile="+formatFloatForLabel(summaryQuantiles[i])+" "+formatFloatForValue(summaryQuantileValues[i])+" "+timestampStr)
	}

	return lines
}
<commit_msg>[chore] fix carbonexporter flacky tests (#14627)<commit_after>// Copyright 2019 OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package carbonexporter

import (
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"go.opentelemetry.io/collector/pdata/pcommon"
	"go.opentelemetry.io/collector/pdata/pmetric"
)

func TestSanitizeTagKey(t *testing.T) {
	tests := []struct {
		name string
		key  string
		want string
	}{
		{
			name: "no_changes",
			key:  "a valid tag key",
			want: "a valid tag key",
		},
		{
			name: "remove_tag_set",
			key:  "a" + tagKeyValueSeparator + "c",
			want: "a" + string(sanitizedRune) + "c",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := sanitizeTagKey(tt.key)
			assert.Equal(t, tt.want, got)
		})
	}
}

func TestSanitizeTagValue(t *testing.T) {
	tests := []struct {
		name  string
		value string
		want  string
	}{
		{
			name:  "no_changes",
			value: "a valid tag value",
			want:  "a valid tag value",
		},
		{
			name:  "replace_tilde",
			value: "a~c",
			want:  "a" + string(sanitizedRune) + "c",
		},
		{
			name:  "replace_semicol",
			value: "a;c",
			want:  "a" + string(sanitizedRune) + "c",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := sanitizeTagValue(tt.value)
			assert.Equal(t, tt.want, got)
		})
	}
}

func TestBuildPath(t *testing.T) {
	tests := []struct {
		name       string
		attributes pcommon.Map
		want       string
	}{
		{
			name: "happy_path",
			attributes: func() pcommon.Map {
				attr := pcommon.NewMap()
				attr.PutString("key0", "val0")
				return attr
			}(),
			want: "happy_path;key0=val0",
		},
		{
			name: "empty_value",
			attributes: func() pcommon.Map {
				attr := pcommon.NewMap()
				attr.PutString("k0", "")
				attr.PutString("k1", "v1")
				return attr
			}(),
			want: "empty_value;k0=" + tagValueEmptyPlaceholder + ";k1=v1",
		},
		{
			name: "int_value",
			attributes: func() pcommon.Map {
				attr := pcommon.NewMap()
				attr.PutInt("k", 1)
				return attr
			}(),
			want: "int_value;k=1",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := buildPath(tt.name, tt.attributes)
			assert.Equal(t, tt.want, got)
		})
	}
}

func TestToPlaintext(t *testing.T) {
	expectedTagsStr := ";k0=v0;k1=v1"

	unixSecs := int64(1574092046)
	expectedUnixSecsStr := strconv.FormatInt(unixSecs, 10)
	unixNSecs := int64(11 * time.Millisecond)
	tsUnix := time.Unix(unixSecs, unixNSecs)

	doubleVal := 1234.5678
	expectedDobuleValStr := strconv.FormatFloat(doubleVal, 'g', -1, 64)
	int64Val := int64(123)
	expectedInt64ValStr := "123"

	distributionCount := uint64(16)
	distributionSum := float64(34.56)
	distributionBounds := []float64{1.5, 2, 4}
	distributionCounts := []uint64{4, 2, 3, 7}

	summaryCount := uint64(11)
	summarySum := float64(111)
	summaryQuantiles := []float64{90, 95, 99, 99.9}
	summaryQuantileValues := []float64{100, 6, 4, 1}
	tests := []struct {
		name                       string
		metricsDataFn              func() pmetric.Metrics
		wantLines                  []string
		wantNumConvertedTimeseries int
		wantNumDroppedTimeseries   int
	}{
		{
			name: "no_dims",
			metricsDataFn: func() pmetric.Metrics {
				md := pmetric.NewMetrics()
				ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()
				ms.AppendEmpty().SetName("gauge_double_no_dims")
				dps1 := ms.At(0).SetEmptyGauge().DataPoints()
				dps1.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps1.At(0).SetDoubleValue(doubleVal)
				ms.AppendEmpty().SetName("gauge_int_no_dims")
				dps2 := ms.At(1).SetEmptyGauge().DataPoints()
				dps2.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps2.At(0).SetIntValue(int64Val)

				ms.AppendEmpty().SetName("cumulative_double_no_dims")
				ms.At(2).SetEmptySum().SetIsMonotonic(true)
				dps3 := ms.At(2).Sum().DataPoints()
				dps3.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps3.At(0).SetDoubleValue(doubleVal)
				ms.AppendEmpty().SetName("cumulative_int_no_dims")
				ms.At(3).SetEmptySum().SetIsMonotonic(true)
				dps4 := ms.At(3).Sum().DataPoints()
				dps4.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps4.At(0).SetIntValue(int64Val)
				return md

			},
			wantLines: []string{
				"gauge_double_no_dims " + expectedDobuleValStr + " " + expectedUnixSecsStr,
				"gauge_int_no_dims " + expectedInt64ValStr + " " + expectedUnixSecsStr,
				"cumulative_double_no_dims " + expectedDobuleValStr + " " + expectedUnixSecsStr,
				"cumulative_int_no_dims " + expectedInt64ValStr + " " + expectedUnixSecsStr,
			},
			wantNumConvertedTimeseries: 4,
		},
		{
			name: "with_dims",
			metricsDataFn: func() pmetric.Metrics {
				md := pmetric.NewMetrics()
				ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()
				ms.AppendEmpty().SetName("gauge_double_with_dims")
				dps1 := ms.At(0).SetEmptyGauge().DataPoints()
				dps1.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps1.At(0).Attributes().PutString("k0", "v0")
				dps1.At(0).Attributes().PutString("k1", "v1")
				dps1.At(0).SetDoubleValue(doubleVal)
				ms.AppendEmpty().SetName("gauge_int_with_dims")
				dps2 := ms.At(1).SetEmptyGauge().DataPoints()
				dps2.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps2.At(0).Attributes().PutString("k0", "v0")
				dps2.At(0).Attributes().PutString("k1", "v1")
				dps2.At(0).SetIntValue(int64Val)

				ms.AppendEmpty().SetName("cumulative_double_with_dims")
				ms.At(2).SetEmptySum().SetIsMonotonic(true)
				dps3 := ms.At(2).Sum().DataPoints()
				dps3.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps3.At(0).Attributes().PutString("k0", "v0")
				dps3.At(0).Attributes().PutString("k1", "v1")
				dps3.At(0).SetDoubleValue(doubleVal)
				ms.AppendEmpty().SetName("cumulative_int_with_dims")
				ms.At(3).SetEmptySum().SetIsMonotonic(true)
				dps4 := ms.At(3).Sum().DataPoints()
				dps4.AppendEmpty().SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dps4.At(0).Attributes().PutString("k0", "v0")
				dps4.At(0).Attributes().PutString("k1", "v1")
				dps4.At(0).SetIntValue(int64Val)
				return md
			},
			wantLines: []string{
				"gauge_double_with_dims" + expectedTagsStr + " " + expectedDobuleValStr + " " + expectedUnixSecsStr,
				"gauge_int_with_dims" + expectedTagsStr + " " + expectedInt64ValStr + " " + expectedUnixSecsStr,
				"cumulative_double_with_dims" + expectedTagsStr + " " + expectedDobuleValStr + " " + expectedUnixSecsStr,
				"cumulative_int_with_dims" + expectedTagsStr + " " + expectedInt64ValStr + " " + expectedUnixSecsStr,
			},
			wantNumConvertedTimeseries: 4,
		},
		{
			name: "distributions",
			metricsDataFn: func() pmetric.Metrics {
				md := pmetric.NewMetrics()
				ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()
				ms.AppendEmpty().SetName("distrib")
				ms.At(0).SetEmptyHistogram().SetAggregationTemporality(pmetric.MetricAggregationTemporalityCumulative)
				dp := ms.At(0).SetEmptyHistogram().DataPoints().AppendEmpty()
				dp.SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dp.Attributes().FromRaw(map[string]interface{}{"k0": "v0", "k1": "v1"})
				dp.Attributes().Sort() // ensures result order
				dp.SetCount(distributionCount)
				dp.SetSum(distributionSum)
				dp.ExplicitBounds().FromRaw(distributionBounds)
				dp.BucketCounts().FromRaw(distributionCounts)
				return md
			},
			wantLines: expectedDistributionLines(
				"distrib", expectedTagsStr, expectedUnixSecsStr,
				distributionSum,
				distributionCount,
				distributionBounds,
				distributionCounts),
			wantNumConvertedTimeseries: 1,
		},
		{
			name: "summary",
			metricsDataFn: func() pmetric.Metrics {
				md := pmetric.NewMetrics()
				ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics()
				ms.AppendEmpty().SetName("summary")
				dp := ms.At(0).SetEmptySummary().DataPoints().AppendEmpty()
				dp.SetTimestamp(pcommon.NewTimestampFromTime(tsUnix))
				dp.Attributes().FromRaw(map[string]interface{}{"k0": "v0", "k1": "v1"})
				dp.Attributes().Sort() // ensures result order
				dp.SetCount(summaryCount)
				dp.SetSum(summarySum)
				for i := range summaryQuantiles {
					qv := dp.QuantileValues().AppendEmpty()
					qv.SetQuantile(summaryQuantiles[i] / 100)
					qv.SetValue(summaryQuantileValues[i])
				}
				return md
			},
			wantLines: expectedSummaryLines(
				"summary", expectedTagsStr, expectedUnixSecsStr,
				summarySum,
				summaryCount,
				summaryQuantiles,
				summaryQuantileValues),
			wantNumConvertedTimeseries: 1,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gotLines := metricDataToPlaintext(tt.metricsDataFn())
			got := strings.Split(gotLines, "\n")
			got = got[:len(got)-1]
			assert.Equal(t, tt.wantLines, got)
		})
	}
}

func expectedDistributionLines(
	metricName, tags, timestampStr string,
	sum float64,
	count uint64,
	bounds []float64,
	counts []uint64,
) []string {
	lines := []string{
		metricName + ".count" + tags + " " + formatInt64(int64(count)) + " " + timestampStr,
		metricName + tags + " " + formatFloatForLabel(sum) + " " + timestampStr,
	}

	for i, bound := range bounds {
		lines = append(lines,
			metricName+".bucket"+tags+";upper_bound="+formatFloatForLabel(bound)+" "+formatInt64(int64(counts[i]))+" "+timestampStr)
	}
	lines = append(lines,
		metricName+".bucket"+tags+";upper_bound=inf "+formatInt64(int64(counts[len(bounds)]))+" "+timestampStr)

	return lines
}

func expectedSummaryLines(
	metricName, tags, timestampStr string,
	sum float64,
	count uint64,
	summaryQuantiles []float64,
	summaryQuantileValues []float64,
) []string {
	lines := []string{
		metricName + ".count" + tags + " " + formatInt64(int64(count)) + " " + timestampStr,
		metricName + tags + " " + formatFloatForValue(sum) + " " + timestampStr,
	}

	for i := range summaryQuantiles {
		lines = append(lines,
			metricName+".quantile"+tags+";quantile="+formatFloatForLabel(summaryQuantiles[i])+" "+formatFloatForValue(summaryQuantileValues[i])+" "+timestampStr)
	}

	return lines
}
<|endoftext|>


<commit_before>package virtctl

import (
	"fmt"
	"os"
	"strings"

	"github.com/spf13/cobra"

	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/virtctl/console"
	"kubevirt.io/kubevirt/pkg/virtctl/expose"
	"kubevirt.io/kubevirt/pkg/virtctl/templates"
	"kubevirt.io/kubevirt/pkg/virtctl/version"
	"kubevirt.io/kubevirt/pkg/virtctl/vm"
	"kubevirt.io/kubevirt/pkg/virtctl/vnc"
)

func NewVirtctlCommand() *cobra.Command {
	rootCmd := &cobra.Command{
		Use:           "virtctl",
		Short:         "virtctl controls virtual machine related operations on your kubernetes cluster.",
		SilenceUsage:  true,
		SilenceErrors: true,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Fprint(cmd.OutOrStderr(), cmd.UsageString())
		},
	}

	optionsCmd := &cobra.Command{
		Use:    "options",
		Hidden: true,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Fprint(cmd.OutOrStderr(), cmd.UsageString())
		},
	}
	optionsCmd.SetUsageTemplate(templates.OptionsUsageTemplate())
	//TODO: Add a ClientConfigFactory which allows substituting the KubeVirt client with a mock for unit testing
	clientConfig := kubecli.DefaultClientConfig(rootCmd.PersistentFlags())
	AddGlogFlags(rootCmd.PersistentFlags())
	rootCmd.SetUsageTemplate(templates.MainUsageTemplate())
	rootCmd.AddCommand(
		console.NewCommand(clientConfig),
		vnc.NewCommand(clientConfig),
		vm.NewStartCommand(clientConfig),
		vm.NewStopCommand(clientConfig),
		expose.NewExposeCommand(clientConfig),
		version.VersionCommand(clientConfig),
		optionsCmd,
	)
	return rootCmd
}

func Execute() {
	if err := NewVirtctlCommand().Execute(); err != nil {
		fmt.Println(strings.TrimSpace(err.Error()))
		os.Exit(1)
	}
}
<commit_msg>Initialize logger for virtctl<commit_after>package virtctl

import (
	"fmt"
	"os"
	"strings"

	"github.com/spf13/cobra"

	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	"kubevirt.io/kubevirt/pkg/virtctl/console"
	"kubevirt.io/kubevirt/pkg/virtctl/expose"
	"kubevirt.io/kubevirt/pkg/virtctl/templates"
	"kubevirt.io/kubevirt/pkg/virtctl/version"
	"kubevirt.io/kubevirt/pkg/virtctl/vm"
	"kubevirt.io/kubevirt/pkg/virtctl/vnc"
)

func NewVirtctlCommand() *cobra.Command {
	rootCmd := &cobra.Command{
		Use:           "virtctl",
		Short:         "virtctl controls virtual machine related operations on your kubernetes cluster.",
		SilenceUsage:  true,
		SilenceErrors: true,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Fprint(cmd.OutOrStderr(), cmd.UsageString())
		},
	}

	optionsCmd := &cobra.Command{
		Use:    "options",
		Hidden: true,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Fprint(cmd.OutOrStderr(), cmd.UsageString())
		},
	}
	optionsCmd.SetUsageTemplate(templates.OptionsUsageTemplate())
	//TODO: Add a ClientConfigFactory which allows substituting the KubeVirt client with a mock for unit testing
	clientConfig := kubecli.DefaultClientConfig(rootCmd.PersistentFlags())
	AddGlogFlags(rootCmd.PersistentFlags())
	rootCmd.SetUsageTemplate(templates.MainUsageTemplate())
	rootCmd.AddCommand(
		console.NewCommand(clientConfig),
		vnc.NewCommand(clientConfig),
		vm.NewStartCommand(clientConfig),
		vm.NewStopCommand(clientConfig),
		expose.NewExposeCommand(clientConfig),
		version.VersionCommand(clientConfig),
		optionsCmd,
	)
	return rootCmd
}

func Execute() {
	log.InitializeLogging("virtctl")
	if err := NewVirtctlCommand().Execute(); err != nil {
		fmt.Println(strings.TrimSpace(err.Error()))
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Adam Tauber
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy

import (
	"context"
	"net/http"
	"net/url"
	"sync/atomic"

	"github.com/gocolly/colly/v2"
)

type roundRobinSwitcher struct {
	proxyURLs []*url.URL
	index     uint32
}

func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) {
	u := r.proxyURLs[r.index%uint32(len(r.proxyURLs))]
	atomic.AddUint32(&r.index, 1)
	ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, u.String())
	*pr = *pr.WithContext(ctx)
	return u, nil
}

// RoundRobinProxySwitcher creates a proxy switcher function which rotates
// ProxyURLs on every request.
// The proxy type is determined by the URL scheme. "http", "https"
// and "socks5" are supported. If the scheme is empty,
// "http" is assumed.
func RoundRobinProxySwitcher(ProxyURLs ...string) (colly.ProxyFunc, error) {
	if len(ProxyURLs) < 1 {
		return nil, colly.ErrEmptyProxyURL
	}
	urls := make([]*url.URL, len(ProxyURLs))
	for i, u := range ProxyURLs {
		parsedU, err := url.Parse(u)
		if err != nil {
			return nil, err
		}
		urls[i] = parsedU
	}
	return (&roundRobinSwitcher{urls, 0}).GetProxy, nil
}
<commit_msg>Update proxy.go<commit_after>// Copyright 2018 Adam Tauber
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy

import (
	"context"
	"net/http"
	"net/url"
	"sync"

	"github.com/gocolly/colly/v2"
)

type roundRobinSwitcher struct {
	proxyURLs []*url.URL
	index     uint32
	mutex     sync.Mutex
}

func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) {
	r.mutex.Lock()
	u := r.proxyURLs[r.index%uint32(len(r.proxyURLs))]
	r.index = r.index + 1
	r.mutex.Unlock()
	ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, u.String())
	*pr = *pr.WithContext(ctx)
	return u, nil
}

// RoundRobinProxySwitcher creates a proxy switcher function which rotates
// ProxyURLs on every request.
// The proxy type is determined by the URL scheme. "http", "https"
// and "socks5" are supported. If the scheme is empty,
// "http" is assumed.
func RoundRobinProxySwitcher(ProxyURLs ...string) (colly.ProxyFunc, error) {
	if len(ProxyURLs) < 1 {
		return nil, colly.ErrEmptyProxyURL
	}
	urls := make([]*url.URL, len(ProxyURLs))
	for i, u := range ProxyURLs {
		parsedU, err := url.Parse(u)
		if err != nil {
			return nil, err
		}
		urls[i] = parsedU
	}
	var mutex sync.Mutex
	return (&roundRobinSwitcher{urls, 0, mutex}).GetProxy, nil
}
<|endoftext|>


<commit_before>// In the name of Allah

// Copyright (c) 2016 Navid Fathollahzade
// This source code is licensed under MIT license that can be found in the LICENSE file.

// Version: 0.1
// Please visit https://github.com/yaa110/go-persian-calendar for more information.

// Package ptime provides functionality for implementation of Persian (Jalali) Calendar.
package ptime

import "time"

// A Month specifies a month of the year in Persian calendar starting from 1.
type Month int

// A Weekday specifies a day of the week in Persian starting from 0.
type Weekday int

// A PersianDate represents a day in Persian (Jalali) Calendar.
type Time struct {
	year int
	month Month
	day int
	hour int
	min int
	sec int
	nsec int
	loc *time.Location
}

const (
	Farvardin Month = 1 + iota
	Ordibehesht
	Khordad
	Tir
	Mordad
	Shahrivar
	Mehr
	Aban
	Azar
	Dey
	Bahman
	Esfand
)

const (
	Shanbe Weekday = iota
	Yekshanbe
	Doshanbe
	Seshanbe
	Chaharshanbe
	Panjshanbe
	Jome
)

const (
	persian_epoch = 226899

	month_count_normal = 0
	month_count_leap = 1
	month_count_normal_before = 2
	month_count_leap_before = 3
)

var months = [...]string{
	"فروردین",
	"اردیبهشت",
	"خرداد",
	"تیر",
	"مرداد",
	"شهریور",
	"مهر",
	"آبان",
	"آذر",
	"دی",
	"بهمن",
	"اسفند",
}

var days = [...]string{
	"شنبه",
	"یک‌شنبه",
	"دوشنبه",
	"سه‌شنبه",
	"چهارشنبه",
	"پنج‌شنبه",
	"جمعه",
}

var p_month_count = [...][...]int {
	{31,     31,      0},       // Farvardin
	{31,     31,      31},      // Ordibehesht
	{31,     31,      62},      // Khordad
	{31,     31,      93},      // Tir
	{31,     31,      124},     // Mordad
	{31,     31,      155},     // Shahrivar
	{30,     30,      186},     // Mehr
	{30,     30,      216},     // Aban
	{30,     30,      246},     // Azar
	{30,     30,      276},     // Dey
	{30,     30,      306},     // Bahman
	{29,     30,      336},     // Esfand
}

var g_month_count = [...][...]int {
	{31,     31,      0,        0},       // Jan
	{28,     29,      31,       31},      // Feb
	{31,     31,      59,       60},      // Mar
	{30,     30,      90,       91},      // Apr
	{31,     31,      120,      121},     // May
	{30,     30,      151,      152},     // Jun
	{31,     31,      181,      182},     // Jul
	{31,     31,      212,      213},     // Aug
	{30,     30,      243,      244},     // Sep
	{31,     31,      273,      274},     // Oct
	{30,     30,      304,      305},     // Nov
	{31,     31,      334,      335},     // Dec
}

// Returns the Persian name of the month.
func (m Month) String() string {
	return months[m - 1]
}

// Returns the Persian name of the day in week.
func (d Weekday) String() string {
	return days[d]
}

func Time(t time.Time) Time {
	// TODO convert time.Time (Gregorian) to Persian Time
	return nil
}

// Returns a new instance of time.Time from t.
func (t Time) Time() time.Time {
	// TODO convert Persian date to time.Time (Gregorian)
	return nil
}

// Returns a new instance of PersianDate.
func Date(year int, month Month, day, hour, min, sec, nsec int, loc *time.Location) Time {
	if loc == nil {
		panic("ptime: the Location must not be nil in call to Date")
	}

	return Time{year, month, day, hour, min, sec, nsec, loc}.normalize()
}

// Returns a new instance of PersianDate from unix timestamp.
// sec seconds and nsec nanoseconds since January 1, 1970 UTC.
func Unix(sec, nsec int64, loc *time.Location) Time {
	if loc == nil {
		panic("ptime: the Location must not be nil in call to Unix")
	}

	return Time(time.Unix(sec, nsec).In(loc))
}

// Returns unix timestamp (the number of seconds) of t.
func (t Time) Unix() int64 {
	return t.Time().Unix()
}

// Returns unix timestamp (the number of nanoseconds) of t.
func (t Time) UnixNano() int64 {
	return t.Time().UnixNano()
}

func Now(loc *time.Location) Time {
	if loc == nil {
		panic("ptime: the Location must not be nil in call to Now")
	}

	return Time(time.Now().In(loc))
}

// Returns the year, month, day of t.
func (t Time) Date() (int, Month, int) {
	return t.year, t.month, t.day
}

// Returns the year of t.
func (t Time) Year() int {
	return t.year
}

// Returns the month of t.
func (t Time) Month() Month {
	return t.month
}

// Returns the day in month of t.
func (t Time) Day() int {
	return t.day
}

// Returns the hour of t in the range [0, 23].
func (t Time) Hour() int {
	return t.hour
}

// Returns the minute offset of t in the range [0, 59].
func (t Time) Minute() int {
	return t.min
}

// Returns the second offset of t in the range [0, 59].
func (t Time) Second() int {
	return t.sec
}

// Returns the nanosecond offset of t in the range [0, 999999999].
func (t Time) Nanosecond() int {
	return t.nsec
}

// Returns the time zone information of t.
// For more information check the documentation of time.Location
func (t Time) Location() *time.Location {
	return t.loc
}

// Returns the day in year of t.
func (t Time) YearDay() int {
	// TODO YearDay of PersianDate
	return 0
}

// Returns the weekday of t.
func (t Time) Weekday() Weekday {
	// TODO Weekday of PersianDate
	return 0
}

// Returns a new instance of Time for t+d.
func (t Time) Add(d time.Duration) Time {
	return Time(t.Time().Add(d))
}

// Returns true if the year of t is a leap year.
func (t Time) IsLeap() bool {
	return IsLeap(t.year)
}

func IsLeap(year int) bool {
	// TODO IsPersianLeap
	return false
}

// Normalizes the year, month and day if they were outside their usual ranges.
func (date Time) normalize() Time {
	// TODO Validate PersianDate
	return date
}<commit_msg>Add all functionalities<commit_after>// In the name of Allah

// Copyright (c) 2016 Navid Fathollahzade
// This source code is licensed under MIT license that can be found in the LICENSE file.

// Version: 0.1
// Please visit https://github.com/yaa110/go-persian-calendar for more information.

// Package ptime provides functionality for implementation of Persian (Jalali) Calendar.
package ptime

import (
	"time"
	"math"
)

// A Month specifies a month of the year in Persian calendar starting from 1.
type Month int

// A Weekday specifies a day of the week in Persian starting from 0.
type Weekday int

// A PersianDate represents a day in Persian (Jalali) Calendar.
type Time struct {
	year int
	month Month
	day int
	hour int
	min int
	sec int
	nsec int
	loc *time.Location
}

const (
	Farvardin Month = 1 + iota
	Ordibehesht
	Khordad
	Tir
	Mordad
	Shahrivar
	Mehr
	Aban
	Azar
	Dey
	Bahman
	Esfand
)

const (
	Shanbe Weekday = iota
	Yekshanbe
	Doshanbe
	Seshanbe
	Chaharshanbe
	Panjshanbe
	Jome
)

const (
	persian_epoch = 226899

	month_count_normal = 0
	month_count_leap = 1
	month_count_normal_before = 2
	month_count_leap_before = 3
)

var months = [...]string{
	"فروردین",
	"اردیبهشت",
	"خرداد",
	"تیر",
	"مرداد",
	"شهریور",
	"مهر",
	"آبان",
	"آذر",
	"دی",
	"بهمن",
	"اسفند",
}

var days = [...]string{
	"شنبه",
	"یک‌شنبه",
	"دوشنبه",
	"سه‌شنبه",
	"چهارشنبه",
	"پنج‌شنبه",
	"جمعه",
}

var p_month_count = [...][...]int {
	{31,     31,      0},       // Farvardin
	{31,     31,      31},      // Ordibehesht
	{31,     31,      62},      // Khordad
	{31,     31,      93},      // Tir
	{31,     31,      124},     // Mordad
	{31,     31,      155},     // Shahrivar
	{30,     30,      186},     // Mehr
	{30,     30,      216},     // Aban
	{30,     30,      246},     // Azar
	{30,     30,      276},     // Dey
	{30,     30,      306},     // Bahman
	{29,     30,      336},     // Esfand
}

var g_month_count = [...][...]int {
	{31,     31,      0,        0},       // Jan
	{28,     29,      31,       31},      // Feb
	{31,     31,      59,       60},      // Mar
	{30,     30,      90,       91},      // Apr
	{31,     31,      120,      121},     // May
	{30,     30,      151,      152},     // Jun
	{31,     31,      181,      182},     // Jul
	{31,     31,      212,      213},     // Aug
	{30,     30,      243,      244},     // Sep
	{31,     31,      273,      274},     // Oct
	{30,     30,      304,      305},     // Nov
	{31,     31,      334,      335},     // Dec
}

// Returns the Persian name of the month.
func (m Month) String() string {
	return months[m - 1]
}

// Returns the Persian name of the day in week.
func (d Weekday) String() string {
	return days[d]
}

func Time(t time.Time) Time {
	// TODO convert time.Time (Gregorian) to Persian Time
	return nil
}

// Returns a new instance of time.Time from t.
func (t Time) Time() time.Time {
	// TODO convert Persian date to time.Time (Gregorian)
	return nil
}

// Returns a new instance of PersianDate.
func Date(year int, month Month, day, hour, min, sec, nsec int, loc *time.Location) Time {
	if loc == nil {
		panic("ptime: the Location must not be nil in call to Date")
	}

	return Time{year, month, day, hour, min, sec, nsec, loc}.normalize()
}

// Returns a new instance of PersianDate from unix timestamp.
// sec seconds and nsec nanoseconds since January 1, 1970 UTC.
func Unix(sec, nsec int64, loc *time.Location) Time {
	if loc == nil {
		panic("ptime: the Location must not be nil in call to Unix")
	}

	return Time(time.Unix(sec, nsec).In(loc))
}

// Returns unix timestamp (the number of seconds) of t.
func (t Time) Unix() int64 {
	return t.Time().Unix()
}

// Returns unix timestamp (the number of nanoseconds) of t.
func (t Time) UnixNano() int64 {
	return t.Time().UnixNano()
}

func Now(loc *time.Location) Time {
	if loc == nil {
		panic("ptime: the Location must not be nil in call to Now")
	}

	return Time(time.Now().In(loc))
}

// Returns the year, month, day of t.
func (t Time) Date() (int, Month, int) {
	return t.year, t.month, t.day
}

// Returns the year of t.
func (t Time) Year() int {
	return t.year
}

// Returns the month of t.
func (t Time) Month() Month {
	return t.month
}

// Returns the day in month of t.
func (t Time) Day() int {
	return t.day
}

// Returns the hour of t in the range [0, 23].
func (t Time) Hour() int {
	return t.hour
}

// Returns the minute offset of t in the range [0, 59].
func (t Time) Minute() int {
	return t.min
}

// Returns the second offset of t in the range [0, 59].
func (t Time) Second() int {
	return t.sec
}

// Returns the nanosecond offset of t in the range [0, 999999999].
func (t Time) Nanosecond() int {
	return t.nsec
}

// Returns the time zone information of t.
// For more information check the documentation of time.Location
func (t Time) Location() *time.Location {
	return t.loc
}

// Returns the day in the year of t.
func (t Time) YearDay() int {
	// TODO YearDay of PersianDate
	return 0
}

// Returns the weekday of t.
func (t Time) Weekday() Weekday {
	// TODO Weekday of PersianDate
	return 0
}

// Returns the number of remaining days in the year of t.
func (t Time) RYearDay() int {
	// TODO RYearDay
	return 0
}

// Returns the number of remaining days in the month of t.
func (t Time) RMonthDay() int {
	// TODO RMonthDay
	return 0
}

// Returns the number of remaining days in the week of t.
func (t Time) RWeekday() int {
	// TODO RWeekday
	return 0
}

func (t Time) FirstDayInWeek() Time {
	// TODO return the first day in the week
	return nil
}

func (t Time) FirstDayInMonth() Time {
	// TODO return the first day in the month
	return nil
}

func (t Time) FirstDayInYear() Time {
	// TODO return the first day in the year
	return nil
}

func (t Time) LastDayInWeek() Time {
	// TODO return the last day in the week
	return nil
}

func (t Time) LastDayInMonth() Time {
	// TODO return the last day in the month
	return nil
}

func (t Time) LastDayInYear() Time {
	// TODO return the last day in the year
	return nil
}

func (t Time) MonthWeek() int {
	// TODO return the week number in the month
	return 0
}

func (t Time) YearWeek() int {
	// TODO return the week number in the year
	return 0
}

func (t Time) Yesterday() int {
	// TODO return Yesterday
	return 0
}

func (t Time) Tomorrow() int {
	// TODO return Tomorrow
	return 0
}

// Returns a new instance of Time for t+d.
func (t Time) Add(d time.Duration) Time {
	return Time(t.Time().Add(d))
}

// Returns the time.Duration between t and t2
func (t Time) Diff(t2 Time) time.Duration {
	return math.Abs(t2.Unix() - t.Unix()) * time.Second
}

// Returns true if the year of t is a leap year.
func (t Time) IsLeap() bool {
	// TODO IsLeap
	return false
}

// Normalizes the year, month and day if they were outside their usual ranges.
func (date Time) normalize() Time {
	// TODO Validate PersianDate
	return date
}<|endoftext|>