<commit_before>package tsm1

import (
	"fmt"
	"sync"
	"sync/atomic"

	"github.com/cespare/xxhash"
	"github.com/influxdata/influxdb/pkg/bytesutil"
)

// partitions is the number of partitions we used in the ring's continuum. It
// basically defines the maximum number of partitions you can have in the ring.
// If a smaller number of partitions are chosen when creating a ring, then
// they're evenly spread across this many partitions in the ring.
const partitions = 4096

// ring is a structure that maps series keys to entries.
//
// ring is implemented as a crude hash ring, in so much that you can have
// variable numbers of members in the ring, and the appropriate member for a
// given series key can always consistently be found. Unlike a true hash ring
// though, this ring is not resizeable—there must be at most 256 members in the
// ring, and the number of members must always be a power of 2.
//
// ring works as follows: Each member of the ring contains a single store, which
// contains a map of series keys to entries. A ring always has 256 partitions,
// and a member takes up one or more of these partitions (depending on how many
// members are specified to be in the ring)
//
// To determine the partition that a series key should be added to, the series
// key is hashed and the first 8 bits are used as an index to the ring.
//
type ring struct {
	// The unique set of partitions in the ring.
	// len(partitions) <= len(continuum)
	partitions []*partition

	// Number of keys within the ring. This is used to provide a hint for
	// allocating the return values in keys(). It will not be perfectly accurate
	// since it doesn't consider adding duplicate keys, or trying to remove non-
	// existent keys.
	keysHint int64
}

// newring returns a new ring initialised with n partitions. n must always be a
// power of 2, and for performance reasons should be larger than the number of
// cores on the host. The supported set of values for n is:
//
//     {1, 2, 4, 8, 16, 32, 64, 128, 256}.
//
func newring(n int) (*ring, error) {
	if n <= 0 || n > partitions {
		return nil, fmt.Errorf("invalid number of paritions: %d", n)
	}

	r := ring{
		partitions: make([]*partition, n), // maximum number of partitions.
	}

	// The trick here is to map N partitions to all points on the continuum,
	// such that the first eight bits of a given hash will map directly to one
	// of the N partitions.
	for i := 0; i < len(r.partitions); i++ {
		r.partitions[i] = &partition{
			store:          make(map[string]*entry),
			entrySizeHints: make(map[uint64]int),
		}
	}
	return &r, nil
}

// reset resets the ring so it can be reused. Before removing references to entries
// within each partition it gathers sizing information to provide hints when
// reallocating entries in partition maps.
//
// reset is not safe for use by multiple goroutines.
func (r *ring) reset() {
	for _, partition := range r.partitions {
		partition.reset()
	}
	r.keysHint = 0
}

// getPartition retrieves the hash ring partition associated with the provided
// key.
func (r *ring) getPartition(key []byte) *partition {
	return r.partitions[int(xxhash.Sum64(key)%partitions)]
}

// entry returns the entry for the given key.
// entry is safe for use by multiple goroutines.
func (r *ring) entry(key []byte) *entry {
	return r.getPartition(key).entry(key)
}

// write writes values to the entry in the ring's partition associated with key.
// If no entry exists for the key then one will be created.
// write is safe for use by multiple goroutines.
func (r *ring) write(key []byte, values Values) (bool, error) {
	return r.getPartition(key).write(key, values)
}

// add adds an entry to the ring.
func (r *ring) add(key []byte, entry *entry) {
	r.getPartition(key).add(key, entry)
	atomic.AddInt64(&r.keysHint, 1)
}

// remove deletes the entry for the given key.
// remove is safe for use by multiple goroutines.
func (r *ring) remove(key []byte) {
	r.getPartition(key).remove(key)
	if r.keysHint > 0 {
		atomic.AddInt64(&r.keysHint, -1)
	}
}

// keys returns all the keys from all partitions in the hash ring. The returned
// keys will be in order if sorted is true.
func (r *ring) keys(sorted bool) [][]byte {
	keys := make([][]byte, 0, atomic.LoadInt64(&r.keysHint))
	for _, p := range r.partitions {
		keys = append(keys, p.keys()...)
	}

	if sorted {
		bytesutil.Sort(keys)
	}
	return keys
}

// apply applies the provided function to every entry in the ring under a read
// lock using a separate goroutine for each partition. The provided function
// will be called with each key and the corresponding entry. The first error
// encountered will be returned, if any. apply is safe for use by multiple
// goroutines.
func (r *ring) apply(f func([]byte, *entry) error) error {

	var (
		wg  sync.WaitGroup
		res = make(chan error, len(r.partitions))
	)

	for _, p := range r.partitions {
		wg.Add(1)

		go func(p *partition) {
			defer wg.Done()

			p.mu.RLock()
			for k, e := range p.store {
				if err := f([]byte(k), e); err != nil {
					res <- err
					p.mu.RUnlock()
					return
				}
			}
			p.mu.RUnlock()
		}(p)
	}

	go func() {
		wg.Wait()
		close(res)
	}()

	// Collect results.
	for err := range res {
		if err != nil {
			return err
		}
	}
	return nil
}

// applySerial is similar to apply, but invokes f on each partition in the same
// goroutine.
// apply is safe for use by multiple goroutines.
func (r *ring) applySerial(f func([]byte, *entry) error) error {
	for _, p := range r.partitions {
		p.mu.RLock()
		for k, e := range p.store {
			if err := f([]byte(k), e); err != nil {
				p.mu.RUnlock()
				return err
			}
		}
		p.mu.RUnlock()
	}
	return nil
}

// partition provides safe access to a map of series keys to entries.
type partition struct {
	mu    sync.RWMutex
	store map[string]*entry

	// entrySizeHints stores hints for appropriate sizes to pre-allocate the
	// []Values in an entry. entrySizeHints will only contain hints for entries
	// that were present prior to the most recent snapshot, preventing unbounded
	// growth over time.
	entrySizeHints map[uint64]int
}

// entry returns the partition's entry for the provided key.
// It's safe for use by multiple goroutines.
func (p *partition) entry(key []byte) *entry {
	p.mu.RLock()
	e := p.store[string(key)]
	p.mu.RUnlock()
	return e
}

// write writes the values to the entry in the partition, creating the entry
// if it does not exist.
// write is safe for use by multiple goroutines.
func (p *partition) write(key []byte, values Values) (bool, error) {
	p.mu.RLock()
	e := p.store[string(key)]
	p.mu.RUnlock()
	if e != nil {
		// Hot path.
		return false, e.add(values)
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	// Check again.
	if e = p.store[string(key)]; e != nil {
		return false, e.add(values)
	}

	// Create a new entry using a preallocated size if we have a hint available.
	hint, _ := p.entrySizeHints[xxhash.Sum64(key)]
	e, err := newEntryValues(values, hint)
	if err != nil {
		return false, err
	}

	p.store[string(key)] = e
	return true, nil
}

// add adds a new entry for key to the partition.
func (p *partition) add(key []byte, entry *entry) {
	p.mu.Lock()
	p.store[string(key)] = entry
	p.mu.Unlock()
}

// remove deletes the entry associated with the provided key.
// remove is safe for use by multiple goroutines.
func (p *partition) remove(key []byte) {
	p.mu.Lock()
	delete(p.store, string(key))
	p.mu.Unlock()
}

// keys returns an unsorted slice of the keys in the partition.
func (p *partition) keys() [][]byte {
	p.mu.RLock()
	keys := make([][]byte, 0, len(p.store))
	for k := range p.store {
		keys = append(keys, []byte(k))
	}
	p.mu.RUnlock()
	return keys
}

// reset resets the partition by reinitialising the store. reset returns hints
// about sizes that the entries within the store could be reallocated with.
func (p *partition) reset() {
	p.mu.Lock()
	defer p.mu.Unlock()

	// Collect the allocated sizes of values for each entry in the store.
	p.entrySizeHints = make(map[uint64]int)
	for k, entry := range p.store {
		// If the capacity is large then there are many values in the entry.
		// Store a hint to pre-allocate the next time we see the same entry.
		entry.mu.RLock()
		if cap(entry.values) > 128 { // 4 x the default entry capacity size.
			p.entrySizeHints[xxhash.Sum64String(k)] = cap(entry.values)
		}
		entry.mu.RUnlock()
	}

	// Reset the store.
	p.store = make(map[string]*entry, len(p.store))
}
<commit_msg>Remove entrySizeHints map<commit_after>package tsm1

import (
	"fmt"
	"sync"
	"sync/atomic"

	"github.com/cespare/xxhash"
	"github.com/influxdata/influxdb/pkg/bytesutil"
)

// partitions is the number of partitions we used in the ring's continuum. It
// basically defines the maximum number of partitions you can have in the ring.
// If a smaller number of partitions are chosen when creating a ring, then
// they're evenly spread across this many partitions in the ring.
const partitions = 4096

// ring is a structure that maps series keys to entries.
//
// ring is implemented as a crude hash ring, in so much that you can have
// variable numbers of members in the ring, and the appropriate member for a
// given series key can always consistently be found. Unlike a true hash ring
// though, this ring is not resizeable—there must be at most 256 members in the
// ring, and the number of members must always be a power of 2.
//
// ring works as follows: Each member of the ring contains a single store, which
// contains a map of series keys to entries. A ring always has 256 partitions,
// and a member takes up one or more of these partitions (depending on how many
// members are specified to be in the ring)
//
// To determine the partition that a series key should be added to, the series
// key is hashed and the first 8 bits are used as an index to the ring.
//
type ring struct {
	// The unique set of partitions in the ring.
	// len(partitions) <= len(continuum)
	partitions []*partition

	// Number of keys within the ring. This is used to provide a hint for
	// allocating the return values in keys(). It will not be perfectly accurate
	// since it doesn't consider adding duplicate keys, or trying to remove non-
	// existent keys.
	keysHint int64
}

// newring returns a new ring initialised with n partitions. n must always be a
// power of 2, and for performance reasons should be larger than the number of
// cores on the host. The supported set of values for n is:
//
//     {1, 2, 4, 8, 16, 32, 64, 128, 256}.
//
func newring(n int) (*ring, error) {
	if n <= 0 || n > partitions {
		return nil, fmt.Errorf("invalid number of paritions: %d", n)
	}

	r := ring{
		partitions: make([]*partition, n), // maximum number of partitions.
	}

	// The trick here is to map N partitions to all points on the continuum,
	// such that the first eight bits of a given hash will map directly to one
	// of the N partitions.
	for i := 0; i < len(r.partitions); i++ {
		r.partitions[i] = &partition{
			store: make(map[string]*entry),
		}
	}
	return &r, nil
}

// reset resets the ring so it can be reused. Before removing references to entries
// within each partition it gathers sizing information to provide hints when
// reallocating entries in partition maps.
//
// reset is not safe for use by multiple goroutines.
func (r *ring) reset() {
	for _, partition := range r.partitions {
		partition.reset()
	}
	r.keysHint = 0
}

// getPartition retrieves the hash ring partition associated with the provided
// key.
func (r *ring) getPartition(key []byte) *partition {
	return r.partitions[int(xxhash.Sum64(key)%partitions)]
}

// entry returns the entry for the given key.
// entry is safe for use by multiple goroutines.
func (r *ring) entry(key []byte) *entry {
	return r.getPartition(key).entry(key)
}

// write writes values to the entry in the ring's partition associated with key.
// If no entry exists for the key then one will be created.
// write is safe for use by multiple goroutines.
func (r *ring) write(key []byte, values Values) (bool, error) {
	return r.getPartition(key).write(key, values)
}

// add adds an entry to the ring.
func (r *ring) add(key []byte, entry *entry) {
	r.getPartition(key).add(key, entry)
	atomic.AddInt64(&r.keysHint, 1)
}

// remove deletes the entry for the given key.
// remove is safe for use by multiple goroutines.
func (r *ring) remove(key []byte) {
	r.getPartition(key).remove(key)
	if r.keysHint > 0 {
		atomic.AddInt64(&r.keysHint, -1)
	}
}

// keys returns all the keys from all partitions in the hash ring. The returned
// keys will be in order if sorted is true.
func (r *ring) keys(sorted bool) [][]byte {
	keys := make([][]byte, 0, atomic.LoadInt64(&r.keysHint))
	for _, p := range r.partitions {
		keys = append(keys, p.keys()...)
	}

	if sorted {
		bytesutil.Sort(keys)
	}
	return keys
}

// apply applies the provided function to every entry in the ring under a read
// lock using a separate goroutine for each partition. The provided function
// will be called with each key and the corresponding entry. The first error
// encountered will be returned, if any. apply is safe for use by multiple
// goroutines.
func (r *ring) apply(f func([]byte, *entry) error) error {

	var (
		wg  sync.WaitGroup
		res = make(chan error, len(r.partitions))
	)

	for _, p := range r.partitions {
		wg.Add(1)

		go func(p *partition) {
			defer wg.Done()

			p.mu.RLock()
			for k, e := range p.store {
				if err := f([]byte(k), e); err != nil {
					res <- err
					p.mu.RUnlock()
					return
				}
			}
			p.mu.RUnlock()
		}(p)
	}

	go func() {
		wg.Wait()
		close(res)
	}()

	// Collect results.
	for err := range res {
		if err != nil {
			return err
		}
	}
	return nil
}

// applySerial is similar to apply, but invokes f on each partition in the same
// goroutine.
// apply is safe for use by multiple goroutines.
func (r *ring) applySerial(f func([]byte, *entry) error) error {
	for _, p := range r.partitions {
		p.mu.RLock()
		for k, e := range p.store {
			if err := f([]byte(k), e); err != nil {
				p.mu.RUnlock()
				return err
			}
		}
		p.mu.RUnlock()
	}
	return nil
}

// partition provides safe access to a map of series keys to entries.
type partition struct {
	mu    sync.RWMutex
	store map[string]*entry
}

// entry returns the partition's entry for the provided key.
// It's safe for use by multiple goroutines.
func (p *partition) entry(key []byte) *entry {
	p.mu.RLock()
	e := p.store[string(key)]
	p.mu.RUnlock()
	return e
}

// write writes the values to the entry in the partition, creating the entry
// if it does not exist.
// write is safe for use by multiple goroutines.
func (p *partition) write(key []byte, values Values) (bool, error) {
	p.mu.RLock()
	e := p.store[string(key)]
	p.mu.RUnlock()
	if e != nil {
		// Hot path.
		return false, e.add(values)
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	// Check again.
	if e = p.store[string(key)]; e != nil {
		return false, e.add(values)
	}

	// Create a new entry using a preallocated size if we have a hint available.
	e, err := newEntryValues(values, 32)
	if err != nil {
		return false, err
	}

	p.store[string(key)] = e
	return true, nil
}

// add adds a new entry for key to the partition.
func (p *partition) add(key []byte, entry *entry) {
	p.mu.Lock()
	p.store[string(key)] = entry
	p.mu.Unlock()
}

// remove deletes the entry associated with the provided key.
// remove is safe for use by multiple goroutines.
func (p *partition) remove(key []byte) {
	p.mu.Lock()
	delete(p.store, string(key))
	p.mu.Unlock()
}

// keys returns an unsorted slice of the keys in the partition.
func (p *partition) keys() [][]byte {
	p.mu.RLock()
	keys := make([][]byte, 0, len(p.store))
	for k := range p.store {
		keys = append(keys, []byte(k))
	}
	p.mu.RUnlock()
	return keys
}

// reset resets the partition by reinitialising the store. reset returns hints
// about sizes that the entries within the store could be reallocated with.
func (p *partition) reset() {
	p.mu.Lock()
	defer p.mu.Unlock()
	for k := range p.store {
		delete(p.store, k)
	}
}
<|endoftext|>


<commit_before>package collector

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	clientaws "github.com/giantswarm/aws-operator/client/aws"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/sync/errgroup"
)

const (
	// labelInstance is the metric's label key that will hold the ec2 instance ID.
	labelInstance = "ec2_instance"

	// labelInstanceState is a label that will contain the instance state string
	labelInstanceState = "state"

	// labelInstanceStatus is a label that will contain the instance status check value
	labelInstanceStatus = "status"

	// subsystemEC2 will become the second part of the metric name, right after namespace.
	subsystemEC2 = "ec2"
)

var (
	ec2InstanceStatus = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystemEC2, "instance_status"),
		"Gauge indicating the status of an EC2 instance. 1 = healthy, 0 = unhealthy",
		[]string{
			labelInstance,
			labelAccount,
			labelCluster,
			labelInstallation,
			labelOrganization,
			labelInstanceState,
			labelInstanceStatus,
		},
		nil,
	)
)

// EC2InstancesConfig is this collector's configuration struct.
type EC2InstancesConfig struct {
	Helper *helper
	Logger micrologger.Logger

	InstallationName string
}

// EC2Instances is the main struct for this collector.
type EC2Instances struct {
	helper *helper
	logger micrologger.Logger

	installationName string
}

// NewEC2Instances creates a new EC2 instance metrics collector.
func NewEC2Instances(config EC2InstancesConfig) (*EC2Instances, error) {
	if config.Helper == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Helper must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}

	if config.InstallationName == "" {
		return nil, microerror.Maskf(invalidConfigError, "%T.InstallationName must not be empty", config)
	}

	e := &EC2Instances{
		helper: config.Helper,
		logger: config.Logger,

		installationName: config.InstallationName,
	}

	return e, nil
}

// Collect is the main metrics collection function.
func (e *EC2Instances) Collect(ch chan<- prometheus.Metric) error {
	awsClientsList, err := e.helper.GetAWSClients()
	if err != nil {
		return microerror.Mask(err)
	}

	var g errgroup.Group

	for _, item := range awsClientsList {
		awsClients := item

		g.Go(func() error {
			err := e.collectForAccount(ch, awsClients)
			if err != nil {
				return microerror.Mask(err)
			}

			return nil
		})
	}

	err = g.Wait()
	if err != nil {
		return microerror.Mask(err)
	}

	return nil
}

// Describe emits the description for the metrics collected here.
func (e *EC2Instances) Describe(ch chan<- *prometheus.Desc) error {
	ch <- ec2InstanceStatus
	return nil
}

// collectForAccount collects and emits our metric for one AWS account.
//
// We gather two separate collections first, then match them by instance ID:
// - instance information, including tags, only for those tagged for our installation
// - instance status information
func (e *EC2Instances) collectForAccount(ch chan<- prometheus.Metric, awsClients clientaws.Clients) error {
	account, err := e.helper.AWSAccountID(awsClients)
	if err != nil {
		return microerror.Mask(err)
	}

	// Collect instance status info.
	// map key will be the instance ID.
	instanceStatuses := map[string]*ec2.InstanceStatus{}
	{
		input := &ec2.DescribeInstanceStatusInput{
			IncludeAllInstances: aws.Bool(true),
			MaxResults:          aws.Int64(1000),
		}

		for {
			o, err := awsClients.EC2.DescribeInstanceStatus(input)
			if err != nil {
				return microerror.Mask(err)
			}

			// collect statuses
			for _, s := range o.InstanceStatuses {
				instanceStatuses[*s.InstanceId] = s
			}

			if o.NextToken == nil {
				break
			}
			input.SetNextToken(*o.NextToken)
		}
	}

	// Collect instance info.
	instances := map[string]*ec2.Instance{}
	{
		input := &ec2.DescribeInstancesInput{
			Filters: []*ec2.Filter{
				&ec2.Filter{
					Name: aws.String(fmt.Sprintf("tag:%s", tagInstallation)),
					Values: []*string{
						aws.String(e.installationName),
					},
				},
			},
			MaxResults: aws.Int64(1000),
		}

		for {
			o, err := awsClients.EC2.DescribeInstances(input)
			if err != nil {
				return microerror.Mask(err)
			}

			for _, reservation := range o.Reservations {
				for _, instance := range reservation.Instances {
					instances[*instance.InstanceId] = instance
				}
			}

			if o.NextToken == nil {
				break
			}
			input.SetNextToken(*o.NextToken)
		}
	}

	// Iterate over found instances and emit metrics.
	for instanceID := range instances {
		// Skip if we don't have a status for this instance.
		statuses, statusesAvailable := instanceStatuses[instanceID]
		if !statusesAvailable {
			continue
		}

		var cluster, installation, organization, state, status string
		for _, tag := range instances[instanceID].Tags {
			switch *tag.Key {
			case tagCluster:
				cluster = *tag.Value
			case tagInstallation:
				installation = *tag.Value
			case tagOrganization:
				organization = *tag.Value
			}
		}

		up := 0
		if statuses.InstanceState.Name != nil {
			state = strings.ToLower(*statuses.InstanceState.Name)
		}
		if statuses.InstanceStatus.Status != nil {
			status = strings.ToLower(*statuses.InstanceStatus.Status)
		}
		if state == "running" && status == "ok" {
			up = 1
		}

		ch <- prometheus.MustNewConstMetric(
			ec2InstanceStatus,
			prometheus.GaugeValue,
			float64(up),
			instanceID,
			account,
			cluster,
			installation,
			organization,
			state,
			status,
		)
	}

	return nil
}
<commit_msg>Add instance type and availability zone as labels (#1319)<commit_after>package collector

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	clientaws "github.com/giantswarm/aws-operator/client/aws"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/sync/errgroup"
)

const (
	// labelAvailabilityZone will contain the AZ name the instance is in
	labelAvailabilityZone = "availability_zone"

	// labelInstance is the metric's label key that will hold the ec2 instance ID.
	labelInstance = "ec2_instance"

	// labelInstanceState is a label that will contain the instance state string
	labelInstanceState = "state"

	// labelInstanceStatus is a label that will contain the instance status check value
	labelInstanceStatus = "status"

	// labelInstanceType will contain the instance type name
	labelInstanceType = "instance_type"

	// subsystemEC2 will become the second part of the metric name, right after namespace.
	subsystemEC2 = "ec2"
)

var (
	ec2InstanceStatus = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystemEC2, "instance_status"),
		"Gauge indicating the status of an EC2 instance. 1 = healthy, 0 = unhealthy",
		[]string{
			labelInstance,
			labelAccount,
			labelCluster,
			labelInstallation,
			labelOrganization,
			labelAvailabilityZone,
			labelInstanceType,
			labelInstanceState,
			labelInstanceStatus,
		},
		nil,
	)
)

// EC2InstancesConfig is this collector's configuration struct.
type EC2InstancesConfig struct {
	Helper *helper
	Logger micrologger.Logger

	InstallationName string
}

// EC2Instances is the main struct for this collector.
type EC2Instances struct {
	helper *helper
	logger micrologger.Logger

	installationName string
}

// NewEC2Instances creates a new EC2 instance metrics collector.
func NewEC2Instances(config EC2InstancesConfig) (*EC2Instances, error) {
	if config.Helper == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Helper must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}

	if config.InstallationName == "" {
		return nil, microerror.Maskf(invalidConfigError, "%T.InstallationName must not be empty", config)
	}

	e := &EC2Instances{
		helper: config.Helper,
		logger: config.Logger,

		installationName: config.InstallationName,
	}

	return e, nil
}

// Collect is the main metrics collection function.
func (e *EC2Instances) Collect(ch chan<- prometheus.Metric) error {
	awsClientsList, err := e.helper.GetAWSClients()
	if err != nil {
		return microerror.Mask(err)
	}

	var g errgroup.Group

	for _, item := range awsClientsList {
		awsClients := item

		g.Go(func() error {
			err := e.collectForAccount(ch, awsClients)
			if err != nil {
				return microerror.Mask(err)
			}

			return nil
		})
	}

	err = g.Wait()
	if err != nil {
		return microerror.Mask(err)
	}

	return nil
}

// Describe emits the description for the metrics collected here.
func (e *EC2Instances) Describe(ch chan<- *prometheus.Desc) error {
	ch <- ec2InstanceStatus
	return nil
}

// collectForAccount collects and emits our metric for one AWS account.
//
// We gather two separate collections first, then match them by instance ID:
// - instance information, including tags, only for those tagged for our installation
// - instance status information
func (e *EC2Instances) collectForAccount(ch chan<- prometheus.Metric, awsClients clientaws.Clients) error {
	account, err := e.helper.AWSAccountID(awsClients)
	if err != nil {
		return microerror.Mask(err)
	}

	// Collect instance status info.
	// map key will be the instance ID.
	instanceStatuses := map[string]*ec2.InstanceStatus{}
	{
		input := &ec2.DescribeInstanceStatusInput{
			IncludeAllInstances: aws.Bool(true),
			MaxResults:          aws.Int64(1000),
		}

		for {
			o, err := awsClients.EC2.DescribeInstanceStatus(input)
			if err != nil {
				return microerror.Mask(err)
			}

			// collect statuses
			for _, s := range o.InstanceStatuses {
				instanceStatuses[*s.InstanceId] = s
			}

			if o.NextToken == nil {
				break
			}
			input.SetNextToken(*o.NextToken)
		}
	}

	// Collect instance info.
	instances := map[string]*ec2.Instance{}
	{
		input := &ec2.DescribeInstancesInput{
			Filters: []*ec2.Filter{
				&ec2.Filter{
					Name: aws.String(fmt.Sprintf("tag:%s", tagInstallation)),
					Values: []*string{
						aws.String(e.installationName),
					},
				},
			},
			MaxResults: aws.Int64(1000),
		}

		for {
			o, err := awsClients.EC2.DescribeInstances(input)
			if err != nil {
				return microerror.Mask(err)
			}

			for _, reservation := range o.Reservations {
				for _, instance := range reservation.Instances {
					instances[*instance.InstanceId] = instance
				}
			}

			if o.NextToken == nil {
				break
			}
			input.SetNextToken(*o.NextToken)
		}
	}

	// Iterate over found instances and emit metrics.
	for instanceID := range instances {
		// Skip if we don't have a status for this instance.
		statuses, statusesAvailable := instanceStatuses[instanceID]
		if !statusesAvailable {
			continue
		}

		var az, cluster, instanceType, installation, organization, state, status string
		for _, tag := range instances[instanceID].Tags {
			switch *tag.Key {
			case tagCluster:
				cluster = *tag.Value
			case tagInstallation:
				installation = *tag.Value
			case tagOrganization:
				organization = *tag.Value
			}
		}

		instanceType = *instances[instanceID].InstanceType

		up := 0
		if statuses.InstanceState.Name != nil {
			state = strings.ToLower(*statuses.InstanceState.Name)
		}
		if statuses.InstanceStatus.Status != nil {
			status = strings.ToLower(*statuses.InstanceStatus.Status)
		}
		if statuses.AvailabilityZone != nil {
			az = strings.ToLower(*statuses.AvailabilityZone)
		}
		if state == "running" && status == "ok" {
			up = 1
		}

		ch <- prometheus.MustNewConstMetric(
			ec2InstanceStatus,
			prometheus.GaugeValue,
			float64(up),
			instanceID,
			account,
			cluster,
			installation,
			organization,
			az,
			instanceType,
			state,
			status,
		)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package v2

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/containerd/containerd/identifiers"
	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/namespaces"
	"github.com/pkg/errors"
)

const configFilename = "config.json"

// LoadBundle loads an existing bundle from disk
func LoadBundle(ctx context.Context, root, id string) (*Bundle, error) {
	ns, err := namespaces.NamespaceRequired(ctx)
	if err != nil {
		return nil, err
	}
	return &Bundle{
		ID:        id,
		Path:      filepath.Join(root, ns, id),
		Namespace: ns,
	}, nil
}

// NewBundle returns a new bundle on disk
func NewBundle(ctx context.Context, root, state, id string, spec []byte) (b *Bundle, err error) {
	if err := identifiers.Validate(id); err != nil {
		return nil, errors.Wrapf(err, "invalid task id %s", id)
	}

	ns, err := namespaces.NamespaceRequired(ctx)
	if err != nil {
		return nil, err
	}
	work := filepath.Join(root, ns, id)
	b = &Bundle{
		ID:        id,
		Path:      filepath.Join(state, ns, id),
		Namespace: ns,
	}
	var paths []string
	defer func() {
		if err != nil {
			for _, d := range paths {
				os.RemoveAll(d)
			}
		}
	}()
	// create state directory for the bundle
	if err := os.MkdirAll(filepath.Dir(b.Path), 0711); err != nil {
		return nil, err
	}
	if err := os.Mkdir(b.Path, 0711); err != nil {
		return nil, err
	}
	paths = append(paths, b.Path)
	// create working directory for the bundle
	if err := os.MkdirAll(filepath.Dir(work), 0711); err != nil {
		return nil, err
	}
	rootfs := filepath.Join(b.Path, "rootfs")
	if err := os.MkdirAll(rootfs, 0711); err != nil {
		return nil, err
	}
	paths = append(paths, rootfs)
	if err := os.Mkdir(work, 0711); err != nil {
		if !os.IsExist(err) {
			return nil, err
		}
		os.RemoveAll(work)
		if err := os.Mkdir(work, 0711); err != nil {
			return nil, err
		}
	}
	paths = append(paths, work)
	// symlink workdir
	if err := os.Symlink(work, filepath.Join(b.Path, "work")); err != nil {
		return nil, err
	}
	// write the spec to the bundle
	err = ioutil.WriteFile(filepath.Join(b.Path, configFilename), spec, 0666)
	return b, err
}

// Bundle represents an OCI bundle
type Bundle struct {
	// ID of the bundle
	ID string
	// Path to the bundle
	Path string
	// Namespace of the bundle
	Namespace string
}

// Delete a bundle atomically
func (b *Bundle) Delete() error {
	work, werr := os.Readlink(filepath.Join(b.Path, "work"))
	rootfs := filepath.Join(b.Path, "rootfs")
	if err := mount.UnmountAll(rootfs, 0); err != nil {
		return errors.Wrapf(err, "unmount rootfs %s", rootfs)
	}
	if err := os.Remove(rootfs); err != nil && os.IsNotExist(err) {
		return errors.Wrap(err, "failed to remove bundle rootfs")
	}
	err := atomicDelete(b.Path)
	if err == nil {
		if werr == nil {
			return atomicDelete(work)
		}
		return nil
	}
	// error removing the bundle path; still attempt removing work dir
	var err2 error
	if werr == nil {
		err2 = atomicDelete(work)
		if err2 == nil {
			return err
		}
	}
	return errors.Wrapf(err, "failed to remove both bundle and workdir locations: %v", err2)
}

// atomicDelete renames the path to a hidden file before removal
func atomicDelete(path string) error {
	// create a hidden dir for an atomic removal
	atomicPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path)))
	if err := os.Rename(path, atomicPath); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	return os.RemoveAll(atomicPath)
}
<commit_msg>runtime: ignore ErrNotExist when remove rootfs<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package v2

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/containerd/containerd/identifiers"
	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/namespaces"
	"github.com/pkg/errors"
)

const configFilename = "config.json"

// LoadBundle loads an existing bundle from disk
func LoadBundle(ctx context.Context, root, id string) (*Bundle, error) {
	ns, err := namespaces.NamespaceRequired(ctx)
	if err != nil {
		return nil, err
	}
	return &Bundle{
		ID:        id,
		Path:      filepath.Join(root, ns, id),
		Namespace: ns,
	}, nil
}

// NewBundle returns a new bundle on disk
func NewBundle(ctx context.Context, root, state, id string, spec []byte) (b *Bundle, err error) {
	if err := identifiers.Validate(id); err != nil {
		return nil, errors.Wrapf(err, "invalid task id %s", id)
	}

	ns, err := namespaces.NamespaceRequired(ctx)
	if err != nil {
		return nil, err
	}
	work := filepath.Join(root, ns, id)
	b = &Bundle{
		ID:        id,
		Path:      filepath.Join(state, ns, id),
		Namespace: ns,
	}
	var paths []string
	defer func() {
		if err != nil {
			for _, d := range paths {
				os.RemoveAll(d)
			}
		}
	}()
	// create state directory for the bundle
	if err := os.MkdirAll(filepath.Dir(b.Path), 0711); err != nil {
		return nil, err
	}
	if err := os.Mkdir(b.Path, 0711); err != nil {
		return nil, err
	}
	paths = append(paths, b.Path)
	// create working directory for the bundle
	if err := os.MkdirAll(filepath.Dir(work), 0711); err != nil {
		return nil, err
	}
	rootfs := filepath.Join(b.Path, "rootfs")
	if err := os.MkdirAll(rootfs, 0711); err != nil {
		return nil, err
	}
	paths = append(paths, rootfs)
	if err := os.Mkdir(work, 0711); err != nil {
		if !os.IsExist(err) {
			return nil, err
		}
		os.RemoveAll(work)
		if err := os.Mkdir(work, 0711); err != nil {
			return nil, err
		}
	}
	paths = append(paths, work)
	// symlink workdir
	if err := os.Symlink(work, filepath.Join(b.Path, "work")); err != nil {
		return nil, err
	}
	// write the spec to the bundle
	err = ioutil.WriteFile(filepath.Join(b.Path, configFilename), spec, 0666)
	return b, err
}

// Bundle represents an OCI bundle
type Bundle struct {
	// ID of the bundle
	ID string
	// Path to the bundle
	Path string
	// Namespace of the bundle
	Namespace string
}

// Delete a bundle atomically
func (b *Bundle) Delete() error {
	work, werr := os.Readlink(filepath.Join(b.Path, "work"))
	rootfs := filepath.Join(b.Path, "rootfs")
	if err := mount.UnmountAll(rootfs, 0); err != nil {
		return errors.Wrapf(err, "unmount rootfs %s", rootfs)
	}
	if err := os.Remove(rootfs); err != nil && !os.IsNotExist(err) {
		return errors.Wrap(err, "failed to remove bundle rootfs")
	}
	err := atomicDelete(b.Path)
	if err == nil {
		if werr == nil {
			return atomicDelete(work)
		}
		return nil
	}
	// error removing the bundle path; still attempt removing work dir
	var err2 error
	if werr == nil {
		err2 = atomicDelete(work)
		if err2 == nil {
			return err
		}
	}
	return errors.Wrapf(err, "failed to remove both bundle and workdir locations: %v", err2)
}

// atomicDelete renames the path to a hidden file before removal
func atomicDelete(path string) error {
	// create a hidden dir for an atomic removal
	atomicPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path)))
	if err := os.Rename(path, atomicPath); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	return os.RemoveAll(atomicPath)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package outputhost

import (
	"errors"
	"strconv"
	"sync/atomic"

	s "github.com/Shopify/sarama"
	"github.com/uber-common/bark"
	"github.com/uber/cherami-server/common"
	"github.com/uber/cherami-server/stream"
	"github.com/uber/cherami-thrift/.generated/go/cherami"
	"github.com/uber/cherami-thrift/.generated/go/store"
)

var (
	errKafkaNilControlFlow = errors.New(`nil or non-positive controlFlow passed to Write()`)
	errKafkaClosed         = errors.New(`closed`)
)

// kafkaStream implements the same interface and similar-enough semantics to the Cherami store interface (BStoreOpenReadStreamOutCall)

type kafkaStream struct {
	creditSemaphore common.UnboundedSemaphore
	kafkaMsgsCh     <-chan *s.ConsumerMessage
	kafkaConverter  KafkaMessageConverter
	logger          bark.Logger
	seqNo           int64
}

// KafkaMessageConverterConfig is used to config customized converter
type KafkaMessageConverterConfig struct {
	// Destination and ConsumerGroup are not needed currently, but may in the future
	// Destination *cherami.DestinationDescription
	// ConsumerGroup *cherami.ConsumerGroupDescription
	KafkaTopics  []string // optional: already contained in destination-description
	KafkaCluster string   // optional: already contained in destination-description
}

// KafkaMessageConverter defines interface to convert a Kafka message to Cherami message
type KafkaMessageConverter func(kMsg *s.ConsumerMessage) (cMsg *store.ReadMessageContent)

// KafkaMessageConverterFactory provides converter from Kafka message to Cherami message
// In the future, it may provide implementations for BStoreOpenReadStreamOutCall
type KafkaMessageConverterFactory interface {
	GetConverter(cfg *KafkaMessageConverterConfig, log bark.Logger) KafkaMessageConverter
}

/*
 * BStoreOpenReadStreamOutCall Interface
 */

// Write grants credits to the Read call
func (k *kafkaStream) Write(arg *cherami.ControlFlow) error {
	if arg == nil || arg.GetCredits() <= 0 {
		return errKafkaNilControlFlow
	}
	k.creditSemaphore.Release(int(arg.GetCredits()))
	return nil
}

// Flush is a no-op
func (k *kafkaStream) Flush() error { return nil }

// Done is a no-op; consumer connection is handled at a higher level
func (k *kafkaStream) Done() error { return nil }

// Read returns the next message, if:
// * a message is available
// * enough credit has been granted
func (k *kafkaStream) Read() (*store.ReadMessageContent, error) {
	m, ok := <-k.kafkaMsgsCh
	if !ok {
		return nil, errKafkaClosed
	}
	k.creditSemaphore.Acquire(1) // TODO: Size-based credits
	return k.kafkaConverter(m), nil
}

// ResponseHeaders returns the response headers sent from the server. This will block until server headers have been received.
func (k *kafkaStream) ResponseHeaders() (map[string]string, error) {
	return nil, errors.New(`unimplemented`)
}

/*
 * Setup & Utility
 */

// OpenKafkaStream opens a store call simulated from the given sarama message stream
func OpenKafkaStream(c <-chan *s.ConsumerMessage, kafkaMessageConverter KafkaMessageConverter, logger bark.Logger) stream.BStoreOpenReadStreamOutCall {
	k := &kafkaStream{
		kafkaMsgsCh:    c,
		logger:         logger,
		kafkaConverter: kafkaMessageConverter,
	}
	if k.kafkaConverter == nil {
		k.kafkaConverter = k.getDefaultKafkaMessageConverter()
	}
	return k
}

func (k *kafkaStream) getDefaultKafkaMessageConverter() KafkaMessageConverter {
	return func(m *s.ConsumerMessage) (c *store.ReadMessageContent) {
		c = &store.ReadMessageContent{
			Type: store.ReadMessageContentTypePtr(store.ReadMessageContentType_MESSAGE),
		}

		c.Message = &store.ReadMessage{
			Address: common.Int64Ptr(
				int64(kafkaAddresser.GetStoreAddress(
					&TopicPartition{
						Topic:     m.Topic,
						Partition: m.Partition,
					},
					m.Offset,
					func() bark.Logger {
						return k.logger.WithFields(bark.Fields{
							`module`:    `kafkaStream`,
							`topic`:     m.Topic,
							`partition`: m.Partition,
						})
					},
				))),
		}

		c.Message.Message = &store.AppendMessage{
			SequenceNumber: common.Int64Ptr(atomic.AddInt64(&k.seqNo, 1)),
		}

		if !m.Timestamp.IsZero() {
			// only set if kafka is version 0.10+
			c.Message.Message.EnqueueTimeUtc = common.Int64Ptr(m.Timestamp.UnixNano())
		}

		c.Message.Message.Payload = &cherami.PutMessage{
			Data: m.Value,
			UserContext: map[string]string{
				`key`:       string(m.Key),
				`topic`:     m.Topic,
				`partition`: strconv.Itoa(int(m.Partition)),
				`offset`:    strconv.Itoa(int(m.Offset)),
			},
			// TODO: Checksum?
		}
		return c
	}
}
<commit_msg>Expose get default kafka message converter (#312)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package outputhost

import (
	"errors"
	"strconv"
	"sync/atomic"

	s "github.com/Shopify/sarama"
	"github.com/uber-common/bark"
	"github.com/uber/cherami-server/common"
	"github.com/uber/cherami-server/stream"
	"github.com/uber/cherami-thrift/.generated/go/cherami"
	"github.com/uber/cherami-thrift/.generated/go/store"
)

var (
	errKafkaNilControlFlow = errors.New(`nil or non-positive controlFlow passed to Write()`)
	errKafkaClosed         = errors.New(`closed`)
)

// kafkaStream implements the same interface and similar-enough semantics to the Cherami store interface (BStoreOpenReadStreamOutCall)

type kafkaStream struct {
	creditSemaphore common.UnboundedSemaphore
	kafkaMsgsCh     <-chan *s.ConsumerMessage
	kafkaConverter  KafkaMessageConverter
	logger          bark.Logger
	seqNo           int64
}

// KafkaMessageConverterConfig is used to config customized converter
type KafkaMessageConverterConfig struct {
	// Destination and ConsumerGroup are not needed currently, but may in the future
	// Destination *cherami.DestinationDescription
	// ConsumerGroup *cherami.ConsumerGroupDescription
	KafkaTopics  []string // optional: already contained in destination-description
	KafkaCluster string   // optional: already contained in destination-description
}

// KafkaMessageConverter defines interface to convert a Kafka message to Cherami message
type KafkaMessageConverter func(kMsg *s.ConsumerMessage) (cMsg *store.ReadMessageContent)

// KafkaMessageConverterFactory provides converter from Kafka message to Cherami message
// In the future, it may provide implementations for BStoreOpenReadStreamOutCall
type KafkaMessageConverterFactory interface {
	GetConverter(cfg *KafkaMessageConverterConfig, log bark.Logger) KafkaMessageConverter
}

/*
 * BStoreOpenReadStreamOutCall Interface
 */

// Write grants credits to the Read call
func (k *kafkaStream) Write(arg *cherami.ControlFlow) error {
	if arg == nil || arg.GetCredits() <= 0 {
		return errKafkaNilControlFlow
	}
	k.creditSemaphore.Release(int(arg.GetCredits()))
	return nil
}

// Flush is a no-op
func (k *kafkaStream) Flush() error { return nil }

// Done is a no-op; consumer connection is handled at a higher level
func (k *kafkaStream) Done() error { return nil }

// Read returns the next message, if:
// * a message is available
// * enough credit has been granted
func (k *kafkaStream) Read() (*store.ReadMessageContent, error) {
	m, ok := <-k.kafkaMsgsCh
	if !ok {
		return nil, errKafkaClosed
	}
	k.creditSemaphore.Acquire(1) // TODO: Size-based credits
	return k.kafkaConverter(m), nil
}

// ResponseHeaders returns the response headers sent from the server. This will block until server headers have been received.
func (k *kafkaStream) ResponseHeaders() (map[string]string, error) {
	return nil, errors.New(`unimplemented`)
}

/*
 * Setup & Utility
 */

// OpenKafkaStream opens a store call simulated from the given sarama message stream
func OpenKafkaStream(c <-chan *s.ConsumerMessage, kafkaMessageConverter KafkaMessageConverter, logger bark.Logger) stream.BStoreOpenReadStreamOutCall {
	k := &kafkaStream{
		kafkaMsgsCh:    c,
		logger:         logger,
		kafkaConverter: kafkaMessageConverter,
	}
	if k.kafkaConverter == nil {
		k.kafkaConverter = GetDefaultKafkaMessageConverter(&k.seqNo, k.logger)
	}
	return k
}

// GetDefaultKafkaMessageConverter returns the default kafka message converter
func GetDefaultKafkaMessageConverter(seqNo *int64, logger bark.Logger) KafkaMessageConverter {
	return func(m *s.ConsumerMessage) (c *store.ReadMessageContent) {
		c = &store.ReadMessageContent{
			Type: store.ReadMessageContentTypePtr(store.ReadMessageContentType_MESSAGE),
		}

		c.Message = &store.ReadMessage{
			Address: common.Int64Ptr(
				int64(kafkaAddresser.GetStoreAddress(
					&TopicPartition{
						Topic:     m.Topic,
						Partition: m.Partition,
					},
					m.Offset,
					func() bark.Logger {
						return logger.WithFields(bark.Fields{
							`module`:    `kafkaStream`,
							`topic`:     m.Topic,
							`partition`: m.Partition,
						})
					},
				))),
		}

		c.Message.Message = &store.AppendMessage{
			SequenceNumber: common.Int64Ptr(atomic.AddInt64(seqNo, 1)),
		}

		if !m.Timestamp.IsZero() {
			// only set if kafka is version 0.10+
			c.Message.Message.EnqueueTimeUtc = common.Int64Ptr(m.Timestamp.UnixNano())
		}

		c.Message.Message.Payload = &cherami.PutMessage{
			Data: m.Value,
			UserContext: map[string]string{
				`key`:       string(m.Key),
				`topic`:     m.Topic,
				`partition`: strconv.Itoa(int(m.Partition)),
				`offset`:    strconv.Itoa(int(m.Offset)),
			},
			// TODO: Checksum?
		}
		return c
	}
}
<|endoftext|>


<commit_before>package types

import "testing"

func TestBlockMetaValidateBasic(t *testing.T) {
	// TODO
}
<commit_msg>types: add tests for blockmeta (#5013)<commit_after>package types

import (
	"testing"

	"github.com/stretchr/testify/require"
	"github.com/tendermint/tendermint/crypto/tmhash"
	tmrand "github.com/tendermint/tendermint/libs/rand"
)

func TestBlockMeta_ToProto(t *testing.T) {
	h := makeRandHeader()
	bi := BlockID{Hash: h.Hash(), PartsHeader: PartSetHeader{Total: 123, Hash: tmrand.Bytes(tmhash.Size)}}

	bm := &BlockMeta{
		BlockID:   bi,
		BlockSize: 200,
		Header:    h,
		NumTxs:    0,
	}

	tests := []struct {
		testName string
		bm       *BlockMeta
		expErr   bool
	}{
		{"success", bm, false},
		{"failure nil", nil, true},
	}

	for _, tt := range tests {
		tt := tt
		t.Run(tt.testName, func(t *testing.T) {
			pb := tt.bm.ToProto()

			bm, err := BlockMetaFromProto(pb)

			if !tt.expErr {
				require.NoError(t, err, tt.testName)
				require.Equal(t, tt.bm, bm, tt.testName)
			} else {
				require.Error(t, err, tt.testName)
			}
		})
	}
}

func TestBlockMeta_ValidateBasic(t *testing.T) {
	h := makeRandHeader()
	bi := BlockID{Hash: h.Hash(), PartsHeader: PartSetHeader{Total: 123, Hash: tmrand.Bytes(tmhash.Size)}}
	bi2 := BlockID{Hash: tmrand.Bytes(tmhash.Size),
		PartsHeader: PartSetHeader{Total: 123, Hash: tmrand.Bytes(tmhash.Size)}}
	bi3 := BlockID{Hash: []byte("incorrect hash"),
		PartsHeader: PartSetHeader{Total: 123, Hash: []byte("incorrect hash")}}

	bm := &BlockMeta{
		BlockID:   bi,
		BlockSize: 200,
		Header:    h,
		NumTxs:    0,
	}

	bm2 := &BlockMeta{
		BlockID:   bi2,
		BlockSize: 200,
		Header:    h,
		NumTxs:    0,
	}

	bm3 := &BlockMeta{
		BlockID:   bi3,
		BlockSize: 200,
		Header:    h,
		NumTxs:    0,
	}

	tests := []struct {
		name    string
		bm      *BlockMeta
		wantErr bool
	}{
		{"success", bm, false},
		{"failure wrong blockID hash", bm2, true},
		{"failure wrong length blockID hash", bm3, true},
	}
	for _, tt := range tests {
		tt := tt
		t.Run(tt.name, func(t *testing.T) {
			if err := tt.bm.ValidateBasic(); (err != nil) != tt.wantErr {
				t.Errorf("BlockMeta.ValidateBasic() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2014, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package main

import (
	"flag"
	"fmt"
	"github.com/percona/percona-agent/agent"
	"github.com/percona/percona-agent/bin/percona-agent-installer/installer"
	"github.com/percona/percona-agent/bin/percona-agent-installer/term"
	"github.com/percona/percona-agent/pct"
	"log"
	"os"
)

const (
	DEFAULT_APP_HOSTNAME = "https://cloud.percona.com"
)

var (
	flagApiHostname             string
	flagApiKey                  string
	flagBasedir                 string
	flagDebug                   bool
	flagCreateMySQLInstance     bool
	flagCreateServerInstance    bool
	flagStartServices           bool
	flagCreateAgent             bool
	flagStartMySQLServices      bool
	flagMySQL                   bool
	flagOldPasswords            bool
	flagPlainPasswords          bool
	flagInteractive             bool
	flagMySQLDefaultsFile       string
	flagAutoDetectMySQL         bool
	flagCreateMySQLUser         bool
	flagMySQLUser               string
	flagMySQLPass               string
	flagMySQLHost               string
	flagMySQLPort               string
	flagMySQLSocket             string
	flagIgnoreFailures          bool
	flagMySQLMaxUserConnections int64
)

func init() {
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)

	flag.StringVar(&flagApiHostname, "api-host", agent.DEFAULT_API_HOSTNAME, "API host")
	flag.StringVar(&flagApiKey, "api-key", "", "API key, it is available at "+DEFAULT_APP_HOSTNAME+"/api-key")
	flag.StringVar(&flagBasedir, "basedir", pct.DEFAULT_BASEDIR, "Agent basedir")
	flag.BoolVar(&flagDebug, "debug", false, "Debug")
	// --
	flag.BoolVar(&flagMySQL, "mysql", true, "Install for MySQL")
	flag.BoolVar(&flagCreateMySQLInstance, "create-mysql-instance", true, "Create MySQL instance")
	flag.BoolVar(&flagCreateServerInstance, "create-server-instance", true, "Create server instance")
	flag.BoolVar(&flagStartServices, "start-services", true, "Start all services")
	flag.BoolVar(&flagStartMySQLServices, "start-mysql-services", true, "Start MySQL services")
	flag.BoolVar(&flagCreateAgent, "create-agent", true, "Create agent")
	flag.BoolVar(&flagOldPasswords, "old-passwords", false, "Old passwords")
	flag.BoolVar(&flagPlainPasswords, "plain-passwords", false, "Plain passwords") // @todo: Workaround used in tests for "stty: standard input: Inappropriate ioctl for device"
	flag.BoolVar(&flagInteractive, "interactive", true, "Prompt for input on STDIN")
	flag.BoolVar(&flagAutoDetectMySQL, "auto-detect-mysql", true, "Auto detect MySQL options")
	flag.BoolVar(&flagCreateMySQLUser, "create-mysql-user", true, "Create MySQL user for agent")
	flag.StringVar(&flagAgentMySQLUser, "agent-mysql-user", "percona-agent", "MySQL username for agent")
	flag.StringVar(&flagAgentMySQLPass, "agent-mysql-pass", "", "MySQL password for agent")
	flag.StringVar(&flagMySQLDefaultsFile, "mysql-defaults-file", "", "Path to my.cnf, used for auto detection of connection details")
	flag.StringVar(&flagMySQLUser, "mysql-user", "", "MySQL username")
	flag.StringVar(&flagMySQLPass, "mysql-pass", "", "MySQL password")
	flag.StringVar(&flagMySQLHost, "mysql-host", "", "MySQL host")
	flag.StringVar(&flagMySQLPort, "mysql-port", "", "MySQL port")
	flag.StringVar(&flagMySQLSocket, "mysql-socket", "", "MySQL socket file")
	flag.Int64Var(&flagMySQLMaxUserConnections, "mysql-max-user-connections", 5, "Max number of MySQL connections")
}

func main() {
	// It flag is unknown it exist with os.Exit(10),
	// so exit code=10 is strictly reserved for flags
	// Don't use it anywhere else, as shell script install.sh depends on it
	// NOTE: standard flag.Parse() was using os.Exit(2)
	//       which was the same as returned with ctrl+c
	if err := flag.CommandLine.Parse(os.Args[1:]); err != nil {
		os.Exit(10)
	}

	agentConfig := &agent.Config{
		ApiHostname: flagApiHostname,
		ApiKey:      flagApiKey,
	}
	// todo: do flags a better way
	if !flagMySQL {
		flagCreateMySQLInstance = false
		flagStartMySQLServices = false
	}

	if flagMySQLSocket != "" && flagMySQLHost != "" {
		log.Println("Options -mysql-socket and -mysql-host are exclusive\n")
		os.Exit(1)
	}

	if flagMySQLSocket != "" && flagMySQLPort != "" {
		log.Println("Options -mysql-socket and -mysql-port are exclusive\n")
		os.Exit(1)
	}

	flags := installer.Flags{
		Bool: map[string]bool{
			"debug":                  flagDebug,
			"create-server-instance": flagCreateServerInstance,
			"start-services":         flagStartServices,
			"create-mysql-instance":  flagCreateMySQLInstance,
			"start-mysql-services":   flagStartMySQLServices,
			"create-agent":           flagCreateAgent,
			"old-passwords":          flagOldPasswords,
			"plain-passwords":        flagPlainPasswords,
			"interactive":            flagInteractive,
			"auto-detect-mysql":      flagAutoDetectMySQL,
			"create-mysql-user":      flagCreateMySQLUser,
			"mysql":                  flagMySQL,
		},
		String: map[string]string{
			"app-host":            DEFAULT_APP_HOSTNAME,
			"mysql-defaults-file": flagMySQLDefaultsFile,
			"mysql-user":          flagMySQLUser,
			"mysql-pass":          flagMySQLPass,
			"mysql-host":          flagMySQLHost,
			"mysql-port":          flagMySQLPort,
			"mysql-socket":        flagMySQLSocket,
			"agent-mysql-user":    flagAgentMySQLUser,
			"agent-mysql-pass":    flagAgentMySQLPass,
		},
		Int64: map[string]int64{
			"mysql-max-user-connections": flagMySQLMaxUserConnections,
		},
	}

	// Agent stores all its files in the basedir.  This must be called first
	// because installer uses pct.Basedir and assumes it's already initialized.
	if err := pct.Basedir.Init(flagBasedir); err != nil {
		log.Printf("Error initializing basedir %s: %s\n", flagBasedir, err)
		os.Exit(1)
	}

	agentInstaller := installer.NewInstaller(term.NewTerminal(os.Stdin, flagInteractive, flagDebug), flagBasedir, pct.NewAPI(), agentConfig, flags)
	fmt.Println("CTRL-C at any time to quit")
	// todo: catch SIGINT and clean up
	if err := agentInstaller.Run(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	os.Exit(0)
}
<commit_msg>changed the order of the flags<commit_after>/*
   Copyright (c) 2014, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package main

import (
	"flag"
	"fmt"
	"github.com/percona/percona-agent/agent"
	"github.com/percona/percona-agent/bin/percona-agent-installer/installer"
	"github.com/percona/percona-agent/bin/percona-agent-installer/term"
	"github.com/percona/percona-agent/pct"
	"log"
	"os"
)

const (
	DEFAULT_APP_HOSTNAME = "https://cloud.percona.com"
)

var (
	flagApiHostname             string
	flagApiKey                  string
	flagBasedir                 string
	flagDebug                   bool
	flagCreateMySQLInstance     bool
	flagCreateServerInstance    bool
	flagStartServices           bool
	flagCreateAgent             bool
	flagStartMySQLServices      bool
	flagMySQL                   bool
	flagOldPasswords            bool
	flagPlainPasswords          bool
	flagInteractive             bool
	flagMySQLDefaultsFile       string
	flagAutoDetectMySQL         bool
	flagCreateMySQLUser         bool
	flagAgentMySQLUser          string
	flagAgentMySQLPass          string
	flagMySQLUser               string
	flagMySQLPass               string
	flagMySQLHost               string
	flagMySQLPort               string
	flagMySQLSocket             string
	flagIgnoreFailures          bool
	flagMySQLMaxUserConnections int64
)

func init() {
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)

	flag.StringVar(&flagApiHostname, "api-host", agent.DEFAULT_API_HOSTNAME, "API host")
	flag.StringVar(&flagApiKey, "api-key", "", "API key, it is available at "+DEFAULT_APP_HOSTNAME+"/api-key")
	flag.StringVar(&flagBasedir, "basedir", pct.DEFAULT_BASEDIR, "Agent basedir")
	flag.BoolVar(&flagDebug, "debug", false, "Debug")
	// --
	flag.BoolVar(&flagMySQL, "mysql", true, "Install for MySQL")
	flag.BoolVar(&flagCreateMySQLInstance, "create-mysql-instance", true, "Create MySQL instance")
	flag.BoolVar(&flagCreateServerInstance, "create-server-instance", true, "Create server instance")
	flag.BoolVar(&flagStartServices, "start-services", true, "Start all services")
	flag.BoolVar(&flagStartMySQLServices, "start-mysql-services", true, "Start MySQL services")
	flag.BoolVar(&flagCreateAgent, "create-agent", true, "Create agent")
	flag.BoolVar(&flagOldPasswords, "old-passwords", false, "Old passwords")
	flag.BoolVar(&flagPlainPasswords, "plain-passwords", false, "Plain passwords") // @todo: Workaround used in tests for "stty: standard input: Inappropriate ioctl for device"
	flag.BoolVar(&flagInteractive, "interactive", true, "Prompt for input on STDIN")
	flag.BoolVar(&flagAutoDetectMySQL, "auto-detect-mysql", true, "Auto detect MySQL options")
	flag.BoolVar(&flagCreateMySQLUser, "create-mysql-user", true, "Create MySQL user for agent")
	flag.StringVar(&flagAgentMySQLUser, "agent-mysql-user", "percona-agent", "MySQL username for agent")
	flag.StringVar(&flagAgentMySQLPass, "agent-mysql-pass", "", "MySQL password for agent")
	flag.StringVar(&flagMySQLDefaultsFile, "mysql-defaults-file", "", "Path to my.cnf, used for auto detection of connection details")
	flag.StringVar(&flagMySQLUser, "mysql-user", "", "MySQL username")
	flag.StringVar(&flagMySQLPass, "mysql-pass", "", "MySQL password")
	flag.StringVar(&flagMySQLHost, "mysql-host", "", "MySQL host")
	flag.StringVar(&flagMySQLPort, "mysql-port", "", "MySQL port")
	flag.StringVar(&flagMySQLSocket, "mysql-socket", "", "MySQL socket file")
	flag.Int64Var(&flagMySQLMaxUserConnections, "mysql-max-user-connections", 5, "Max number of MySQL connections")
}

func main() {
	// It flag is unknown it exist with os.Exit(10),
	// so exit code=10 is strictly reserved for flags
	// Don't use it anywhere else, as shell script install.sh depends on it
	// NOTE: standard flag.Parse() was using os.Exit(2)
	//       which was the same as returned with ctrl+c
	if err := flag.CommandLine.Parse(os.Args[1:]); err != nil {
		os.Exit(10)
	}

	agentConfig := &agent.Config{
		ApiHostname: flagApiHostname,
		ApiKey:      flagApiKey,
	}
	// todo: do flags a better way
	if !flagMySQL {
		flagCreateMySQLInstance = false
		flagStartMySQLServices = false
	}

	if flagMySQLSocket != "" && flagMySQLHost != "" {
		log.Println("Options -mysql-socket and -mysql-host are exclusive\n")
		os.Exit(1)
	}

	if flagMySQLSocket != "" && flagMySQLPort != "" {
		log.Println("Options -mysql-socket and -mysql-port are exclusive\n")
		os.Exit(1)
	}

	flags := installer.Flags{
		Bool: map[string]bool{
			"debug":                  flagDebug,
			"create-server-instance": flagCreateServerInstance,
			"start-services":         flagStartServices,
			"create-mysql-instance":  flagCreateMySQLInstance,
			"start-mysql-services":   flagStartMySQLServices,
			"create-agent":           flagCreateAgent,
			"old-passwords":          flagOldPasswords,
			"plain-passwords":        flagPlainPasswords,
			"interactive":            flagInteractive,
			"auto-detect-mysql":      flagAutoDetectMySQL,
			"create-mysql-user":      flagCreateMySQLUser,
			"mysql":                  flagMySQL,
		},
		String: map[string]string{
			"app-host":            DEFAULT_APP_HOSTNAME,
			"mysql-defaults-file": flagMySQLDefaultsFile,
			"agent-mysql-user":    flagAgentMySQLUser,
			"agent-mysql-pass":    flagAgentMySQLPass,
			"mysql-user":          flagMySQLUser,
			"mysql-pass":          flagMySQLPass,
			"mysql-host":          flagMySQLHost,
			"mysql-port":          flagMySQLPort,
			"mysql-socket":        flagMySQLSocket,
		},
		Int64: map[string]int64{
			"mysql-max-user-connections": flagMySQLMaxUserConnections,
		},
	}

	// Agent stores all its files in the basedir.  This must be called first
	// because installer uses pct.Basedir and assumes it's already initialized.
	if err := pct.Basedir.Init(flagBasedir); err != nil {
		log.Printf("Error initializing basedir %s: %s\n", flagBasedir, err)
		os.Exit(1)
	}

	agentInstaller := installer.NewInstaller(term.NewTerminal(os.Stdin, flagInteractive, flagDebug), flagBasedir, pct.NewAPI(), agentConfig, flags)
	fmt.Println("CTRL-C at any time to quit")
	// todo: catch SIGINT and clean up
	if err := agentInstaller.Run(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	os.Exit(0)
}
<|endoftext|>


<commit_before>package azure

import (
	"encoding/base64"
	"io"
	"io/ioutil"
	"math/rand"
	"strings"
	"time"

	"github.com/cenkalti/backoff"

	"github.com/Azure/go-autorest/autorest"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Azure/go-autorest/autorest/azure"

	"net/http"

	"fmt"

	bitsgo "github.com/cloudfoundry-incubator/bits-service"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/validate"
	"github.com/cloudfoundry-incubator/bits-service/config"
	"github.com/cloudfoundry-incubator/bits-service/logger"
	"github.com/pkg/errors"
)

type Blobstore struct {
	containerName  string
	client         storage.BlobStorageClient
	putBlockSize   int64
	maxListResults uint
}

func NewBlobstore(config config.AzureBlobstoreConfig) *Blobstore {
	return NewBlobstoreWithDetails(config, 50<<20, 5000)
}

// NetworkErrorRetryingSender is a replacement for the storage.DefaultSender.
// storage.DefaultSender does not retry on network errors which is rarely what we want in
// a production system.
type NetworkErrorRetryingSender struct{}

var retryableStatusCodes = []int{
	http.StatusRequestTimeout,      // 408
	http.StatusInternalServerError, // 500
	http.StatusBadGateway,          // 502
	http.StatusServiceUnavailable,  // 503
	http.StatusGatewayTimeout,      // 504
}

// Send is mostly emulating storage.DefaultSender.Send, so most code was copied.
// But we use the backoff library for convenience and retry on errors returned from
// HTTPClient.Do.
func (sender *NetworkErrorRetryingSender) Send(c *storage.Client, req *http.Request) (*http.Response, error) {
	rr := autorest.NewRetriableRequest(req)
	var resp *http.Response

	err := backoff.Retry(func() error {
		err := rr.Prepare()
		if err != nil {
			return backoff.Permanent(err)
		}
		resp, err = c.HTTPClient.Do(rr.Request())
		// We deliberately mark errors *not* as permanent, because an error
		// here means network connectivity or similar. This is different to the
		// storage.DefaultSender which on any error.
		if err != nil {
			return err
		}
		if !autorest.ResponseHasStatusCode(resp, retryableStatusCodes...) {
			return backoff.Permanent(err)
		}
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
		return nil
	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 5))

	return resp, err
}

func NewBlobstoreWithDetails(config config.AzureBlobstoreConfig, putBlockSize int64, maxListResults uint) *Blobstore {
	validate.NotEmpty(config.AccountKey)
	validate.NotEmpty(config.AccountName)
	validate.NotEmpty(config.ContainerName)
	validate.NotEmpty(config.EnvironmentName())

	environment, e := azure.EnvironmentFromName(config.EnvironmentName())
	if e != nil {
		logger.Log.Fatalw("Could not get Azure Environment from Name", "error", e, "environment", config.EnvironmentName())
	}
	client, e := storage.NewBasicClientOnSovereignCloud(config.AccountName, config.AccountKey, environment)
	if e != nil {
		logger.Log.Fatalw("Could not instantiate Azure Basic Client", "error", e)
	}
	client.Sender = &NetworkErrorRetryingSender{}
	return &Blobstore{
		client:         client.GetBlobService(),
		containerName:  config.ContainerName,
		putBlockSize:   putBlockSize,
		maxListResults: maxListResults,
	}
}

func (blobstore *Blobstore) Exists(path string) (bool, error) {
	exists, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Exists()
	if e != nil {
		return false, errors.Wrapf(e, "Failed to check for %v/%v", blobstore.containerName, path)
	}
	return exists, nil
}

func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) {
	logger.Log.Debugw("Get", "bucket", blobstore.containerName, "path", path)

	reader, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Get(nil)
	if e != nil {
		return nil, blobstore.handleError(e, "Path %v", path)
	}
	return reader, nil
}

func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) {
	signedUrl, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).GetSASURI(storage.BlobSASOptions{
		BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Read: true},
		SASOptions:                storage.SASOptions{Expiry: time.Now().Add(time.Hour)},
	})
	return nil, signedUrl, e
}

func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error {
	putRequestID := rand.Int63()
	l := logger.Log.With("put-request-id", putRequestID)
	l.Debugw("Put", "bucket", blobstore.containerName, "path", path)

	blob := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path)

	e := blob.CreateBlockBlob(nil)
	if e != nil {
		return errors.Wrapf(e, "create block blob failed. container: %v, path: %v, put-request-id: %v", blobstore.containerName, path, putRequestID)
	}

	uncommittedBlocksList := make([]storage.Block, 0)
	eof := false
	for i := 0; !eof; i++ {
		// using information from https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs
		if i >= 50000 {
			return errors.Errorf("block blob cannot have more than 50,000 blocks. path: %v, put-request-id: %v", path, putRequestID)
		}
		block := storage.Block{
			ID:     base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%05d", i))),
			Status: storage.BlockStatusUncommitted,
		}
		data := make([]byte, blobstore.putBlockSize)
		numBytesRead, e := src.Read(data)
		if e != nil {
			if e.Error() == "EOF" {
				eof = true
			} else {
				return errors.Wrapf(e, "put block failed. path: %v, put-request-id: %v", path, putRequestID)
			}
		}
		if numBytesRead == 0 {
			l.Debugw("Empty read", "block-index", i, "block-id", block.ID, "is-eof", eof)
			continue
		}
		l.Debugw("PutBlock", "block-index", i, "block-id", block.ID, "block-size", numBytesRead, "is-eof", eof)
		e = blob.PutBlock(block.ID, data[:numBytesRead], nil)
		if e != nil {
			return errors.Wrapf(e, "put block failed. path: %v, put-request-id: %v", path, putRequestID)
		}
		uncommittedBlocksList = append(uncommittedBlocksList, block)
	}
	l.Debugw("PutBlockList", "uncommitted-block-list", uncommittedBlocksList)
	e = blob.PutBlockList(uncommittedBlocksList, nil)
	if e != nil {
		return errors.Wrapf(e, "put block list failed. path: %v, put-request-id: %v", path, putRequestID)
	}

	return nil
}

func (blobstore *Blobstore) Copy(src, dest string) error {
	logger.Log.Debugw("Copy in Azure", "container", blobstore.containerName, "src", src, "dest", dest)
	e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(dest).Copy(
		blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(src).GetURL(), nil)

	if e != nil {
		blobstore.handleError(e, "Error while trying to copy src %v to dest %v in bucket %v", src, dest, blobstore.containerName)
	}
	return nil
}

func (blobstore *Blobstore) Delete(path string) error {
	deleted, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).DeleteIfExists(nil)
	if e != nil {
		return errors.Wrapf(e, "Path %v", path)
	}
	if !deleted {
		return bitsgo.NewNotFoundError()
	}
	return nil
}

func (blobstore *Blobstore) DeleteDir(prefix string) error {
	deletionErrs := []error{}
	marker := ""
	for {
		response, e := blobstore.client.GetContainerReference(blobstore.containerName).ListBlobs(storage.ListBlobsParameters{
			Prefix:     prefix,
			MaxResults: blobstore.maxListResults,
			Marker:     marker,
		})
		if e != nil {
			return errors.Wrapf(e, "Prefix %v", prefix)
		}
		for _, blob := range response.Blobs {
			e = blobstore.Delete(blob.Name)
			if e != nil {
				if _, isNotFoundError := e.(*bitsgo.NotFoundError); !isNotFoundError {
					deletionErrs = append(deletionErrs, e)
				}
			}
		}
		if response.NextMarker == "" {
			break
		}
		marker = response.NextMarker
	}

	if len(deletionErrs) != 0 {
		return errors.Errorf("Prefix %v, errors from deleting: %v", prefix, deletionErrs)
	}
	return nil
}

func (blobstore *Blobstore) Sign(resource string, method string, expirationTime time.Time) (signedURL string) {
	var e error
	switch strings.ToLower(method) {
	case "put":
		signedURL, e = blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(resource).GetSASURI(storage.BlobSASOptions{
			BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Write: true, Create: true},
			SASOptions:                storage.SASOptions{Expiry: expirationTime},
		})
	case "get":
		signedURL, e = blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(resource).GetSASURI(storage.BlobSASOptions{
			BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Read: true},
			SASOptions:                storage.SASOptions{Expiry: expirationTime},
		})
	default:
		panic("The only supported methods are 'put' and 'get'")
	}
	if e != nil {
		panic(e)
	}
	return
}

func (blobstore *Blobstore) handleError(e error, context string, args ...interface{}) error {
	if azse, ok := e.(storage.AzureStorageServiceError); ok && azse.StatusCode == http.StatusNotFound {
		exists, e := blobstore.client.GetContainerReference(blobstore.containerName).Exists()
		if e != nil {
			return errors.Wrapf(e, context, args...)
		}
		if !exists {
			return errors.Errorf("Container does not exist '%v", blobstore.containerName)
		}
		return bitsgo.NewNotFoundError()
	}
	return errors.Wrapf(e, context, args...)
}
<commit_msg>Fix comment in Azure blobstore client<commit_after>package azure

import (
	"encoding/base64"
	"io"
	"io/ioutil"
	"math/rand"
	"strings"
	"time"

	"github.com/cenkalti/backoff"

	"github.com/Azure/go-autorest/autorest"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Azure/go-autorest/autorest/azure"

	"net/http"

	"fmt"

	bitsgo "github.com/cloudfoundry-incubator/bits-service"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/validate"
	"github.com/cloudfoundry-incubator/bits-service/config"
	"github.com/cloudfoundry-incubator/bits-service/logger"
	"github.com/pkg/errors"
)

type Blobstore struct {
	containerName  string
	client         storage.BlobStorageClient
	putBlockSize   int64
	maxListResults uint
}

func NewBlobstore(config config.AzureBlobstoreConfig) *Blobstore {
	return NewBlobstoreWithDetails(config, 50<<20, 5000)
}

// NetworkErrorRetryingSender is a replacement for the storage.DefaultSender.
// storage.DefaultSender does not retry on network errors which is rarely what we want in
// a production system.
type NetworkErrorRetryingSender struct{}

var retryableStatusCodes = []int{
	http.StatusRequestTimeout,      // 408
	http.StatusInternalServerError, // 500
	http.StatusBadGateway,          // 502
	http.StatusServiceUnavailable,  // 503
	http.StatusGatewayTimeout,      // 504
}

// Send is mostly emulating storage.DefaultSender.Send, so most code was copied.
// But we use the backoff library for convenience and retry on errors returned from
// HTTPClient.Do.
func (sender *NetworkErrorRetryingSender) Send(c *storage.Client, req *http.Request) (*http.Response, error) {
	rr := autorest.NewRetriableRequest(req)
	var resp *http.Response

	err := backoff.Retry(func() error {
		err := rr.Prepare()
		if err != nil {
			return backoff.Permanent(err)
		}
		resp, err = c.HTTPClient.Do(rr.Request())
		// We deliberately mark errors *not* as permanent, because an error
		// here means network connectivity or similar. This is different to the
		// storage.DefaultSender which stops on any error.
		if err != nil {
			return err
		}
		if !autorest.ResponseHasStatusCode(resp, retryableStatusCodes...) {
			return backoff.Permanent(err)
		}
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
		return nil
	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 5))

	return resp, err
}

func NewBlobstoreWithDetails(config config.AzureBlobstoreConfig, putBlockSize int64, maxListResults uint) *Blobstore {
	validate.NotEmpty(config.AccountKey)
	validate.NotEmpty(config.AccountName)
	validate.NotEmpty(config.ContainerName)
	validate.NotEmpty(config.EnvironmentName())

	environment, e := azure.EnvironmentFromName(config.EnvironmentName())
	if e != nil {
		logger.Log.Fatalw("Could not get Azure Environment from Name", "error", e, "environment", config.EnvironmentName())
	}
	client, e := storage.NewBasicClientOnSovereignCloud(config.AccountName, config.AccountKey, environment)
	if e != nil {
		logger.Log.Fatalw("Could not instantiate Azure Basic Client", "error", e)
	}
	client.Sender = &NetworkErrorRetryingSender{}
	return &Blobstore{
		client:         client.GetBlobService(),
		containerName:  config.ContainerName,
		putBlockSize:   putBlockSize,
		maxListResults: maxListResults,
	}
}

func (blobstore *Blobstore) Exists(path string) (bool, error) {
	exists, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Exists()
	if e != nil {
		return false, errors.Wrapf(e, "Failed to check for %v/%v", blobstore.containerName, path)
	}
	return exists, nil
}

func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) {
	logger.Log.Debugw("Get", "bucket", blobstore.containerName, "path", path)

	reader, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Get(nil)
	if e != nil {
		return nil, blobstore.handleError(e, "Path %v", path)
	}
	return reader, nil
}

func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) {
	signedUrl, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).GetSASURI(storage.BlobSASOptions{
		BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Read: true},
		SASOptions:                storage.SASOptions{Expiry: time.Now().Add(time.Hour)},
	})
	return nil, signedUrl, e
}

func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error {
	putRequestID := rand.Int63()
	l := logger.Log.With("put-request-id", putRequestID)
	l.Debugw("Put", "bucket", blobstore.containerName, "path", path)

	blob := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path)

	e := blob.CreateBlockBlob(nil)
	if e != nil {
		return errors.Wrapf(e, "create block blob failed. container: %v, path: %v, put-request-id: %v", blobstore.containerName, path, putRequestID)
	}

	uncommittedBlocksList := make([]storage.Block, 0)
	eof := false
	for i := 0; !eof; i++ {
		// using information from https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs
		if i >= 50000 {
			return errors.Errorf("block blob cannot have more than 50,000 blocks. path: %v, put-request-id: %v", path, putRequestID)
		}
		block := storage.Block{
			ID:     base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%05d", i))),
			Status: storage.BlockStatusUncommitted,
		}
		data := make([]byte, blobstore.putBlockSize)
		numBytesRead, e := src.Read(data)
		if e != nil {
			if e.Error() == "EOF" {
				eof = true
			} else {
				return errors.Wrapf(e, "put block failed. path: %v, put-request-id: %v", path, putRequestID)
			}
		}
		if numBytesRead == 0 {
			l.Debugw("Empty read", "block-index", i, "block-id", block.ID, "is-eof", eof)
			continue
		}
		l.Debugw("PutBlock", "block-index", i, "block-id", block.ID, "block-size", numBytesRead, "is-eof", eof)
		e = blob.PutBlock(block.ID, data[:numBytesRead], nil)
		if e != nil {
			return errors.Wrapf(e, "put block failed. path: %v, put-request-id: %v", path, putRequestID)
		}
		uncommittedBlocksList = append(uncommittedBlocksList, block)
	}
	l.Debugw("PutBlockList", "uncommitted-block-list", uncommittedBlocksList)
	e = blob.PutBlockList(uncommittedBlocksList, nil)
	if e != nil {
		return errors.Wrapf(e, "put block list failed. path: %v, put-request-id: %v", path, putRequestID)
	}

	return nil
}

func (blobstore *Blobstore) Copy(src, dest string) error {
	logger.Log.Debugw("Copy in Azure", "container", blobstore.containerName, "src", src, "dest", dest)
	e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(dest).Copy(
		blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(src).GetURL(), nil)

	if e != nil {
		blobstore.handleError(e, "Error while trying to copy src %v to dest %v in bucket %v", src, dest, blobstore.containerName)
	}
	return nil
}

func (blobstore *Blobstore) Delete(path string) error {
	deleted, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).DeleteIfExists(nil)
	if e != nil {
		return errors.Wrapf(e, "Path %v", path)
	}
	if !deleted {
		return bitsgo.NewNotFoundError()
	}
	return nil
}

func (blobstore *Blobstore) DeleteDir(prefix string) error {
	deletionErrs := []error{}
	marker := ""
	for {
		response, e := blobstore.client.GetContainerReference(blobstore.containerName).ListBlobs(storage.ListBlobsParameters{
			Prefix:     prefix,
			MaxResults: blobstore.maxListResults,
			Marker:     marker,
		})
		if e != nil {
			return errors.Wrapf(e, "Prefix %v", prefix)
		}
		for _, blob := range response.Blobs {
			e = blobstore.Delete(blob.Name)
			if e != nil {
				if _, isNotFoundError := e.(*bitsgo.NotFoundError); !isNotFoundError {
					deletionErrs = append(deletionErrs, e)
				}
			}
		}
		if response.NextMarker == "" {
			break
		}
		marker = response.NextMarker
	}

	if len(deletionErrs) != 0 {
		return errors.Errorf("Prefix %v, errors from deleting: %v", prefix, deletionErrs)
	}
	return nil
}

func (blobstore *Blobstore) Sign(resource string, method string, expirationTime time.Time) (signedURL string) {
	var e error
	switch strings.ToLower(method) {
	case "put":
		signedURL, e = blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(resource).GetSASURI(storage.BlobSASOptions{
			BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Write: true, Create: true},
			SASOptions:                storage.SASOptions{Expiry: expirationTime},
		})
	case "get":
		signedURL, e = blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(resource).GetSASURI(storage.BlobSASOptions{
			BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Read: true},
			SASOptions:                storage.SASOptions{Expiry: expirationTime},
		})
	default:
		panic("The only supported methods are 'put' and 'get'")
	}
	if e != nil {
		panic(e)
	}
	return
}

func (blobstore *Blobstore) handleError(e error, context string, args ...interface{}) error {
	if azse, ok := e.(storage.AzureStorageServiceError); ok && azse.StatusCode == http.StatusNotFound {
		exists, e := blobstore.client.GetContainerReference(blobstore.containerName).Exists()
		if e != nil {
			return errors.Wrapf(e, context, args...)
		}
		if !exists {
			return errors.Errorf("Container does not exist '%v", blobstore.containerName)
		}
		return bitsgo.NewNotFoundError()
	}
	return errors.Wrapf(e, context, args...)
}
<|endoftext|>


<commit_before>type sub struct {
	last int
	length int
}

type subs []sub

func (s subs) Len() int { return len(s) }

func (s subs) Less(i, j int) bool {
	if s[i].last == s[j].last {
		return s[i].length < s[j].length
	}
	return s[i].last < s[j].last
}

func (s subs) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

func (s *subs) Push(x interface{}) {
	*s = append(*s, x.(sub))
}

func (s *subs) Pop() interface{} {
	old := *s
	n := len(old)
	x := old[n-1]
	*s = old[:n-1]
	return x
}

func isPossible(nums []int) bool {
	hp := &subs{}
	for _, n := range nums {
		for {
			if hp.Len() > 0 && (*hp)[0].last + 1 < n {
				if (*hp)[0].length < 3 {
					return false
				} else {
					heap.Pop(hp)
				}
			} else {
				break
			}
		}
		if hp.Len() > 0 && (*hp)[0].last + 1 == n {
			s := heap.Pop(hp).(sub)
			s.last++
			s.length++
			heap.Push(hp, s)
		} else {
			heap.Push(hp, sub{n, 1})
		}
	}
	for _, s := range *hp {
		if s.length < 3 {
			return false
		}
	}
	return true
}
<commit_msg>LeetCode: 659. Split Array into Consecutive Subsequences (update)<commit_after>package main

import (
	"container/heap"
	"fmt"
)

type sub struct {
	last   int
	length int
}

type subs []sub

func (s subs) Len() int { return len(s) }

func (s subs) Less(i, j int) bool {
	if s[i].last == s[j].last {
		return s[i].length < s[j].length
	}
	return s[i].last < s[j].last
}

func (s subs) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

func (s *subs) Push(x interface{}) {
	*s = append(*s, x.(sub))
}

func (s *subs) Pop() interface{} {
	old := *s
	n := len(old)
	x := old[n-1]
	*s = old[:n-1]
	return x
}

func isPossible(nums []int) bool {
	hp := &subs{}
	for _, n := range nums {
		for hp.Len() > 0 && (*hp)[0].last+1 < n {
			if (*hp)[0].length < 3 {
				return false
			} else {
				heap.Pop(hp)
			}
		}
		if hp.Len() > 0 && (*hp)[0].last+1 == n {
			s := heap.Pop(hp).(sub)
			s.last++
			s.length++
			heap.Push(hp, s)
		} else {
			heap.Push(hp, sub{n, 1})
		}
	}
	for _, s := range *hp {
		if s.length < 3 {
			return false
		}
	}
	return true
}

func main() {
	fmt.Println(isPossible([]int{1, 2, 3, 3, 4, 5}))
}
<|endoftext|>