<commit_before>/*
Copyright 2019 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package status

import (
	"context"
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"sync"
	"sync/atomic"
	"time"

	"go.uber.org/zap"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/workqueue"

	"knative.dev/pkg/network/prober"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/pkg/network"
	"knative.dev/serving/pkg/network/ingress"
)

const (
	// probeConcurrency defines how many probing calls can be issued simultaneously
	probeConcurrency = 15
	// stateExpiration defines how long after being last accessed a state expires
	stateExpiration = 5 * time.Minute
	// cleanupPeriod defines how often states are cleaned up
	cleanupPeriod = 1 * time.Minute
)

// TODO(https://github.com/knative/serving/issues/6407):  Default timeouts may lead to hanging probes.
var dialContext = (&net.Dialer{}).DialContext

// ingressState represents the probing state of an Ingress
type ingressState struct {
	hash string
	ing  *v1alpha1.Ingress

	// pendingCount is the number of pods that haven't been successfully probed yet
	pendingCount int32
	lastAccessed time.Time

	cancel func()
}

// podState represents the probing state of a Pod (for a specific Ingress)
type podState struct {
	// successCount is the number of successful probes
	successCount int32

	cancel func()
}

// cancelContext is a pair of a Context and its cancel function
type cancelContext struct {
	context context.Context
	cancel  func()
}

type workItem struct {
	ingressState *ingressState
	podState     *podState
	context      context.Context
	url          *url.URL
	podIP        string
	podPort      string
}

// ProbeTarget contains the URLs to probes for a set of Pod IPs serving out of the same port.
type ProbeTarget struct {
	PodIPs sets.String
	Port   string
	URLs   []*url.URL
}

// ProbeTargetLister lists all the targets that requires probing.
type ProbeTargetLister interface {
	// ListProbeTargets returns the target to be probed as a map from podIP -> port -> urls.
	ListProbeTargets(ctx context.Context, ingress *v1alpha1.Ingress) ([]ProbeTarget, error)
}

// Manager provides a way to check if an Ingress is ready
type Manager interface {
	IsReady(ctx context.Context, ing *v1alpha1.Ingress) (bool, error)
}

// Prober provides a way to check if a VirtualService is ready by probing the Envoy pods
// handling that VirtualService.
type Prober struct {
	logger *zap.SugaredLogger

	// mu guards ingressStates and podContexts
	mu            sync.Mutex
	ingressStates map[string]*ingressState
	podContexts   map[string]cancelContext

	workQueue workqueue.RateLimitingInterface

	targetLister ProbeTargetLister

	readyCallback func(*v1alpha1.Ingress)

	probeConcurrency int
	stateExpiration  time.Duration
	cleanupPeriod    time.Duration
}

// NewProber creates a new instance of Prober
func NewProber(
	logger *zap.SugaredLogger,
	targetLister ProbeTargetLister,
	readyCallback func(*v1alpha1.Ingress)) *Prober {
	return &Prober{
		logger:        logger,
		ingressStates: make(map[string]*ingressState),
		podContexts:   make(map[string]cancelContext),
		workQueue: workqueue.NewNamedRateLimitingQueue(
			workqueue.DefaultControllerRateLimiter(),
			"ProbingQueue"),
		targetLister:     targetLister,
		readyCallback:    readyCallback,
		probeConcurrency: probeConcurrency,
		stateExpiration:  stateExpiration,
		cleanupPeriod:    cleanupPeriod,
	}
}

func ingressKey(ing *v1alpha1.Ingress) string {
	return fmt.Sprintf("%s/%s", ing.GetNamespace(), ing.GetName())
}

// IsReady checks if the provided Ingress is ready, i.e. the Envoy pods serving the Ingress
// have all been updated. This function is designed to be used by the Ingress controller, i.e. it
// will be called in the order of reconciliation. This means that if IsReady is called on an Ingress,
// this Ingress is the latest known version and therefore anything related to older versions can be ignored.
// Also, it means that IsReady is not called concurrently.
func (m *Prober) IsReady(ctx context.Context, ing *v1alpha1.Ingress) (bool, error) {
	ingressKey := ingressKey(ing)

	bytes, err := ingress.ComputeHash(ing)
	if err != nil {
		return false, fmt.Errorf("failed to compute the hash of the Ingress: %w", err)
	}
	hash := fmt.Sprintf("%x", bytes)

	if ready, ok := func() (bool, bool) {
		m.mu.Lock()
		defer m.mu.Unlock()
		if state, ok := m.ingressStates[ingressKey]; ok {
			if state.hash == hash {
				state.lastAccessed = time.Now()
				return atomic.LoadInt32(&state.pendingCount) == 0, true
			}

			// Cancel the polling for the outdated version
			state.cancel()
			delete(m.ingressStates, ingressKey)
		}
		return false, false
	}(); ok {
		return ready, nil
	}

	ingCtx, cancel := context.WithCancel(context.Background())
	ingressState := &ingressState{
		hash:         hash,
		ing:          ing,
		lastAccessed: time.Now(),
		cancel:       cancel,
	}

	targets, err := m.targetLister.ListProbeTargets(ctx, ing)
	if err != nil {
		return false, err
	}

	// First, group the targets by IPs.
	workItems := make(map[string][]*workItem)
	for _, target := range targets {
		for ip := range target.PodIPs {
			for _, url := range target.URLs {
				workItems[ip] = append(workItems[ip], &workItem{
					ingressState: ingressState,
					url:          url,
					podIP:        ip,
					podPort:      target.Port,
				})
			}
		}
	}
	for ip, ipWorkItems := range workItems {
		// Get or create the context for that IP
		ipCtx := func() context.Context {
			m.mu.Lock()
			defer m.mu.Unlock()
			cancelCtx, ok := m.podContexts[ip]
			if !ok {
				ctx, cancel := context.WithCancel(context.Background())
				cancelCtx = cancelContext{
					context: ctx,
					cancel:  cancel,
				}
				m.podContexts[ip] = cancelCtx
			}
			return cancelCtx.context
		}()

		podCtx, cancel := context.WithCancel(ingCtx)
		podState := &podState{
			successCount: 0,
			cancel:       cancel,
		}

		// Quick and dirty way to join two contexts (i.e. podCtx is cancelled when either ingCtx or ipCtx are cancelled)
		go func() {
			select {
			case <-podCtx.Done():
				// This is the actual context, there is nothing to do except
				// break to avoid leaking this goroutine.
				break
			case <-ipCtx.Done():
				// Cancel podCtx
				cancel()
			}
		}()

		// Update the states when probing is successful or cancelled
		go func() {
			<-podCtx.Done()
			m.updateStates(ingressState, podState)
		}()

		for _, wi := range ipWorkItems {
			wi.podState = podState
			wi.context = podCtx
			m.workQueue.AddRateLimited(wi)
			m.logger.Infof("Queuing probe for %s, IP: %s:%s (depth: %d)",
				wi.url, wi.podIP, wi.podPort, m.workQueue.Len())
		}
	}
	ingressState.pendingCount += int32(len(workItems))
	func() {
		m.mu.Lock()
		defer m.mu.Unlock()
		m.ingressStates[ingressKey] = ingressState
	}()
	return len(workItems) == 0, nil
}

// Start starts the Manager background operations
func (m *Prober) Start(done <-chan struct{}) {
	// Start the worker goroutines
	for i := 0; i < m.probeConcurrency; i++ {
		go func() {
			for m.processWorkItem() {
			}
		}()
	}

	// Cleanup the states periodically
	go wait.Until(m.expireOldStates, m.cleanupPeriod, done)

	// Stop processing the queue when cancelled
	go func() {
		<-done
		m.workQueue.ShutDown()
	}()
}

// CancelIngressProbing cancels probing of the provided Ingress
// TODO(#6270): Use cache.DeletedFinalStateUnknown.
func (m *Prober) CancelIngressProbing(obj interface{}) {
	if ing, ok := obj.(*v1alpha1.Ingress); ok {
		key := ingressKey(ing)

		m.mu.Lock()
		defer m.mu.Unlock()
		if state, ok := m.ingressStates[key]; ok {
			state.cancel()
			delete(m.ingressStates, key)
		}
	}
}

// CancelPodProbing cancels probing of the provided Pod IP.
//
// TODO(#6269): make this cancelation based on Pod x port instead of just Pod.
func (m *Prober) CancelPodProbing(obj interface{}) {
	if pod, ok := obj.(*corev1.Pod); ok {
		m.mu.Lock()
		defer m.mu.Unlock()

		if ctx, ok := m.podContexts[pod.Status.PodIP]; ok {
			ctx.cancel()
			delete(m.podContexts, pod.Status.PodIP)
		}
	}
}

// expireOldStates removes the states that haven't been accessed in a while.
func (m *Prober) expireOldStates() {
	m.mu.Lock()
	defer m.mu.Unlock()
	for key, state := range m.ingressStates {
		if time.Since(state.lastAccessed) > m.stateExpiration {
			state.cancel()
			delete(m.ingressStates, key)
		}
	}
}

// processWorkItem processes a single work item from workQueue.
// It returns false when there is no more items to process, true otherwise.
func (m *Prober) processWorkItem() bool {
	obj, shutdown := m.workQueue.Get()
	if shutdown {
		return false
	}

	defer m.workQueue.Done(obj)

	// Crash if the item is not of the expected type
	item, ok := obj.(*workItem)
	if !ok {
		m.logger.Fatalf("Unexpected work item type: want: %s, got: %s\n",
			reflect.TypeOf(&workItem{}).Name(), reflect.TypeOf(obj).Name())
	}
	m.logger.Infof("Processing probe for %s, IP: %s:%s (depth: %d)",
		item.url, item.podIP, item.podPort, m.workQueue.Len())

	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			// We only want to know that the Gateway is configured, not that the configuration is valid.
			// Therefore, we can safely ignore any TLS certificate validation.
			InsecureSkipVerify: true,
		},
		DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, e error) {
			// Requests with the IP as hostname and the Host header set do no pass client-side validation
			// because the HTTP client validates that the hostname (not the Host header) matches the server
			// TLS certificate Common Name or Alternative Names. Therefore, http.Request.URL is set to the
			// hostname and it is substituted it here with the target IP.
			return dialContext(ctx, network, net.JoinHostPort(item.podIP, item.podPort))
		}}

	ok, err := prober.Do(
		item.context,
		transport,
		item.url.String(),
		prober.WithHeader(network.ProbeHeaderName, network.ProbeHeaderValue),
		prober.ExpectsStatusCodes([]int{http.StatusOK}),
		prober.ExpectsHeader(network.HashHeaderName, item.ingressState.hash))

	// In case of cancellation, drop the work item
	select {
	case <-item.context.Done():
		m.workQueue.Forget(obj)
		return true
	default:
	}

	if err != nil || !ok {
		// In case of error, enqueue for retry
		m.workQueue.AddRateLimited(obj)
		m.logger.Errorf("Probing of %s failed, IP: %s:%s, ready: %t, error: %v (depth: %d)",
			item.url, item.podIP, item.podPort, ok, err, m.workQueue.Len())
	} else {
		m.updateStates(item.ingressState, item.podState)
	}
	return true
}

func (m *Prober) updateStates(ingressState *ingressState, podState *podState) {
	if atomic.AddInt32(&podState.successCount, 1) == 1 {
		// This is the first successful probe call for the pod, cancel all other work items for this pod
		podState.cancel()

		// This is the last pod being successfully probed, the Ingress is ready
		if atomic.AddInt32(&ingressState.pendingCount, -1) == 0 {
			m.readyCallback(ingressState.ing)
		}
	}
}
<commit_msg>Update doc to match signature. (#6470)<commit_after>/*
Copyright 2019 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package status

import (
	"context"
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"sync"
	"sync/atomic"
	"time"

	"go.uber.org/zap"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/workqueue"

	"knative.dev/pkg/network/prober"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/pkg/network"
	"knative.dev/serving/pkg/network/ingress"
)

const (
	// probeConcurrency defines how many probing calls can be issued simultaneously
	probeConcurrency = 15
	// stateExpiration defines how long after being last accessed a state expires
	stateExpiration = 5 * time.Minute
	// cleanupPeriod defines how often states are cleaned up
	cleanupPeriod = 1 * time.Minute
)

// TODO(https://github.com/knative/serving/issues/6407):  Default timeouts may lead to hanging probes.
var dialContext = (&net.Dialer{}).DialContext

// ingressState represents the probing state of an Ingress
type ingressState struct {
	hash string
	ing  *v1alpha1.Ingress

	// pendingCount is the number of pods that haven't been successfully probed yet
	pendingCount int32
	lastAccessed time.Time

	cancel func()
}

// podState represents the probing state of a Pod (for a specific Ingress)
type podState struct {
	// successCount is the number of successful probes
	successCount int32

	cancel func()
}

// cancelContext is a pair of a Context and its cancel function
type cancelContext struct {
	context context.Context
	cancel  func()
}

type workItem struct {
	ingressState *ingressState
	podState     *podState
	context      context.Context
	url          *url.URL
	podIP        string
	podPort      string
}

// ProbeTarget contains the URLs to probes for a set of Pod IPs serving out of the same port.
type ProbeTarget struct {
	PodIPs sets.String
	Port   string
	URLs   []*url.URL
}

// ProbeTargetLister lists all the targets that requires probing.
type ProbeTargetLister interface {
	// ListProbeTargets returns a list of targets to be probed.
	ListProbeTargets(ctx context.Context, ingress *v1alpha1.Ingress) ([]ProbeTarget, error)
}

// Manager provides a way to check if an Ingress is ready
type Manager interface {
	IsReady(ctx context.Context, ing *v1alpha1.Ingress) (bool, error)
}

// Prober provides a way to check if a VirtualService is ready by probing the Envoy pods
// handling that VirtualService.
type Prober struct {
	logger *zap.SugaredLogger

	// mu guards ingressStates and podContexts
	mu            sync.Mutex
	ingressStates map[string]*ingressState
	podContexts   map[string]cancelContext

	workQueue workqueue.RateLimitingInterface

	targetLister ProbeTargetLister

	readyCallback func(*v1alpha1.Ingress)

	probeConcurrency int
	stateExpiration  time.Duration
	cleanupPeriod    time.Duration
}

// NewProber creates a new instance of Prober
func NewProber(
	logger *zap.SugaredLogger,
	targetLister ProbeTargetLister,
	readyCallback func(*v1alpha1.Ingress)) *Prober {
	return &Prober{
		logger:        logger,
		ingressStates: make(map[string]*ingressState),
		podContexts:   make(map[string]cancelContext),
		workQueue: workqueue.NewNamedRateLimitingQueue(
			workqueue.DefaultControllerRateLimiter(),
			"ProbingQueue"),
		targetLister:     targetLister,
		readyCallback:    readyCallback,
		probeConcurrency: probeConcurrency,
		stateExpiration:  stateExpiration,
		cleanupPeriod:    cleanupPeriod,
	}
}

func ingressKey(ing *v1alpha1.Ingress) string {
	return fmt.Sprintf("%s/%s", ing.GetNamespace(), ing.GetName())
}

// IsReady checks if the provided Ingress is ready, i.e. the Envoy pods serving the Ingress
// have all been updated. This function is designed to be used by the Ingress controller, i.e. it
// will be called in the order of reconciliation. This means that if IsReady is called on an Ingress,
// this Ingress is the latest known version and therefore anything related to older versions can be ignored.
// Also, it means that IsReady is not called concurrently.
func (m *Prober) IsReady(ctx context.Context, ing *v1alpha1.Ingress) (bool, error) {
	ingressKey := ingressKey(ing)

	bytes, err := ingress.ComputeHash(ing)
	if err != nil {
		return false, fmt.Errorf("failed to compute the hash of the Ingress: %w", err)
	}
	hash := fmt.Sprintf("%x", bytes)

	if ready, ok := func() (bool, bool) {
		m.mu.Lock()
		defer m.mu.Unlock()
		if state, ok := m.ingressStates[ingressKey]; ok {
			if state.hash == hash {
				state.lastAccessed = time.Now()
				return atomic.LoadInt32(&state.pendingCount) == 0, true
			}

			// Cancel the polling for the outdated version
			state.cancel()
			delete(m.ingressStates, ingressKey)
		}
		return false, false
	}(); ok {
		return ready, nil
	}

	ingCtx, cancel := context.WithCancel(context.Background())
	ingressState := &ingressState{
		hash:         hash,
		ing:          ing,
		lastAccessed: time.Now(),
		cancel:       cancel,
	}

	targets, err := m.targetLister.ListProbeTargets(ctx, ing)
	if err != nil {
		return false, err
	}

	// First, group the targets by IPs.
	workItems := make(map[string][]*workItem)
	for _, target := range targets {
		for ip := range target.PodIPs {
			for _, url := range target.URLs {
				workItems[ip] = append(workItems[ip], &workItem{
					ingressState: ingressState,
					url:          url,
					podIP:        ip,
					podPort:      target.Port,
				})
			}
		}
	}
	for ip, ipWorkItems := range workItems {
		// Get or create the context for that IP
		ipCtx := func() context.Context {
			m.mu.Lock()
			defer m.mu.Unlock()
			cancelCtx, ok := m.podContexts[ip]
			if !ok {
				ctx, cancel := context.WithCancel(context.Background())
				cancelCtx = cancelContext{
					context: ctx,
					cancel:  cancel,
				}
				m.podContexts[ip] = cancelCtx
			}
			return cancelCtx.context
		}()

		podCtx, cancel := context.WithCancel(ingCtx)
		podState := &podState{
			successCount: 0,
			cancel:       cancel,
		}

		// Quick and dirty way to join two contexts (i.e. podCtx is cancelled when either ingCtx or ipCtx are cancelled)
		go func() {
			select {
			case <-podCtx.Done():
				// This is the actual context, there is nothing to do except
				// break to avoid leaking this goroutine.
				break
			case <-ipCtx.Done():
				// Cancel podCtx
				cancel()
			}
		}()

		// Update the states when probing is successful or cancelled
		go func() {
			<-podCtx.Done()
			m.updateStates(ingressState, podState)
		}()

		for _, wi := range ipWorkItems {
			wi.podState = podState
			wi.context = podCtx
			m.workQueue.AddRateLimited(wi)
			m.logger.Infof("Queuing probe for %s, IP: %s:%s (depth: %d)",
				wi.url, wi.podIP, wi.podPort, m.workQueue.Len())
		}
	}
	ingressState.pendingCount += int32(len(workItems))
	func() {
		m.mu.Lock()
		defer m.mu.Unlock()
		m.ingressStates[ingressKey] = ingressState
	}()
	return len(workItems) == 0, nil
}

// Start starts the Manager background operations
func (m *Prober) Start(done <-chan struct{}) {
	// Start the worker goroutines
	for i := 0; i < m.probeConcurrency; i++ {
		go func() {
			for m.processWorkItem() {
			}
		}()
	}

	// Cleanup the states periodically
	go wait.Until(m.expireOldStates, m.cleanupPeriod, done)

	// Stop processing the queue when cancelled
	go func() {
		<-done
		m.workQueue.ShutDown()
	}()
}

// CancelIngressProbing cancels probing of the provided Ingress
// TODO(#6270): Use cache.DeletedFinalStateUnknown.
func (m *Prober) CancelIngressProbing(obj interface{}) {
	if ing, ok := obj.(*v1alpha1.Ingress); ok {
		key := ingressKey(ing)

		m.mu.Lock()
		defer m.mu.Unlock()
		if state, ok := m.ingressStates[key]; ok {
			state.cancel()
			delete(m.ingressStates, key)
		}
	}
}

// CancelPodProbing cancels probing of the provided Pod IP.
//
// TODO(#6269): make this cancelation based on Pod x port instead of just Pod.
func (m *Prober) CancelPodProbing(obj interface{}) {
	if pod, ok := obj.(*corev1.Pod); ok {
		m.mu.Lock()
		defer m.mu.Unlock()

		if ctx, ok := m.podContexts[pod.Status.PodIP]; ok {
			ctx.cancel()
			delete(m.podContexts, pod.Status.PodIP)
		}
	}
}

// expireOldStates removes the states that haven't been accessed in a while.
func (m *Prober) expireOldStates() {
	m.mu.Lock()
	defer m.mu.Unlock()
	for key, state := range m.ingressStates {
		if time.Since(state.lastAccessed) > m.stateExpiration {
			state.cancel()
			delete(m.ingressStates, key)
		}
	}
}

// processWorkItem processes a single work item from workQueue.
// It returns false when there is no more items to process, true otherwise.
func (m *Prober) processWorkItem() bool {
	obj, shutdown := m.workQueue.Get()
	if shutdown {
		return false
	}

	defer m.workQueue.Done(obj)

	// Crash if the item is not of the expected type
	item, ok := obj.(*workItem)
	if !ok {
		m.logger.Fatalf("Unexpected work item type: want: %s, got: %s\n",
			reflect.TypeOf(&workItem{}).Name(), reflect.TypeOf(obj).Name())
	}
	m.logger.Infof("Processing probe for %s, IP: %s:%s (depth: %d)",
		item.url, item.podIP, item.podPort, m.workQueue.Len())

	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			// We only want to know that the Gateway is configured, not that the configuration is valid.
			// Therefore, we can safely ignore any TLS certificate validation.
			InsecureSkipVerify: true,
		},
		DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, e error) {
			// Requests with the IP as hostname and the Host header set do no pass client-side validation
			// because the HTTP client validates that the hostname (not the Host header) matches the server
			// TLS certificate Common Name or Alternative Names. Therefore, http.Request.URL is set to the
			// hostname and it is substituted it here with the target IP.
			return dialContext(ctx, network, net.JoinHostPort(item.podIP, item.podPort))
		}}

	ok, err := prober.Do(
		item.context,
		transport,
		item.url.String(),
		prober.WithHeader(network.ProbeHeaderName, network.ProbeHeaderValue),
		prober.ExpectsStatusCodes([]int{http.StatusOK}),
		prober.ExpectsHeader(network.HashHeaderName, item.ingressState.hash))

	// In case of cancellation, drop the work item
	select {
	case <-item.context.Done():
		m.workQueue.Forget(obj)
		return true
	default:
	}

	if err != nil || !ok {
		// In case of error, enqueue for retry
		m.workQueue.AddRateLimited(obj)
		m.logger.Errorf("Probing of %s failed, IP: %s:%s, ready: %t, error: %v (depth: %d)",
			item.url, item.podIP, item.podPort, ok, err, m.workQueue.Len())
	} else {
		m.updateStates(item.ingressState, item.podState)
	}
	return true
}

func (m *Prober) updateStates(ingressState *ingressState, podState *podState) {
	if atomic.AddInt32(&podState.successCount, 1) == 1 {
		// This is the first successful probe call for the pod, cancel all other work items for this pod
		podState.cancel()

		// This is the last pod being successfully probed, the Ingress is ready
		if atomic.AddInt32(&ingressState.pendingCount, -1) == 0 {
			m.readyCallback(ingressState.ing)
		}
	}
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"errors"
	"math"
	"regexp"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
	"github.com/gophercloud/gophercloud/pagination"
	"github.com/sapcc/go-bits/logg"
	"github.com/sapcc/limes/pkg/core"
)

type capacityNovaPlugin struct {
	cfg core.CapacitorConfiguration
}

func init() {
	core.RegisterCapacityPlugin(func(c core.CapacitorConfiguration, scrapeSubcapacities map[string]map[string]bool) core.CapacityPlugin {
		return &capacityNovaPlugin{c}
	})
}

//Init implements the core.CapacityPlugin interface.
func (p *capacityNovaPlugin) Init(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts) error {
	return nil
}

func (p *capacityNovaPlugin) ID() string {
	return "nova"
}

//Scrape implements the core.CapacityPlugin interface.
func (p *capacityNovaPlugin) Scrape(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts, clusterID string) (map[string]map[string]core.CapacityData, error) {
	var hypervisorTypeRx *regexp.Regexp
	if p.cfg.Nova.HypervisorTypePattern != "" {
		var err error
		hypervisorTypeRx, err = regexp.Compile(p.cfg.Nova.HypervisorTypePattern)
		if err != nil {
			return nil, errors.New("invalid value for hypervisor_type: " + err.Error())
		}
	}

	client, err := openstack.NewComputeV2(provider, eo)
	if err != nil {
		return nil, err
	}

	var result gophercloud.Result

	//enumerate hypervisors
	url := client.ServiceURL("os-hypervisors", "detail")
	_, err = client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}
	var hypervisorData struct {
		Hypervisors []struct {
			Type     string `json:"hypervisor_type"`
			Vcpus    uint64 `json:"vcpus"`
			MemoryMb uint64 `json:"memory_mb"`
			LocalGb  uint64 `json:"local_gb"`
		} `json:"hypervisors"`
	}
	err = result.ExtractInto(&hypervisorData)
	if err != nil {
		return nil, err
	}

	//compute sum of cores and RAM for matching hypervisors
	var (
		totalVcpus    uint64
		totalMemoryMb uint64
		totalLocalGb  uint64
	)
	for _, hypervisor := range hypervisorData.Hypervisors {
		if hypervisorTypeRx != nil {
			if !hypervisorTypeRx.MatchString(hypervisor.Type) {
				continue
			}
		}

		totalVcpus += hypervisor.Vcpus
		totalMemoryMb += hypervisor.MemoryMb
		totalLocalGb += hypervisor.LocalGb
	}

	//Get availability zones
	url = client.ServiceURL("os-availability-zone")
	_, err = client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}
	var availabilityZoneData struct {
		AvailabilityZoneInfo []struct {
			ZoneName  string `json:"zoneName"`
			ZoneState struct {
				Available bool `json:"available"`
			} `json:"zoneState"`
		} `json:"availabilityZoneInfo"`
	}
	err = result.ExtractInto(&availabilityZoneData)
	if err != nil {
		return nil, err
	}

	//list all flavors and get max(flavor_size)
	pages, maxFlavorSize := 0, 0.0
	err = flavors.ListDetail(client, nil).EachPage(func(page pagination.Page) (bool, error) {
		pages++
		f, err := flavors.ExtractFlavors(page)
		if err != nil {
			return false, err
		}
		for _, element := range f {
			extras, err := getFlavorExtras(client, element.ID)
			if err != nil {
				logg.Debug("Failed to get extra specs for flavor: %s.", element.ID)
				return false, err
			}

			//necessary to be able to ignore huge baremetal flavors
			//consider only flavors as defined in extra specs
			var extraSpecs map[string]string
			if p.cfg.Nova.ExtraSpecs != nil {
				extraSpecs = p.cfg.Nova.ExtraSpecs
			}

			matches := true
			for key, value := range extraSpecs {
				if value != extras[key] {
					matches = false
					break
				}
			}
			if matches {
				logg.Debug("FlavorName: %s, FlavorID: %s, FlavorSize: %d GiB", element.Name, element.ID, element.Disk)
				maxFlavorSize = math.Max(maxFlavorSize, float64(element.Disk))
			}
		}

		return true, nil
	})
	if err != nil {
		return nil, err
	}

	var azCount int

	//count availability zones
	for _, element := range availabilityZoneData.AvailabilityZoneInfo {
		if element.ZoneState.Available {
			azCount++
		}
	}

	capacity := map[string]map[string]core.CapacityData{
		"compute": {
			"cores": core.CapacityData{Capacity: totalVcpus},
			"ram":   core.CapacityData{Capacity: totalMemoryMb},
		},
	}

	if maxFlavorSize != 0 {
		instanceCapacity := uint64(math.Min(float64(10000*azCount), float64(totalLocalGb)/maxFlavorSize))
		capacity["compute"]["instances"] = core.CapacityData{Capacity: instanceCapacity}
	} else {
		logg.Error("Nova Capacity: Maximal flavor size is 0. Not reporting instances.")
	}

	return capacity, nil

}

//get flavor extra-specs
//result contains
//{ "vmware:hv_enabled" : 'True' }
//which identifies a VM flavor
func getFlavorExtras(client *gophercloud.ServiceClient, flavorUUID string) (map[string]string, error) {
	var result gophercloud.Result
	var extraSpecs struct {
		ExtraSpecs map[string]string `json:"extra_specs"`
	}

	url := client.ServiceURL("flavors", flavorUUID, "os-extra_specs")
	_, err := client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}

	err = result.ExtractInto(&extraSpecs)
	if err != nil {
		return nil, err
	}

	return extraSpecs.ExtraSpecs, nil
}
<commit_msg>respect the VCenter HA reserve<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"errors"
	"math"
	"regexp"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
	"github.com/gophercloud/gophercloud/pagination"
	"github.com/sapcc/go-bits/logg"
	"github.com/sapcc/limes/pkg/core"
)

type capacityNovaPlugin struct {
	cfg core.CapacitorConfiguration
}

func init() {
	core.RegisterCapacityPlugin(func(c core.CapacitorConfiguration, scrapeSubcapacities map[string]map[string]bool) core.CapacityPlugin {
		return &capacityNovaPlugin{c}
	})
}

//Init implements the core.CapacityPlugin interface.
func (p *capacityNovaPlugin) Init(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts) error {
	return nil
}

func (p *capacityNovaPlugin) ID() string {
	return "nova"
}

//Scrape implements the core.CapacityPlugin interface.
func (p *capacityNovaPlugin) Scrape(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts, clusterID string) (map[string]map[string]core.CapacityData, error) {
	var hypervisorTypeRx *regexp.Regexp
	if p.cfg.Nova.HypervisorTypePattern != "" {
		var err error
		hypervisorTypeRx, err = regexp.Compile(p.cfg.Nova.HypervisorTypePattern)
		if err != nil {
			return nil, errors.New("invalid value for hypervisor_type: " + err.Error())
		}
	}

	client, err := openstack.NewComputeV2(provider, eo)
	if err != nil {
		return nil, err
	}

	var result gophercloud.Result

	//enumerate hypervisors
	url := client.ServiceURL("os-hypervisors", "detail")
	_, err = client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}
	var hypervisorData struct {
		Hypervisors []struct {
			Type     string `json:"hypervisor_type"`
			Vcpus    uint64 `json:"vcpus"`
			MemoryMb uint64 `json:"memory_mb"`
			LocalGb  uint64 `json:"local_gb"`
		} `json:"hypervisors"`
	}
	err = result.ExtractInto(&hypervisorData)
	if err != nil {
		return nil, err
	}

	//compute sum of cores and RAM for matching hypervisors
	var (
		totalVcpus    uint64
		totalMemoryMb uint64
		totalLocalGb  uint64
	)
	for _, hypervisor := range hypervisorData.Hypervisors {
		if hypervisorTypeRx != nil {
			if !hypervisorTypeRx.MatchString(hypervisor.Type) {
				continue
			}
		}

		totalVcpus += hypervisor.Vcpus
		totalMemoryMb += hypervisor.MemoryMb
		totalLocalGb += hypervisor.LocalGb
	}

	//Get availability zones
	url = client.ServiceURL("os-availability-zone")
	_, err = client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}
	var availabilityZoneData struct {
		AvailabilityZoneInfo []struct {
			ZoneName  string `json:"zoneName"`
			ZoneState struct {
				Available bool `json:"available"`
			} `json:"zoneState"`
		} `json:"availabilityZoneInfo"`
	}
	err = result.ExtractInto(&availabilityZoneData)
	if err != nil {
		return nil, err
	}

	//list all flavors and get max(flavor_size)
	pages, maxFlavorSize := 0, 0.0
	err = flavors.ListDetail(client, nil).EachPage(func(page pagination.Page) (bool, error) {
		pages++
		f, err := flavors.ExtractFlavors(page)
		if err != nil {
			return false, err
		}
		for _, element := range f {
			extras, err := getFlavorExtras(client, element.ID)
			if err != nil {
				logg.Debug("Failed to get extra specs for flavor: %s.", element.ID)
				return false, err
			}

			//necessary to be able to ignore huge baremetal flavors
			//consider only flavors as defined in extra specs
			var extraSpecs map[string]string
			if p.cfg.Nova.ExtraSpecs != nil {
				extraSpecs = p.cfg.Nova.ExtraSpecs
			}

			matches := true
			for key, value := range extraSpecs {
				if value != extras[key] {
					matches = false
					break
				}
			}
			if matches {
				logg.Debug("FlavorName: %s, FlavorID: %s, FlavorSize: %d GiB", element.Name, element.ID, element.Disk)
				maxFlavorSize = math.Max(maxFlavorSize, float64(element.Disk))
			}
		}

		return true, nil
	})
	if err != nil {
		return nil, err
	}

	var azCount int

	//count availability zones
	for _, element := range availabilityZoneData.AvailabilityZoneInfo {
		if element.ZoneState.Available {
			azCount++
		}
	}

	//preserve the VCenter HA reserve, which is reported via Nova,
	//but not accesible
	//TODO: This must be replaced with a more generic attribute or solved in Nova
	totalVcpus = uint64(float64(totalVcpus) * 0.9)
	totalMemoryMb = uint64(float64(totalMemoryMb) * 0.9)

	capacity := map[string]map[string]core.CapacityData{
		"compute": {
			"cores": core.CapacityData{Capacity: totalVcpus},
			"ram":   core.CapacityData{Capacity: totalMemoryMb},
		},
	}

	if maxFlavorSize != 0 {
		instanceCapacity := uint64(math.Min(float64(10000*azCount), float64(totalLocalGb)/maxFlavorSize))
		capacity["compute"]["instances"] = core.CapacityData{Capacity: instanceCapacity}
	} else {
		logg.Error("Nova Capacity: Maximal flavor size is 0. Not reporting instances.")
	}

	return capacity, nil

}

//get flavor extra-specs
//result contains
//{ "vmware:hv_enabled" : 'True' }
//which identifies a VM flavor
func getFlavorExtras(client *gophercloud.ServiceClient, flavorUUID string) (map[string]string, error) {
	var result gophercloud.Result
	var extraSpecs struct {
		ExtraSpecs map[string]string `json:"extra_specs"`
	}

	url := client.ServiceURL("flavors", flavorUUID, "os-extra_specs")
	_, err := client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}

	err = result.ExtractInto(&extraSpecs)
	if err != nil {
		return nil, err
	}

	return extraSpecs.ExtraSpecs, nil
}
<|endoftext|>


<commit_before>// +build linux freebsd

package system // import "github.com/docker/docker/pkg/system"

import (
	"os"
	"syscall"
	"testing"

	"gotest.tools/v3/assert"
)

// TestFromStatT tests fromStatT for a tempfile
func TestFromStatT(t *testing.T) {
	file, _, _, dir := prepareFiles(t)
	defer os.RemoveAll(dir)

	stat := &syscall.Stat_t{}
	err := syscall.Lstat(file, stat)
	assert.NilError(t, err)

	s, err := fromStatT(stat)
	assert.NilError(t, err)

	if stat.Mode != s.Mode() {
		t.Fatal("got invalid mode")
	}
	if stat.Uid != s.UID() {
		t.Fatal("got invalid uid")
	}
	if stat.Gid != s.GID() {
		t.Fatal("got invalid gid")
	}
	if stat.Rdev != s.Rdev() {
		t.Fatal("got invalid rdev")
	}
	if stat.Mtim != s.Mtim() {
		t.Fatal("got invalid mtim")
	}
}
<commit_msg>fix "stat.Rdev" invalid operation mismatched types on mips64el compile error the "stat.Rdev" variable and "s.Rdev" mismatched types on mips64el convert "stat.Rdev" type to uint64 explicitly<commit_after>// +build linux freebsd

package system // import "github.com/docker/docker/pkg/system"

import (
	"os"
	"syscall"
	"testing"

	"gotest.tools/v3/assert"
)

// TestFromStatT tests fromStatT for a tempfile
func TestFromStatT(t *testing.T) {
	file, _, _, dir := prepareFiles(t)
	defer os.RemoveAll(dir)

	stat := &syscall.Stat_t{}
	err := syscall.Lstat(file, stat)
	assert.NilError(t, err)

	s, err := fromStatT(stat)
	assert.NilError(t, err)

	if stat.Mode != s.Mode() {
		t.Fatal("got invalid mode")
	}
	if stat.Uid != s.UID() {
		t.Fatal("got invalid uid")
	}
	if stat.Gid != s.GID() {
		t.Fatal("got invalid gid")
	}
	//nolint:unconvert // conversion needed to fix mismatch types on mips64el
	if uint64(stat.Rdev) != s.Rdev() {
		t.Fatal("got invalid rdev")
	}
	if stat.Mtim != s.Mtim() {
		t.Fatal("got invalid mtim")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blueflood

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/square/metrics/api"
)

type httpClient interface {
	// our own client to mock out the standard golang HTTP Client.
	Get(url string) (resp *http.Response, err error)
}

type Blueflood struct {
	api      api.API
	baseUrl  string
	tenantId string
	client   httpClient
}

type QueryResponse struct {
	Values []MetricPoint `json:"values"`
}

type MetricPoint struct {
	Points    int     `json:"numPoints"`
	Timestamp int64   `json:"timestamp"`
	Average   float64 `json:"average"`
	Max       float64 `json:"max"`
	Min       float64 `json:"min"`
	Variance  float64 `json:"variance"`
}

const (
	ResolutionFull    = "FULL"
	Resolution5Min    = "MIN5"
	Resolution20Min   = "MIN20"
	Resolution60Min   = "MIN60"
	Resolution240Min  = "MIN240"
	Resolution1440Min = "MIN1440"
)

func NewBlueflood(api api.API, baseUrl string, tenantId string) *Blueflood {
	return &Blueflood{api: api, baseUrl: baseUrl, tenantId: tenantId, client: http.DefaultClient}
}

func (b *Blueflood) Api() api.API {
	return b.api
}

func (b *Blueflood) FetchSeries(metric api.TaggedMetric, predicate api.Predicate, sampleMethod api.SampleMethod, timerange api.Timerange) (*api.SeriesList, error) {
	metricTagSets, err := b.Api().GetAllTags(metric.MetricKey)
	if err != nil {
		return nil, err
	}

	// TODO: Be a little smarter about this?
	resultSeries := []api.Timeseries{}
	for _, ts := range metricTagSets {
		if predicate == nil || predicate.Apply(ts) {
			graphiteName, err := b.Api().ToGraphiteName(api.TaggedMetric{
				MetricKey: metric.MetricKey,
				TagSet:    ts,
			})
			if err != nil {
				return nil, err
			}

			queryResult, err := b.fetchSingleSeries(
				graphiteName,
				sampleMethod,
				timerange,
			)
			// TODO: Be more tolerant of errors fetching a single metric?
			// Though I guess this behavior is fine since skipping fetches
			// that fail would end up in a result set that you don't quite
			// expect.
			if err != nil {
				return nil, err
			}

			resultSeries = append(resultSeries, api.Timeseries{
				Values: queryResult,
				TagSet: ts,
			})
		}
	}

	return &api.SeriesList{
		Series:    resultSeries,
		Timerange: timerange,
	}, nil
}

func addMetricPoint(metricPoint MetricPoint, field func(MetricPoint) float64, timerange api.Timerange, buckets [][]float64) error {
	// TODO: check the result of .FieldByName against nil (or a panic will occur)
	value := field(metricPoint)
	// TODO: check the result of .FieldByName against nil (or a panic will occur)
	timestamp := metricPoint.Timestamp
	// The index to assign within the array is computed using the timestamp.
	// It rounds to the nearest index.
	index := (timestamp - timerange.Start() + timerange.Resolution()/2) / timerange.Resolution()
	if index < 0 || index >= int64(timerange.Slots()) {
		return errors.New("index out of range")
	}
	buckets[index] = append(buckets[index], value)
	return nil
}

func bucketsFromMetricPoints(metricPoints []MetricPoint, resultField func(MetricPoint) float64, timerange api.Timerange) [][]float64 {
	buckets := make([][]float64, timerange.Slots())
	// Make the buckets:
	for i := range buckets {
		buckets[i] = []float64{}
	}
	for _, point := range metricPoints {
		addMetricPoint(point, resultField, timerange, buckets)
	}
	return buckets
}

func (b *Blueflood) fetchSingleSeries(metric api.GraphiteMetric, sampleMethod api.SampleMethod, timerange api.Timerange) ([]float64, error) {
	// Use this lowercase of this as the select query param. Use the actual value
	// to reflect into result MetricPoints to fetch the correct field.
	var (
		fieldExtractor    string
		selectResultField func(MetricPoint) float64
	)
	switch sampleMethod {
	case api.SampleMean:
		fieldExtractor = "average"
		selectResultField = func(point MetricPoint) float64 { return point.Average }
	case api.SampleMin:
		fieldExtractor = "min"
		selectResultField = func(point MetricPoint) float64 { return point.Min }
	case api.SampleMax:
		fieldExtractor = "max"
		selectResultField = func(point MetricPoint) float64 { return point.Max }
	default:
		return nil, errors.New(fmt.Sprintf("Unsupported SampleMethod %d", sampleMethod))
	}

	// Issue GET to fetch metrics
	queryUrl, err := url.Parse(fmt.Sprintf("%s/v2.0/%s/views/%s",
		b.baseUrl,
		b.tenantId,
		metric))
	if err != nil {
		return nil, err
	}

	params := url.Values{}
	params.Set("from", strconv.FormatInt(timerange.Start(), 10))
	params.Set("to", strconv.FormatInt(timerange.End(), 10))
	params.Set("resolution", bluefloodResolution(timerange.Resolution()))
	params.Set("select", fmt.Sprintf("numPoints,%s", strings.ToLower(fieldExtractor)))

	queryUrl.RawQuery = params.Encode()

	log.Printf("Blueflood fetch: %s", queryUrl.String())
	resp, err := b.client.Get(queryUrl.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	log.Printf("Fetch result: %s", string(body))

	var result QueryResponse
	err = json.Unmarshal(body, &result)
	if err != nil {
		return nil, err
	}

	// Construct a Timeseries from the result

	buckets := bucketsFromMetricPoints(result.Values, selectResultField, timerange)

	values := make([]float64, timerange.Slots())

	bucketSamplers := map[api.SampleMethod]func([]float64) float64{
		api.SampleMean: func(bucket []float64) float64 {
			value := 0.0
			for _, v := range bucket {
				value += v
			}
			return value / float64(len(bucket))
		},
		api.SampleMin: func(bucket []float64) float64 { // These functions may assume bucket is non-empty.
			value := bucket[0]
			for _, v := range bucket {
				value = math.Min(value, v)
			}
			return value
		},
		api.SampleMax: func(bucket []float64) float64 {
			value := bucket[0]
			for _, v := range bucket {
				value = math.Max(value, v)
			}
			return value
		},
	}

	for i, bucket := range buckets {
		if len(bucket) == 0 {
			values[i] = math.NaN()
			continue
		}
		values[i] = bucketSamplers[sampleMethod](bucket)
	}

	log.Printf("Constructed timeseries from result: %v", values)

	// TODO: Resample to the requested resolution

	return values, nil
}

// Blueflood keys the resolution param to a java enum, so we have to convert
// between them.
func bluefloodResolution(r int64) string {
	switch {
	case r < 5*60*1000:
		return ResolutionFull
	case r < 20*60*1000:
		return Resolution5Min
	case r < 60*60*1000:
		return Resolution20Min
	case r < 240*60*1000:
		return Resolution60Min
	case r < 1440*60*1000:
		return Resolution240Min
	}
	return Resolution1440Min
}

func resample(points []float64, currentResolution int64, expectedTimerange api.Timerange, sampleMethod api.SampleMethod) ([]float64, error) {
	return nil, errors.New("Not implemented")
}

var _ api.Backend = (*Blueflood)(nil)
<commit_msg>improve documentary comments<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blueflood

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/square/metrics/api"
)

type httpClient interface {
	// our own client to mock out the standard golang HTTP Client.
	Get(url string) (resp *http.Response, err error)
}

type Blueflood struct {
	api      api.API
	baseUrl  string
	tenantId string
	client   httpClient
}

type QueryResponse struct {
	Values []MetricPoint `json:"values"`
}

type MetricPoint struct {
	Points    int     `json:"numPoints"`
	Timestamp int64   `json:"timestamp"`
	Average   float64 `json:"average"`
	Max       float64 `json:"max"`
	Min       float64 `json:"min"`
	Variance  float64 `json:"variance"`
}

const (
	ResolutionFull    = "FULL"
	Resolution5Min    = "MIN5"
	Resolution20Min   = "MIN20"
	Resolution60Min   = "MIN60"
	Resolution240Min  = "MIN240"
	Resolution1440Min = "MIN1440"
)

func NewBlueflood(api api.API, baseUrl string, tenantId string) *Blueflood {
	return &Blueflood{api: api, baseUrl: baseUrl, tenantId: tenantId, client: http.DefaultClient}
}

func (b *Blueflood) Api() api.API {
	return b.api
}

func (b *Blueflood) FetchSeries(metric api.TaggedMetric, predicate api.Predicate, sampleMethod api.SampleMethod, timerange api.Timerange) (*api.SeriesList, error) {
	metricTagSets, err := b.Api().GetAllTags(metric.MetricKey)
	if err != nil {
		return nil, err
	}

	// TODO: Be a little smarter about this?
	resultSeries := []api.Timeseries{}
	for _, ts := range metricTagSets {
		if predicate == nil || predicate.Apply(ts) {
			graphiteName, err := b.Api().ToGraphiteName(api.TaggedMetric{
				MetricKey: metric.MetricKey,
				TagSet:    ts,
			})
			if err != nil {
				return nil, err
			}

			queryResult, err := b.fetchSingleSeries(
				graphiteName,
				sampleMethod,
				timerange,
			)
			// TODO: Be more tolerant of errors fetching a single metric?
			// Though I guess this behavior is fine since skipping fetches
			// that fail would end up in a result set that you don't quite
			// expect.
			if err != nil {
				return nil, err
			}

			resultSeries = append(resultSeries, api.Timeseries{
				Values: queryResult,
				TagSet: ts,
			})
		}
	}

	return &api.SeriesList{
		Series:    resultSeries,
		Timerange: timerange,
	}, nil
}

func addMetricPoint(metricPoint MetricPoint, field func(MetricPoint) float64, timerange api.Timerange, buckets [][]float64) error {
	// TODO: check the result of .FieldByName against nil (or a panic will occur)
	value := field(metricPoint)
	// TODO: check the result of .FieldByName against nil (or a panic will occur)
	timestamp := metricPoint.Timestamp
	// The index to assign within the array is computed using the timestamp.
	// It rounds to the nearest index.
	index := (timestamp - timerange.Start() + timerange.Resolution()/2) / timerange.Resolution()
	if index < 0 || index >= int64(timerange.Slots()) {
		return errors.New("index out of range")
	}
	buckets[index] = append(buckets[index], value)
	return nil
}

func bucketsFromMetricPoints(metricPoints []MetricPoint, resultField func(MetricPoint) float64, timerange api.Timerange) [][]float64 {
	buckets := make([][]float64, timerange.Slots())
	// Make the buckets:
	for i := range buckets {
		buckets[i] = []float64{}
	}
	for _, point := range metricPoints {
		addMetricPoint(point, resultField, timerange, buckets)
	}
	return buckets
}

func (b *Blueflood) fetchSingleSeries(metric api.GraphiteMetric, sampleMethod api.SampleMethod, timerange api.Timerange) ([]float64, error) {
	// Use this lowercase of this as the select query param. Use the actual value
	// to reflect into result MetricPoints to fetch the correct field.
	var (
		fieldExtractor    string
		selectResultField func(MetricPoint) float64
	)
	switch sampleMethod {
	case api.SampleMean:
		fieldExtractor = "average"
		selectResultField = func(point MetricPoint) float64 { return point.Average }
	case api.SampleMin:
		fieldExtractor = "min"
		selectResultField = func(point MetricPoint) float64 { return point.Min }
	case api.SampleMax:
		fieldExtractor = "max"
		selectResultField = func(point MetricPoint) float64 { return point.Max }
	default:
		return nil, errors.New(fmt.Sprintf("Unsupported SampleMethod %d", sampleMethod))
	}

	// Issue GET to fetch metrics
	queryUrl, err := url.Parse(fmt.Sprintf("%s/v2.0/%s/views/%s",
		b.baseUrl,
		b.tenantId,
		metric))
	if err != nil {
		return nil, err
	}

	params := url.Values{}
	params.Set("from", strconv.FormatInt(timerange.Start(), 10))
	params.Set("to", strconv.FormatInt(timerange.End(), 10))
	params.Set("resolution", bluefloodResolution(timerange.Resolution()))
	params.Set("select", fmt.Sprintf("numPoints,%s", strings.ToLower(fieldExtractor)))

	queryUrl.RawQuery = params.Encode()

	log.Printf("Blueflood fetch: %s", queryUrl.String())
	resp, err := b.client.Get(queryUrl.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	log.Printf("Fetch result: %s", string(body))

	var result QueryResponse
	err = json.Unmarshal(body, &result)
	if err != nil {
		return nil, err
	}

	// Construct a Timeseries from the result:

	// buckets are each filled with from the points stored in result.Values, according to their timestamps.
	buckets := bucketsFromMetricPoints(result.Values, selectResultField, timerange)

	// values will hold the final values to be returned as the series.
	values := make([]float64, timerange.Slots())

	// bucketSamplers is a map describing how to transform { bucket []float64 => values[i] float64 }.
	// each function may assume that the bucket it is given is non-empty (so indexing 0 is valid),
	// which is enforced by the caller (who substitutes NaN for such empty buckets).
	bucketSamplers := map[api.SampleMethod]func([]float64) float64{
		api.SampleMean: func(bucket []float64) float64 {
			value := 0.0
			for _, v := range bucket {
				value += v
			}
			return value / float64(len(bucket))
		},
		api.SampleMin: func(bucket []float64) float64 {
			value := bucket[0]
			for _, v := range bucket {
				value = math.Min(value, v)
			}
			return value
		},
		api.SampleMax: func(bucket []float64) float64 {
			value := bucket[0]
			for _, v := range bucket {
				value = math.Max(value, v)
			}
			return value
		},
	}
	// sampler is the desired function based on the given sampleMethod.
	sampler := bucketSamplers[sampleMethod]
	for i, bucket := range buckets {
		if len(bucket) == 0 {
			values[i] = math.NaN()
			continue
		}
		values[i] = sampler(bucket)
	}

	log.Printf("Constructed timeseries from result: %v", values)

	// TODO: Resample to the requested resolution

	return values, nil
}

// Blueflood keys the resolution param to a java enum, so we have to convert
// between them.
func bluefloodResolution(r int64) string {
	switch {
	case r < 5*60*1000:
		return ResolutionFull
	case r < 20*60*1000:
		return Resolution5Min
	case r < 60*60*1000:
		return Resolution20Min
	case r < 240*60*1000:
		return Resolution60Min
	case r < 1440*60*1000:
		return Resolution240Min
	}
	return Resolution1440Min
}

func resample(points []float64, currentResolution int64, expectedTimerange api.Timerange, sampleMethod api.SampleMethod) ([]float64, error) {
	return nil, errors.New("Not implemented")
}

var _ api.Backend = (*Blueflood)(nil)
<|endoftext|>


<commit_before>package gerrittest

import (
	"os/exec"

	log "github.com/Sirupsen/logrus"
)

// Diff is a struct which represents a single commit to the
// repository.
type Diff struct {
	// Error should be set whenever
	Error   error
	Content []byte
}

// ApplyToRoot will apply the given diff to the provided repository root.
func (d *Diff) ApplyToRoot(root string) error {
	if d.Error != nil {
		return d.Error
	}

	cmd := exec.Command(
		"git", "-C", root, "apply")
	writer, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	if _, err := writer.Write(d.Content); err != nil {
		return err
	}
	if err := writer.Close(); err != nil {
		return err
	}
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.WithError(err).WithField("out", string(out)).Error()
		return err
	}
	return nil
}
<commit_msg>added ApplyToRepository(), more logging<commit_after>package gerrittest

import (
	"os/exec"

	log "github.com/Sirupsen/logrus"
)

// Diff is a struct which represents a single commit to the
// repository.
type Diff struct {
	// Error should be set whenever
	Error   error
	Content []byte
	Commit  string
}

// ApplyToRoot will apply the given diff to the provided repository root.
func (d *Diff) ApplyToRoot(root string) error {
	if d.Error != nil {
		return d.Error
	}
	logger := log.WithFields(log.Fields{
		"phase":  "apply-diff",
		"patch":  string(d.Content),
		"action": "apply-to-root",
	})
	cmd := exec.Command(
		"git", "-C", root, "apply", "-v")
	writer, err := cmd.StdinPipe()
	if err != nil {
		logger.WithError(err).Error()
		return err
	}
	if _, err := writer.Write(d.Content); err != nil {
		logger.WithError(err).Error()
		return err
	}
	if err := writer.Close(); err != nil {
		logger.WithError(err).Error()
		return err
	}
	out, err := cmd.CombinedOutput()
	logger = logger.WithField("output", string(out))
	if err != nil {
		logger.WithError(err).Error()
		return err
	}
	return nil
}

// ApplyToRepository takes the diff and applies it to the given
// repository using ApplyToRoot() after which this function will amend
// the current commit. It's up to the caller to push the change.
func (d *Diff) ApplyToRepository(repo *Repository) error {
	if err := d.ApplyToRoot(repo.Root); err != nil {
		return err
	}
	logger := log.WithFields(log.Fields{
		"phase":  "apply-diff",
		"action": "add-all",
	})
	if _, _, err := repo.Run([]string{"add", "--all"}); err != nil {
		logger.WithError(err).Error()
		return err
	}
	logger.Debug()
	return repo.Amend()
}
<|endoftext|>


<commit_before>package matchers

import (
	"fmt"
	"strings"

	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/onsi/gomega"
)

type SliceMatcher struct {
	expected      [][]string
	failedAtIndex int
}

func ContainSubstrings(substrings ...[]string) gomega.OmegaMatcher {
	return &SliceMatcher{expected: substrings}
}

func (matcher *SliceMatcher) Match(actual interface{}) (success bool, err error) {
	actualStrings, ok := actual.([]string)
	if !ok {
		return false, nil
	}

	matcher.failedAtIndex = 0
	for _, actualValue := range actualStrings {
		allStringsFound := true
		for _, expectedValue := range matcher.expected[matcher.failedAtIndex] {
			allStringsFound = allStringsFound && strings.Contains(terminal.Decolorize(actualValue), expectedValue)
		}

		if allStringsFound {
			matcher.failedAtIndex++
			if matcher.failedAtIndex == len(matcher.expected) {
				return true, nil
			}
		}
	}

	return false, nil
}

func (matcher *SliceMatcher) FailureMessage(actual interface{}) string {
	actualStrings, ok := actual.([]string)
	if !ok {
		return fmt.Sprintf("Expected actual to be a slice of strings, but it's actually a %T", actual)
	}

	return fmt.Sprintf("expected to find \"%s\" in actual:\n'%s'\n", matcher.expected[matcher.failedAtIndex], strings.Join(actualStrings, "\n"))
}

func (matcher *SliceMatcher) NegatedFailureMessage(actual interface{}) string {
	actualStrings, ok := actual.([]string)
	if !ok {
		return fmt.Sprintf("Expected actual to be a slice of strings, but it's actually a %T", actual)
	}
	return fmt.Sprintf("expected to not find \"%s\" in actual:\n'%s'\n", matcher.expected[matcher.failedAtIndex], strings.Join(actualStrings, "\n"))
}
<commit_msg>fix ContainSubstrings to properly handle the negative case<commit_after>package matchers

import (
	"fmt"
	"strings"

	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/onsi/gomega"
)

type SliceMatcher struct {
	expected      [][]string
	failedAtIndex int
}

func ContainSubstrings(substrings ...[]string) gomega.OmegaMatcher {
	return &SliceMatcher{expected: substrings}
}

func (matcher *SliceMatcher) Match(actual interface{}) (success bool, err error) {
	actualStrings, ok := actual.([]string)
	if !ok {
		return false, nil
	}

	matcher.failedAtIndex = 0
	for _, actualValue := range actualStrings {
		allStringsFound := true
		for _, expectedValue := range matcher.expected[matcher.failedAtIndex] {
			allStringsFound = allStringsFound && strings.Contains(terminal.Decolorize(actualValue), expectedValue)
		}

		if allStringsFound {
			matcher.failedAtIndex++
			if matcher.failedAtIndex == len(matcher.expected) {
				matcher.failedAtIndex--
				return true, nil
			}
		}
	}

	return false, nil
}

func (matcher *SliceMatcher) FailureMessage(actual interface{}) string {
	actualStrings, ok := actual.([]string)
	if !ok {
		return fmt.Sprintf("Expected actual to be a slice of strings, but it's actually a %T", actual)
	}

	return fmt.Sprintf("expected to find \"%s\" in actual:\n'%s'\n", matcher.expected[matcher.failedAtIndex], strings.Join(actualStrings, "\n"))
}

func (matcher *SliceMatcher) NegatedFailureMessage(actual interface{}) string {
	actualStrings, ok := actual.([]string)
	if !ok {
		return fmt.Sprintf("Expected actual to be a slice of strings, but it's actually a %T", actual)
	}
	return fmt.Sprintf("expected to not find \"%s\" in actual:\n'%s'\n", matcher.expected[matcher.failedAtIndex], strings.Join(actualStrings, "\n"))
}
<|endoftext|>


<commit_before>package MQTTg

type Edge interface {
	recvConnectMessage(*ConnectMessage, *Client) error
	recvConnackMessage(*ConnackMessage, *Client) error
	recvPublishMessage(*PublishMessage, *Client) error
	recvPubackMessage(*PubackMessage, *Client) error
	recvPubrecMessage(*PubrecMessage, *Client) error
	recvPubrelMessage(*PubrelMessage, *Client) error
	recvPubcompMessage(*PubcompMessage, *Client) error
	recvSubscribeMessage(*SubscribeMessage, *Client) error
	recvSubackMessage(*SubackMessage, *Client) error
	recvUnsubscribeMessage(*UnsubscribeMessage, *Client) error
	recvUnsubackMessage(*UnsubackMessage, *Client) error
	recvPingreqMessage(*PingreqMessage, *Client) error
	recvPingrespMessage(*PingrespMessage, *Client) error
	recvDisconnectMessage(*DisconnectMessage, *Client) error
	ReadMessage() (Message, error)
}

func ReadLoop(edge Edge, c *Client) error {
	for {
		m, err := edge.ReadMessage()
		if err != nil {
			EmitError(err)
			continue
		}
		switch m := m.(type) {
		case *ConnectMessage:
			err = edge.recvConnectMessage(m, c)
		case *ConnackMessage:
			err = edge.recvConnackMessage(m, c)
		case *PublishMessage:
			err = edge.recvPublishMessage(m, c)
		case *PubackMessage:
			err = edge.recvPubackMessage(m, c)
		case *PubrecMessage:
			err = edge.recvPubrecMessage(m, c)
		case *PubrelMessage:
			err = edge.recvPubrelMessage(m, c)
		case *PubcompMessage:
			err = edge.recvPubcompMessage(m, c)
		case *SubscribeMessage:
			err = edge.recvSubscribeMessage(m, c)
		case *SubackMessage:
			err = edge.recvSubackMessage(m, c)
		case *UnsubscribeMessage:
			err = edge.recvUnsubscribeMessage(m, c)
		case *UnsubackMessage:
			err = edge.recvUnsubackMessage(m, c)
		case *PingreqMessage:
			err = edge.recvPingreqMessage(m, c)
		case *PingrespMessage:
			err = edge.recvPingrespMessage(m, c)
		case *DisconnectMessage:
			err = edge.recvDisconnectMessage(m, c)

		}
		EmitError(err)
	}
}
<commit_msg>fix bug<commit_after>package MQTTg

type Edge interface {
	recvConnectMessage(*ConnectMessage, *Client) error
	recvConnackMessage(*ConnackMessage, *Client) error
	recvPublishMessage(*PublishMessage, *Client) error
	recvPubackMessage(*PubackMessage, *Client) error
	recvPubrecMessage(*PubrecMessage, *Client) error
	recvPubrelMessage(*PubrelMessage, *Client) error
	recvPubcompMessage(*PubcompMessage, *Client) error
	recvSubscribeMessage(*SubscribeMessage, *Client) error
	recvSubackMessage(*SubackMessage, *Client) error
	recvUnsubscribeMessage(*UnsubscribeMessage, *Client) error
	recvUnsubackMessage(*UnsubackMessage, *Client) error
	recvPingreqMessage(*PingreqMessage, *Client) error
	recvPingrespMessage(*PingrespMessage, *Client) error
	recvDisconnectMessage(*DisconnectMessage, *Client) error
	ReadMessage() (Message, error)
}

func ReadLoop(edge Edge, c *Client) (err error) {
	var m Message
	for {
		EmitError(err)
		// TODO: not cool
		if c == nil {
			m, err = edge.ReadMessage()
		} else {
			m, err = c.ReadMessage()
		}
		if err != nil {
			EmitError(err)
			continue
		}
		switch m := m.(type) {
		case *ConnectMessage:
			err = edge.recvConnectMessage(m, c)
		case *ConnackMessage:
			err = edge.recvConnackMessage(m, c)
		case *PublishMessage:
			err = edge.recvPublishMessage(m, c)
		case *PubackMessage:
			err = edge.recvPubackMessage(m, c)
		case *PubrecMessage:
			err = edge.recvPubrecMessage(m, c)
		case *PubrelMessage:
			err = edge.recvPubrelMessage(m, c)
		case *PubcompMessage:
			err = edge.recvPubcompMessage(m, c)
		case *SubscribeMessage:
			err = edge.recvSubscribeMessage(m, c)
		case *SubackMessage:
			err = edge.recvSubackMessage(m, c)
		case *UnsubscribeMessage:
			err = edge.recvUnsubscribeMessage(m, c)
		case *UnsubackMessage:
			err = edge.recvUnsubackMessage(m, c)
		case *PingreqMessage:
			err = edge.recvPingreqMessage(m, c)
		case *PingrespMessage:
			err = edge.recvPingrespMessage(m, c)
		case *DisconnectMessage:
			err = edge.recvDisconnectMessage(m, c)

		}
	}
}
<|endoftext|>


<commit_before>// EDNS0
//
// EDNS0 is an extension mechanism for the DNS defined in RFC 2671. It defines a
// standard RR type, the OPT RR, which is then completely abused.
// Basic use pattern for creating an (empty) OPT RR:
//
//	o := new(dns.OPT)
//	o.Hdr.Name = "." // MUST be the root zone, per definition.
//	o.Hdr.Rrtype = dns.TypeOPT
//
// The rdata of an OPT RR consists out of a slice of EDNS0 interfaces. Currently
// only a few have been standardized: EDNS0_NSID (RFC 5001) and EDNS0_SUBNET (draft). Note that
// these options may be combined in an OPT RR.
// Basic use pattern for a server to check if (and which) options are set:
//
//	// o is a dns.OPT
//	for _, s := range o.Option {
//		switch e := s.(type) {
//		case *dns.EDNS0_NSID:
//			// do stuff with e.Nsid
//		case *dns.EDNS0_SUBNET:
//			// access e.Family, e.Address, etc.
//		}
//	}
package dns

import (
	"encoding/hex"
	"errors"
	"net"
	"strconv"
)

// EDNS0 Option codes.
const (
	EDNS0LLQ         = 0x1    // not used
	EDNS0UL          = 0x2    // not used
	EDNS0UPDATELEASE = 0x2    // update lease draft
	EDNS0NSID        = 0x3    // nsid (RFC5001)
	EDNS0SUBNET      = 0x50fa // client-subnet draft
	_DO              = 1 << 7 // dnssec ok
)

type OPT struct {
	Hdr    RR_Header
	Option []EDNS0 `dns:"opt"`
}

func (rr *OPT) Header() *RR_Header {
	return &rr.Hdr
}

func (rr *OPT) String() string {
	s := "\n;; OPT PSEUDOSECTION:\n; EDNS: version " + strconv.Itoa(int(rr.Version())) + "; "
	if rr.Do() {
		s += "flags: do; "
	} else {
		s += "flags: ; "
	}
	s += "udp: " + strconv.Itoa(int(rr.UDPSize()))

	for _, o := range rr.Option {
		switch o.(type) {
		case *EDNS0_NSID:
			s += "\n; NSID: " + o.String()
			h, e := o.pack()
			var r string
			if e == nil {
				for _, c := range h {
					r += "(" + string(c) + ")"
				}
				s += "  " + r
			}
		case *EDNS0_SUBNET:
			s += "\n; SUBNET: " + o.String()
		case *EDNS0_UPDATE_LEASE:
			s += "\n; LEASE: " + o.String()
		}
	}
	return s
}

func (rr *OPT) Len() int {
	l := rr.Hdr.Len()
	for i := 0; i < len(rr.Option); i++ {
		lo, _ := rr.Option[i].pack()
		l += 2 + len(lo)
	}
	return l
}

func (rr *OPT) Copy() RR {
	return &OPT{*rr.Hdr.CopyHeader(), rr.Option}
}

// Version returns the EDNS version used. Only zero is defined.
func (rr *OPT) Version() uint8 {
	return uint8(rr.Hdr.Ttl & 0x00FF00FFFF)
}

// SetVersion sets the version of EDNS. This is usually zero.
func (rr *OPT) SetVersion(v uint8) {
	rr.Hdr.Ttl = rr.Hdr.Ttl&0xFF00FFFF | uint32(v)
}

// UDPSize returns the UDP buffer size.
func (rr *OPT) UDPSize() uint16 {
	return rr.Hdr.Class
}

// SetUDPSize sets the UDP buffer size.
func (rr *OPT) SetUDPSize(size uint16) {
	rr.Hdr.Class = size
}

// Do returns the value of the DO (DNSSEC OK) bit.
func (rr *OPT) Do() bool {
	return byte(rr.Hdr.Ttl>>8)&_DO == _DO
}

// SetDo sets the DO (DNSSEC OK) bit.
func (rr *OPT) SetDo() {
	b1 := byte(rr.Hdr.Ttl >> 24)
	b2 := byte(rr.Hdr.Ttl >> 16)
	b3 := byte(rr.Hdr.Ttl >> 8)
	b4 := byte(rr.Hdr.Ttl)
	b3 |= _DO // Set it
	rr.Hdr.Ttl = uint32(b1)<<24 | uint32(b2)<<16 | uint32(b3)<<8 | uint32(b4)
}

// EDNS0 defines an EDNS0 Option. An OPT RR can have multiple options appended to
// it. Basic use pattern for adding an option to and OPT RR:
//
//	// o is the OPT RR, e is the EDNS0 option
//	o.Option = append(o.Option, e)
type EDNS0 interface {
	// Option returns the option code for the option.
	Option() uint16
	// pack returns the bytes of the option data.
	pack() ([]byte, error)
	// unpack sets the data as found in the buffer. Is also sets
	// the length of the slice as the length of the option data.
	unpack([]byte)
	// String returns the string representation of the option.
	String() string
}

// The nsid EDNS0 option is used to retrieve some sort of nameserver
// identifier. When seding a request Nsid must be set to the empty string
// The identifier is an opaque string encoded as hex.
// Basic use pattern for creating an nsid option:
//
//	o := new(dns.OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_NSID)
//	e.Code = dns.EDNS0NSID
//	o.Option = append(o.Option, e)
type EDNS0_NSID struct {
	Code uint16 // Always EDNS0NSID
	Nsid string // This string needs to be hex encoded
}

func (e *EDNS0_NSID) Option() uint16 {
	return EDNS0NSID
}

func (e *EDNS0_NSID) pack() ([]byte, error) {
	h, err := hex.DecodeString(e.Nsid)
	if err != nil {
		return nil, err
	}
	return h, nil
}

func (e *EDNS0_NSID) unpack(b []byte) {
	e.Nsid = hex.EncodeToString(b)
}

func (e *EDNS0_NSID) String() string {
	return string(e.Nsid)
}

// The subnet EDNS0 option is used to give the remote nameserver
// an idea of where the client lives. It can then give back a different
// answer depending on the location or network topology.
// Basic use pattern for creating an subnet option:
//
//	o := new(dns.OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_SUBNET)
//	e.Code = dns.EDNS0SUBNET
//	e.Family = 1	// 1 for IPv4 source address, 2 for IPv6
//	e.NetMask = 32	// 32 for IPV4, 128 for IPv6
//	e.SourceScope = 0
//	e.Address = net.ParseIP("127.0.0.1").To4()	// for IPv4
//	// e.Address = net.ParseIP("2001:7b8:32a::2")	// for IPV6
//	o.Option = append(o.Option, e)
type EDNS0_SUBNET struct {
	Code          uint16 // Always EDNS0SUBNET
	Family        uint16 // 1 for IP, 2 for IP6
	SourceNetmask uint8
	SourceScope   uint8
	Address       net.IP
}

func (e *EDNS0_SUBNET) Option() uint16 {
	return EDNS0SUBNET
}

func (e *EDNS0_SUBNET) pack() ([]byte, error) {
	b := make([]byte, 4)
	b[0], b[1] = packUint16(e.Family)
	b[2] = e.SourceNetmask
	b[3] = e.SourceScope
	switch e.Family {
	case 1:
		if e.SourceNetmask > net.IPv4len*8 {
			return nil, errors.New("dns: bad netmask")
		}
		ip := make([]byte, net.IPv4len)
		a := e.Address.To4().Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv4len*8))
		for i := 0; i < net.IPv4len; i++ {
			if i+1 > len(e.Address) {
				break
			}
			ip[i] = a[i]
		}
		b = append(b, ip...)
	case 2:
		if e.SourceNetmask > net.IPv6len*8 {
			return nil, errors.New("dns: bad netmask")
		}
		ip := make([]byte, net.IPv6len)
		a := e.Address.Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv6len*8))
		for i := 0; i < net.IPv6len; i++ {
			if i+1 > len(e.Address) {
				break
			}
			ip[i] = a[i]
		}
		b = append(b, ip...)
	default:
		return nil, errors.New("dns: bad address family")
	}
	return b, nil
}

func (e *EDNS0_SUBNET) unpack(b []byte) {
	lb := len(b)
	if lb < 4 {
		return
	}
	e.Family, _ = unpackUint16(b, 0)
	e.SourceNetmask = b[2]
	e.SourceScope = b[3]
	switch e.Family {
	case 1:
		addr := make([]byte, 4)
		for i := 0; i < int(e.SourceNetmask/8); i++ {
			if 4+i > len(b) {
				break
			}
			addr[i] = b[4+i]
		}
		e.Address = net.IPv4(addr[0], addr[1], addr[2], addr[3])
	case 2:
		addr := make([]byte, 16)
		for i := 0; i < int(e.SourceNetmask/8); i++ {
			if 4+i > len(b) {
				break
			}
			addr[i] = b[4+i]
		}
		e.Address = net.IP{addr[0], addr[1], addr[2], addr[3], addr[4],
			addr[5], addr[6], addr[7], addr[8], addr[9], addr[10],
			addr[11], addr[12], addr[13], addr[14], addr[15]}
	}
	return
}

func (e *EDNS0_SUBNET) String() (s string) {
	if e.Address == nil {
		s = "<nil>"
	} else if e.Address.To4() != nil {
		s = e.Address.String()
	} else {
		s = "[" + e.Address.String() + "]"
	}
	s += "/" + strconv.Itoa(int(e.SourceNetmask)) + "/" + strconv.Itoa(int(e.SourceScope))
	return
}

// The UPDATE_LEASE EDNS0 (draft RFC) option is used to tell the server to set
// an expiration on an update RR. This is helpful for clients that cannot clean
// up after themselves. This is a draft RFC and more information can be found at
// http://files.dns-sd.org/draft-sekar-dns-ul.txt
//
//	o := new(dns.OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_UPDATE_LEASE)
//	e.Code = dns.EDNS0UPDATELEASE
//	e.Lease = 120 // in seconds
//	o.Option = append(o.Option, e)

type EDNS0_UPDATE_LEASE struct {
	Code  uint16 // Always EDNS0UPDATELEASE
	Lease uint32
}

func (e *EDNS0_UPDATE_LEASE) Option() uint16 {
	return EDNS0UPDATELEASE
}

// Copied: http://golang.org/src/pkg/net/dnsmsg.go
func (e *EDNS0_UPDATE_LEASE) pack() ([]byte, error) {
	b := make([]byte, 4)
	b[0] = byte(e.Lease >> 24)
	b[1] = byte(e.Lease >> 16)
	b[2] = byte(e.Lease >> 8)
	b[3] = byte(e.Lease)
	return b, nil
}

func (e *EDNS0_UPDATE_LEASE) unpack(b []byte) {
	e.Lease = uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
}

func (e *EDNS0_UPDATE_LEASE) String() string {
	return strconv.Itoa(int(e.Lease))
}
<commit_msg>add todo<commit_after>// EDNS0
//
// EDNS0 is an extension mechanism for the DNS defined in RFC 2671. It defines a
// standard RR type, the OPT RR, which is then completely abused.
// Basic use pattern for creating an (empty) OPT RR:
//
//	o := new(dns.OPT)
//	o.Hdr.Name = "." // MUST be the root zone, per definition.
//	o.Hdr.Rrtype = dns.TypeOPT
//
// The rdata of an OPT RR consists out of a slice of EDNS0 interfaces. Currently
// only a few have been standardized: EDNS0_NSID (RFC 5001) and EDNS0_SUBNET (draft). Note that
// these options may be combined in an OPT RR.
// Basic use pattern for a server to check if (and which) options are set:
//
//	// o is a dns.OPT
//	for _, s := range o.Option {
//		switch e := s.(type) {
//		case *dns.EDNS0_NSID:
//			// do stuff with e.Nsid
//		case *dns.EDNS0_SUBNET:
//			// access e.Family, e.Address, etc.
//		}
//	}
package dns

import (
	"encoding/hex"
	"errors"
	"net"
	"strconv"
)

// EDNS0 Option codes.
const (
	EDNS0LLQ         = 0x1    // not used
	EDNS0UL          = 0x2    // not used
	EDNS0UPDATELEASE = 0x2    // update lease draft
	EDNS0NSID        = 0x3    // nsid (RFC5001)
	EDNS0SUBNET      = 0x50fa // client-subnet draft
	_DO              = 1 << 7 // dnssec ok
)

type OPT struct {
	Hdr    RR_Header
	Option []EDNS0 `dns:"opt"`
}

func (rr *OPT) Header() *RR_Header {
	return &rr.Hdr
}

func (rr *OPT) String() string {
	s := "\n;; OPT PSEUDOSECTION:\n; EDNS: version " + strconv.Itoa(int(rr.Version())) + "; "
	if rr.Do() {
		s += "flags: do; "
	} else {
		s += "flags: ; "
	}
	s += "udp: " + strconv.Itoa(int(rr.UDPSize()))

	for _, o := range rr.Option {
		switch o.(type) {
		case *EDNS0_NSID:
			s += "\n; NSID: " + o.String()
			h, e := o.pack()
			var r string
			if e == nil {
				for _, c := range h {
					r += "(" + string(c) + ")"
				}
				s += "  " + r
			}
		case *EDNS0_SUBNET:
			s += "\n; SUBNET: " + o.String()
		case *EDNS0_UPDATE_LEASE:
			s += "\n; LEASE: " + o.String()
		}
	}
	return s
}

func (rr *OPT) Len() int {
	l := rr.Hdr.Len()
	for i := 0; i < len(rr.Option); i++ {
		lo, _ := rr.Option[i].pack()
		l += 2 + len(lo)
	}
	return l
}

func (rr *OPT) Copy() RR {
	return &OPT{*rr.Hdr.CopyHeader(), rr.Option}
}

// Version returns the EDNS version used. Only zero is defined.
func (rr *OPT) Version() uint8 {
	return uint8(rr.Hdr.Ttl & 0x00FF00FFFF)
}

// SetVersion sets the version of EDNS. This is usually zero.
func (rr *OPT) SetVersion(v uint8) {
	rr.Hdr.Ttl = rr.Hdr.Ttl&0xFF00FFFF | uint32(v)
}

// UDPSize returns the UDP buffer size.
func (rr *OPT) UDPSize() uint16 {
	return rr.Hdr.Class
}

// SetUDPSize sets the UDP buffer size.
func (rr *OPT) SetUDPSize(size uint16) {
	rr.Hdr.Class = size
}

// Do returns the value of the DO (DNSSEC OK) bit.
func (rr *OPT) Do() bool {
	return byte(rr.Hdr.Ttl>>8)&_DO == _DO
}

// SetDo sets the DO (DNSSEC OK) bit.
func (rr *OPT) SetDo() {
	b1 := byte(rr.Hdr.Ttl >> 24)
	b2 := byte(rr.Hdr.Ttl >> 16)
	b3 := byte(rr.Hdr.Ttl >> 8)
	b4 := byte(rr.Hdr.Ttl)
	b3 |= _DO // Set it
	rr.Hdr.Ttl = uint32(b1)<<24 | uint32(b2)<<16 | uint32(b3)<<8 | uint32(b4)
}

// EDNS0 defines an EDNS0 Option. An OPT RR can have multiple options appended to
// it. Basic use pattern for adding an option to and OPT RR:
//
//	// o is the OPT RR, e is the EDNS0 option
//	o.Option = append(o.Option, e)
type EDNS0 interface {
	// Option returns the option code for the option.
	Option() uint16
	// pack returns the bytes of the option data.
	pack() ([]byte, error)
	// unpack sets the data as found in the buffer. Is also sets
	// the length of the slice as the length of the option data.
	unpack([]byte)
	// String returns the string representation of the option.
	String() string
}

// The nsid EDNS0 option is used to retrieve some sort of nameserver
// identifier. When seding a request Nsid must be set to the empty string
// The identifier is an opaque string encoded as hex.
// Basic use pattern for creating an nsid option:
//
//	o := new(dns.OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_NSID)
//	e.Code = dns.EDNS0NSID
//	o.Option = append(o.Option, e)
type EDNS0_NSID struct {
	Code uint16 // Always EDNS0NSID
	Nsid string // This string needs to be hex encoded
}

func (e *EDNS0_NSID) Option() uint16 {
	return EDNS0NSID
}

func (e *EDNS0_NSID) pack() ([]byte, error) {
	h, err := hex.DecodeString(e.Nsid)
	if err != nil {
		return nil, err
	}
	return h, nil
}

func (e *EDNS0_NSID) unpack(b []byte) {
	e.Nsid = hex.EncodeToString(b)
}

func (e *EDNS0_NSID) String() string {
	return string(e.Nsid)
}

// The subnet EDNS0 option is used to give the remote nameserver
// an idea of where the client lives. It can then give back a different
// answer depending on the location or network topology.
// Basic use pattern for creating an subnet option:
//
//	o := new(dns.OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_SUBNET)
//	e.Code = dns.EDNS0SUBNET
//	e.Family = 1	// 1 for IPv4 source address, 2 for IPv6
//	e.NetMask = 32	// 32 for IPV4, 128 for IPv6
//	e.SourceScope = 0
//	e.Address = net.ParseIP("127.0.0.1").To4()	// for IPv4
//	// e.Address = net.ParseIP("2001:7b8:32a::2")	// for IPV6
//	o.Option = append(o.Option, e)
type EDNS0_SUBNET struct {
	Code          uint16 // Always EDNS0SUBNET
	Family        uint16 // 1 for IP, 2 for IP6
	SourceNetmask uint8
	SourceScope   uint8
	Address       net.IP
}

func (e *EDNS0_SUBNET) Option() uint16 {
	return EDNS0SUBNET
}

func (e *EDNS0_SUBNET) pack() ([]byte, error) {
	b := make([]byte, 4)
	b[0], b[1] = packUint16(e.Family)
	b[2] = e.SourceNetmask
	b[3] = e.SourceScope
	switch e.Family {
	case 1:
		if e.SourceNetmask > net.IPv4len*8 {
			return nil, errors.New("dns: bad netmask")
		}
		ip := make([]byte, net.IPv4len)
		a := e.Address.To4().Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv4len*8))
		for i := 0; i < net.IPv4len; i++ {
			if i+1 > len(e.Address) {
				break
			}
			ip[i] = a[i]
		}
		b = append(b, ip...)
	case 2:
		if e.SourceNetmask > net.IPv6len*8 {
			return nil, errors.New("dns: bad netmask")
		}
		ip := make([]byte, net.IPv6len)
		a := e.Address.Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv6len*8))
		for i := 0; i < net.IPv6len; i++ {
			if i+1 > len(e.Address) {
				break
			}
			ip[i] = a[i]
		}
		// chop off ip a SourceNetmask/8: ip = ip[:e.SourceNetmask/8] ?
		b = append(b, ip...)
	default:
		return nil, errors.New("dns: bad address family")
	}
	return b, nil
}

func (e *EDNS0_SUBNET) unpack(b []byte) {
	lb := len(b)
	if lb < 4 {
		return
	}
	e.Family, _ = unpackUint16(b, 0)
	e.SourceNetmask = b[2]
	e.SourceScope = b[3]
	switch e.Family {
	case 1:
		addr := make([]byte, 4)
		for i := 0; i < int(e.SourceNetmask/8); i++ {
			if 4+i > len(b) {
				break
			}
			addr[i] = b[4+i]
		}
		e.Address = net.IPv4(addr[0], addr[1], addr[2], addr[3])
	case 2:
		addr := make([]byte, 16)
		for i := 0; i < int(e.SourceNetmask/8); i++ {
			if 4+i > len(b) {
				break
			}
			addr[i] = b[4+i]
		}
		e.Address = net.IP{addr[0], addr[1], addr[2], addr[3], addr[4],
			addr[5], addr[6], addr[7], addr[8], addr[9], addr[10],
			addr[11], addr[12], addr[13], addr[14], addr[15]}
	}
	return
}

func (e *EDNS0_SUBNET) String() (s string) {
	if e.Address == nil {
		s = "<nil>"
	} else if e.Address.To4() != nil {
		s = e.Address.String()
	} else {
		s = "[" + e.Address.String() + "]"
	}
	s += "/" + strconv.Itoa(int(e.SourceNetmask)) + "/" + strconv.Itoa(int(e.SourceScope))
	return
}

// The UPDATE_LEASE EDNS0 (draft RFC) option is used to tell the server to set
// an expiration on an update RR. This is helpful for clients that cannot clean
// up after themselves. This is a draft RFC and more information can be found at
// http://files.dns-sd.org/draft-sekar-dns-ul.txt
//
//	o := new(dns.OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_UPDATE_LEASE)
//	e.Code = dns.EDNS0UPDATELEASE
//	e.Lease = 120 // in seconds
//	o.Option = append(o.Option, e)

type EDNS0_UPDATE_LEASE struct {
	Code  uint16 // Always EDNS0UPDATELEASE
	Lease uint32
}

func (e *EDNS0_UPDATE_LEASE) Option() uint16 {
	return EDNS0UPDATELEASE
}

// Copied: http://golang.org/src/pkg/net/dnsmsg.go
func (e *EDNS0_UPDATE_LEASE) pack() ([]byte, error) {
	b := make([]byte, 4)
	b[0] = byte(e.Lease >> 24)
	b[1] = byte(e.Lease >> 16)
	b[2] = byte(e.Lease >> 8)
	b[3] = byte(e.Lease)
	return b, nil
}

func (e *EDNS0_UPDATE_LEASE) unpack(b []byte) {
	e.Lease = uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
}

func (e *EDNS0_UPDATE_LEASE) String() string {
	return strconv.Itoa(int(e.Lease))
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"github.com/jacobsa/aws/exp/sdb"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"math/rand"
	"sync"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type integrationTest struct {
	db sdb.SimpleDB
}

func (t *integrationTest) SetUp(i *TestInfo) {
	var err error

	// Open a connection.
	t.db, err = sdb.NewSimpleDB(g_region, g_accessKey)
	AssertEq(nil, err)
}

// Generate an item name likely to be unique.
func (t *integrationTest) makeItemName() sdb.ItemName {
	return sdb.ItemName(fmt.Sprintf("item.%16x", uint64(rand.Int63())))
}

func sortByName(attrs []sdb.Attribute) []sdb.Attribute

////////////////////////////////////////////////////////////////////////
// Domains
////////////////////////////////////////////////////////////////////////

var g_domainsTestDb sdb.SimpleDB
var g_domainsTestDomain0 sdb.Domain
var g_domainsTestDomain1 sdb.Domain

type DomainsTest struct {
	integrationTest

	mutex           sync.Mutex
	domainsToDelete []sdb.Domain  // Protected by mutex
}

func init() { RegisterTestSuite(&DomainsTest{}) }

func (t *DomainsTest) SetUpTestSuite() {
	var err error

	// Open a connection.
	g_domainsTestDb, err = sdb.NewSimpleDB(g_region, g_accessKey)
	if err != nil {
		panic(err)
	}

	// Create domain 0.
	g_domainsTestDomain0, err = g_domainsTestDb.OpenDomain("DomainsTest.domain0")
	if err != nil {
		panic(err)
	}

	// Create domain 1.
	g_domainsTestDomain1, err = g_domainsTestDb.OpenDomain("DomainsTest.domain1")
	if err != nil {
		panic(err)
	}
}

func (t *DomainsTest) TearDownTestSuite() {
	// Delete both domains.
	if err := g_domainsTestDb.DeleteDomain(g_domainsTestDomain0); err != nil {
		panic(err)
	}

	if err := g_domainsTestDb.DeleteDomain(g_domainsTestDomain1); err != nil {
		panic(err)
	}

	// Clear variables.
	g_domainsTestDb = nil
	g_domainsTestDomain0 = nil
	g_domainsTestDomain1 = nil
}

func (t *DomainsTest) TearDown() {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	// Delete each of the domains created during the test.
	for _, d := range t.domainsToDelete {
		ExpectEq(nil, t.db.DeleteDomain(d), "Domain: %s", d.Name())
	}
}

func (t *DomainsTest) InvalidAccessKey() {
	// Open a connection with an unknown key ID.
	wrongKey := g_accessKey
	wrongKey.Id += "taco"

	db, err := sdb.NewSimpleDB(g_region, wrongKey)
	AssertEq(nil, err)

	// Attempt to create a domain.
	_, err = db.OpenDomain("some_domain")

	ExpectThat(err, Error(HasSubstr("403")))
	ExpectThat(err, Error(HasSubstr("Key Id")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *DomainsTest) SeparatelyNamedDomainsHaveIndependentItems() {
	var err error

	// Set up an item in the first domain.
	itemName := t.makeItemName()
	err = g_domainsTestDomain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in the other domain. There should be
	// none.
	attrs, err := g_domainsTestDomain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(attrs, ElementsAre())
}

func (t *DomainsTest) IdenticallyNamedDomainsHaveIdenticalItems() {
	var err error

	// Set up an item in the first domain.
	itemName := t.makeItemName()
	err = g_domainsTestDomain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in another domain object opened with the
	// same name.
	domain1, err := t.db.OpenDomain(g_domainsTestDomain0.Name())
	AssertEq(nil, err)

	attrs, err := domain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(
		attrs,
		ElementsAre(
			DeepEquals(sdb.Attribute{Name: "enchilada", Value: "queso"}),
		),
	)
}

func (t *DomainsTest) Delete() {
	var err error
	domainName := "DomainsTest.Delete"

	// Create a domain, then delete it.
	domain, err := t.db.OpenDomain(domainName)
	AssertEq(nil, err)

	err = t.db.DeleteDomain(domain)
	AssertEq(nil, err)

	// Delete again; nothing should go wrong.
	err = t.db.DeleteDomain(domain)
	AssertEq(nil, err)

	// Attempt to write to the domain.
	err = domain.PutAttributes(
		"some_item",
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "foo", Value: "bar"},
		},
		[]sdb.Precondition{},
	)

	ExpectThat(err, Error(HasSubstr("NoSuchDomain")))
	ExpectThat(err, Error(HasSubstr("domain")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

////////////////////////////////////////////////////////////////////////
// Items
////////////////////////////////////////////////////////////////////////

var g_itemsTestDb     sdb.SimpleDB
var g_itemsTestDomain sdb.Domain

type ItemsTest struct {
	integrationTest
}

func init() { RegisterTestSuite(&ItemsTest{}) }

func (t *ItemsTest) SetUpTestSuite() {
	var err error

	// Open a connection.
	g_itemsTestDb, err = sdb.NewSimpleDB(g_region, g_accessKey)
	if err != nil {
		panic(err)
	}

	// Create a domain.
	g_itemsTestDomain, err = g_itemsTestDb.OpenDomain("ItemsTest.domain")
	if err != nil {
		panic(err)
	}
}

func (t *ItemsTest) TearDownTestSuite() {
	// Delete the domain.
	if err := g_itemsTestDb.DeleteDomain(g_itemsTestDomain); err != nil {
		panic(err)
	}

	// Clear variables.
	g_itemsTestDb = nil
	g_itemsTestDomain = nil
}

func (t *ItemsTest) PutThenGet() {
	var err error
	item := t.makeItemName()

	// Put
	err = g_itemsTestDomain.PutAttributes(
		item,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "foo", Value: "taco"},
			sdb.PutUpdate{Name: "bar", Value: "burrito"},
			sdb.PutUpdate{Name: "baz", Value: "enchilada"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get
	attrs, err := g_itemsTestDomain.GetAttributes(item, true, nil)

	AssertEq(nil, err)
	ExpectThat(
		sortByName(attrs),
		ElementsAre(
			DeepEquals(sdb.Attribute{Name: "bar", Value: "burrito"}),
			DeepEquals(sdb.Attribute{Name: "baz", Value: "enchilada"}),
			DeepEquals(sdb.Attribute{Name: "foo", Value: "taco"}),
		),
	)
}

func (t *ItemsTest) BatchPutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenBatchGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetForNonExistentItem() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetForNonExistentItems() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedValuePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedNonExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SuccessfulPreconditions() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteAllAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchDelete() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidSelectQuery() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectAll() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectCount() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithPredicates() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithSortOrder() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithLimit() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectEmptyResultSet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectLargeResultSet() {
	ExpectEq("TODO", "")
}
<commit_msg>Implemented sortByName.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"github.com/jacobsa/aws/exp/sdb"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"math/rand"
	"sort"
	"sync"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type integrationTest struct {
	db sdb.SimpleDB
}

func (t *integrationTest) SetUp(i *TestInfo) {
	var err error

	// Open a connection.
	t.db, err = sdb.NewSimpleDB(g_region, g_accessKey)
	AssertEq(nil, err)
}

// Generate an item name likely to be unique.
func (t *integrationTest) makeItemName() sdb.ItemName {
	return sdb.ItemName(fmt.Sprintf("item.%16x", uint64(rand.Int63())))
}

type nameSortedAttrList []sdb.Attribute

func (l nameSortedAttrList) Len() int           { return len(l) }
func (l nameSortedAttrList) Less(i, j int) bool { return l[i].Name < l[j].Name }
func (l nameSortedAttrList) Swap(i, j int)      { l[j], l[i] = l[i], l[j] }

func sortByName(attrs []sdb.Attribute) []sdb.Attribute {
	res := make(nameSortedAttrList, len(attrs))
	copy(res, attrs)
	sort.Sort(res)
	return res
}

////////////////////////////////////////////////////////////////////////
// Domains
////////////////////////////////////////////////////////////////////////

var g_domainsTestDb sdb.SimpleDB
var g_domainsTestDomain0 sdb.Domain
var g_domainsTestDomain1 sdb.Domain

type DomainsTest struct {
	integrationTest

	mutex           sync.Mutex
	domainsToDelete []sdb.Domain  // Protected by mutex
}

func init() { RegisterTestSuite(&DomainsTest{}) }

func (t *DomainsTest) SetUpTestSuite() {
	var err error

	// Open a connection.
	g_domainsTestDb, err = sdb.NewSimpleDB(g_region, g_accessKey)
	if err != nil {
		panic(err)
	}

	// Create domain 0.
	g_domainsTestDomain0, err = g_domainsTestDb.OpenDomain("DomainsTest.domain0")
	if err != nil {
		panic(err)
	}

	// Create domain 1.
	g_domainsTestDomain1, err = g_domainsTestDb.OpenDomain("DomainsTest.domain1")
	if err != nil {
		panic(err)
	}
}

func (t *DomainsTest) TearDownTestSuite() {
	// Delete both domains.
	if err := g_domainsTestDb.DeleteDomain(g_domainsTestDomain0); err != nil {
		panic(err)
	}

	if err := g_domainsTestDb.DeleteDomain(g_domainsTestDomain1); err != nil {
		panic(err)
	}

	// Clear variables.
	g_domainsTestDb = nil
	g_domainsTestDomain0 = nil
	g_domainsTestDomain1 = nil
}

func (t *DomainsTest) TearDown() {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	// Delete each of the domains created during the test.
	for _, d := range t.domainsToDelete {
		ExpectEq(nil, t.db.DeleteDomain(d), "Domain: %s", d.Name())
	}
}

func (t *DomainsTest) InvalidAccessKey() {
	// Open a connection with an unknown key ID.
	wrongKey := g_accessKey
	wrongKey.Id += "taco"

	db, err := sdb.NewSimpleDB(g_region, wrongKey)
	AssertEq(nil, err)

	// Attempt to create a domain.
	_, err = db.OpenDomain("some_domain")

	ExpectThat(err, Error(HasSubstr("403")))
	ExpectThat(err, Error(HasSubstr("Key Id")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *DomainsTest) SeparatelyNamedDomainsHaveIndependentItems() {
	var err error

	// Set up an item in the first domain.
	itemName := t.makeItemName()
	err = g_domainsTestDomain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in the other domain. There should be
	// none.
	attrs, err := g_domainsTestDomain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(attrs, ElementsAre())
}

func (t *DomainsTest) IdenticallyNamedDomainsHaveIdenticalItems() {
	var err error

	// Set up an item in the first domain.
	itemName := t.makeItemName()
	err = g_domainsTestDomain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in another domain object opened with the
	// same name.
	domain1, err := t.db.OpenDomain(g_domainsTestDomain0.Name())
	AssertEq(nil, err)

	attrs, err := domain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(
		attrs,
		ElementsAre(
			DeepEquals(sdb.Attribute{Name: "enchilada", Value: "queso"}),
		),
	)
}

func (t *DomainsTest) Delete() {
	var err error
	domainName := "DomainsTest.Delete"

	// Create a domain, then delete it.
	domain, err := t.db.OpenDomain(domainName)
	AssertEq(nil, err)

	err = t.db.DeleteDomain(domain)
	AssertEq(nil, err)

	// Delete again; nothing should go wrong.
	err = t.db.DeleteDomain(domain)
	AssertEq(nil, err)

	// Attempt to write to the domain.
	err = domain.PutAttributes(
		"some_item",
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "foo", Value: "bar"},
		},
		[]sdb.Precondition{},
	)

	ExpectThat(err, Error(HasSubstr("NoSuchDomain")))
	ExpectThat(err, Error(HasSubstr("domain")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

////////////////////////////////////////////////////////////////////////
// Items
////////////////////////////////////////////////////////////////////////

var g_itemsTestDb     sdb.SimpleDB
var g_itemsTestDomain sdb.Domain

type ItemsTest struct {
	integrationTest
}

func init() { RegisterTestSuite(&ItemsTest{}) }

func (t *ItemsTest) SetUpTestSuite() {
	var err error

	// Open a connection.
	g_itemsTestDb, err = sdb.NewSimpleDB(g_region, g_accessKey)
	if err != nil {
		panic(err)
	}

	// Create a domain.
	g_itemsTestDomain, err = g_itemsTestDb.OpenDomain("ItemsTest.domain")
	if err != nil {
		panic(err)
	}
}

func (t *ItemsTest) TearDownTestSuite() {
	// Delete the domain.
	if err := g_itemsTestDb.DeleteDomain(g_itemsTestDomain); err != nil {
		panic(err)
	}

	// Clear variables.
	g_itemsTestDb = nil
	g_itemsTestDomain = nil
}

func (t *ItemsTest) PutThenGet() {
	var err error
	item := t.makeItemName()

	// Put
	err = g_itemsTestDomain.PutAttributes(
		item,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "foo", Value: "taco"},
			sdb.PutUpdate{Name: "bar", Value: "burrito"},
			sdb.PutUpdate{Name: "baz", Value: "enchilada"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get
	attrs, err := g_itemsTestDomain.GetAttributes(item, true, nil)

	AssertEq(nil, err)
	ExpectThat(
		sortByName(attrs),
		ElementsAre(
			DeepEquals(sdb.Attribute{Name: "bar", Value: "burrito"}),
			DeepEquals(sdb.Attribute{Name: "baz", Value: "enchilada"}),
			DeepEquals(sdb.Attribute{Name: "foo", Value: "taco"}),
		),
	)
}

func (t *ItemsTest) BatchPutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenBatchGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetForNonExistentItem() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetForNonExistentItems() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedValuePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedNonExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SuccessfulPreconditions() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteAllAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchDelete() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidSelectQuery() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectAll() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectCount() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithPredicates() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithSortOrder() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithLimit() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectEmptyResultSet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectLargeResultSet() {
	ExpectEq("TODO", "")
}
<|endoftext|>