<commit_before>package volumewatcher

import (
	"context"
	"sync"

	log "github.com/hashicorp/go-hclog"
	memdb "github.com/hashicorp/go-memdb"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/nomad/state"
	"github.com/hashicorp/nomad/nomad/structs"
)

// volumeWatcher is used to watch a single volume and trigger the
// scheduler when allocation health transitions.
type volumeWatcher struct {
	// v is the volume being watched
	v *structs.CSIVolume

	// state is the state that is watched for state changes.
	state *state.StateStore

	// server interface for CSI client RPCs
	rpc CSIVolumeRPC

	// the ACL needed to send RPCs
	leaderAcl string

	logger      log.Logger
	shutdownCtx context.Context // parent context
	ctx         context.Context // own context
	exitFn      context.CancelFunc

	// updateCh is triggered when there is an updated volume
	updateCh chan *structs.CSIVolume

	wLock   sync.RWMutex
	running bool
}

// newVolumeWatcher returns a volume watcher that is used to watch
// volumes
func newVolumeWatcher(parent *Watcher, vol *structs.CSIVolume) *volumeWatcher {

	w := &volumeWatcher{
		updateCh:    make(chan *structs.CSIVolume, 1),
		v:           vol,
		state:       parent.state,
		rpc:         parent.rpc,
		leaderAcl:   parent.leaderAcl,
		logger:      parent.logger.With("volume_id", vol.ID, "namespace", vol.Namespace),
		shutdownCtx: parent.ctx,
	}

	// Start the long lived watcher that scans for allocation updates
	w.Start()
	return w
}

// Notify signals an update to the tracked volume.
func (vw *volumeWatcher) Notify(v *structs.CSIVolume) {
	if !vw.isRunning() {
		vw.Start()
	}
	select {
	case vw.updateCh <- v:
	case <-vw.shutdownCtx.Done(): // prevent deadlock if we stopped
	case <-vw.ctx.Done(): // prevent deadlock if we stopped
	}
}

func (vw *volumeWatcher) Start() {
	vw.logger.Trace("starting watcher")
	vw.wLock.Lock()
	defer vw.wLock.Unlock()
	vw.running = true
	ctx, exitFn := context.WithCancel(vw.shutdownCtx)
	vw.ctx = ctx
	vw.exitFn = exitFn
	go vw.watch()
}

// Stop stops watching the volume. This should be called whenever a
// volume's claims are fully reaped or the watcher is no longer needed.
func (vw *volumeWatcher) Stop() {
	vw.logger.Trace("no more claims")
	vw.exitFn()
}

func (vw *volumeWatcher) isRunning() bool {
	vw.wLock.RLock()
	defer vw.wLock.RUnlock()
	select {
	case <-vw.shutdownCtx.Done():
		return false
	case <-vw.ctx.Done():
		return false
	default:
		return vw.running
	}
}

// watch is the long-running function that watches for changes to a volume.
// Each pass steps the volume's claims through the various states of reaping
// until the volume has no more claims eligible to be reaped.
func (vw *volumeWatcher) watch() {
	for {
		select {
		// TODO(tgross): currently server->client RPC have no cancellation
		// context, so we can't stop the long-runner RPCs gracefully
		case <-vw.shutdownCtx.Done():
			return
		case <-vw.ctx.Done():
			return
		case vol := <-vw.updateCh:
			// while we won't make raft writes if we get a stale update,
			// we can still fire extra CSI RPC calls if we don't check this
			if vol.ModifyIndex >= vw.v.ModifyIndex {
				vol = vw.getVolume(vol)
				if vol == nil {
					return
				}
				vw.volumeReap(vol)
			}
		default:
			vw.Stop() // no pending work
			return
		}
	}
}

// getVolume returns the tracked volume, fully populated with the current
// state
func (vw *volumeWatcher) getVolume(vol *structs.CSIVolume) *structs.CSIVolume {
	vw.wLock.RLock()
	defer vw.wLock.RUnlock()

	var err error
	ws := memdb.NewWatchSet()

	vol, err = vw.state.CSIVolumeDenormalizePlugins(ws, vol.Copy())
	if err != nil {
		vw.logger.Error("could not query plugins for volume", "error", err)
		return nil
	}

	vol, err = vw.state.CSIVolumeDenormalize(ws, vol)
	if err != nil {
		vw.logger.Error("could not query allocs for volume", "error", err)
		return nil
	}
	vw.v = vol
	return vol
}

// volumeReap collects errors for logging but doesn't return them
// to the main loop.
func (vw *volumeWatcher) volumeReap(vol *structs.CSIVolume) {
	vw.logger.Trace("releasing unused volume claims")
	err := vw.volumeReapImpl(vol)
	if err != nil {
		vw.logger.Error("error releasing volume claims", "error", err)
	}
	if vw.isUnclaimed(vol) {
		vw.Stop()
	}
}

func (vw *volumeWatcher) isUnclaimed(vol *structs.CSIVolume) bool {
	return len(vol.ReadClaims) == 0 && len(vol.WriteClaims) == 0 && len(vol.PastClaims) == 0
}

func (vw *volumeWatcher) volumeReapImpl(vol *structs.CSIVolume) error {

	// PastClaims written by a volume GC core job will have no allocation,
	// so we need to find out which allocs are eligible for cleanup.
	for _, claim := range vol.PastClaims {
		if claim.AllocationID == "" {
			vol = vw.collectPastClaims(vol)
		}
	}

	var result *multierror.Error
	for _, claim := range vol.PastClaims {
		err := vw.unpublish(vol, claim)
		if err != nil {
			result = multierror.Append(result, err)
		}
	}

	return result.ErrorOrNil()

}

func (vw *volumeWatcher) collectPastClaims(vol *structs.CSIVolume) *structs.CSIVolume {

	collect := func(allocs map[string]*structs.Allocation,
		claims map[string]*structs.CSIVolumeClaim) {

		for allocID, alloc := range allocs {
			if alloc == nil {
				_, exists := vol.PastClaims[allocID]
				if !exists {
					vol.PastClaims[allocID] = &structs.CSIVolumeClaim{
						AllocationID: allocID,
						State:        structs.CSIVolumeClaimStateReadyToFree,
					}
				}
			} else if alloc.Terminated() {
				// don't overwrite the PastClaim if we've seen it before,
				// so that we can track state between subsequent calls
				_, exists := vol.PastClaims[allocID]
				if !exists {
					claim, ok := claims[allocID]
					if !ok {
						claim = &structs.CSIVolumeClaim{
							AllocationID: allocID,
							NodeID:       alloc.NodeID,
						}
					}
					claim.State = structs.CSIVolumeClaimStateTaken
					vol.PastClaims[allocID] = claim
				}
			}
		}
	}

	collect(vol.ReadAllocs, vol.ReadClaims)
	collect(vol.WriteAllocs, vol.WriteClaims)
	return vol
}

func (vw *volumeWatcher) unpublish(vol *structs.CSIVolume, claim *structs.CSIVolumeClaim) error {
	req := &structs.CSIVolumeUnpublishRequest{
		VolumeID: vol.ID,
		Claim:    claim,
		WriteRequest: structs.WriteRequest{
			Namespace: vol.Namespace,
			Region:    vw.state.Config().Region,
			AuthToken: vw.leaderAcl,
		},
	}
	err := vw.rpc.Unpublish(req, &structs.CSIVolumeUnpublishResponse{})
	if err != nil {
		return err
	}
	claim.State = structs.CSIVolumeClaimStateReadyToFree
	return nil
}
<commit_msg>csi: volumewatcher only needs one pass to collect past claims<commit_after>package volumewatcher

import (
	"context"
	"sync"

	log "github.com/hashicorp/go-hclog"
	memdb "github.com/hashicorp/go-memdb"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/nomad/state"
	"github.com/hashicorp/nomad/nomad/structs"
)

// volumeWatcher is used to watch a single volume and trigger the
// scheduler when allocation health transitions.
type volumeWatcher struct {
	// v is the volume being watched
	v *structs.CSIVolume

	// state is the state that is watched for state changes.
	state *state.StateStore

	// server interface for CSI client RPCs
	rpc CSIVolumeRPC

	// the ACL needed to send RPCs
	leaderAcl string

	logger      log.Logger
	shutdownCtx context.Context // parent context
	ctx         context.Context // own context
	exitFn      context.CancelFunc

	// updateCh is triggered when there is an updated volume
	updateCh chan *structs.CSIVolume

	wLock   sync.RWMutex
	running bool
}

// newVolumeWatcher returns a volume watcher that is used to watch
// volumes
func newVolumeWatcher(parent *Watcher, vol *structs.CSIVolume) *volumeWatcher {

	w := &volumeWatcher{
		updateCh:    make(chan *structs.CSIVolume, 1),
		v:           vol,
		state:       parent.state,
		rpc:         parent.rpc,
		leaderAcl:   parent.leaderAcl,
		logger:      parent.logger.With("volume_id", vol.ID, "namespace", vol.Namespace),
		shutdownCtx: parent.ctx,
	}

	// Start the long lived watcher that scans for allocation updates
	w.Start()
	return w
}

// Notify signals an update to the tracked volume.
func (vw *volumeWatcher) Notify(v *structs.CSIVolume) {
	if !vw.isRunning() {
		vw.Start()
	}
	select {
	case vw.updateCh <- v:
	case <-vw.shutdownCtx.Done(): // prevent deadlock if we stopped
	case <-vw.ctx.Done(): // prevent deadlock if we stopped
	}
}

func (vw *volumeWatcher) Start() {
	vw.logger.Trace("starting watcher")
	vw.wLock.Lock()
	defer vw.wLock.Unlock()
	vw.running = true
	ctx, exitFn := context.WithCancel(vw.shutdownCtx)
	vw.ctx = ctx
	vw.exitFn = exitFn
	go vw.watch()
}

// Stop stops watching the volume. This should be called whenever a
// volume's claims are fully reaped or the watcher is no longer needed.
func (vw *volumeWatcher) Stop() {
	vw.logger.Trace("no more claims")
	vw.exitFn()
}

func (vw *volumeWatcher) isRunning() bool {
	vw.wLock.RLock()
	defer vw.wLock.RUnlock()
	select {
	case <-vw.shutdownCtx.Done():
		return false
	case <-vw.ctx.Done():
		return false
	default:
		return vw.running
	}
}

// watch is the long-running function that watches for changes to a volume.
// Each pass steps the volume's claims through the various states of reaping
// until the volume has no more claims eligible to be reaped.
func (vw *volumeWatcher) watch() {
	for {
		select {
		// TODO(tgross): currently server->client RPC have no cancellation
		// context, so we can't stop the long-runner RPCs gracefully
		case <-vw.shutdownCtx.Done():
			return
		case <-vw.ctx.Done():
			return
		case vol := <-vw.updateCh:
			// while we won't make raft writes if we get a stale update,
			// we can still fire extra CSI RPC calls if we don't check this
			if vol.ModifyIndex >= vw.v.ModifyIndex {
				vol = vw.getVolume(vol)
				if vol == nil {
					return
				}
				vw.volumeReap(vol)
			}
		default:
			vw.Stop() // no pending work
			return
		}
	}
}

// getVolume returns the tracked volume, fully populated with the current
// state
func (vw *volumeWatcher) getVolume(vol *structs.CSIVolume) *structs.CSIVolume {
	vw.wLock.RLock()
	defer vw.wLock.RUnlock()

	var err error
	ws := memdb.NewWatchSet()

	vol, err = vw.state.CSIVolumeDenormalizePlugins(ws, vol.Copy())
	if err != nil {
		vw.logger.Error("could not query plugins for volume", "error", err)
		return nil
	}

	vol, err = vw.state.CSIVolumeDenormalize(ws, vol)
	if err != nil {
		vw.logger.Error("could not query allocs for volume", "error", err)
		return nil
	}
	vw.v = vol
	return vol
}

// volumeReap collects errors for logging but doesn't return them
// to the main loop.
func (vw *volumeWatcher) volumeReap(vol *structs.CSIVolume) {
	vw.logger.Trace("releasing unused volume claims")
	err := vw.volumeReapImpl(vol)
	if err != nil {
		vw.logger.Error("error releasing volume claims", "error", err)
	}
	if vw.isUnclaimed(vol) {
		vw.Stop()
	}
}

func (vw *volumeWatcher) isUnclaimed(vol *structs.CSIVolume) bool {
	return len(vol.ReadClaims) == 0 && len(vol.WriteClaims) == 0 && len(vol.PastClaims) == 0
}

func (vw *volumeWatcher) volumeReapImpl(vol *structs.CSIVolume) error {

	// PastClaims written by a volume GC core job will have no allocation,
	// so we need to find out which allocs are eligible for cleanup.
	for _, claim := range vol.PastClaims {
		if claim.AllocationID == "" {
			vol = vw.collectPastClaims(vol)
			break // only need to collect once
		}
	}

	var result *multierror.Error
	for _, claim := range vol.PastClaims {
		err := vw.unpublish(vol, claim)
		if err != nil {
			result = multierror.Append(result, err)
		}
	}

	return result.ErrorOrNil()

}

func (vw *volumeWatcher) collectPastClaims(vol *structs.CSIVolume) *structs.CSIVolume {

	collect := func(allocs map[string]*structs.Allocation,
		claims map[string]*structs.CSIVolumeClaim) {

		for allocID, alloc := range allocs {
			if alloc == nil {
				_, exists := vol.PastClaims[allocID]
				if !exists {
					vol.PastClaims[allocID] = &structs.CSIVolumeClaim{
						AllocationID: allocID,
						State:        structs.CSIVolumeClaimStateReadyToFree,
					}
				}
			} else if alloc.Terminated() {
				// don't overwrite the PastClaim if we've seen it before,
				// so that we can track state between subsequent calls
				_, exists := vol.PastClaims[allocID]
				if !exists {
					claim, ok := claims[allocID]
					if !ok {
						claim = &structs.CSIVolumeClaim{
							AllocationID: allocID,
							NodeID:       alloc.NodeID,
						}
					}
					claim.State = structs.CSIVolumeClaimStateTaken
					vol.PastClaims[allocID] = claim
				}
			}
		}
	}

	collect(vol.ReadAllocs, vol.ReadClaims)
	collect(vol.WriteAllocs, vol.WriteClaims)
	return vol
}

func (vw *volumeWatcher) unpublish(vol *structs.CSIVolume, claim *structs.CSIVolumeClaim) error {
	req := &structs.CSIVolumeUnpublishRequest{
		VolumeID: vol.ID,
		Claim:    claim,
		WriteRequest: structs.WriteRequest{
			Namespace: vol.Namespace,
			Region:    vw.state.Config().Region,
			AuthToken: vw.leaderAcl,
		},
	}
	err := vw.rpc.Unpublish(req, &structs.CSIVolumeUnpublishResponse{})
	if err != nil {
		return err
	}
	claim.State = structs.CSIVolumeClaimStateReadyToFree
	return nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package indexer

import (
	"errors"
	"github.com/couchbase/indexing/secondary/logging"
	"sync/atomic"
	"time"
	"unsafe"
)

//MutationQueue interface specifies methods which a mutation queue for indexer
//needs to implement
type MutationQueue interface {

	//enqueue a mutation reference based on vbucket
	Enqueue(mutation *MutationKeys, vbucket Vbucket) error

	//dequeue a vbucket's mutation and keep sending on a channel until stop signal
	Dequeue(vbucket Vbucket) (<-chan *MutationKeys, chan<- bool, error)
	//dequeue a vbucket's mutation upto seqno(wait if not available)
	DequeueUptoSeqno(vbucket Vbucket, seqno Seqno) (<-chan *MutationKeys, error)
	//dequeue single element for a vbucket and return
	DequeueSingleElement(vbucket Vbucket) *MutationKeys

	//return reference to a vbucket's mutation at Tail of queue without dequeue
	PeekTail(vbucket Vbucket) *MutationKeys
	//return reference to a vbucket's mutation at Head of queue without dequeue
	PeekHead(vbucket Vbucket) *MutationKeys

	//return size of queue per vbucket
	GetSize(vbucket Vbucket) int64

	//returns the numbers of vbuckets for the queue
	GetNumVbuckets() uint16
}

//AtomicMutationQueue is a lock-free multi-queue with internal queue per
//vbucket for storing mutation references. This is loosely based on
//http://www.drdobbs.com/parallel/writing-lock-free-code-a-corrected-queue/210604448?pgno=1
//with the main difference being that free nodes are being reused here to reduce GC.
//
//It doesn't copy the mutation and its caller's responsiblity
//to allocate/deallocate KeyVersions struct. A mutation which is currently in queue
//shouldn't be freed.
//
//This implementation uses Go "atomic" pkg to provide safe concurrent access
//for a single reader and writer per vbucket queue without using mutex locks.
//
//It provides safe concurrent read/write access across vbucket queues.

type atomicMutationQueue struct {
	// IMPORTANT: should be 64 bit aligned.
	head   []unsafe.Pointer //head pointer per vbucket queue
	tail   []unsafe.Pointer //tail pointer per vbucket queue
	size   []int64          //size of queue per vbucket
	maxLen int64            //max length of queue per vbucket

	free        []*node //free pointer per vbucket queue
	numVbuckets uint16  //num vbuckets for the queue
}

//NewAtomicMutationQueue allocates a new Atomic Mutation Queue and initializes it
func NewAtomicMutationQueue(numVbuckets uint16, maxLenPerVb int64) *atomicMutationQueue {

	q := &atomicMutationQueue{head: make([]unsafe.Pointer, numVbuckets),
		tail:        make([]unsafe.Pointer, numVbuckets),
		free:        make([]*node, numVbuckets),
		size:        make([]int64, numVbuckets),
		numVbuckets: numVbuckets,
		maxLen:      maxLenPerVb,
	}

	var x uint16
	for x = 0; x < numVbuckets; x++ {
		node := &node{} //sentinel node for the queue
		q.head[x] = unsafe.Pointer(node)
		q.tail[x] = unsafe.Pointer(node)
		q.free[x] = node
	}

	return q

}

//Node represents a single element in the queue
type node struct {
	mutation *MutationKeys
	next     *node
}

//Poll Interval for dequeue thread
const DEQUEUE_POLL_INTERVAL = 20
const ALLOC_POLL_INTERVAL = 30
const MAX_VB_QUEUE_LENGTH = 1000

//Enqueue will enqueue the mutation reference for given vbucket.
//Caller should not free the mutation till it is dequeued.
//Mutation will not be copied internally by the queue.
func (q *atomicMutationQueue) Enqueue(mutation *MutationKeys, vbucket Vbucket) error {

	if vbucket < 0 || vbucket > Vbucket(q.numVbuckets)-1 {
		return errors.New("vbucket out of range")
	}

	//create a new node
	n := q.allocNode(vbucket)
	n.mutation = mutation
	n.next = nil

	//point tail's next to new node
	tail := (*node)(atomic.LoadPointer(&q.tail[vbucket]))
	tail.next = n
	//update tail to new node
	atomic.StorePointer(&q.tail[vbucket], unsafe.Pointer(tail.next))

	atomic.AddInt64(&q.size[vbucket], 1)

	return nil

}

//DequeueUptoSeqno returns a channel on which it will return mutation reference
//for specified vbucket upto the sequence number specified.
//This function will keep polling till mutations upto seqno are available
//to be sent. It terminates when it finds a mutation with seqno higher than
//the one specified as argument. This allow for multiple mutations with same
//seqno (e.g. in case of multiple indexes)
//It closes the mutation channel to indicate its done.
func (q *atomicMutationQueue) DequeueUptoSeqno(vbucket Vbucket, seqno Seqno) (
	<-chan *MutationKeys, error) {

	datach := make(chan *MutationKeys)

	go q.dequeueUptoSeqno(vbucket, seqno, datach)

	return datach, nil

}

func (q *atomicMutationQueue) dequeueUptoSeqno(vbucket Vbucket, seqno Seqno,
	datach chan *MutationKeys) {

	//every DEQUEUE_POLL_INTERVAL milliseconds, check for new mutations
	ticker := time.NewTicker(time.Millisecond * DEQUEUE_POLL_INTERVAL)

	var dequeueSeq Seqno

	for _ = range ticker.C {
		for atomic.LoadPointer(&q.head[vbucket]) !=
			atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty

			head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
			//copy the mutation pointer
			m := head.next.mutation
			if seqno >= m.meta.seqno {
				//free mutation pointer
				head.next.mutation = nil
				//move head to next
				atomic.StorePointer(&q.head[vbucket], unsafe.Pointer(head.next))
				atomic.AddInt64(&q.size[vbucket], -1)
				//send mutation to caller
				dequeueSeq = m.meta.seqno
				datach <- m
			}

			//once the seqno is reached, close the channel
			if seqno <= dequeueSeq {
				ticker.Stop()
				close(datach)
				return
			}
		}
	}
}

//Dequeue returns a channel on which it will return mutation reference for specified vbucket.
//This function will keep polling and send mutations as those become available.
//It returns a stop channel on which caller can signal it to stop.
func (q *atomicMutationQueue) Dequeue(vbucket Vbucket) (<-chan *MutationKeys,
	chan<- bool, error) {

	datach := make(chan *MutationKeys)
	stopch := make(chan bool)

	//every DEQUEUE_POLL_INTERVAL milliseconds, check for new mutations
	ticker := time.NewTicker(time.Millisecond * DEQUEUE_POLL_INTERVAL)

	go func() {
		for {
			select {
			case <-ticker.C:
				q.dequeue(vbucket, datach)
			case <-stopch:
				ticker.Stop()
				close(datach)
				return
			}
		}
	}()

	return datach, stopch, nil

}

func (q *atomicMutationQueue) dequeue(vbucket Vbucket, datach chan *MutationKeys) {

	//keep dequeuing till list is empty
	for {
		m := q.DequeueSingleElement(vbucket)
		if m == nil {
			return
		}
		//send mutation to caller
		datach <- m
	}

}

//DequeueSingleElement dequeues a single element and returns.
//Returns nil in case of empty queue.
func (q *atomicMutationQueue) DequeueSingleElement(vbucket Vbucket) *MutationKeys {

	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty

		head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
		//copy the mutation pointer
		m := head.next.mutation
		//move head to next
		atomic.StorePointer(&q.head[vbucket], unsafe.Pointer(head.next))
		atomic.AddInt64(&q.size[vbucket], -1)
		return m
	}
	return nil
}

//PeekTail returns reference to a vbucket's mutation at tail of queue without dequeue
func (q *atomicMutationQueue) PeekTail(vbucket Vbucket) *MutationKeys {
	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty
		tail := (*node)(atomic.LoadPointer(&q.tail[vbucket]))
		return tail.mutation
	}
	return nil
}

//PeekHead returns reference to a vbucket's mutation at head of queue without dequeue
func (q *atomicMutationQueue) PeekHead(vbucket Vbucket) *MutationKeys {
	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty
		head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
		return head.mutation
	}
	return nil
}

//GetSize returns the size of the vbucket queue
func (q *atomicMutationQueue) GetSize(vbucket Vbucket) int64 {
	return atomic.LoadInt64(&q.size[vbucket])
}

//GetNumVbuckets returns the numbers of vbuckets for the queue
func (q *atomicMutationQueue) GetNumVbuckets() uint16 {
	return q.numVbuckets
}

//allocNode tries to get node from freelist, otherwise allocates a new node and returns
func (q *atomicMutationQueue) allocNode(vbucket Vbucket) *node {

	//get node from freelist
	n := q.popFreeList(vbucket)
	if n != nil {
		return n
	} else {
		currLen := atomic.LoadInt64(&q.size[vbucket])
		if currLen < q.maxLen {
			//allocate new node and return
			return &node{}
		}
	}

	//every ALLOC_POLL_INTERVAL milliseconds, check for free nodes
	ticker := time.NewTicker(time.Millisecond * ALLOC_POLL_INTERVAL)

	var totalWait int
	for _ = range ticker.C {
		totalWait += ALLOC_POLL_INTERVAL
		n = q.popFreeList(vbucket)
		if n != nil {
			return n
		}
		if totalWait > 5000 {
			logging.Warnf("Indexer::MutationQueue Waiting for Node "+
				"Alloc for %v Milliseconds Vbucket %v", totalWait, vbucket)
		}
	}

	return nil

}

//popFreeList removes a node from freelist and returns to caller.
//if freelist is empty, it returns nil.
func (q *atomicMutationQueue) popFreeList(vbucket Vbucket) *node {

	if q.free[vbucket] != (*node)(atomic.LoadPointer(&q.head[vbucket])) {
		n := q.free[vbucket]
		q.free[vbucket] = q.free[vbucket].next
		n.mutation = nil
		n.next = nil
		return n
	} else {
		return nil
	}

}
<commit_msg>MB-14796 Add Destroy to Mutation Queue<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package indexer

import (
	"errors"
	"github.com/couchbase/indexing/secondary/logging"
	"sync/atomic"
	"time"
	"unsafe"
)

//MutationQueue interface specifies methods which a mutation queue for indexer
//needs to implement
type MutationQueue interface {

	//enqueue a mutation reference based on vbucket
	Enqueue(mutation *MutationKeys, vbucket Vbucket) error

	//dequeue a vbucket's mutation and keep sending on a channel until stop signal
	Dequeue(vbucket Vbucket) (<-chan *MutationKeys, chan<- bool, error)
	//dequeue a vbucket's mutation upto seqno(wait if not available)
	DequeueUptoSeqno(vbucket Vbucket, seqno Seqno) (<-chan *MutationKeys, error)
	//dequeue single element for a vbucket and return
	DequeueSingleElement(vbucket Vbucket) *MutationKeys

	//return reference to a vbucket's mutation at Tail of queue without dequeue
	PeekTail(vbucket Vbucket) *MutationKeys
	//return reference to a vbucket's mutation at Head of queue without dequeue
	PeekHead(vbucket Vbucket) *MutationKeys

	//return size of queue per vbucket
	GetSize(vbucket Vbucket) int64

	//returns the numbers of vbuckets for the queue
	GetNumVbuckets() uint16

	//destroy the resources
	Destroy()
}

//AtomicMutationQueue is a lock-free multi-queue with internal queue per
//vbucket for storing mutation references. This is loosely based on
//http://www.drdobbs.com/parallel/writing-lock-free-code-a-corrected-queue/210604448?pgno=1
//with the main difference being that free nodes are being reused here to reduce GC.
//
//It doesn't copy the mutation and its caller's responsiblity
//to allocate/deallocate KeyVersions struct. A mutation which is currently in queue
//shouldn't be freed.
//
//This implementation uses Go "atomic" pkg to provide safe concurrent access
//for a single reader and writer per vbucket queue without using mutex locks.
//
//It provides safe concurrent read/write access across vbucket queues.

type atomicMutationQueue struct {
	// IMPORTANT: should be 64 bit aligned.
	head   []unsafe.Pointer //head pointer per vbucket queue
	tail   []unsafe.Pointer //tail pointer per vbucket queue
	size   []int64          //size of queue per vbucket
	maxLen int64            //max length of queue per vbucket

	free        []*node //free pointer per vbucket queue
	stopch      []StopChannel
	numVbuckets uint16 //num vbuckets for the queue
	isDestroyed bool
}

//NewAtomicMutationQueue allocates a new Atomic Mutation Queue and initializes it
func NewAtomicMutationQueue(numVbuckets uint16, maxLenPerVb int64) *atomicMutationQueue {

	q := &atomicMutationQueue{head: make([]unsafe.Pointer, numVbuckets),
		tail:        make([]unsafe.Pointer, numVbuckets),
		free:        make([]*node, numVbuckets),
		size:        make([]int64, numVbuckets),
		numVbuckets: numVbuckets,
		maxLen:      maxLenPerVb,
		stopch:      make([]StopChannel, numVbuckets),
	}

	var x uint16
	for x = 0; x < numVbuckets; x++ {
		node := &node{} //sentinel node for the queue
		q.head[x] = unsafe.Pointer(node)
		q.tail[x] = unsafe.Pointer(node)
		q.free[x] = node
		q.stopch[x] = make(StopChannel)
	}

	return q

}

//Node represents a single element in the queue
type node struct {
	mutation *MutationKeys
	next     *node
}

//Poll Interval for dequeue thread
const DEQUEUE_POLL_INTERVAL = 20
const ALLOC_POLL_INTERVAL = 30
const MAX_VB_QUEUE_LENGTH = 1000

//Enqueue will enqueue the mutation reference for given vbucket.
//Caller should not free the mutation till it is dequeued.
//Mutation will not be copied internally by the queue.
func (q *atomicMutationQueue) Enqueue(mutation *MutationKeys, vbucket Vbucket) error {

	if vbucket < 0 || vbucket > Vbucket(q.numVbuckets)-1 {
		return errors.New("vbucket out of range")
	}

	//no more requests are taken once queue
	//is marked as destroyed
	if q.isDestroyed {
		return nil
	}

	//create a new node
	n := q.allocNode(vbucket)
	if n == nil {
		return nil
	}

	n.mutation = mutation
	n.next = nil

	//point tail's next to new node
	tail := (*node)(atomic.LoadPointer(&q.tail[vbucket]))
	tail.next = n
	//update tail to new node
	atomic.StorePointer(&q.tail[vbucket], unsafe.Pointer(tail.next))

	atomic.AddInt64(&q.size[vbucket], 1)

	return nil

}

//DequeueUptoSeqno returns a channel on which it will return mutation reference
//for specified vbucket upto the sequence number specified.
//This function will keep polling till mutations upto seqno are available
//to be sent. It terminates when it finds a mutation with seqno higher than
//the one specified as argument. This allow for multiple mutations with same
//seqno (e.g. in case of multiple indexes)
//It closes the mutation channel to indicate its done.
func (q *atomicMutationQueue) DequeueUptoSeqno(vbucket Vbucket, seqno Seqno) (
	<-chan *MutationKeys, error) {

	datach := make(chan *MutationKeys)

	go q.dequeueUptoSeqno(vbucket, seqno, datach)

	return datach, nil

}

func (q *atomicMutationQueue) dequeueUptoSeqno(vbucket Vbucket, seqno Seqno,
	datach chan *MutationKeys) {

	//every DEQUEUE_POLL_INTERVAL milliseconds, check for new mutations
	ticker := time.NewTicker(time.Millisecond * DEQUEUE_POLL_INTERVAL)

	var dequeueSeq Seqno

	for _ = range ticker.C {
		for atomic.LoadPointer(&q.head[vbucket]) !=
			atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty

			head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
			//copy the mutation pointer
			m := head.next.mutation
			if seqno >= m.meta.seqno {
				//free mutation pointer
				head.next.mutation = nil
				//move head to next
				atomic.StorePointer(&q.head[vbucket], unsafe.Pointer(head.next))
				atomic.AddInt64(&q.size[vbucket], -1)
				//send mutation to caller
				dequeueSeq = m.meta.seqno
				datach <- m
			}

			//once the seqno is reached, close the channel
			if seqno <= dequeueSeq {
				ticker.Stop()
				close(datach)
				return
			}
		}
	}
}

//Dequeue returns a channel on which it will return mutation reference for specified vbucket.
//This function will keep polling and send mutations as those become available.
//It returns a stop channel on which caller can signal it to stop.
func (q *atomicMutationQueue) Dequeue(vbucket Vbucket) (<-chan *MutationKeys,
	chan<- bool, error) {

	datach := make(chan *MutationKeys)
	stopch := make(chan bool)

	//every DEQUEUE_POLL_INTERVAL milliseconds, check for new mutations
	ticker := time.NewTicker(time.Millisecond * DEQUEUE_POLL_INTERVAL)

	go func() {
		for {
			select {
			case <-ticker.C:
				q.dequeue(vbucket, datach)
			case <-stopch:
				ticker.Stop()
				close(datach)
				return
			}
		}
	}()

	return datach, stopch, nil

}

func (q *atomicMutationQueue) dequeue(vbucket Vbucket, datach chan *MutationKeys) {

	//keep dequeuing till list is empty
	for {
		m := q.DequeueSingleElement(vbucket)
		if m == nil {
			return
		}
		//send mutation to caller
		datach <- m
	}

}

//DequeueSingleElement dequeues a single element and returns.
//Returns nil in case of empty queue.
func (q *atomicMutationQueue) DequeueSingleElement(vbucket Vbucket) *MutationKeys {

	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty

		head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
		//copy the mutation pointer
		m := head.next.mutation
		//free mutation pointer
		head.next.mutation = nil
		//move head to next
		atomic.StorePointer(&q.head[vbucket], unsafe.Pointer(head.next))
		atomic.AddInt64(&q.size[vbucket], -1)
		return m
	}
	return nil
}

//PeekTail returns reference to a vbucket's mutation at tail of queue without dequeue
func (q *atomicMutationQueue) PeekTail(vbucket Vbucket) *MutationKeys {
	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty
		tail := (*node)(atomic.LoadPointer(&q.tail[vbucket]))
		return tail.mutation
	}
	return nil
}

//PeekHead returns reference to a vbucket's mutation at head of queue without dequeue
func (q *atomicMutationQueue) PeekHead(vbucket Vbucket) *MutationKeys {
	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty
		head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
		return head.mutation
	}
	return nil
}

//GetSize returns the size of the vbucket queue
func (q *atomicMutationQueue) GetSize(vbucket Vbucket) int64 {
	return atomic.LoadInt64(&q.size[vbucket])
}

//GetNumVbuckets returns the numbers of vbuckets for the queue
func (q *atomicMutationQueue) GetNumVbuckets() uint16 {
	return q.numVbuckets
}

//allocNode tries to get node from freelist, otherwise allocates a new node and returns
func (q *atomicMutationQueue) allocNode(vbucket Vbucket) *node {

	//get node from freelist
	n := q.popFreeList(vbucket)
	if n != nil {
		return n
	} else {
		currLen := atomic.LoadInt64(&q.size[vbucket])
		if currLen < q.maxLen {
			//allocate new node and return
			return &node{}
		}
	}

	//every ALLOC_POLL_INTERVAL milliseconds, check for free nodes
	ticker := time.NewTicker(time.Millisecond * ALLOC_POLL_INTERVAL)

	var totalWait int
	for {
		select {
		case <-ticker.C:
			totalWait += ALLOC_POLL_INTERVAL
			n = q.popFreeList(vbucket)
			if n != nil {
				return n
			}
			if totalWait > 5000 {
				logging.Warnf("Indexer::MutationQueue Waiting for Node "+
					"Alloc for %v Milliseconds Vbucket %v", totalWait, vbucket)
			}

		case <-q.stopch[vbucket]:
			return nil

		}
	}

	return nil

}

//popFreeList removes a node from freelist and returns to caller.
//if freelist is empty, it returns nil.
func (q *atomicMutationQueue) popFreeList(vbucket Vbucket) *node {

	if q.free[vbucket] != (*node)(atomic.LoadPointer(&q.head[vbucket])) {
		n := q.free[vbucket]
		q.free[vbucket] = q.free[vbucket].next
		n.mutation = nil
		n.next = nil
		return n
	} else {
		return nil
	}

}

//Destroy will free up all resources of the queue.
//Importantly it will free up pending mutations as well.
//Once destroy have been called, further enqueue operations
//will be no-op.
func (q *atomicMutationQueue) Destroy() {

	//set the flag so no more Enqueue requests
	//are taken on this queue
	q.isDestroyed = true

	//ensure all pending allocs get stopped
	var i uint16
	for i = 0; i < q.numVbuckets; i++ {
		close(q.stopch[i])
	}

	//dequeue all the items in the queue and free
	for i = 0; i < q.numVbuckets; i++ {
		mutch := make(chan *MutationKeys)
		go func() {
			for mutk := range mutch {
				mutk.Free()
			}
		}()
		q.dequeue(Vbucket(i), mutch)
		close(mutch)
	}

}
<|endoftext|>


<commit_before>package security_groups_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	. "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry-incubator/cf-test-helpers/workflowhelpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name"
)

var _ = SecurityGroupsDescribe("Security Groups", func() {

	type AppResource struct {
		Metadata struct {
			Url string
		}
	}
	type AppsResponse struct {
		Resources []AppResource
	}

	type Stat struct {
		Stats struct {
			Host string
			Port int
		}
	}
	type StatsResponse map[string]Stat

	type DoraCurlResponse struct {
		Stdout     string
		Stderr     string
		ReturnCode int `json:"return_code"`
	}

	var serverAppName, securityGroupName, privateHost string
	var privatePort int

	BeforeEach(func() {
		serverAppName = random_name.CATSRandomName("APP")
		Expect(cf.Cf("push",
			serverAppName,
			"--no-start",
			"-b", Config.RubyBuildpackName,
			"-m", DEFAULT_MEMORY_LIMIT,
			"-p", assets.NewAssets().Dora,
			"-d", Config.AppsDomain).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
		app_helpers.SetBackend(serverAppName)
		Expect(cf.Cf("start", serverAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))

		// gather app url
		var appsResponse AppsResponse
		cfResponse := cf.Cf("curl", fmt.Sprintf("/v2/apps?q=name:%s", serverAppName)).Wait(Config.DefaultTimeoutDuration()).Out.Contents()
		json.Unmarshal(cfResponse, &appsResponse)
		serverAppUrl := appsResponse.Resources[0].Metadata.Url

		// gather app stats for execution vm ip and app port
		var statsResponse StatsResponse
		cfResponse = cf.Cf("curl", fmt.Sprintf("%s/stats", serverAppUrl)).Wait(Config.DefaultTimeoutDuration()).Out.Contents()
		json.Unmarshal(cfResponse, &statsResponse)

		privateHost = statsResponse["0"].Stats.Host
		privatePort = statsResponse["0"].Stats.Port
	})

	AfterEach(func() {
		app_helpers.AppReport(serverAppName, Config.DefaultTimeoutDuration())

		Expect(cf.Cf("delete", serverAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
	})

	// This test assumes the default running security groups block access to the vms that execute apps.
	// The test takes advantage of the fact that the execution vm's ip address and internal container ip address
	//   are discoverable via the cc api and dora's myip endpoint.
	It("allows previously-blocked ip traffic after applying a security group, and re-blocks it when the group is removed", func() {
		clientAppName := random_name.CATSRandomName("APP")
		Expect(cf.Cf("push", clientAppName, "--no-start", "-b", Config.RubyBuildpackName, "-m", DEFAULT_MEMORY_LIMIT, "-p", assets.NewAssets().Dora, "-d", Config.AppsDomain).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		app_helpers.SetBackend(clientAppName)
		Expect(cf.Cf("start", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
		defer func() { cf.Cf("delete", clientAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration()) }()

		curlResponse := helpers.CurlApp(serverAppName, "/myip")
		containerIp := strings.TrimSpace(curlResponse)

		curlResponse = helpers.CurlApp(serverAppName, "/env/VCAP_APPLICATION")
		var env map[string]interface{}
		err := json.Unmarshal([]byte(curlResponse), &env)
		Expect(err).NotTo(HaveOccurred())
		containerPort := int(env["port"].(float64))

		By("Asserting default running security-group configuration")
		var doraCurlResponse DoraCurlResponse
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).ToNot(Equal(0), "Expected running security groups not to allow internal communication between app containers. Configure your running security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.")

		By("Applying security group")
		rules := fmt.Sprintf(
			`[{"destination":"%s","ports":"%d","protocol":"tcp"},
			{"destination":"%s","ports":"%d","protocol":"tcp"}]`,
			privateHost, privatePort, containerIp, containerPort)

		file, _ := ioutil.TempFile(os.TempDir(), "CATS-sg-rules")
		defer os.Remove(file.Name())
		file.WriteString(rules)

		rulesPath := file.Name()
		securityGroupName = random_name.CATSRandomName("SG")

		workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
			Expect(cf.Cf("create-security-group", securityGroupName, rulesPath).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			Expect(
				cf.Cf("bind-security-group",
					securityGroupName,
					TestSetup.RegularUserContext().Org,
					TestSetup.RegularUserContext().Space).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		})
		defer func() {
			workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
				Expect(cf.Cf("delete-security-group", securityGroupName, "-f").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			})
		}()

		Expect(cf.Cf("restart", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))

		By("Testing app egress rules")
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).To(Equal(0))

		By("Unapplying security group")
		workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
			Expect(cf.Cf("unbind-security-group", securityGroupName, TestSetup.RegularUserContext().Org, TestSetup.RegularUserContext().Space).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		})
		Expect(cf.Cf("restart", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))

		By("Testing app egress rules")
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).ToNot(Equal(0))
	})

	It("allows external and denies internal traffic during staging based on default staging security rules", func() {
		buildpack := random_name.CATSRandomName("BPK")
		testAppName := random_name.CATSRandomName("APP")
		privateUri := fmt.Sprintf("%s:%d", privateHost, privatePort)

		buildpackZip := assets.NewAssets().SecurityGroupBuildpack

		workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
			Expect(cf.Cf("create-buildpack", buildpack, buildpackZip, "999").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		})
		defer func() {
			workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
				Expect(cf.Cf("delete-buildpack", buildpack, "-f").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			})
		}()

		Expect(cf.Cf("push", testAppName, "--no-start", "-b", Config.RubyBuildpackName, "-m", DEFAULT_MEMORY_LIMIT, "-b", buildpack, "-p", assets.NewAssets().HelloWorld, "-d", Config.AppsDomain).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
		app_helpers.SetBackend(testAppName)
		defer func() { cf.Cf("delete", testAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration()) }()

		Expect(cf.Cf("set-env", testAppName, "TESTURI", privateUri).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		Expect(cf.Cf("restart", testAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(1))
		Eventually(func() *Session {
			appLogsSession := cf.Cf("logs", "--recent", testAppName)
			Expect(appLogsSession.Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			return appLogsSession
		}, 5).Should(Say("CURL_EXIT=[^0]"), "Expected staging security groups not to allow internal communication between app containers. Configure your staging security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.")

		Expect(cf.Cf("set-env", testAppName, "TESTURI", "www.google.com").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		Expect(cf.Cf("start", testAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(1))
		Eventually(func() *Session {
			appLogsSession := cf.Cf("logs", "--recent", testAppName)
			Expect(appLogsSession.Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			return appLogsSession
		}, 5).Should(Say("CURL_EXIT=0"))
	})
})
<commit_msg>Fixup flawed command usages in security-groups<commit_after>package security_groups_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	. "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry-incubator/cf-test-helpers/workflowhelpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name"
)

var _ = SecurityGroupsDescribe("Security Groups", func() {

	type AppResource struct {
		Metadata struct {
			Url string
		}
	}
	type AppsResponse struct {
		Resources []AppResource
	}

	type Stat struct {
		Stats struct {
			Host string
			Port int
		}
	}
	type StatsResponse map[string]Stat

	type DoraCurlResponse struct {
		Stdout     string
		Stderr     string
		ReturnCode int `json:"return_code"`
	}

	var serverAppName, securityGroupName, privateHost string
	var privatePort int

	BeforeEach(func() {
		serverAppName = random_name.CATSRandomName("APP")
		Expect(cf.Cf("push",
			serverAppName,
			"--no-start",
			"-b", Config.RubyBuildpackName,
			"-m", DEFAULT_MEMORY_LIMIT,
			"-p", assets.NewAssets().Dora,
			"-d", Config.AppsDomain).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
		app_helpers.SetBackend(serverAppName)
		Expect(cf.Cf("start", serverAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))

		// gather app url
		var appsResponse AppsResponse
		cfResponse := cf.Cf("curl", fmt.Sprintf("/v2/apps?q=name:%s", serverAppName)).Wait(Config.DefaultTimeoutDuration()).Out.Contents()
		json.Unmarshal(cfResponse, &appsResponse)
		serverAppUrl := appsResponse.Resources[0].Metadata.Url

		// gather app stats for execution vm ip and app port
		var statsResponse StatsResponse
		cfResponse = cf.Cf("curl", fmt.Sprintf("%s/stats", serverAppUrl)).Wait(Config.DefaultTimeoutDuration()).Out.Contents()
		json.Unmarshal(cfResponse, &statsResponse)

		privateHost = statsResponse["0"].Stats.Host
		privatePort = statsResponse["0"].Stats.Port
	})

	AfterEach(func() {
		app_helpers.AppReport(serverAppName, Config.DefaultTimeoutDuration())

		Expect(cf.Cf("delete", serverAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
	})

	// This test assumes the default running security groups block access to the vms that execute apps.
	// The test takes advantage of the fact that the execution vm's ip address and internal container ip address
	//   are discoverable via the cc api and dora's myip endpoint.
	It("allows previously-blocked ip traffic after applying a security group, and re-blocks it when the group is removed", func() {
		clientAppName := random_name.CATSRandomName("APP")
		Expect(cf.Cf("push", clientAppName, "--no-start", "-b", Config.RubyBuildpackName, "-m", DEFAULT_MEMORY_LIMIT, "-p", assets.NewAssets().Dora, "-d", Config.AppsDomain).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		app_helpers.SetBackend(clientAppName)
		Expect(cf.Cf("start", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
		defer func() { cf.Cf("delete", clientAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration()) }()

		curlResponse := helpers.CurlApp(serverAppName, "/myip")
		containerIp := strings.TrimSpace(curlResponse)

		curlResponse = helpers.CurlApp(serverAppName, "/env/VCAP_APPLICATION")
		var env map[string]interface{}
		err := json.Unmarshal([]byte(curlResponse), &env)
		Expect(err).NotTo(HaveOccurred())
		containerPort := int(env["port"].(float64))

		By("Asserting default running security-group configuration")
		var doraCurlResponse DoraCurlResponse
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).ToNot(Equal(0), "Expected running security groups not to allow internal communication between app containers. Configure your running security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.")

		By("Applying security group")
		rules := fmt.Sprintf(
			`[{"destination":"%s","ports":"%d","protocol":"tcp"},
			{"destination":"%s","ports":"%d","protocol":"tcp"}]`,
			privateHost, privatePort, containerIp, containerPort)

		file, _ := ioutil.TempFile(os.TempDir(), "CATS-sg-rules")
		defer os.Remove(file.Name())
		file.WriteString(rules)

		rulesPath := file.Name()
		securityGroupName = random_name.CATSRandomName("SG")

		workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
			Expect(cf.Cf("create-security-group", securityGroupName, rulesPath).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			Expect(
				cf.Cf("bind-security-group",
					securityGroupName,
					TestSetup.RegularUserContext().Org,
					TestSetup.RegularUserContext().Space).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		})
		defer func() {
			workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
				Expect(cf.Cf("delete-security-group", securityGroupName, "-f").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			})
		}()

		Expect(cf.Cf("restart", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))

		By("Testing app egress rules")
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).To(Equal(0))

		By("Unapplying security group")
		workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
			Expect(cf.Cf("unbind-security-group", securityGroupName, TestSetup.RegularUserContext().Org, TestSetup.RegularUserContext().Space).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		})
		Expect(cf.Cf("restart", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))

		By("Testing app egress rules")
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).ToNot(Equal(0))
	})

	It("allows external and denies internal traffic during staging based on default staging security rules", func() {
		buildpack := random_name.CATSRandomName("BPK")
		testAppName := random_name.CATSRandomName("APP")
		privateUri := fmt.Sprintf("%s:%d", privateHost, privatePort)

		buildpackZip := assets.NewAssets().SecurityGroupBuildpack

		workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
			Expect(cf.Cf("create-buildpack", buildpack, buildpackZip, "999").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		})
		defer func() {
			workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
				Expect(cf.Cf("delete-buildpack", buildpack, "-f").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			})
		}()

		Expect(cf.Cf("push", testAppName, "--no-start", "-b", Config.RubyBuildpackName, "-m", DEFAULT_MEMORY_LIMIT, "-b", buildpack, "-p", assets.NewAssets().HelloWorld, "-d", Config.AppsDomain).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
		app_helpers.SetBackend(testAppName)
		defer func() { cf.Cf("delete", testAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration()) }()

		Expect(cf.Cf("set-env", testAppName, "TESTURI", privateUri).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		Expect(cf.Cf("restart", testAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(1))
		Eventually(func() *Session {
			appLogsSession := cf.Cf("logs", "--recent", testAppName)
			Expect(appLogsSession.Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			return appLogsSession
		}, 5).Should(Say("CURL_EXIT=[^0]"), "Expected staging security groups not to allow internal communication between app containers. Configure your staging security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.")

		Expect(cf.Cf("set-env", testAppName, "TESTURI", "www.google.com").Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
		Expect(cf.Cf("restart", testAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(1))
		Eventually(func() *Session {
			appLogsSession := cf.Cf("logs", "--recent", testAppName)
			Expect(appLogsSession.Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
			return appLogsSession
		}, 5).Should(Say("CURL_EXIT=0"))
	})
})
<|endoftext|>


<commit_before>// Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
// Licensed under the Mozilla Public License v2.0

package oci

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"

	"github.com/oracle/oci-go-sdk/common"
	oci_mysql "github.com/oracle/oci-go-sdk/mysql"

	"github.com/terraform-providers/terraform-provider-oci/httpreplay"
)

var (
	mysqlConfigurationSingularDataSourceRepresentation = map[string]interface{}{
		"configuration_id": Representation{repType: Required, create: `${var.MysqlConfigurationOCID[var.region]}`},
	}

	mysqlConfigurationDataSourceRepresentation = map[string]interface{}{
		"compartment_id":   Representation{repType: Required, create: `${var.compartment_id}`},
		"configuration_id": Representation{repType: Optional, create: `${var.MysqlConfigurationOCID[var.region]}`},
		"display_name":     Representation{repType: Optional, create: `VM.Standard.E2.2.Built-in`},
		"shape_name":       Representation{repType: Optional, create: `VM.Standard.E2.2`},
		"state":            Representation{repType: Optional, create: `ACTIVE`},
		"type":             Representation{repType: Optional, create: []string{`DEFAULT`}},
	}

	MysqlConfigurationResourceConfig = MysqlConfigurationIdVariable
)

func TestMysqlMysqlConfigurationResource_basic(t *testing.T) {
	httpreplay.SetScenario("TestMysqlMysqlConfigurationResource_basic")
	defer httpreplay.SaveScenario()

	provider := testAccProvider
	config := testProviderConfig()

	compartmentId := getEnvSettingWithBlankDefault("compartment_ocid")
	compartmentIdVariableStr := fmt.Sprintf("variable \"compartment_id\" { default = \"%s\" }\n", compartmentId)

	datasourceName := "data.oci_mysql_mysql_configurations.test_mysql_configurations"
	singularDatasourceName := "data.oci_mysql_mysql_configuration.test_mysql_configuration"

	resource.Test(t, resource.TestCase{
		PreCheck: func() { testAccPreCheck(t) },
		Providers: map[string]terraform.ResourceProvider{
			"oci": provider,
		},
		Steps: []resource.TestStep{
			// verify datasource
			{
				Config: config +
					generateDataSourceFromRepresentationMap("oci_mysql_mysql_configurations", "test_mysql_configurations", Required, Create, mysqlConfigurationDataSourceRepresentation) +
					compartmentIdVariableStr + MysqlConfigurationResourceConfig,
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestCheckResourceAttr(datasourceName, "compartment_id", compartmentId),

					resource.TestCheckResourceAttrSet(datasourceName, "configurations.#"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.id"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.shape_name"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.state"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.time_created"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.type"),
				),
			},
			// verify singular datasource
			{
				Config: config +
					generateDataSourceFromRepresentationMap("oci_mysql_mysql_configuration", "test_mysql_configuration", Required, Create, mysqlConfigurationSingularDataSourceRepresentation) +
					compartmentIdVariableStr + MysqlConfigurationResourceConfig,
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestCheckResourceAttrSet(singularDatasourceName, "configuration_id"),

					resource.TestCheckResourceAttr(singularDatasourceName, "defined_tags.%", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "description", "Default configuration for the VM.Standard.E2.2 MySQL Shape"),
					resource.TestCheckResourceAttr(singularDatasourceName, "display_name", "VM.Standard.E2.2.Built-in"),
					resource.TestCheckResourceAttr(singularDatasourceName, "freeform_tags.%", "0"),
					resource.TestCheckResourceAttrSet(singularDatasourceName, "id"),
					resource.TestCheckResourceAttrSet(singularDatasourceName, "state"),
					resource.TestCheckResourceAttrSet(singularDatasourceName, "time_created"),
					resource.TestCheckResourceAttrSet(singularDatasourceName, "type"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.#", "1"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.autocommit", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.completion_type", ""),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.connect_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.cte_max_recursion_depth", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.default_authentication_plugin", ""),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.foreign_key_checks", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.generated_random_password_length", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.information_schema_stats_expiry", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_buffer_pool_instances", "4"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_buffer_pool_size", "10200547328"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_enable_stopword", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_max_token_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_min_token_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_num_word_optimize", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_result_cache_limit", "33554432"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_server_stopword_table", ""),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_lock_wait_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_max_purge_lag", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_max_purge_lag_delay", "300000"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.local_infile", "true"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mandatory_roles", "public"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.max_connections", "2000"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.max_execution_time", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.max_prepared_stmt_count", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysql_firewall_mode", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_connect_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_deflate_default_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_deflate_max_client_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_document_id_unique_prefix", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_enable_hello_notice", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_idle_worker_thread_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_interactive_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_lz4default_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_lz4max_client_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_max_allowed_packet", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_min_worker_threads", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_read_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_wait_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_write_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_zstd_max_client_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.parser_max_mem_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.query_alloc_block_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.query_prealloc_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.sql_mode", ""),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.sql_require_primary_key", "true"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.sql_warnings", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.transaction_isolation", ""),
				),
			},
		},
	})
}

func init() {
	if DependencyGraph == nil {
		initDependencyGraph()
	}
	if !inSweeperExcludeList("MysqlMysqlConfiguration") {
		resource.AddTestSweepers("MysqlMysqlConfiguration", &resource.Sweeper{
			Name:         "MysqlMysqlConfiguration",
			Dependencies: DependencyGraph["mysqlConfiguration"],
			F:            sweepMysqlMysqlConfigurationResource,
		})
	}
}

func sweepMysqlMysqlConfigurationResource(compartment string) error {
	mysqlaasClient := GetTestClients(&schema.ResourceData{}).mysqlaasClient()
	mysqlConfigurationIds, err := getMysqlConfigurationIds(compartment)
	if err != nil {
		return err
	}
	for _, mysqlConfigurationId := range mysqlConfigurationIds {
		if ok := SweeperDefaultResourceId[mysqlConfigurationId]; !ok {
			deleteConfigurationRequest := oci_mysql.DeleteConfigurationRequest{}
			deleteConfigurationRequest.ConfigurationId = &mysqlConfigurationId

			deleteConfigurationRequest.RequestMetadata.RetryPolicy = getRetryPolicy(true, "mysql")
			_, error := mysqlaasClient.DeleteConfiguration(context.Background(), deleteConfigurationRequest)
			if error != nil {
				fmt.Printf("Error deleting MysqlConfiguration %s %s, It is possible that the resource is already deleted. Please verify manually \n", mysqlConfigurationId, error)
				continue
			}
			waitTillCondition(testAccProvider, &mysqlConfigurationId, mysqlConfigurationSweepWaitCondition, time.Duration(3*time.Minute),
				mysqlConfigurationSweepResponseFetchOperation, "mysql", true)
		}
	}
	return nil
}

func getMysqlConfigurationIds(compartment string) ([]string, error) {
	ids := getResourceIdsToSweep(compartment, "MysqlConfigurationId")
	if ids != nil {
		return ids, nil
	}
	var resourceIds []string
	compartmentId := compartment
	mysqlaasClient := GetTestClients(&schema.ResourceData{}).mysqlaasClient()

	listConfigurationsRequest := oci_mysql.ListConfigurationsRequest{}
	listConfigurationsRequest.CompartmentId = &compartmentId
	listConfigurationsRequest.LifecycleState = oci_mysql.ConfigurationLifecycleStateActive
	listConfigurationsResponse, err := mysqlaasClient.ListConfigurations(context.Background(), listConfigurationsRequest)

	if err != nil {
		return resourceIds, fmt.Errorf("Error getting MysqlConfiguration list for compartment id : %s , %s \n", compartmentId, err)
	}
	for _, mysqlConfiguration := range listConfigurationsResponse.Items {
		id := *mysqlConfiguration.Id
		resourceIds = append(resourceIds, id)
		addResourceIdToSweeperResourceIdMap(compartmentId, "MysqlConfigurationId", id)
	}
	return resourceIds, nil
}

func mysqlConfigurationSweepWaitCondition(response common.OCIOperationResponse) bool {
	// Only stop if the resource is available beyond 3 mins. As there could be an issue for the sweeper to delete the resource and manual intervention required.
	if mysqlConfigurationResponse, ok := response.Response.(oci_mysql.GetConfigurationResponse); ok {
		return mysqlConfigurationResponse.LifecycleState != oci_mysql.ConfigurationLifecycleStateDeleted
	}
	return false
}

func mysqlConfigurationSweepResponseFetchOperation(client *OracleClients, resourceId *string, retryPolicy *common.RetryPolicy) error {
	_, err := client.mysqlaasClient().GetConfiguration(context.Background(), oci_mysql.GetConfigurationRequest{RequestMetadata: common.RequestMetadata{
		RetryPolicy: retryPolicy,
	},
	})
	return err
}
<commit_msg>Mysql configuration test fix<commit_after>// Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
// Licensed under the Mozilla Public License v2.0

package oci

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"

	"github.com/oracle/oci-go-sdk/common"
	oci_mysql "github.com/oracle/oci-go-sdk/mysql"

	"github.com/terraform-providers/terraform-provider-oci/httpreplay"
)

var (
	mysqlConfigurationSingularDataSourceRepresentation = map[string]interface{}{
		"configuration_id": Representation{repType: Required, create: `${var.MysqlConfigurationOCID[var.region]}`},
	}

	mysqlConfigurationDataSourceRepresentation = map[string]interface{}{
		"compartment_id":   Representation{repType: Required, create: `${var.compartment_id}`},
		"configuration_id": Representation{repType: Optional, create: `${var.MysqlConfigurationOCID[var.region]}`},
		"display_name":     Representation{repType: Optional, create: `VM.Standard.E2.2.Built-in`},
		"shape_name":       Representation{repType: Optional, create: `VM.Standard.E2.2`},
		"state":            Representation{repType: Optional, create: `ACTIVE`},
		"type":             Representation{repType: Optional, create: []string{`DEFAULT`}},
	}

	MysqlConfigurationResourceConfig = MysqlConfigurationIdVariable
)

func TestMysqlMysqlConfigurationResource_basic(t *testing.T) {
	httpreplay.SetScenario("TestMysqlMysqlConfigurationResource_basic")
	defer httpreplay.SaveScenario()

	provider := testAccProvider
	config := testProviderConfig()

	compartmentId := getEnvSettingWithBlankDefault("compartment_ocid")
	compartmentIdVariableStr := fmt.Sprintf("variable \"compartment_id\" { default = \"%s\" }\n", compartmentId)

	datasourceName := "data.oci_mysql_mysql_configurations.test_mysql_configurations"
	singularDatasourceName := "data.oci_mysql_mysql_configuration.test_mysql_configuration"

	resource.Test(t, resource.TestCase{
		PreCheck: func() { testAccPreCheck(t) },
		Providers: map[string]terraform.ResourceProvider{
			"oci": provider,
		},
		Steps: []resource.TestStep{
			// verify datasource
			{
				Config: config +
					generateDataSourceFromRepresentationMap("oci_mysql_mysql_configurations", "test_mysql_configurations", Required, Create, mysqlConfigurationDataSourceRepresentation) +
					compartmentIdVariableStr + MysqlConfigurationResourceConfig,
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestCheckResourceAttr(datasourceName, "compartment_id", compartmentId),

					resource.TestCheckResourceAttrSet(datasourceName, "configurations.#"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.id"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.shape_name"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.state"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.time_created"),
					resource.TestCheckResourceAttrSet(datasourceName, "configurations.0.type"),
				),
			},
			// verify singular datasource
			{
				Config: config +
					generateDataSourceFromRepresentationMap("oci_mysql_mysql_configuration", "test_mysql_configuration", Required, Create, mysqlConfigurationSingularDataSourceRepresentation) +
					compartmentIdVariableStr + MysqlConfigurationResourceConfig,
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestCheckResourceAttrSet(singularDatasourceName, "configuration_id"),

					resource.TestCheckResourceAttr(singularDatasourceName, "defined_tags.%", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "description", "Default configuration for the VM.Standard.E2.2 MySQL Shape"),
					resource.TestCheckResourceAttr(singularDatasourceName, "display_name", "VM.Standard.E2.2.Built-in"),
					resource.TestCheckResourceAttr(singularDatasourceName, "freeform_tags.%", "0"),
					resource.TestCheckResourceAttrSet(singularDatasourceName, "id"),
					resource.TestCheckResourceAttrSet(singularDatasourceName, "state"),
					resource.TestCheckResourceAttrSet(singularDatasourceName, "time_created"),
					resource.TestCheckResourceAttrSet(singularDatasourceName, "type"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.#", "1"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.autocommit", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.completion_type", ""),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.connect_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.cte_max_recursion_depth", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.default_authentication_plugin", ""),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.foreign_key_checks", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.generated_random_password_length", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.information_schema_stats_expiry", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_buffer_pool_instances", "4"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_buffer_pool_size", "10200547328"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_enable_stopword", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_max_token_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_min_token_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_num_word_optimize", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_result_cache_limit", "33554432"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_ft_server_stopword_table", ""),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_lock_wait_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_max_purge_lag", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.innodb_max_purge_lag_delay", "300000"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.local_infile", "true"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mandatory_roles", "public"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.max_connections", "2000"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.max_execution_time", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.max_prepared_stmt_count", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysql_firewall_mode", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_connect_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_deflate_default_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_deflate_max_client_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_document_id_unique_prefix", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_enable_hello_notice", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_idle_worker_thread_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_interactive_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_lz4default_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_lz4max_client_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_max_allowed_packet", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_min_worker_threads", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_read_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_wait_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_write_timeout", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.mysqlx_zstd_max_client_compression_level", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.parser_max_mem_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.query_alloc_block_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.query_prealloc_size", "0"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.sql_mode", ""),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.sql_require_primary_key", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.sql_warnings", "false"),
					resource.TestCheckResourceAttr(singularDatasourceName, "variables.0.transaction_isolation", ""),
				),
			},
		},
	})
}

func init() {
	if DependencyGraph == nil {
		initDependencyGraph()
	}
	if !inSweeperExcludeList("MysqlMysqlConfiguration") {
		resource.AddTestSweepers("MysqlMysqlConfiguration", &resource.Sweeper{
			Name:         "MysqlMysqlConfiguration",
			Dependencies: DependencyGraph["mysqlConfiguration"],
			F:            sweepMysqlMysqlConfigurationResource,
		})
	}
}

func sweepMysqlMysqlConfigurationResource(compartment string) error {
	mysqlaasClient := GetTestClients(&schema.ResourceData{}).mysqlaasClient()
	mysqlConfigurationIds, err := getMysqlConfigurationIds(compartment)
	if err != nil {
		return err
	}
	for _, mysqlConfigurationId := range mysqlConfigurationIds {
		if ok := SweeperDefaultResourceId[mysqlConfigurationId]; !ok {
			deleteConfigurationRequest := oci_mysql.DeleteConfigurationRequest{}
			deleteConfigurationRequest.ConfigurationId = &mysqlConfigurationId

			deleteConfigurationRequest.RequestMetadata.RetryPolicy = getRetryPolicy(true, "mysql")
			_, error := mysqlaasClient.DeleteConfiguration(context.Background(), deleteConfigurationRequest)
			if error != nil {
				fmt.Printf("Error deleting MysqlConfiguration %s %s, It is possible that the resource is already deleted. Please verify manually \n", mysqlConfigurationId, error)
				continue
			}
			waitTillCondition(testAccProvider, &mysqlConfigurationId, mysqlConfigurationSweepWaitCondition, time.Duration(3*time.Minute),
				mysqlConfigurationSweepResponseFetchOperation, "mysql", true)
		}
	}
	return nil
}

func getMysqlConfigurationIds(compartment string) ([]string, error) {
	ids := getResourceIdsToSweep(compartment, "MysqlConfigurationId")
	if ids != nil {
		return ids, nil
	}
	var resourceIds []string
	compartmentId := compartment
	mysqlaasClient := GetTestClients(&schema.ResourceData{}).mysqlaasClient()

	listConfigurationsRequest := oci_mysql.ListConfigurationsRequest{}
	listConfigurationsRequest.CompartmentId = &compartmentId
	listConfigurationsRequest.LifecycleState = oci_mysql.ConfigurationLifecycleStateActive
	listConfigurationsResponse, err := mysqlaasClient.ListConfigurations(context.Background(), listConfigurationsRequest)

	if err != nil {
		return resourceIds, fmt.Errorf("Error getting MysqlConfiguration list for compartment id : %s , %s \n", compartmentId, err)
	}
	for _, mysqlConfiguration := range listConfigurationsResponse.Items {
		id := *mysqlConfiguration.Id
		resourceIds = append(resourceIds, id)
		addResourceIdToSweeperResourceIdMap(compartmentId, "MysqlConfigurationId", id)
	}
	return resourceIds, nil
}

func mysqlConfigurationSweepWaitCondition(response common.OCIOperationResponse) bool {
	// Only stop if the resource is available beyond 3 mins. As there could be an issue for the sweeper to delete the resource and manual intervention required.
	if mysqlConfigurationResponse, ok := response.Response.(oci_mysql.GetConfigurationResponse); ok {
		return mysqlConfigurationResponse.LifecycleState != oci_mysql.ConfigurationLifecycleStateDeleted
	}
	return false
}

func mysqlConfigurationSweepResponseFetchOperation(client *OracleClients, resourceId *string, retryPolicy *common.RetryPolicy) error {
	_, err := client.mysqlaasClient().GetConfiguration(context.Background(), oci_mysql.GetConfigurationRequest{RequestMetadata: common.RequestMetadata{
		RetryPolicy: retryPolicy,
	},
	})
	return err
}
<|endoftext|>


<commit_before>package openapi3

import (
	"testing"

	"github.com/stretchr/testify/require"
)

func TestLoaderSupportsRecursiveReference(t *testing.T) {
	loader := NewLoader()
	loader.IsExternalRefsAllowed = true
	doc, err := loader.LoadFromFile("testdata/recursiveRef/openapi.yml")
	require.NoError(t, err)
	require.NotNil(t, doc)
	require.NoError(t, doc.Validate(loader.Context))
	require.Equal(t, "bar", doc.Paths["/foo"].Get.Responses.Get(200).Value.Content.Get("application/json").Schema.Value.Properties["foo2"].Value.Properties["foo"].Value.Properties["bar"].Value.Example)
}
<commit_msg>try reproducing #447 (#448)<commit_after>package openapi3

import (
	"testing"

	"github.com/stretchr/testify/require"
)

func TestLoaderSupportsRecursiveReference(t *testing.T) {
	loader := NewLoader()
	loader.IsExternalRefsAllowed = true
	doc, err := loader.LoadFromFile("testdata/recursiveRef/openapi.yml")
	require.NoError(t, err)
	err = doc.Validate(loader.Context)
	require.NoError(t, err)
	require.Equal(t, "bar", doc.Paths["/foo"].Get.Responses.Get(200).Value.Content.Get("application/json").Schema.Value.Properties["foo2"].Value.Properties["foo"].Value.Properties["bar"].Value.Example)
}

func TestIssue447(t *testing.T) {
	loader := NewLoader()
	doc, err := loader.LoadFromData([]byte(`
openapi: 3.0.1
info:
  title: Recursive refs example
  version: "1.0"
paths: {}
components:
  schemas:
    Complex:
      type: object
      properties:
        parent:
          $ref: '#/components/schemas/Complex'
`))
	require.NoError(t, err)
	err = doc.Validate(loader.Context)
	require.NoError(t, err)
	require.Equal(t, "object", doc.Components.
		// Complex
		Schemas["Complex"].
		// parent
		Value.Properties["parent"].
		// parent
		Value.Properties["parent"].
		// parent
		Value.Properties["parent"].
		// type
		Value.Type)
}
<|endoftext|>


<commit_before>// +build integration

/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package opentsdb

import (
	"bytes"
	"encoding/gob"
	"os"
	"testing"
	"time"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/core/ctypes"

	. "github.com/smartystreets/goconvey/convey"
)

func TestOpentsdbPublish(t *testing.T) {
	config := make(map[string]ctypes.ConfigValue)

	Convey("Snap Plugin integration testing with OpenTSDB", t, func() {
		var buf bytes.Buffer
		buf.Reset()
		enc := gob.NewEncoder(&buf)

		config["host"] = ctypes.ConfigValueStr{Value: os.Getenv("SNAP_OPENTSDB_HOST")}
		config["port"] = ctypes.ConfigValueInt{Value: 4242}

		op := NewOpentsdbPublisher()
		cp, _ := op.GetConfigPolicy()
		cfg, _ := cp.Get([]string{""}).Process(config)

		Convey("Publish float metrics to OpenTSDB", func() {
			metrics := []plugin.PluginMetricType{
				*plugin.NewPluginMetricType([]string{"/psutil/load/load15"}, time.Now(), "mac1", 23.1),
				*plugin.NewPluginMetricType([]string{"/psutil/vm/available"}, time.Now().Add(2*time.Second), "mac2", 23.2),
				*plugin.NewPluginMetricType([]string{"/psutil/load/load1"}, time.Now().Add(3*time.Second), "linux3", 23.3),
			}
			enc.Encode(metrics)

			err := op.Publish(plugin.SnapGOBContentType, buf.Bytes(), *cfg)
			So(err, ShouldBeNil)
		})

		Convey("Publish int metrics to OpenTSDB", func() {
			metrics := []plugin.PluginMetricType{
				*plugin.NewPluginMetricType([]string{"/psutil/vm/free"}, time.Now().Add(5*time.Second), "linux7", 23),
			}
			enc.Encode(metrics)

			err := op.Publish(plugin.SnapGOBContentType, buf.Bytes(), *cfg)
			So(err, ShouldBeNil)
		})
	})
}
<commit_msg>Fixed unit tests<commit_after>// +build integration

/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package opentsdb

import (
	"bytes"
	"encoding/gob"
	"os"
	"testing"
	"time"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/core/ctypes"

	. "github.com/smartystreets/goconvey/convey"
)

func TestOpentsdbPublish(t *testing.T) {
	config := make(map[string]ctypes.ConfigValue)

	Convey("Snap Plugin integration testing with OpenTSDB", t, func() {
		var buf bytes.Buffer
		buf.Reset()
		enc := gob.NewEncoder(&buf)

		config["host"] = ctypes.ConfigValueStr{Value: os.Getenv("SNAP_OPENTSDB_HOST")}
		config["port"] = ctypes.ConfigValueInt{Value: 4242}

		op := NewOpentsdbPublisher()
		cp, _ := op.GetConfigPolicy()
		cfg, _ := cp.Get([]string{""}).Process(config)

		Convey("Publish float metrics to OpenTSDB", func() {
			metrics := []plugin.PluginMetricType{
				*plugin.NewPluginMetricType([]string{"/psutil/load/load15"}, time.Now(), "mac1", nil, nil, 23.1),
				*plugin.NewPluginMetricType([]string{"/psutil/vm/available"}, time.Now().Add(2*time.Second), "mac2", nil, nil, 23.2),
				*plugin.NewPluginMetricType([]string{"/psutil/load/load1"}, time.Now().Add(3*time.Second), "linux3", nil, nil, 23.3),
			}
			enc.Encode(metrics)

			err := op.Publish(plugin.SnapGOBContentType, buf.Bytes(), *cfg)
			So(err, ShouldBeNil)
		})

		Convey("Publish int metrics to OpenTSDB", func() {
			metrics := []plugin.PluginMetricType{
				*plugin.NewPluginMetricType([]string{"/psutil/vm/free"}, time.Now().Add(5*time.Second), "linux7", nil, nil, 23),
			}
			enc.Encode(metrics)

			err := op.Publish(plugin.SnapGOBContentType, buf.Bytes(), *cfg)
			So(err, ShouldBeNil)
		})
	})
}
<|endoftext|>