<commit_before>// Package bitseq provides a structure and utilities for representing long bitmask
// as sequence of run-lenght encoded blocks. It operates direclty on the encoded
// representation, it does not decode/encode.
package bitseq

import (
	"fmt"
	"sync"

	"github.com/docker/libnetwork/datastore"
	"github.com/docker/libnetwork/netutils"
)

// Block Sequence constants
// If needed we can think of making these configurable
const (
	blockLen      = 32
	blockBytes    = blockLen / 8
	blockMAX      = 1<<blockLen - 1
	blockFirstBit = 1 << (blockLen - 1)
)

// Handle contains the sequece representing the bitmask and its identifier
type Handle struct {
	bits       uint32
	unselected uint32
	head       *Sequence
	app        string
	id         string
	dbIndex    uint64
	dbExists   bool
	store      datastore.DataStore
	sync.Mutex
}

// NewHandle returns a thread-safe instance of the bitmask handler
func NewHandle(app string, ds datastore.DataStore, id string, numElements uint32) (*Handle, error) {
	h := &Handle{
		app:        app,
		id:         id,
		store:      ds,
		bits:       numElements,
		unselected: numElements,
		head: &Sequence{
			Block: 0x0,
			Count: getNumBlocks(numElements),
		},
	}

	if h.store == nil {
		return h, nil
	}

	// Register for status changes
	h.watchForChanges()

	// Get the initial status from the ds if present.
	err := h.store.GetObject(datastore.Key(h.Key()...), h)
	if err != datastore.ErrKeyNotFound {
		return nil, err
	}

	return h, err
}

// Sequence reresents a recurring sequence of 32 bits long bitmasks
type Sequence struct {
	Block uint32    // block representing 4 byte long allocation bitmask
	Count uint32    // number of consecutive blocks
	Next  *Sequence // next sequence
}

// NewSequence returns a sequence initialized to represent a bitmaks of numElements bits
func NewSequence(numElements uint32) *Sequence {
	return &Sequence{Block: 0x0, Count: getNumBlocks(numElements), Next: nil}
}

// String returns a string representation of the block sequence starting from this block
func (s *Sequence) String() string {
	var nextBlock string
	if s.Next == nil {
		nextBlock = "end"
	} else {
		nextBlock = s.Next.String()
	}
	return fmt.Sprintf("(0x%x, %d)->%s", s.Block, s.Count, nextBlock)
}

// GetAvailableBit returns the position of the first unset bit in the bitmask represented by this sequence
func (s *Sequence) GetAvailableBit() (bytePos, bitPos int) {
	if s.Block == blockMAX || s.Count == 0 {
		return -1, -1
	}
	bits := 0
	bitSel := uint32(blockFirstBit)
	for bitSel > 0 && s.Block&bitSel != 0 {
		bitSel >>= 1
		bits++
	}
	return bits / 8, bits % 8
}

// GetCopy returns a copy of the linked list rooted at this node
func (s *Sequence) GetCopy() *Sequence {
	n := &Sequence{Block: s.Block, Count: s.Count}
	pn := n
	ps := s.Next
	for ps != nil {
		pn.Next = &Sequence{Block: ps.Block, Count: ps.Count}
		pn = pn.Next
		ps = ps.Next
	}
	return n
}

// Equal checks if this sequence is equal to the passed one
func (s *Sequence) Equal(o *Sequence) bool {
	this := s
	other := o
	for this != nil {
		if other == nil {
			return false
		}
		if this.Block != other.Block || this.Count != other.Count {
			return false
		}
		this = this.Next
		other = other.Next
	}
	// Check if other is longer than this
	if other != nil {
		return false
	}
	return true
}

// ToByteArray converts the sequence into a byte array
// TODO (aboch): manage network/host order stuff
func (s *Sequence) ToByteArray() ([]byte, error) {
	var bb []byte

	p := s
	for p != nil {
		bb = append(bb, netutils.U32ToA(p.Block)...)
		bb = append(bb, netutils.U32ToA(p.Count)...)
		p = p.Next
	}

	return bb, nil
}

// FromByteArray construct the sequence from the byte array
// TODO (aboch): manage network/host order stuff
func (s *Sequence) FromByteArray(data []byte) error {
	l := len(data)
	if l%8 != 0 {
		return fmt.Errorf("cannot deserialize byte sequence of lenght %d (%v)", l, data)
	}

	p := s
	i := 0
	for {
		p.Block = netutils.ATo32(data[i : i+4])
		p.Count = netutils.ATo32(data[i+4 : i+8])
		i += 8
		if i == l {
			break
		}
		p.Next = &Sequence{}
		p = p.Next
	}

	return nil
}

// GetFirstAvailable returns the byte and bit position of the first unset bit
func (h *Handle) GetFirstAvailable() (int, int, error) {
	h.Lock()
	defer h.Unlock()
	return GetFirstAvailable(h.head)
}

// CheckIfAvailable checks if the bit correspondent to the specified ordinal is unset
// If the ordinal is beyond the Sequence limits, a negative response is returned
func (h *Handle) CheckIfAvailable(ordinal int) (int, int, error) {
	h.Lock()
	defer h.Unlock()
	return CheckIfAvailable(h.head, ordinal)
}

// PushReservation pushes the bit reservation inside the bitmask.
func (h *Handle) PushReservation(bytePos, bitPos int, release bool) error {
	// Create a copy of the current handler
	h.Lock()
	nh := &Handle{
		app:      h.app,
		id:       h.id,
		store:    h.store,
		dbIndex:  h.dbIndex,
		head:     h.head.GetCopy(),
		dbExists: h.dbExists,
	}
	h.Unlock()

	nh.head = PushReservation(bytePos, bitPos, nh.head, release)

	err := nh.writeToStore()
	if err == nil {
		// Commit went through, save locally
		h.Lock()
		h.head = nh.head
		if release {
			h.unselected++
		} else {
			h.unselected--
		}
		// Can't use SetIndex() since we're locked.
		h.dbIndex = nh.Index()
		h.dbExists = true
		h.Unlock()
	}

	return err
}

// Destroy removes from the datastore the data belonging to this handle
func (h *Handle) Destroy() {
	h.deleteFromStore()
}

// ToByteArray converts this handle's data into a byte array
func (h *Handle) ToByteArray() ([]byte, error) {
	ba := make([]byte, 8)

	h.Lock()
	defer h.Unlock()
	copy(ba[0:4], netutils.U32ToA(h.bits))
	copy(ba[4:8], netutils.U32ToA(h.unselected))
	bm, err := h.head.ToByteArray()
	if err != nil {
		return nil, fmt.Errorf("failed to serialize head: %s", err.Error())
	}
	ba = append(ba, bm...)

	return ba, nil
}

// FromByteArray reads his handle's data from a byte array
func (h *Handle) FromByteArray(ba []byte) error {
	if ba == nil {
		return fmt.Errorf("nil byte array")
	}

	nh := &Sequence{}
	err := nh.FromByteArray(ba[8:])
	if err != nil {
		return fmt.Errorf("failed to deserialize head: %s", err.Error())
	}

	h.Lock()
	h.head = nh
	h.bits = netutils.ATo32(ba[0:4])
	h.unselected = netutils.ATo32(ba[4:8])
	h.Unlock()

	return nil
}

// Bits returns the length of the bit sequence
func (h *Handle) Bits() uint32 {
	return h.bits
}

// Unselected returns the number of bits which are not selected
func (h *Handle) Unselected() uint32 {
	h.Lock()
	defer h.Unlock()
	return h.unselected
}

// GetFirstAvailable looks for the first unset bit in passed mask
func GetFirstAvailable(head *Sequence) (int, int, error) {
	byteIndex := 0
	current := head
	for current != nil {
		if current.Block != blockMAX {
			bytePos, bitPos := current.GetAvailableBit()
			return byteIndex + bytePos, bitPos, nil
		}
		byteIndex += int(current.Count * blockBytes)
		current = current.Next
	}
	return -1, -1, fmt.Errorf("no bit available")
}

// CheckIfAvailable checks if the bit correspondent to the specified ordinal is unset
// If the ordinal is beyond the Sequence limits, a negative response is returned
func CheckIfAvailable(head *Sequence, ordinal int) (int, int, error) {
	bytePos := ordinal / 8
	bitPos := ordinal % 8

	// Find the Sequence containing this byte
	current, _, _, inBlockBytePos := findSequence(head, bytePos)

	if current != nil {
		// Check whether the bit corresponding to the ordinal address is unset
		bitSel := uint32(blockFirstBit >> uint(inBlockBytePos*8+bitPos))
		if current.Block&bitSel == 0 {
			return bytePos, bitPos, nil
		}
	}

	return -1, -1, fmt.Errorf("requested bit is not available")
}

// Given the byte position and the sequences list head, return the pointer to the
// sequence containing the byte (current), the pointer to the previous sequence,
// the number of blocks preceding the block containing the byte inside the current sequence.
// If bytePos is outside of the list, function will return (nil, nil, 0, -1)
func findSequence(head *Sequence, bytePos int) (*Sequence, *Sequence, uint32, int) {
	// Find the Sequence containing this byte
	previous := head
	current := head
	n := bytePos
	for current.Next != nil && n >= int(current.Count*blockBytes) { // Nil check for less than 32 addresses masks
		n -= int(current.Count * blockBytes)
		previous = current
		current = current.Next
	}

	// If byte is outside of the list, let caller know
	if n >= int(current.Count*blockBytes) {
		return nil, nil, 0, -1
	}

	// Find the byte position inside the block and the number of blocks
	// preceding the block containing the byte inside this sequence
	precBlocks := uint32(n / blockBytes)
	inBlockBytePos := bytePos % blockBytes

	return current, previous, precBlocks, inBlockBytePos
}

// PushReservation pushes the bit reservation inside the bitmask.
// Given byte and bit positions, identify the sequence (current) which holds the block containing the affected bit.
// Create a new block with the modified bit according to the operation (allocate/release).
// Create a new Sequence containing the new Block and insert it in the proper position.
// Remove current sequence if empty.
// Check if new Sequence can be merged with neighbour (previous/Next) sequences.
//
//
// Identify "current" Sequence containing block:
//                                      [prev seq] [current seq] [Next seq]
//
// Based on block position, resulting list of sequences can be any of three forms:
//
//        Block position                        Resulting list of sequences
// A) Block is first in current:         [prev seq] [new] [modified current seq] [Next seq]
// B) Block is last in current:          [prev seq] [modified current seq] [new] [Next seq]
// C) Block is in the middle of current: [prev seq] [curr pre] [new] [curr post] [Next seq]
func PushReservation(bytePos, bitPos int, head *Sequence, release bool) *Sequence {
	// Store list's head
	newHead := head

	// Find the Sequence containing this byte
	current, previous, precBlocks, inBlockBytePos := findSequence(head, bytePos)
	if current == nil {
		return newHead
	}

	// Construct updated block
	bitSel := uint32(blockFirstBit >> uint(inBlockBytePos*8+bitPos))
	newBlock := current.Block
	if release {
		newBlock &^= bitSel
	} else {
		newBlock |= bitSel
	}

	// Quit if it was a redundant request
	if current.Block == newBlock {
		return newHead
	}

	// Current Sequence inevitably looses one block, upadate Count
	current.Count--

	// Create new sequence
	newSequence := &Sequence{Block: newBlock, Count: 1}

	// Insert the new sequence in the list based on block position
	if precBlocks == 0 { // First in sequence (A)
		newSequence.Next = current
		if current == head {
			newHead = newSequence
			previous = newHead
		} else {
			previous.Next = newSequence
		}
		removeCurrentIfEmpty(&newHead, newSequence, current)
		mergeSequences(previous)
	} else if precBlocks == current.Count-2 { // Last in sequence (B)
		newSequence.Next = current.Next
		current.Next = newSequence
		mergeSequences(current)
	} else { // In between the sequence (C)
		currPre := &Sequence{Block: current.Block, Count: precBlocks, Next: newSequence}
		currPost := current
		currPost.Count -= precBlocks
		newSequence.Next = currPost
		if currPost == head {
			newHead = currPre
		} else {
			previous.Next = currPre
		}
		// No merging or empty current possible here
	}

	return newHead
}

// Removes the current sequence from the list if empty, adjusting the head pointer if needed
func removeCurrentIfEmpty(head **Sequence, previous, current *Sequence) {
	if current.Count == 0 {
		if current == *head {
			*head = current.Next
		} else {
			previous.Next = current.Next
			current = current.Next
		}
	}
}

// Given a pointer to a Sequence, it checks if it can be merged with any following sequences
// It stops when no more merging is possible.
// TODO: Optimization: only attempt merge from start to end sequence, no need to scan till the end of the list
func mergeSequences(seq *Sequence) {
	if seq != nil {
		// Merge all what possible from seq
		for seq.Next != nil && seq.Block == seq.Next.Block {
			seq.Count += seq.Next.Count
			seq.Next = seq.Next.Next
		}
		// Move to Next
		mergeSequences(seq.Next)
	}
}

func getNumBlocks(numBits uint32) uint32 {
	numBlocks := numBits / blockLen
	if numBits%blockLen != 0 {
		numBlocks++
	}
	return numBlocks
}
<commit_msg>Fix incorrect error handling in bitseq constructor<commit_after>// Package bitseq provides a structure and utilities for representing long bitmask
// as sequence of run-lenght encoded blocks. It operates direclty on the encoded
// representation, it does not decode/encode.
package bitseq

import (
	"fmt"
	"sync"

	"github.com/docker/libnetwork/datastore"
	"github.com/docker/libnetwork/netutils"
)

// Block Sequence constants
// If needed we can think of making these configurable
const (
	blockLen      = 32
	blockBytes    = blockLen / 8
	blockMAX      = 1<<blockLen - 1
	blockFirstBit = 1 << (blockLen - 1)
)

// Handle contains the sequece representing the bitmask and its identifier
type Handle struct {
	bits       uint32
	unselected uint32
	head       *Sequence
	app        string
	id         string
	dbIndex    uint64
	dbExists   bool
	store      datastore.DataStore
	sync.Mutex
}

// NewHandle returns a thread-safe instance of the bitmask handler
func NewHandle(app string, ds datastore.DataStore, id string, numElements uint32) (*Handle, error) {
	h := &Handle{
		app:        app,
		id:         id,
		store:      ds,
		bits:       numElements,
		unselected: numElements,
		head: &Sequence{
			Block: 0x0,
			Count: getNumBlocks(numElements),
		},
	}

	if h.store == nil {
		return h, nil
	}

	// Register for status changes
	h.watchForChanges()

	// Get the initial status from the ds if present.
	if err := h.store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound {
		return nil, err
	}

	return h, nil
}

// Sequence reresents a recurring sequence of 32 bits long bitmasks
type Sequence struct {
	Block uint32    // block representing 4 byte long allocation bitmask
	Count uint32    // number of consecutive blocks
	Next  *Sequence // next sequence
}

// NewSequence returns a sequence initialized to represent a bitmaks of numElements bits
func NewSequence(numElements uint32) *Sequence {
	return &Sequence{Block: 0x0, Count: getNumBlocks(numElements), Next: nil}
}

// String returns a string representation of the block sequence starting from this block
func (s *Sequence) String() string {
	var nextBlock string
	if s.Next == nil {
		nextBlock = "end"
	} else {
		nextBlock = s.Next.String()
	}
	return fmt.Sprintf("(0x%x, %d)->%s", s.Block, s.Count, nextBlock)
}

// GetAvailableBit returns the position of the first unset bit in the bitmask represented by this sequence
func (s *Sequence) GetAvailableBit() (bytePos, bitPos int) {
	if s.Block == blockMAX || s.Count == 0 {
		return -1, -1
	}
	bits := 0
	bitSel := uint32(blockFirstBit)
	for bitSel > 0 && s.Block&bitSel != 0 {
		bitSel >>= 1
		bits++
	}
	return bits / 8, bits % 8
}

// GetCopy returns a copy of the linked list rooted at this node
func (s *Sequence) GetCopy() *Sequence {
	n := &Sequence{Block: s.Block, Count: s.Count}
	pn := n
	ps := s.Next
	for ps != nil {
		pn.Next = &Sequence{Block: ps.Block, Count: ps.Count}
		pn = pn.Next
		ps = ps.Next
	}
	return n
}

// Equal checks if this sequence is equal to the passed one
func (s *Sequence) Equal(o *Sequence) bool {
	this := s
	other := o
	for this != nil {
		if other == nil {
			return false
		}
		if this.Block != other.Block || this.Count != other.Count {
			return false
		}
		this = this.Next
		other = other.Next
	}
	// Check if other is longer than this
	if other != nil {
		return false
	}
	return true
}

// ToByteArray converts the sequence into a byte array
// TODO (aboch): manage network/host order stuff
func (s *Sequence) ToByteArray() ([]byte, error) {
	var bb []byte

	p := s
	for p != nil {
		bb = append(bb, netutils.U32ToA(p.Block)...)
		bb = append(bb, netutils.U32ToA(p.Count)...)
		p = p.Next
	}

	return bb, nil
}

// FromByteArray construct the sequence from the byte array
// TODO (aboch): manage network/host order stuff
func (s *Sequence) FromByteArray(data []byte) error {
	l := len(data)
	if l%8 != 0 {
		return fmt.Errorf("cannot deserialize byte sequence of lenght %d (%v)", l, data)
	}

	p := s
	i := 0
	for {
		p.Block = netutils.ATo32(data[i : i+4])
		p.Count = netutils.ATo32(data[i+4 : i+8])
		i += 8
		if i == l {
			break
		}
		p.Next = &Sequence{}
		p = p.Next
	}

	return nil
}

// GetFirstAvailable returns the byte and bit position of the first unset bit
func (h *Handle) GetFirstAvailable() (int, int, error) {
	h.Lock()
	defer h.Unlock()
	return GetFirstAvailable(h.head)
}

// CheckIfAvailable checks if the bit correspondent to the specified ordinal is unset
// If the ordinal is beyond the Sequence limits, a negative response is returned
func (h *Handle) CheckIfAvailable(ordinal int) (int, int, error) {
	h.Lock()
	defer h.Unlock()
	return CheckIfAvailable(h.head, ordinal)
}

// PushReservation pushes the bit reservation inside the bitmask.
func (h *Handle) PushReservation(bytePos, bitPos int, release bool) error {
	// Create a copy of the current handler
	h.Lock()
	nh := &Handle{
		app:      h.app,
		id:       h.id,
		store:    h.store,
		dbIndex:  h.dbIndex,
		head:     h.head.GetCopy(),
		dbExists: h.dbExists,
	}
	h.Unlock()

	nh.head = PushReservation(bytePos, bitPos, nh.head, release)

	err := nh.writeToStore()
	if err == nil {
		// Commit went through, save locally
		h.Lock()
		h.head = nh.head
		if release {
			h.unselected++
		} else {
			h.unselected--
		}
		// Can't use SetIndex() since we're locked.
		h.dbIndex = nh.Index()
		h.dbExists = true
		h.Unlock()
	}

	return err
}

// Destroy removes from the datastore the data belonging to this handle
func (h *Handle) Destroy() {
	h.deleteFromStore()
}

// ToByteArray converts this handle's data into a byte array
func (h *Handle) ToByteArray() ([]byte, error) {
	ba := make([]byte, 8)

	h.Lock()
	defer h.Unlock()
	copy(ba[0:4], netutils.U32ToA(h.bits))
	copy(ba[4:8], netutils.U32ToA(h.unselected))
	bm, err := h.head.ToByteArray()
	if err != nil {
		return nil, fmt.Errorf("failed to serialize head: %s", err.Error())
	}
	ba = append(ba, bm...)

	return ba, nil
}

// FromByteArray reads his handle's data from a byte array
func (h *Handle) FromByteArray(ba []byte) error {
	if ba == nil {
		return fmt.Errorf("nil byte array")
	}

	nh := &Sequence{}
	err := nh.FromByteArray(ba[8:])
	if err != nil {
		return fmt.Errorf("failed to deserialize head: %s", err.Error())
	}

	h.Lock()
	h.head = nh
	h.bits = netutils.ATo32(ba[0:4])
	h.unselected = netutils.ATo32(ba[4:8])
	h.Unlock()

	return nil
}

// Bits returns the length of the bit sequence
func (h *Handle) Bits() uint32 {
	return h.bits
}

// Unselected returns the number of bits which are not selected
func (h *Handle) Unselected() uint32 {
	h.Lock()
	defer h.Unlock()
	return h.unselected
}

// GetFirstAvailable looks for the first unset bit in passed mask
func GetFirstAvailable(head *Sequence) (int, int, error) {
	byteIndex := 0
	current := head
	for current != nil {
		if current.Block != blockMAX {
			bytePos, bitPos := current.GetAvailableBit()
			return byteIndex + bytePos, bitPos, nil
		}
		byteIndex += int(current.Count * blockBytes)
		current = current.Next
	}
	return -1, -1, fmt.Errorf("no bit available")
}

// CheckIfAvailable checks if the bit correspondent to the specified ordinal is unset
// If the ordinal is beyond the Sequence limits, a negative response is returned
func CheckIfAvailable(head *Sequence, ordinal int) (int, int, error) {
	bytePos := ordinal / 8
	bitPos := ordinal % 8

	// Find the Sequence containing this byte
	current, _, _, inBlockBytePos := findSequence(head, bytePos)

	if current != nil {
		// Check whether the bit corresponding to the ordinal address is unset
		bitSel := uint32(blockFirstBit >> uint(inBlockBytePos*8+bitPos))
		if current.Block&bitSel == 0 {
			return bytePos, bitPos, nil
		}
	}

	return -1, -1, fmt.Errorf("requested bit is not available")
}

// Given the byte position and the sequences list head, return the pointer to the
// sequence containing the byte (current), the pointer to the previous sequence,
// the number of blocks preceding the block containing the byte inside the current sequence.
// If bytePos is outside of the list, function will return (nil, nil, 0, -1)
func findSequence(head *Sequence, bytePos int) (*Sequence, *Sequence, uint32, int) {
	// Find the Sequence containing this byte
	previous := head
	current := head
	n := bytePos
	for current.Next != nil && n >= int(current.Count*blockBytes) { // Nil check for less than 32 addresses masks
		n -= int(current.Count * blockBytes)
		previous = current
		current = current.Next
	}

	// If byte is outside of the list, let caller know
	if n >= int(current.Count*blockBytes) {
		return nil, nil, 0, -1
	}

	// Find the byte position inside the block and the number of blocks
	// preceding the block containing the byte inside this sequence
	precBlocks := uint32(n / blockBytes)
	inBlockBytePos := bytePos % blockBytes

	return current, previous, precBlocks, inBlockBytePos
}

// PushReservation pushes the bit reservation inside the bitmask.
// Given byte and bit positions, identify the sequence (current) which holds the block containing the affected bit.
// Create a new block with the modified bit according to the operation (allocate/release).
// Create a new Sequence containing the new Block and insert it in the proper position.
// Remove current sequence if empty.
// Check if new Sequence can be merged with neighbour (previous/Next) sequences.
//
//
// Identify "current" Sequence containing block:
//                                      [prev seq] [current seq] [Next seq]
//
// Based on block position, resulting list of sequences can be any of three forms:
//
//        Block position                        Resulting list of sequences
// A) Block is first in current:         [prev seq] [new] [modified current seq] [Next seq]
// B) Block is last in current:          [prev seq] [modified current seq] [new] [Next seq]
// C) Block is in the middle of current: [prev seq] [curr pre] [new] [curr post] [Next seq]
func PushReservation(bytePos, bitPos int, head *Sequence, release bool) *Sequence {
	// Store list's head
	newHead := head

	// Find the Sequence containing this byte
	current, previous, precBlocks, inBlockBytePos := findSequence(head, bytePos)
	if current == nil {
		return newHead
	}

	// Construct updated block
	bitSel := uint32(blockFirstBit >> uint(inBlockBytePos*8+bitPos))
	newBlock := current.Block
	if release {
		newBlock &^= bitSel
	} else {
		newBlock |= bitSel
	}

	// Quit if it was a redundant request
	if current.Block == newBlock {
		return newHead
	}

	// Current Sequence inevitably looses one block, upadate Count
	current.Count--

	// Create new sequence
	newSequence := &Sequence{Block: newBlock, Count: 1}

	// Insert the new sequence in the list based on block position
	if precBlocks == 0 { // First in sequence (A)
		newSequence.Next = current
		if current == head {
			newHead = newSequence
			previous = newHead
		} else {
			previous.Next = newSequence
		}
		removeCurrentIfEmpty(&newHead, newSequence, current)
		mergeSequences(previous)
	} else if precBlocks == current.Count-2 { // Last in sequence (B)
		newSequence.Next = current.Next
		current.Next = newSequence
		mergeSequences(current)
	} else { // In between the sequence (C)
		currPre := &Sequence{Block: current.Block, Count: precBlocks, Next: newSequence}
		currPost := current
		currPost.Count -= precBlocks
		newSequence.Next = currPost
		if currPost == head {
			newHead = currPre
		} else {
			previous.Next = currPre
		}
		// No merging or empty current possible here
	}

	return newHead
}

// Removes the current sequence from the list if empty, adjusting the head pointer if needed
func removeCurrentIfEmpty(head **Sequence, previous, current *Sequence) {
	if current.Count == 0 {
		if current == *head {
			*head = current.Next
		} else {
			previous.Next = current.Next
			current = current.Next
		}
	}
}

// Given a pointer to a Sequence, it checks if it can be merged with any following sequences
// It stops when no more merging is possible.
// TODO: Optimization: only attempt merge from start to end sequence, no need to scan till the end of the list
func mergeSequences(seq *Sequence) {
	if seq != nil {
		// Merge all what possible from seq
		for seq.Next != nil && seq.Block == seq.Next.Block {
			seq.Count += seq.Next.Count
			seq.Next = seq.Next.Next
		}
		// Move to Next
		mergeSequences(seq.Next)
	}
}

func getNumBlocks(numBits uint32) uint32 {
	numBlocks := numBits / blockLen
	if numBits%blockLen != 0 {
		numBlocks++
	}
	return numBlocks
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"k8s.io/test-infra/mungegithub/features"
	"k8s.io/test-infra/mungegithub/github"
	"strings"
)

type SigMentionHandler struct{}

func init() {
	h := &SigMentionHandler{}
	RegisterMungerOrDie(h)
}

// Name is the name usable in --pr-mungers
func (*SigMentionHandler) Name() string { return "sig-mention-handler" }

// RequiredFeatures is a slice of 'features' that must be provided
func (*SigMentionHandler) RequiredFeatures() []string {
	return []string{}
}

// Initialize will initialize the munger
func (s *SigMentionHandler) Initialize(config *github.Config, features *features.Features) error {
	return nil
}

// EachLoop is called at the start of every munge loop
func (*SigMentionHandler) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (*SigMentionHandler) AddFlags(cmd *cobra.Command, config *github.Config) {}

func (*SigMentionHandler) HasSigLabel(obj *github.MungeObject) bool {
	labels := obj.Issue.Labels

	for i := range labels {
		if labels[i].Name != nil && strings.HasPrefix(*labels[i].Name, "sig/") {
			return true
		}
	}

	return false
}

func (*SigMentionHandler) HasNeedsSigLabel(obj *github.MungeObject) bool {
	labels := obj.Issue.Labels

	for i := range labels {
		if labels[i].Name != nil && strings.Compare(*labels[i].Name, "needs-sig") == 0 {
			return true
		}
	}

	return false
}

// Munge is the workhorse notifying issue owner to add a @kubernetes/sig mention if there is none
// The algorithm:
// (1) return if it is a PR and/or the issue is closed
// (2) find if the issue has a sig label
// (3) find if the issue has a needs-sig label
// (4) if the issue has both the sig and needs-sig labels, remove the needs-sig label
// (5) if the issue has none of the labels, add the needs-sig label and comment
// (6) if the issue has only the sig label, do nothing
// (7) if the issue has only the needs-sig label, do nothing
func (s *SigMentionHandler) Munge(obj *github.MungeObject) {
	if obj.Issue == nil || obj.IsPR() || obj.Issue.State == nil || *obj.Issue.State == "closed" {
		return
	}

	hasSigLabel := s.HasSigLabel(obj)
	hasNeedsSigLabel := s.HasNeedsSigLabel(obj)

	if hasSigLabel && hasNeedsSigLabel {
		if err := obj.RemoveLabel("needs-sig"); err != nil {
			glog.Errorf("failed to remove needs-sig label for issue #%v", *obj.Issue.Number)
		}
	} else if !hasSigLabel && !hasNeedsSigLabel {
		if err := obj.AddLabel("needs-sig"); err != nil {
			glog.Errorf("failed to add needs-sig label for issue #%v", *obj.Issue.Number)
			return
		}

		msg := fmt.Sprintf("@%s There are no sig labels on this issue. Please [add a sig label](https://github.com/kubernetes/test-infra/blob/master/commands.md) by:
(1) mentioning a sig: `@kubernetes/sig-<team-name>-misc`
(2) specifying the label manually: `/sig <label>`

_Note: method (1) will trigger a notification to the team. You can find the team list [here](https://github.com/kubernetes/community/blob/master/sig-list.md)._", *obj.Issue.User.Login)

		if err := obj.WriteComment(msg); err != nil {
			glog.Errorf("failed to leave comment for %s that issue #%v needs sig label", *obj.Issue.User.Login, *obj.Issue.Number)
		}
	}
}
<commit_msg>add link to label list (#2927)<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"k8s.io/test-infra/mungegithub/features"
	"k8s.io/test-infra/mungegithub/github"
	"strings"
)

type SigMentionHandler struct{}

func init() {
	h := &SigMentionHandler{}
	RegisterMungerOrDie(h)
}

// Name is the name usable in --pr-mungers
func (*SigMentionHandler) Name() string { return "sig-mention-handler" }

// RequiredFeatures is a slice of 'features' that must be provided
func (*SigMentionHandler) RequiredFeatures() []string {
	return []string{}
}

// Initialize will initialize the munger
func (s *SigMentionHandler) Initialize(config *github.Config, features *features.Features) error {
	return nil
}

// EachLoop is called at the start of every munge loop
func (*SigMentionHandler) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (*SigMentionHandler) AddFlags(cmd *cobra.Command, config *github.Config) {}

func (*SigMentionHandler) HasSigLabel(obj *github.MungeObject) bool {
	labels := obj.Issue.Labels

	for i := range labels {
		if labels[i].Name != nil && strings.HasPrefix(*labels[i].Name, "sig/") {
			return true
		}
	}

	return false
}

func (*SigMentionHandler) HasNeedsSigLabel(obj *github.MungeObject) bool {
	labels := obj.Issue.Labels

	for i := range labels {
		if labels[i].Name != nil && strings.Compare(*labels[i].Name, "needs-sig") == 0 {
			return true
		}
	}

	return false
}

// Munge is the workhorse notifying issue owner to add a @kubernetes/sig mention if there is none
// The algorithm:
// (1) return if it is a PR and/or the issue is closed
// (2) find if the issue has a sig label
// (3) find if the issue has a needs-sig label
// (4) if the issue has both the sig and needs-sig labels, remove the needs-sig label
// (5) if the issue has none of the labels, add the needs-sig label and comment
// (6) if the issue has only the sig label, do nothing
// (7) if the issue has only the needs-sig label, do nothing
func (s *SigMentionHandler) Munge(obj *github.MungeObject) {
	if obj.Issue == nil || obj.IsPR() || obj.Issue.State == nil || *obj.Issue.State == "closed" {
		return
	}

	hasSigLabel := s.HasSigLabel(obj)
	hasNeedsSigLabel := s.HasNeedsSigLabel(obj)

	if hasSigLabel && hasNeedsSigLabel {
		if err := obj.RemoveLabel("needs-sig"); err != nil {
			glog.Errorf("failed to remove needs-sig label for issue #%v", *obj.Issue.Number)
		}
	} else if !hasSigLabel && !hasNeedsSigLabel {
		if err := obj.AddLabel("needs-sig"); err != nil {
			glog.Errorf("failed to add needs-sig label for issue #%v", *obj.Issue.Number)
			return
		}

		msg := fmt.Sprintf("@%s There are no sig labels on this issue. Please [add a sig label](https://github.com/kubernetes/test-infra/blob/master/commands.md) by:
(1) mentioning a sig: `@kubernetes/sig-<team-name>-misc`
(2) specifying the label manually: `/sig <label>`

_Note: method (1) will trigger a notification to the team. You can find the team list [here](https://github.com/kubernetes/community/blob/master/sig-list.md) and label list [here](https://github.com/kubernetes/kubernetes/labels)_", *obj.Issue.User.Login)

		if err := obj.WriteComment(msg); err != nil {
			glog.Errorf("failed to leave comment for %s that issue #%v needs sig label", *obj.Issue.User.Login, *obj.Issue.Number)
		}
	}
}
<|endoftext|>


<commit_before>package datadog

import (
	"bytes"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/zorkian/go-datadog-api"

	"github.com/hashicorp/terraform/helper/schema"
)

// resourceDatadogOutlierAlert is a Datadog monitor resource
func resourceDatadogOutlierAlert() *schema.Resource {
	return &schema.Resource{
		Create: resourceDatadogOutlierAlertCreate,
		Read:   resourceDatadogOutlierAlertRead,
		Update: resourceDatadogOutlierAlertUpdate,
		Delete: resourceDatadogOutlierAlertDelete,
		Exists: resourceDatadogOutlierAlertExists,

		Schema: map[string]*schema.Schema{
			// TODO: import this shit from a shared resource (prevent duplication)
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"metric": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"tags": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
			"keys": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
			"time_aggr": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"time_window": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"space_aggr": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"operator": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"message": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			// Alert Settings
			"warning": &schema.Schema{
				Type:     schema.TypeMap,
				Required: true,
			},
			"critical": &schema.Schema{
				Type:     schema.TypeMap,
				Required: true,
			},

			// Additional Settings
			"notify_no_data": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  true,
			},

			"no_data_timeframe": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
			},
			// TODO: Add outlier specific data here
			// Options: algo (currently dbscan or mad) (encorce check?)
			"algorithm": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Default:  "dbscan",
			},
			"tolerance": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  3,
			},
		},
	}
}

// buildMonitorStruct returns a monitor struct
func buildOutlierAlertStruct(d *schema.ResourceData, typeStr string) *datadog.Monitor {
	name := d.Get("name").(string)
	message := d.Get("message").(string)
	timeAggr := d.Get("time_aggr").(string)
	timeWindow := d.Get("time_window").(string)
	spaceAggr := d.Get("space_aggr").(string)
	metric := d.Get("metric").(string)
	algorithm := d.Get("algorithm").(string)
	tolerance := strconv.Itoa(d.Get("tolerance").(int))

	// Tags are are no separate resource/gettable, so some trickery is needed
	var buffer bytes.Buffer
	if raw, ok := d.GetOk("tags"); ok {
		list := raw.([]interface{})
		length := (len(list) - 1)
		for i, v := range list {
			buffer.WriteString(fmt.Sprintf("%s", v))
			if i != length {
				buffer.WriteString(",")
			}

		}
	}

	tagsParsed := buffer.String()

	// Keys are used for multi alerts
	var b bytes.Buffer
	if raw, ok := d.GetOk("keys"); ok {
		list := raw.([]interface{})
		b.WriteString("by {")
		length := (len(list) - 1)
		for i, v := range list {
			b.WriteString(fmt.Sprintf("%s", v))
			if i != length {
				b.WriteString(",")
			}

		}
		b.WriteString("}")
	}

	keys := b.String()

	operator := d.Get("operator").(string)

	query := fmt.Sprintf("%s(%s):outliers(%s:%s{%s} %s, '%s',%s) %s %s", timeAggr,
		timeWindow,
		spaceAggr,
		metric,
		tagsParsed,
		keys,
		algorithm,
		tolerance,
		operator,
		d.Get(fmt.Sprintf("%s.threshold", typeStr)))

	log.Print(fmt.Sprintf("[DEBUG] submitting query: %s", query))

	o := datadog.Options{
		NotifyNoData:    d.Get("notify_no_data").(bool),
		NoDataTimeframe: d.Get("no_data_timeframe").(int),
	}

	m := datadog.Monitor{
		Type:    "query alert",
		Query:   query,
		Name:    fmt.Sprintf("[%s] %s", typeStr, name),
		Message: fmt.Sprintf("%s %s", message, d.Get(fmt.Sprintf("%s.notify", typeStr))),
		Options: o,
	}

	return &m
}

// resourceDatadogOutlierAlertCreate creates a monitor.
func resourceDatadogOutlierAlertCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*datadog.Client)

	w, err := client.CreateMonitor(buildOutlierAlertStruct(d, "warning"))

	if err != nil {
		return fmt.Errorf("error creating warning: %s", err)
	}

	c, cErr := client.CreateMonitor(buildOutlierAlertStruct(d, "critical"))

	if cErr != nil {
		return fmt.Errorf("error creating warning: %s", cErr)
	}

	log.Printf("[DEBUG] Saving IDs: %s__%s", strconv.Itoa(w.Id), strconv.Itoa(c.Id))

	d.SetId(fmt.Sprintf("%s__%s", strconv.Itoa(w.Id), strconv.Itoa(c.Id)))

	return nil
}

// resourceDatadogOutlierAlertDelete deletes a monitor.
func resourceDatadogOutlierAlertDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*datadog.Client)

	for _, v := range strings.Split(d.Id(), "__") {
		if v == "" {
			return fmt.Errorf("Id not set.")
		}
		ID, iErr := strconv.Atoi(v)

		if iErr != nil {
			return iErr
		}

		err := client.DeleteMonitor(ID)
		if err != nil {
			return err
		}
	}
	return nil
}

// resourceDatadogOutlierAlertExists verifies a monitor exists.
func resourceDatadogOutlierAlertExists(d *schema.ResourceData, meta interface{}) (b bool, e error) {
	// Exists - This is called to verify a resource still exists. It is called prior to Read,
	// and lowers the burden of Read to be able to assume the resource exists.

	client := meta.(*datadog.Client)

	exists := false
	for _, v := range strings.Split(d.Id(), "__") {
		if v == "" {
			log.Printf("[DEBUG] Could not parse IDs: %s", v)
			return false, fmt.Errorf("Id not set.")
		}
		ID, iErr := strconv.Atoi(v)

		if iErr != nil {
			log.Printf("[DEBUG] Received error converting string: %s", iErr)
			return false, iErr
		}
		_, err := client.GetMonitor(ID)
		if err != nil {
			if strings.EqualFold(err.Error(), "API error: 404 Not Found") {
				log.Printf("[DEBUG] monitor does not exist: %s", err)
				exists = false
				continue
			} else {
				e = err
				continue
			}
		}
		exists = true
	}

	return exists, nil
}

// resourceDatadogOutlierAlertRead synchronises Datadog and local state .
func resourceDatadogOutlierAlertRead(d *schema.ResourceData, meta interface{}) error {
	// TODO: add support for this a read function.
	/* Read - This is called to resync the local state with the remote state.
	Terraform guarantees that an existing ID will be set. This ID should be
	used to look up the resource. Any remote data should be updated into the
	local data. No changes to the remote resource are to be made.
	*/

	return nil
}

// resourceDatadogOutlierAlertUpdate updates a monitor.
func resourceDatadogOutlierAlertUpdate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] running update.")

	split := strings.Split(d.Id(), "__")

	wID, cID := split[0], split[1]

	if wID == "" {
		return fmt.Errorf("Id not set.")
	}

	if cID == "" {
		return fmt.Errorf("Id not set.")
	}

	warningID, iErr := strconv.Atoi(wID)

	if iErr != nil {
		return iErr
	}

	criticalID, iErr := strconv.Atoi(cID)

	if iErr != nil {
		return iErr
	}

	client := meta.(*datadog.Client)

	warningBody := buildOutlierAlertStruct(d, "warning")
	criticalBody := buildOutlierAlertStruct(d, "critical")

	warningBody.Id = warningID
	criticalBody.Id = criticalID

	wErr := client.UpdateMonitor(warningBody)

	if wErr != nil {
		return fmt.Errorf("error updating warning: %s", wErr.Error())
	}

	cErr := client.UpdateMonitor(criticalBody)

	if cErr != nil {
		return fmt.Errorf("error updating critical: %s", cErr.Error())
	}

	return nil
}
<commit_msg>35: Remove TODO/notes.<commit_after>package datadog

import (
	"bytes"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/zorkian/go-datadog-api"

	"github.com/hashicorp/terraform/helper/schema"
)

// resourceDatadogOutlierAlert is a Datadog monitor resource
func resourceDatadogOutlierAlert() *schema.Resource {
	return &schema.Resource{
		Create: resourceDatadogOutlierAlertCreate,
		Read:   resourceDatadogOutlierAlertRead,
		Update: resourceDatadogOutlierAlertUpdate,
		Delete: resourceDatadogOutlierAlertDelete,
		Exists: resourceDatadogOutlierAlertExists,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"metric": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"tags": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
			"keys": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
			"time_aggr": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"time_window": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"space_aggr": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"operator": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"message": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			// Alert Settings
			"warning": &schema.Schema{
				Type:     schema.TypeMap,
				Required: true,
			},
			"critical": &schema.Schema{
				Type:     schema.TypeMap,
				Required: true,
			},

			// Additional Settings
			"notify_no_data": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  true,
			},

			"no_data_timeframe": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
			},

			"algorithm": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Default:  "dbscan",
			},
			"tolerance": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Default:  3,
			},
		},
	}
}

// buildMonitorStruct returns a monitor struct
func buildOutlierAlertStruct(d *schema.ResourceData, typeStr string) *datadog.Monitor {
	name := d.Get("name").(string)
	message := d.Get("message").(string)
	timeAggr := d.Get("time_aggr").(string)
	timeWindow := d.Get("time_window").(string)
	spaceAggr := d.Get("space_aggr").(string)
	metric := d.Get("metric").(string)
	algorithm := d.Get("algorithm").(string)
	tolerance := strconv.Itoa(d.Get("tolerance").(int))

	// Tags are are no separate resource/gettable, so some trickery is needed
	var buffer bytes.Buffer
	if raw, ok := d.GetOk("tags"); ok {
		list := raw.([]interface{})
		length := (len(list) - 1)
		for i, v := range list {
			buffer.WriteString(fmt.Sprintf("%s", v))
			if i != length {
				buffer.WriteString(",")
			}

		}
	}

	tagsParsed := buffer.String()

	// Keys are used for multi alerts
	var b bytes.Buffer
	if raw, ok := d.GetOk("keys"); ok {
		list := raw.([]interface{})
		b.WriteString("by {")
		length := (len(list) - 1)
		for i, v := range list {
			b.WriteString(fmt.Sprintf("%s", v))
			if i != length {
				b.WriteString(",")
			}

		}
		b.WriteString("}")
	}

	keys := b.String()

	operator := d.Get("operator").(string)

	query := fmt.Sprintf("%s(%s):outliers(%s:%s{%s} %s, '%s',%s) %s %s", timeAggr,
		timeWindow,
		spaceAggr,
		metric,
		tagsParsed,
		keys,
		algorithm,
		tolerance,
		operator,
		d.Get(fmt.Sprintf("%s.threshold", typeStr)))

	log.Print(fmt.Sprintf("[DEBUG] submitting query: %s", query))

	o := datadog.Options{
		NotifyNoData:    d.Get("notify_no_data").(bool),
		NoDataTimeframe: d.Get("no_data_timeframe").(int),
	}

	m := datadog.Monitor{
		Type:    "query alert",
		Query:   query,
		Name:    fmt.Sprintf("[%s] %s", typeStr, name),
		Message: fmt.Sprintf("%s %s", message, d.Get(fmt.Sprintf("%s.notify", typeStr))),
		Options: o,
	}

	return &m
}

// resourceDatadogOutlierAlertCreate creates a monitor.
func resourceDatadogOutlierAlertCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*datadog.Client)

	w, err := client.CreateMonitor(buildOutlierAlertStruct(d, "warning"))

	if err != nil {
		return fmt.Errorf("error creating warning: %s", err)
	}

	c, cErr := client.CreateMonitor(buildOutlierAlertStruct(d, "critical"))

	if cErr != nil {
		return fmt.Errorf("error creating warning: %s", cErr)
	}

	log.Printf("[DEBUG] Saving IDs: %s__%s", strconv.Itoa(w.Id), strconv.Itoa(c.Id))

	d.SetId(fmt.Sprintf("%s__%s", strconv.Itoa(w.Id), strconv.Itoa(c.Id)))

	return nil
}

// resourceDatadogOutlierAlertDelete deletes a monitor.
func resourceDatadogOutlierAlertDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*datadog.Client)

	for _, v := range strings.Split(d.Id(), "__") {
		if v == "" {
			return fmt.Errorf("Id not set.")
		}
		ID, iErr := strconv.Atoi(v)

		if iErr != nil {
			return iErr
		}

		err := client.DeleteMonitor(ID)
		if err != nil {
			return err
		}
	}
	return nil
}

// resourceDatadogOutlierAlertExists verifies a monitor exists.
func resourceDatadogOutlierAlertExists(d *schema.ResourceData, meta interface{}) (b bool, e error) {
	// Exists - This is called to verify a resource still exists. It is called prior to Read,
	// and lowers the burden of Read to be able to assume the resource exists.

	client := meta.(*datadog.Client)

	exists := false
	for _, v := range strings.Split(d.Id(), "__") {
		if v == "" {
			log.Printf("[DEBUG] Could not parse IDs: %s", v)
			return false, fmt.Errorf("Id not set.")
		}
		ID, iErr := strconv.Atoi(v)

		if iErr != nil {
			log.Printf("[DEBUG] Received error converting string: %s", iErr)
			return false, iErr
		}
		_, err := client.GetMonitor(ID)
		if err != nil {
			if strings.EqualFold(err.Error(), "API error: 404 Not Found") {
				log.Printf("[DEBUG] monitor does not exist: %s", err)
				exists = false
				continue
			} else {
				e = err
				continue
			}
		}
		exists = true
	}

	return exists, nil
}

// resourceDatadogOutlierAlertRead synchronises Datadog and local state .
func resourceDatadogOutlierAlertRead(d *schema.ResourceData, meta interface{}) error {
	// TODO: add support for this a read function.
	/* Read - This is called to resync the local state with the remote state.
	Terraform guarantees that an existing ID will be set. This ID should be
	used to look up the resource. Any remote data should be updated into the
	local data. No changes to the remote resource are to be made.
	*/

	return nil
}

// resourceDatadogOutlierAlertUpdate updates a monitor.
func resourceDatadogOutlierAlertUpdate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] running update.")

	split := strings.Split(d.Id(), "__")

	wID, cID := split[0], split[1]

	if wID == "" {
		return fmt.Errorf("Id not set.")
	}

	if cID == "" {
		return fmt.Errorf("Id not set.")
	}

	warningID, iErr := strconv.Atoi(wID)

	if iErr != nil {
		return iErr
	}

	criticalID, iErr := strconv.Atoi(cID)

	if iErr != nil {
		return iErr
	}

	client := meta.(*datadog.Client)

	warningBody := buildOutlierAlertStruct(d, "warning")
	criticalBody := buildOutlierAlertStruct(d, "critical")

	warningBody.Id = warningID
	criticalBody.Id = criticalID

	wErr := client.UpdateMonitor(warningBody)

	if wErr != nil {
		return fmt.Errorf("error updating warning: %s", wErr.Error())
	}

	cErr := client.UpdateMonitor(criticalBody)

	if cErr != nil {
		return fmt.Errorf("error updating critical: %s", cErr.Error())
	}

	return nil
}
<|endoftext|>


<commit_before>// Bounded is Consistent hashing with bounded loads.
// It acheives that by adding a capacity counter on every host,
// and when a host gets picked it, checks its capacity to see if it's below
// the Average Load per Host.
//
// All opertaions in bounded are concurrency-safe.
//
// Average Load Per Host is defined as follows:
//
// (totalLoad/number_of_hosts)*imbalance_constant
//
// totalLoad = sum of all hosts load
//
// load = the number of active requests
//
// imbalance_constant = is the imbalance constant, which is 1.25 in our case
//
// it bounds the load imabalnce to be at most 25% more than (totalLoad/number_of_hosts).
//
//
// For more info:
// https://medium.com/vimeo-engineering-blog/improving-load-balancing-with-a-new-consistent-hashing-algorithm-9f1bd75709ed
//
// https://research.googleblog.com/2017/04/consistent-hashing-with-bounded-loads.html
package bounded

import (
	"github.com/lafikl/consistent"
	"github.com/lafikl/liblb"
)

type bhost struct {
	load   uint64
	weight int
}

type Bounded struct {
	ch *consistent.Consistent
}

func New(hosts ...string) *Bounded {
	c := &Bounded{
		ch: consistent.New(),
	}
	for _, h := range hosts {
		c.Add(h)
	}
	return c
}

func (b *Bounded) Add(host string) {
	b.ch.Add(host)
}

func (b *Bounded) Remove(host string) {
	b.ch.Remove(host)
}

// err can be either liblb.ErrNoHost if there's no added hosts.
func (b *Bounded) Balance(key string) (host string, err error) {
	if len(b.ch.Hosts()) == 0 {
		return "", liblb.ErrNoHost
	}

	host, err = b.ch.GetLeast(key)
	return
}

// It should be called once a request is assigned to a host,
// obtained from b.Balance.
func (b *Bounded) Inc(host string) {
	b.ch.Inc(host)
}

// should be called when an assigned request to host is finished.
func (b *Bounded) Done(host string) {
	b.ch.Done(host)
}

func (b *Bounded) Loads() map[string]int64 {
	return b.ch.GetLoads()
}

// Max load of a host is (Average Load Per Host*1.25)
func (b *Bounded) MaxLoad() int64 {
	return b.ch.MaxLoad()
}
<commit_msg>Minor Typo Correction (removed word 'either')<commit_after>// Bounded is Consistent hashing with bounded loads.
// It acheives that by adding a capacity counter on every host,
// and when a host gets picked it, checks its capacity to see if it's below
// the Average Load per Host.
//
// All opertaions in bounded are concurrency-safe.
//
// Average Load Per Host is defined as follows:
//
// (totalLoad/number_of_hosts)*imbalance_constant
//
// totalLoad = sum of all hosts load
//
// load = the number of active requests
//
// imbalance_constant = is the imbalance constant, which is 1.25 in our case
//
// it bounds the load imabalnce to be at most 25% more than (totalLoad/number_of_hosts).
//
//
// For more info:
// https://medium.com/vimeo-engineering-blog/improving-load-balancing-with-a-new-consistent-hashing-algorithm-9f1bd75709ed
//
// https://research.googleblog.com/2017/04/consistent-hashing-with-bounded-loads.html
package bounded

import (
	"github.com/lafikl/consistent"
	"github.com/lafikl/liblb"
)

type bhost struct {
	load   uint64
	weight int
}

type Bounded struct {
	ch *consistent.Consistent
}

func New(hosts ...string) *Bounded {
	c := &Bounded{
		ch: consistent.New(),
	}
	for _, h := range hosts {
		c.Add(h)
	}
	return c
}

func (b *Bounded) Add(host string) {
	b.ch.Add(host)
}

func (b *Bounded) Remove(host string) {
	b.ch.Remove(host)
}

// err can be liblb.ErrNoHost if there's no added hosts.
func (b *Bounded) Balance(key string) (host string, err error) {
	if len(b.ch.Hosts()) == 0 {
		return "", liblb.ErrNoHost
	}

	host, err = b.ch.GetLeast(key)
	return
}

// It should be called once a request is assigned to a host,
// obtained from b.Balance.
func (b *Bounded) Inc(host string) {
	b.ch.Inc(host)
}

// should be called when an assigned request to host is finished.
func (b *Bounded) Done(host string) {
	b.ch.Done(host)
}

func (b *Bounded) Loads() map[string]int64 {
	return b.ch.GetLoads()
}

// Max load of a host is (Average Load Per Host*1.25)
func (b *Bounded) MaxLoad() int64 {
	return b.ch.MaxLoad()
}
<|endoftext|>


<commit_before>// Copyright © 2016 Mark Bates <mark@markbates.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"errors"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	"github.com/gobuffalo/envy"
	"github.com/gobuffalo/velvet"
	"github.com/markbates/inflect"
	"github.com/spf13/cobra"
)

var force bool
var verbose bool
var skipPop bool
var skipWebpack bool
var dbType = "postgres"

var newCmd = &cobra.Command{
	Use:   "new [name]",
	Short: "Creates a new Buffalo application",
	RunE: func(cmd *cobra.Command, args []string) error {
		if !validDbType() {
			return fmt.Errorf("Unknown db-type %s expecting one of postgres, mysql or sqlite3", dbType)
		}

		if len(args) == 0 {
			return errors.New("you must enter a name for your new application")
		}

		name := args[0]

		err := validateInGoPath(name)
		if err != nil {
			return err
		}

		rootPath, err := rootPath(name)
		if err != nil {
			return err
		}

		s, _ := os.Stat(rootPath)
		if s != nil {
			if force {
				os.RemoveAll(rootPath)
			} else {
				return fmt.Errorf("%s already exists! Either delete it or use the -f flag to force", name)
			}
		}

		return genNewFiles(name, rootPath)
	},
}

func validDbType() bool {
	return dbType == "postgres" || dbType == "mysql" || dbType == "sqlite3"
}

func validateInGoPath(name string) error {
	gp, err := envy.MustGet("GOPATH")
	if err != nil {
		fmt.Println(noGoPath)
		os.Exit(-1)
	}

	root, err := rootPath(name)
	if err != nil {
		return err
	}

	if !strings.HasPrefix(root, filepath.Join(gp, "src")) {
		u, err := user.Current()
		if err != nil {
			return err
		}
		t, err := velvet.Render(notInGoWorkspace, velvet.NewContextWith(map[string]interface{}{
			"name":     name,
			"gopath":   gp,
			"current":  root,
			"username": u.Username,
		}))
		if err != nil {
			return err
		}
		fmt.Println(t)
		os.Exit(-1)
	}
	return nil
}

func rootPath(name string) (string, error) {
	pwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	rootPath := filepath.Join(pwd, name)
	return rootPath, nil
}

func packagePath(rootPath string) string {
	gosrcpath := strings.Replace(filepath.Join(os.Getenv("GOPATH"), "src"), "\\", "/", -1)
	rootPath = strings.Replace(rootPath, "\\", "/", -1)
	return strings.Replace(rootPath, gosrcpath+"/", "", 2)
}

func genNewFiles(name, rootPath string) error {
	packagePath := packagePath(rootPath)

	data := map[string]interface{}{
		"name":        name,
		"titleName":   inflect.Titleize(name),
		"packagePath": packagePath,
		"actionsPath": packagePath + "/actions",
		"modelsPath":  packagePath + "/models",
		"withPop":     !skipPop,
		"withWebpack": !skipWebpack,
		"dbType":      dbType,
		"version":     Version,
	}

	g := newAppGenerator(data)
	return g.Run(rootPath, data)
}

func init() {
	RootCmd.AddCommand(newCmd)
	newCmd.Flags().BoolVarP(&force, "force", "f", false, "delete and remake if the app already exists")
	newCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "verbosely print out the go get/install commands")
	newCmd.Flags().BoolVar(&skipPop, "skip-pop", false, "skips adding pop/soda to your app")
	newCmd.Flags().BoolVar(&skipWebpack, "skip-webpack", false, "skips adding Webpack to your app")
	newCmd.Flags().StringVar(&dbType, "db-type", "postgres", "specify the type of database you want to use [postgres, mysql, sqlite3]")
}

const notInGoWorkspace = `Oops! It would appear that you are not in your Go Workspace.

Your $GOPATH is set to "{{gopath}}".

You are currently in "{{current}}".

The standard location for putting Go projects is something along the lines of "$GOPATH/src/github.com/{{username}}/{{name}}" (adjust accordingly).

We recommend you go to "$GOPATH/src/github.com/{{username}}/" and try "buffalo new {{name}}" again.`

const noGoPath = `You do not have a $GOPATH set. In order to work with Go, you must set up your $GOPATH and your Go Workspace.

We recommend reading this tutorial on setting everything up: https://www.goinggo.net/2016/05/installing-go-and-your-workspace.html

When you're ready come back and try again. Don't worry, Buffalo will be right here waiting for you. :)`
<commit_msg>Fixed issue #203: multiple GOPATH are not supported by buffalo new.<commit_after>// Copyright © 2016 Mark Bates <mark@markbates.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"errors"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/gobuffalo/envy"
	"github.com/gobuffalo/velvet"
	"github.com/markbates/inflect"
	"github.com/spf13/cobra"
)

var force bool
var verbose bool
var skipPop bool
var skipWebpack bool
var dbType = "postgres"
var goPath string

var newCmd = &cobra.Command{
	Use:   "new [name]",
	Short: "Creates a new Buffalo application",
	RunE: func(cmd *cobra.Command, args []string) error {
		if !validDbType() {
			return fmt.Errorf("Unknown db-type %s expecting one of postgres, mysql or sqlite3", dbType)
		}

		if len(args) == 0 {
			return errors.New("you must enter a name for your new application")
		}

		name := args[0]

		err := validateInGoPath(name)
		if err != nil {
			return err
		}

		rootPath, err := rootPath(name)
		if err != nil {
			return err
		}

		s, _ := os.Stat(rootPath)
		if s != nil {
			if force {
				os.RemoveAll(rootPath)
			} else {
				return fmt.Errorf("%s already exists! Either delete it or use the -f flag to force", name)
			}
		}

		return genNewFiles(name, rootPath)
	},
}

func validDbType() bool {
	return dbType == "postgres" || dbType == "mysql" || dbType == "sqlite3"
}

func validateInGoPath(name string) error {
	gp, err := envy.MustGet("GOPATH")
	if err != nil {
		fmt.Println(noGoPath)
		os.Exit(-1)
	}

	root, err := rootPath(name)
	if err != nil {
		return err
	}

	var gpMultiple []string

	if runtime.GOOS == "windows" {
		gpMultiple = strings.Split(gp, ";") // Windows uses a different separator
	} else {
		gpMultiple = strings.Split(gp, ":")
	}
	gpMultipleLen := len(gpMultiple)
	foundInPath := false

	for i := 0; i < gpMultipleLen; i++ {
		if strings.HasPrefix(root, filepath.Join(gpMultiple[i], "src")) {
			goPath = gpMultiple[i]
			foundInPath = true
			break
		}
	}

	if !foundInPath {
		u, err := user.Current()
		if err != nil {
			return err
		}
		t, err := velvet.Render(notInGoWorkspace, velvet.NewContextWith(map[string]interface{}{
			"name":     name,
			"gopath":   gp,
			"current":  root,
			"username": u.Username,
		}))
		if err != nil {
			return err
		}
		fmt.Println(t)
		os.Exit(-1)
	}
	return nil
}

func rootPath(name string) (string, error) {
	pwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	rootPath := filepath.Join(pwd, name)
	return rootPath, nil
}

func packagePath(rootPath string) string {
	gosrcpath := strings.Replace(filepath.Join(goPath, "src"), "\\", "/", -1)
	rootPath = strings.Replace(rootPath, "\\", "/", -1)
	return strings.Replace(rootPath, gosrcpath+"/", "", 2)
}

func genNewFiles(name, rootPath string) error {
	packagePath := packagePath(rootPath)

	data := map[string]interface{}{
		"name":        name,
		"titleName":   inflect.Titleize(name),
		"packagePath": packagePath,
		"actionsPath": packagePath + "/actions",
		"modelsPath":  packagePath + "/models",
		"withPop":     !skipPop,
		"withWebpack": !skipWebpack,
		"dbType":      dbType,
		"version":     Version,
	}

	g := newAppGenerator(data)
	return g.Run(rootPath, data)
}

func init() {
	RootCmd.AddCommand(newCmd)
	newCmd.Flags().BoolVarP(&force, "force", "f", false, "delete and remake if the app already exists")
	newCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "verbosely print out the go get/install commands")
	newCmd.Flags().BoolVar(&skipPop, "skip-pop", false, "skips adding pop/soda to your app")
	newCmd.Flags().BoolVar(&skipWebpack, "skip-webpack", false, "skips adding Webpack to your app")
	newCmd.Flags().StringVar(&dbType, "db-type", "postgres", "specify the type of database you want to use [postgres, mysql, sqlite3]")
}

const notInGoWorkspace = `Oops! It would appear that you are not in your Go Workspace.

Your $GOPATH is set to "{{gopath}}".

You are currently in "{{current}}".

The standard location for putting Go projects is something along the lines of "$GOPATH/src/github.com/{{username}}/{{name}}" (adjust accordingly).

We recommend you go to "$GOPATH/src/github.com/{{username}}/" and try "buffalo new {{name}}" again.`

const noGoPath = `You do not have a $GOPATH set. In order to work with Go, you must set up your $GOPATH and your Go Workspace.

We recommend reading this tutorial on setting everything up: https://www.goinggo.net/2016/05/installing-go-and-your-workspace.html

When you're ready come back and try again. Don't worry, Buffalo will be right here waiting for you. :)`
<|endoftext|>


<commit_before>package disruption

import (
	"encoding/xml"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	g "github.com/onsi/ginkgo"

	"k8s.io/kubernetes/test/e2e/chaosmonkey"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/framework/ginkgowrapper"
	"k8s.io/kubernetes/test/e2e/upgrades"
	"k8s.io/kubernetes/test/utils/junit"
)

// flakeSummary is a test summary type that allows upgrades to report violations
// without failing the upgrade test.
type flakeSummary string

func (s flakeSummary) PrintHumanReadable() string { return string(s) }
func (s flakeSummary) SummaryKind() string        { return "Flake" }
func (s flakeSummary) PrintJSON() string          { return `{"type":"Flake"}` }

// Flakef records a flake on the current framework.
func Flakef(f *framework.Framework, format string, options ...interface{}) {
	framework.Logf(format, options...)
	f.TestSummaries = append(f.TestSummaries, flakeSummary(fmt.Sprintf(format, options...)))
}

// TestData is passed to the invariant tests executed during the upgrade. The default UpgradeType
// is MasterUpgrade.
type TestData struct {
	UpgradeType    upgrades.UpgradeType
	UpgradeContext upgrades.UpgradeContext
}

// Run executes the provided fn in a test context, ensuring that invariants are preserved while the
// test is being executed. Description is used to populate the JUnit suite name, and testname is
// used to define the overall test that will be run.
func Run(description, testname string, adapter TestData, invariants []upgrades.Test, fn func()) {
	testSuite := &junit.TestSuite{Name: description, Package: testname}
	test := &junit.TestCase{Name: testname, Classname: testname}
	testSuite.TestCases = append(testSuite.TestCases, test)
	cm := chaosmonkey.New(func() {
		start := time.Now()
		defer finalizeTest(start, test, nil)
		fn()
	})
	runChaosmonkey(cm, adapter, invariants, testSuite)
}

func runChaosmonkey(
	cm *chaosmonkey.Chaosmonkey,
	testData TestData,
	tests []upgrades.Test,
	testSuite *junit.TestSuite,
) {
	testFrameworks := createTestFrameworks(tests)
	for _, t := range tests {
		testCase := &junit.TestCase{
			Name:      t.Name(),
			Classname: "disruption_tests",
		}
		testSuite.TestCases = append(testSuite.TestCases, testCase)

		f, ok := testFrameworks[t.Name()]
		if !ok {
			panic(fmt.Sprintf("can't find test framework for %q", t.Name()))
		}
		cma := chaosMonkeyAdapter{
			TestData:   testData,
			framework:  f,
			test:       t,
			testReport: testCase,
		}
		cm.Register(cma.Test)
	}

	start := time.Now()
	defer func() {
		testSuite.Update()
		testSuite.Time = time.Since(start).Seconds()
		if framework.TestContext.ReportDir != "" {
			fname := filepath.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%s_%d.xml", testSuite.Package, time.Now().Unix()))
			f, err := os.Create(fname)
			if err != nil {
				return
			}
			defer f.Close()
			xml.NewEncoder(f).Encode(testSuite)
		}
	}()
	cm.Do()
}

type chaosMonkeyAdapter struct {
	TestData

	test       upgrades.Test
	testReport *junit.TestCase
	framework  *framework.Framework
}

func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) {
	start := time.Now()
	var once sync.Once
	ready := func() {
		once.Do(func() {
			sem.Ready()
		})
	}
	defer finalizeTest(start, cma.testReport, cma.framework)
	defer ready()
	if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.UpgradeContext) {
		g.By("skipping test " + cma.test.Name())
		cma.testReport.Skipped = "skipping test " + cma.test.Name()
		return
	}
	fmt.Printf("DEBUG: starting test\n")
	cma.framework.BeforeEach()
	fmt.Printf("DEBUG: starting test, setup\n")
	cma.test.Setup(cma.framework)
	defer cma.test.Teardown(cma.framework)
	ready()
	cma.test.Test(cma.framework, sem.StopCh, cma.UpgradeType)
}

func finalizeTest(start time.Time, tc *junit.TestCase, f *framework.Framework) {
	tc.Time = time.Since(start).Seconds()
	r := recover()
	if r == nil {
		if f != nil {
			for _, summary := range f.TestSummaries {
				if summary.SummaryKind() == "Flake" {
					tc.Failures = append(tc.Failures, &junit.Failure{
						Message: summary.PrintHumanReadable(),
						Type:    "Failure",
						Value:   summary.PrintHumanReadable(),
					})
				}
			}
		}
		return
	}

	switch r := r.(type) {
	case ginkgowrapper.FailurePanic:
		tc.Failures = []*junit.Failure{
			{
				Message: r.Message,
				Type:    "Failure",
				Value:   fmt.Sprintf("%s\n\n%s", r.Message, r.FullStackTrace),
			},
		}
	case ginkgowrapper.SkipPanic:
		tc.Skipped = fmt.Sprintf("%s:%d %q", r.Filename, r.Line, r.Message)
	default:
		tc.Errors = []*junit.Error{
			{
				Message: fmt.Sprintf("%v", r),
				Type:    "Panic",
				Value:   fmt.Sprintf("%v\n\n%s", r, debug.Stack()),
			},
		}
	}
}

// TODO: accept a default framework
func createTestFrameworks(tests []upgrades.Test) map[string]*framework.Framework {
	nsFilter := regexp.MustCompile("[^[:word:]-]+") // match anything that's not a word character or hyphen
	testFrameworks := map[string]*framework.Framework{}
	for _, t := range tests {
		ns := nsFilter.ReplaceAllString(t.Name(), "-") // and replace with a single hyphen
		ns = strings.Trim(ns, "-")
		// identify tests that come from kube as strictly e2e tests so they get the correct semantics
		if strings.Contains(reflect.ValueOf(t).Elem().Type().PkgPath(), "/kubernetes/test/e2e/") {
			ns = "e2e-k8s-" + ns
		}
		testFrameworks[t.Name()] = &framework.Framework{
			BaseName:                 ns,
			AddonResourceConstraints: make(map[string]framework.ResourceConstraint),
			Options: framework.Options{
				ClientQPS:   20,
				ClientBurst: 50,
			},
		}
	}
	return testFrameworks
}
<commit_msg>test: Remove debug statements<commit_after>package disruption

import (
	"encoding/xml"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	g "github.com/onsi/ginkgo"

	"k8s.io/kubernetes/test/e2e/chaosmonkey"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/framework/ginkgowrapper"
	"k8s.io/kubernetes/test/e2e/upgrades"
	"k8s.io/kubernetes/test/utils/junit"
)

// flakeSummary is a test summary type that allows upgrades to report violations
// without failing the upgrade test.
type flakeSummary string

func (s flakeSummary) PrintHumanReadable() string { return string(s) }
func (s flakeSummary) SummaryKind() string        { return "Flake" }
func (s flakeSummary) PrintJSON() string          { return `{"type":"Flake"}` }

// Flakef records a flake on the current framework.
func Flakef(f *framework.Framework, format string, options ...interface{}) {
	framework.Logf(format, options...)
	f.TestSummaries = append(f.TestSummaries, flakeSummary(fmt.Sprintf(format, options...)))
}

// TestData is passed to the invariant tests executed during the upgrade. The default UpgradeType
// is MasterUpgrade.
type TestData struct {
	UpgradeType    upgrades.UpgradeType
	UpgradeContext upgrades.UpgradeContext
}

// Run executes the provided fn in a test context, ensuring that invariants are preserved while the
// test is being executed. Description is used to populate the JUnit suite name, and testname is
// used to define the overall test that will be run.
func Run(description, testname string, adapter TestData, invariants []upgrades.Test, fn func()) {
	testSuite := &junit.TestSuite{Name: description, Package: testname}
	test := &junit.TestCase{Name: testname, Classname: testname}
	testSuite.TestCases = append(testSuite.TestCases, test)
	cm := chaosmonkey.New(func() {
		start := time.Now()
		defer finalizeTest(start, test, nil)
		fn()
	})
	runChaosmonkey(cm, adapter, invariants, testSuite)
}

func runChaosmonkey(
	cm *chaosmonkey.Chaosmonkey,
	testData TestData,
	tests []upgrades.Test,
	testSuite *junit.TestSuite,
) {
	testFrameworks := createTestFrameworks(tests)
	for _, t := range tests {
		testCase := &junit.TestCase{
			Name:      t.Name(),
			Classname: "disruption_tests",
		}
		testSuite.TestCases = append(testSuite.TestCases, testCase)

		f, ok := testFrameworks[t.Name()]
		if !ok {
			panic(fmt.Sprintf("can't find test framework for %q", t.Name()))
		}
		cma := chaosMonkeyAdapter{
			TestData:   testData,
			framework:  f,
			test:       t,
			testReport: testCase,
		}
		cm.Register(cma.Test)
	}

	start := time.Now()
	defer func() {
		testSuite.Update()
		testSuite.Time = time.Since(start).Seconds()
		if framework.TestContext.ReportDir != "" {
			fname := filepath.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%s_%d.xml", testSuite.Package, time.Now().Unix()))
			f, err := os.Create(fname)
			if err != nil {
				return
			}
			defer f.Close()
			xml.NewEncoder(f).Encode(testSuite)
		}
	}()
	cm.Do()
}

type chaosMonkeyAdapter struct {
	TestData

	test       upgrades.Test
	testReport *junit.TestCase
	framework  *framework.Framework
}

func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) {
	start := time.Now()
	var once sync.Once
	ready := func() {
		once.Do(func() {
			sem.Ready()
		})
	}
	defer finalizeTest(start, cma.testReport, cma.framework)
	defer ready()
	if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.UpgradeContext) {
		g.By("skipping test " + cma.test.Name())
		cma.testReport.Skipped = "skipping test " + cma.test.Name()
		return
	}
	cma.framework.BeforeEach()
	cma.test.Setup(cma.framework)
	defer cma.test.Teardown(cma.framework)
	ready()
	cma.test.Test(cma.framework, sem.StopCh, cma.UpgradeType)
}

func finalizeTest(start time.Time, tc *junit.TestCase, f *framework.Framework) {
	tc.Time = time.Since(start).Seconds()
	r := recover()
	if r == nil {
		if f != nil {
			for _, summary := range f.TestSummaries {
				if summary.SummaryKind() == "Flake" {
					tc.Failures = append(tc.Failures, &junit.Failure{
						Message: summary.PrintHumanReadable(),
						Type:    "Failure",
						Value:   summary.PrintHumanReadable(),
					})
				}
			}
		}
		return
	}

	switch r := r.(type) {
	case ginkgowrapper.FailurePanic:
		tc.Failures = []*junit.Failure{
			{
				Message: r.Message,
				Type:    "Failure",
				Value:   fmt.Sprintf("%s\n\n%s", r.Message, r.FullStackTrace),
			},
		}
	case ginkgowrapper.SkipPanic:
		tc.Skipped = fmt.Sprintf("%s:%d %q", r.Filename, r.Line, r.Message)
	default:
		tc.Errors = []*junit.Error{
			{
				Message: fmt.Sprintf("%v", r),
				Type:    "Panic",
				Value:   fmt.Sprintf("%v\n\n%s", r, debug.Stack()),
			},
		}
	}
}

// TODO: accept a default framework
func createTestFrameworks(tests []upgrades.Test) map[string]*framework.Framework {
	nsFilter := regexp.MustCompile("[^[:word:]-]+") // match anything that's not a word character or hyphen
	testFrameworks := map[string]*framework.Framework{}
	for _, t := range tests {
		ns := nsFilter.ReplaceAllString(t.Name(), "-") // and replace with a single hyphen
		ns = strings.Trim(ns, "-")
		// identify tests that come from kube as strictly e2e tests so they get the correct semantics
		if strings.Contains(reflect.ValueOf(t).Elem().Type().PkgPath(), "/kubernetes/test/e2e/") {
			ns = "e2e-k8s-" + ns
		}
		testFrameworks[t.Name()] = &framework.Framework{
			BaseName:                 ns,
			AddonResourceConstraints: make(map[string]framework.ResourceConstraint),
			Options: framework.Options{
				ClientQPS:   20,
				ClientBurst: 50,
			},
		}
	}
	return testFrameworks
}
<|endoftext|>


<commit_before>package s3api

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/golang/protobuf/jsonpb"
	"github.com/gorilla/mux"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
)

type Action string

const (
	ACTION_READ  = "Read"
	ACTION_WRITE = "Write"
	ACTION_ADMIN = "Admin"
)

type Iam interface {
	Check(f http.HandlerFunc, actions ...Action) http.HandlerFunc
}

type IdentityAccessManagement struct {
	identities []*Identity
	domain     string
}

type Identity struct {
	Name        string
	Credentials []*Credential
	Actions     []Action
}

type Credential struct {
	AccessKey string
	SecretKey string
}

func NewIdentityAccessManagement(fileName string, domain string) *IdentityAccessManagement {
	iam := &IdentityAccessManagement{
		domain: domain,
	}
	if fileName == "" {
		return iam
	}
	if err := iam.loadS3ApiConfiguration(fileName); err != nil {
		glog.Fatalf("fail to load config file %s: %v", fileName, err)
	}
	return iam
}

func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) error {

	s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}

	rawData, readErr := ioutil.ReadFile(fileName)
	if readErr != nil {
		glog.Warningf("fail to read %s : %v", fileName, readErr)
		return fmt.Errorf("fail to read %s : %v", fileName, readErr)
	}

	glog.V(1).Infof("maybeLoadVolumeInfo Unmarshal volume info %v", fileName)
	if err := jsonpb.Unmarshal(bytes.NewReader(rawData), s3ApiConfiguration); err != nil {
		glog.Warningf("unmarshal error: %v", err)
		return fmt.Errorf("unmarshal %s error: %v", fileName, err)
	}

	for _, ident := range s3ApiConfiguration.Identities {
		t := &Identity{
			Name:        ident.Name,
			Credentials: nil,
			Actions:     nil,
		}
		for _, action := range ident.Actions {
			t.Actions = append(t.Actions, Action(action))
		}
		for _, cred := range ident.Credentials {
			t.Credentials = append(t.Credentials, &Credential{
				AccessKey: cred.AccessKey,
				SecretKey: cred.SecretKey,
			})
		}
		iam.identities = append(iam.identities, t)
	}

	return nil
}

func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) {
	for _, ident := range iam.identities {
		for _, cred := range ident.Credentials {
			if cred.AccessKey == accessKey {
				return ident, cred, true
			}
		}
	}
	return nil, nil, false
}

func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc {

	if len(iam.identities) == 0 {
		return f
	}

	return func(w http.ResponseWriter, r *http.Request) {
		errCode := iam.authRequest(r, action)
		if errCode == ErrNone {
			f(w, r)
			return
		}
		writeErrorResponse(w, errCode, r.URL)
	}
}

// check whether the request has valid access keys
func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) ErrorCode {
	var identity *Identity
	var s3Err ErrorCode
	switch getRequestAuthType(r) {
	case authTypeStreamingSigned:
		return ErrNone
	case authTypeUnknown:
		glog.V(3).Infof("unknown auth type")
		return ErrAccessDenied
	case authTypePresignedV2, authTypeSignedV2:
		glog.V(3).Infof("v2 auth type")
		identity, s3Err = iam.isReqAuthenticatedV2(r)
	case authTypeSigned, authTypePresigned:
		glog.V(3).Infof("v4 auth type")
		identity, s3Err = iam.reqSignatureV4Verify(r)
	case authTypePostPolicy:
		glog.V(3).Infof("post policy auth type")
		return ErrNotImplemented
	case authTypeJWT:
		glog.V(3).Infof("jwt auth type")
		return ErrNotImplemented
	case authTypeAnonymous:
		return ErrAccessDenied
	default:
		return ErrNotImplemented
	}

	glog.V(3).Infof("auth error: %v", s3Err)
	if s3Err != ErrNone {
		return s3Err
	}

	glog.V(3).Infof("user name: %v actions: %v", identity.Name, identity.Actions)

	vars := mux.Vars(r)
	bucket := vars["bucket"]

	if !identity.canDo(action, bucket) {
		return ErrAccessDenied
	}

	return ErrNone

}

func (identity *Identity) canDo(action Action, bucket string) bool {
	for _, a := range identity.Actions {
		if a == "Admin" {
			return true
		}
	}
	for _, a := range identity.Actions {
		if a == action {
			return true
		}
	}
	if bucket == "" {
		return false
	}
	limitedByBucket := string(action) + ":" + bucket
	for _, a := range identity.Actions {
		if string(a) == limitedByBucket {
			return true
		}
	}
	return false
}
<commit_msg>refactoring<commit_after>package s3api

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/golang/protobuf/jsonpb"
	"github.com/gorilla/mux"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
)

type Action string

const (
	ACTION_READ  = "Read"
	ACTION_WRITE = "Write"
	ACTION_ADMIN = "Admin"
)

type Iam interface {
	Check(f http.HandlerFunc, actions ...Action) http.HandlerFunc
}

type IdentityAccessManagement struct {
	identities []*Identity
	domain     string
}

type Identity struct {
	Name        string
	Credentials []*Credential
	Actions     []Action
}

type Credential struct {
	AccessKey string
	SecretKey string
}

func NewIdentityAccessManagement(fileName string, domain string) *IdentityAccessManagement {
	iam := &IdentityAccessManagement{
		domain: domain,
	}
	if fileName == "" {
		return iam
	}
	if err := iam.loadS3ApiConfiguration(fileName); err != nil {
		glog.Fatalf("fail to load config file %s: %v", fileName, err)
	}
	return iam
}

func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) error {

	s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}

	rawData, readErr := ioutil.ReadFile(fileName)
	if readErr != nil {
		glog.Warningf("fail to read %s : %v", fileName, readErr)
		return fmt.Errorf("fail to read %s : %v", fileName, readErr)
	}

	glog.V(1).Infof("maybeLoadVolumeInfo Unmarshal volume info %v", fileName)
	if err := jsonpb.Unmarshal(bytes.NewReader(rawData), s3ApiConfiguration); err != nil {
		glog.Warningf("unmarshal error: %v", err)
		return fmt.Errorf("unmarshal %s error: %v", fileName, err)
	}

	for _, ident := range s3ApiConfiguration.Identities {
		t := &Identity{
			Name:        ident.Name,
			Credentials: nil,
			Actions:     nil,
		}
		for _, action := range ident.Actions {
			t.Actions = append(t.Actions, Action(action))
		}
		for _, cred := range ident.Credentials {
			t.Credentials = append(t.Credentials, &Credential{
				AccessKey: cred.AccessKey,
				SecretKey: cred.SecretKey,
			})
		}
		iam.identities = append(iam.identities, t)
	}

	return nil
}

func (iam *IdentityAccessManagement) isEnabled() bool {

	return len(iam.identities) > 0
}

func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) {

	for _, ident := range iam.identities {
		for _, cred := range ident.Credentials {
			if cred.AccessKey == accessKey {
				return ident, cred, true
			}
		}
	}
	return nil, nil, false
}

func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc {

	if iam.isEnabled() {
		return f
	}

	return func(w http.ResponseWriter, r *http.Request) {
		errCode := iam.authRequest(r, action)
		if errCode == ErrNone {
			f(w, r)
			return
		}
		writeErrorResponse(w, errCode, r.URL)
	}
}

// check whether the request has valid access keys
func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) ErrorCode {
	var identity *Identity
	var s3Err ErrorCode
	switch getRequestAuthType(r) {
	case authTypeStreamingSigned:
		return ErrNone
	case authTypeUnknown:
		glog.V(3).Infof("unknown auth type")
		return ErrAccessDenied
	case authTypePresignedV2, authTypeSignedV2:
		glog.V(3).Infof("v2 auth type")
		identity, s3Err = iam.isReqAuthenticatedV2(r)
	case authTypeSigned, authTypePresigned:
		glog.V(3).Infof("v4 auth type")
		identity, s3Err = iam.reqSignatureV4Verify(r)
	case authTypePostPolicy:
		glog.V(3).Infof("post policy auth type")
		return ErrNotImplemented
	case authTypeJWT:
		glog.V(3).Infof("jwt auth type")
		return ErrNotImplemented
	case authTypeAnonymous:
		return ErrAccessDenied
	default:
		return ErrNotImplemented
	}

	glog.V(3).Infof("auth error: %v", s3Err)
	if s3Err != ErrNone {
		return s3Err
	}

	glog.V(3).Infof("user name: %v actions: %v", identity.Name, identity.Actions)

	vars := mux.Vars(r)
	bucket := vars["bucket"]

	if !identity.canDo(action, bucket) {
		return ErrAccessDenied
	}

	return ErrNone

}

func (identity *Identity) canDo(action Action, bucket string) bool {
	for _, a := range identity.Actions {
		if a == "Admin" {
			return true
		}
	}
	for _, a := range identity.Actions {
		if a == action {
			return true
		}
	}
	if bucket == "" {
		return false
	}
	limitedByBucket := string(action) + ":" + bucket
	for _, a := range identity.Actions {
		if string(a) == limitedByBucket {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2012-2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package runner

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils/clock"
	utilexec "github.com/juju/utils/exec"

	"github.com/juju/juju/core/actions"
	"github.com/juju/juju/worker/uniter/runner/context"
	"github.com/juju/juju/worker/uniter/runner/debug"
	"github.com/juju/juju/worker/uniter/runner/jujuc"
	jujuos "github.com/juju/utils/os"
)

var logger = loggo.GetLogger("juju.worker.uniter.runner")

// Runner is responsible for invoking commands in a context.
type Runner interface {

	// Context returns the context against which the runner executes.
	Context() Context

	// RunHook executes the hook with the supplied name.
	RunHook(name string) error

	// RunAction executes the action with the supplied name.
	RunAction(name string) error

	// RunCommands executes the supplied script.
	RunCommands(commands string) (*utilexec.ExecResponse, error)
}

// Context exposes jujuc.Context, and additional methods needed by Runner.
type Context interface {
	jujuc.Context
	Id() string
	HookVars(paths context.Paths) ([]string, error)
	ActionData() (*context.ActionData, error)
	SetProcess(process context.HookProcess)
	HasExecutionSetUnitStatus() bool
	ResetExecutionSetUnitStatus()

	Prepare() error
	Flush(badge string, failure error) error
}

// NewRunner returns a Runner backed by the supplied context and paths.
func NewRunner(context Context, paths context.Paths) Runner {
	return &runner{context, paths}
}

// runner implements Runner.
type runner struct {
	context Context
	paths   context.Paths
}

func (runner *runner) Context() Context {
	return runner.context
}

// RunCommands exists to satisfy the Runner interface.
func (runner *runner) RunCommands(commands string) (*utilexec.ExecResponse, error) {
	result, err := runner.runCommandsWithTimeout(commands, 0, clock.WallClock)
	return result, runner.context.Flush("run commands", err)
}

// runCommandsWithTimeout is a helper to abstract common code between run commands and
// juju-run as an action
func (runner *runner) runCommandsWithTimeout(commands string, timeout time.Duration, clock clock.Clock) (*utilexec.ExecResponse, error) {
	srv, err := runner.startJujucServer()
	if err != nil {
		return nil, err
	}
	defer srv.Close()

	env, err := runner.context.HookVars(runner.paths)
	if err != nil {
		return nil, errors.Trace(err)
	}
	command := utilexec.RunParams{
		Commands:    commands,
		WorkingDir:  runner.paths.GetCharmDir(),
		Environment: env,
		Clock:       clock,
	}

	err = command.Run()
	if err != nil {
		return nil, err
	}
	runner.context.SetProcess(hookProcess{command.Process()})

	var cancel chan struct{}
	if timeout != 0 {
		cancel = make(chan struct{})
		go func() {
			<-clock.After(timeout)
			close(cancel)
		}()
	}

	// Block and wait for process to finish
	return command.WaitWithCancel(cancel)
}

// runJujuRunAction is the function that executes when a juju-run action is ran.
func (runner *runner) runJujuRunAction() (err error) {
	params, err := runner.context.ActionParams()
	if err != nil {
		return errors.Trace(err)
	}
	command, ok := params["command"].(string)
	if !ok {
		return errors.New("no command parameter to juju-run action")
	}

	// The timeout is passed in in nanoseconds(which are represented in go as int64)
	// But due to serialization it comes out as float64
	timeout, ok := params["timeout"].(float64)
	if !ok {
		logger.Debugf("unable to read juju-run action timeout, will continue running action without one")
	}

	results, err := runner.runCommandsWithTimeout(command, time.Duration(timeout), clock.WallClock)

	if err != nil {
		return runner.context.Flush("juju-run", err)
	}

	if err := runner.context.UpdateActionResults([]string{"Code"}, fmt.Sprintf("%d", results.Code)); err != nil {
		return runner.context.Flush("juju-run", err)
	}
	if err := runner.context.UpdateActionResults([]string{"Stdout"}, fmt.Sprintf("%s", results.Stdout)); err != nil {
		return runner.context.Flush("juju-run", err)
	}
	if err := runner.context.UpdateActionResults([]string{"Stderr"}, fmt.Sprintf("%s", results.Stderr)); err != nil {
		return runner.context.Flush("juju-run", err)
	}

	return runner.context.Flush("juju-run", nil)
}

// RunAction exists to satisfy the Runner interface.
func (runner *runner) RunAction(actionName string) error {
	if _, err := runner.context.ActionData(); err != nil {
		return errors.Trace(err)
	}
	if actionName == actions.JujuRunActionName {
		return runner.runJujuRunAction()
	}
	return runner.runCharmHookWithLocation(actionName, "actions")
}

// RunHook exists to satisfy the Runner interface.
func (runner *runner) RunHook(hookName string) error {
	return runner.runCharmHookWithLocation(hookName, "hooks")
}

func (runner *runner) runCharmHookWithLocation(hookName, charmLocation string) error {
	srv, err := runner.startJujucServer()
	if err != nil {
		return err
	}
	defer srv.Close()

	env, err := runner.context.HookVars(runner.paths)
	if err != nil {
		return errors.Trace(err)
	}
	if jujuos.HostOS() == jujuos.Windows {
		// TODO(fwereade): somehow consolidate with utils/exec?
		// We don't do this on the other code path, which uses exec.RunCommands,
		// because that already has handling for windows environment requirements.
		env = mergeWindowsEnvironment(env, os.Environ())
	}

	debugctx := debug.NewHooksContext(runner.context.UnitName())
	if session, _ := debugctx.FindSession(); session != nil && session.MatchHook(hookName) {
		logger.Infof("executing %s via debug-hooks", hookName)
		err = session.RunHook(hookName, runner.paths.GetCharmDir(), env)
	} else {
		err = runner.runCharmHook(hookName, env, charmLocation)
	}
	return runner.context.Flush(hookName, err)
}

func (runner *runner) runCharmHook(hookName string, env []string, charmLocation string) error {
	charmDir := runner.paths.GetCharmDir()
	hook, err := searchHook(charmDir, filepath.Join(charmLocation, hookName))
	if err != nil {
		return err
	}
	hookCmd := hookCommand(hook)
	ps := exec.Command(hookCmd[0], hookCmd[1:]...)
	ps.Env = env
	ps.Dir = charmDir
	outReader, outWriter, err := os.Pipe()
	if err != nil {
		return errors.Errorf("cannot make logging pipe: %v", err)
	}
	ps.Stdout = outWriter
	ps.Stderr = outWriter
	hookLogger := &hookLogger{
		r:      outReader,
		done:   make(chan struct{}),
		logger: runner.getLogger(hookName),
	}
	go hookLogger.run()
	err = ps.Start()
	outWriter.Close()
	if err == nil {
		// Record the *os.Process of the hook
		runner.context.SetProcess(hookProcess{ps.Process})
		// Block until execution finishes
		err = ps.Wait()
	}
	hookLogger.stop()
	return errors.Trace(err)
}

func (runner *runner) startJujucServer() (*jujuc.Server, error) {
	// Prepare server.
	getCmd := func(ctxId, cmdName string) (cmd.Command, error) {
		if ctxId != runner.context.Id() {
			return nil, errors.Errorf("expected context id %q, got %q", runner.context.Id(), ctxId)
		}
		return jujuc.NewCommand(runner.context, cmdName)
	}
	srv, err := jujuc.NewServer(getCmd, runner.paths.GetJujucSocket())
	if err != nil {
		return nil, err
	}
	go srv.Run()
	return srv, nil
}

func (runner *runner) getLogger(hookName string) loggo.Logger {
	return loggo.GetLogger(fmt.Sprintf("unit.%s.%s", runner.context.UnitName(), hookName))
}

type hookProcess struct {
	*os.Process
}

func (p hookProcess) Pid() int {
	return p.Process.Pid
}
<commit_msg>encode if not utf8<commit_after>// Copyright 2012-2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package runner

import (
	"encoding/base64"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"time"
	"unicode/utf8"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils/clock"
	utilexec "github.com/juju/utils/exec"

	"github.com/juju/juju/core/actions"
	"github.com/juju/juju/worker/uniter/runner/context"
	"github.com/juju/juju/worker/uniter/runner/debug"
	"github.com/juju/juju/worker/uniter/runner/jujuc"
	jujuos "github.com/juju/utils/os"
)

var logger = loggo.GetLogger("juju.worker.uniter.runner")

// Runner is responsible for invoking commands in a context.
type Runner interface {

	// Context returns the context against which the runner executes.
	Context() Context

	// RunHook executes the hook with the supplied name.
	RunHook(name string) error

	// RunAction executes the action with the supplied name.
	RunAction(name string) error

	// RunCommands executes the supplied script.
	RunCommands(commands string) (*utilexec.ExecResponse, error)
}

// Context exposes jujuc.Context, and additional methods needed by Runner.
type Context interface {
	jujuc.Context
	Id() string
	HookVars(paths context.Paths) ([]string, error)
	ActionData() (*context.ActionData, error)
	SetProcess(process context.HookProcess)
	HasExecutionSetUnitStatus() bool
	ResetExecutionSetUnitStatus()

	Prepare() error
	Flush(badge string, failure error) error
}

// NewRunner returns a Runner backed by the supplied context and paths.
func NewRunner(context Context, paths context.Paths) Runner {
	return &runner{context, paths}
}

// runner implements Runner.
type runner struct {
	context Context
	paths   context.Paths
}

func (runner *runner) Context() Context {
	return runner.context
}

// RunCommands exists to satisfy the Runner interface.
func (runner *runner) RunCommands(commands string) (*utilexec.ExecResponse, error) {
	result, err := runner.runCommandsWithTimeout(commands, 0, clock.WallClock)
	return result, runner.context.Flush("run commands", err)
}

// runCommandsWithTimeout is a helper to abstract common code between run commands and
// juju-run as an action
func (runner *runner) runCommandsWithTimeout(commands string, timeout time.Duration, clock clock.Clock) (*utilexec.ExecResponse, error) {
	srv, err := runner.startJujucServer()
	if err != nil {
		return nil, err
	}
	defer srv.Close()

	env, err := runner.context.HookVars(runner.paths)
	if err != nil {
		return nil, errors.Trace(err)
	}
	command := utilexec.RunParams{
		Commands:    commands,
		WorkingDir:  runner.paths.GetCharmDir(),
		Environment: env,
		Clock:       clock,
	}

	err = command.Run()
	if err != nil {
		return nil, err
	}
	runner.context.SetProcess(hookProcess{command.Process()})

	var cancel chan struct{}
	if timeout != 0 {
		cancel = make(chan struct{})
		go func() {
			<-clock.After(timeout)
			close(cancel)
		}()
	}

	// Block and wait for process to finish
	return command.WaitWithCancel(cancel)
}

// runJujuRunAction is the function that executes when a juju-run action is ran.
func (runner *runner) runJujuRunAction() (err error) {
	params, err := runner.context.ActionParams()
	if err != nil {
		return errors.Trace(err)
	}
	command, ok := params["command"].(string)
	if !ok {
		return errors.New("no command parameter to juju-run action")
	}

	// The timeout is passed in in nanoseconds(which are represented in go as int64)
	// But due to serialization it comes out as float64
	timeout, ok := params["timeout"].(float64)
	if !ok {
		logger.Debugf("unable to read juju-run action timeout, will continue running action without one")
	}

	results, err := runner.runCommandsWithTimeout(command, time.Duration(timeout), clock.WallClock)

	if err != nil {
		return runner.context.Flush("juju-run", err)
	}

	if err := runner.updateActionResults(results); err != nil {
		return runner.context.Flush("juju-run", err)
	}

	return runner.context.Flush("juju-run", nil)
}

func encodeBytes(input []byte) (value string, encoding string) {
	if utf8.Valid(input) {
		value = string(input)
		encoding = "utf8"
	} else {
		value = base64.StdEncoding.EncodeToString(input)
		encoding = "base64"
	}
	return value, encoding
}

func (runner *runner) updateActionResults(results *utilexec.ExecResponse) error {
	if err := runner.context.UpdateActionResults([]string{"Code"}, fmt.Sprintf("%d", results.Code)); err != nil {
		return errors.Trace(err)
	}

	stdout, encoding := encodeBytes(results.Stdout)
	if err := runner.context.UpdateActionResults([]string{"Stdout"}, stdout); err != nil {
		return errors.Trace(err)
	}
	if encoding != "utf8" {
		if err := runner.context.UpdateActionResults([]string{"Stdout.encoding"}, encoding); err != nil {
			return errors.Trace(err)
		}
	}

	stderr, encoding := encodeBytes(results.Stderr)
	if err := runner.context.UpdateActionResults([]string{"Stderr"}, stderr); err != nil {
		return errors.Trace(err)
	}
	if encoding != "utf8" {
		if err := runner.context.UpdateActionResults([]string{"Stderr.encoding"}, encoding); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

// RunAction exists to satisfy the Runner interface.
func (runner *runner) RunAction(actionName string) error {
	if _, err := runner.context.ActionData(); err != nil {
		return errors.Trace(err)
	}
	if actionName == actions.JujuRunActionName {
		return runner.runJujuRunAction()
	}
	return runner.runCharmHookWithLocation(actionName, "actions")
}

// RunHook exists to satisfy the Runner interface.
func (runner *runner) RunHook(hookName string) error {
	return runner.runCharmHookWithLocation(hookName, "hooks")
}

func (runner *runner) runCharmHookWithLocation(hookName, charmLocation string) error {
	srv, err := runner.startJujucServer()
	if err != nil {
		return err
	}
	defer srv.Close()

	env, err := runner.context.HookVars(runner.paths)
	if err != nil {
		return errors.Trace(err)
	}
	if jujuos.HostOS() == jujuos.Windows {
		// TODO(fwereade): somehow consolidate with utils/exec?
		// We don't do this on the other code path, which uses exec.RunCommands,
		// because that already has handling for windows environment requirements.
		env = mergeWindowsEnvironment(env, os.Environ())
	}

	debugctx := debug.NewHooksContext(runner.context.UnitName())
	if session, _ := debugctx.FindSession(); session != nil && session.MatchHook(hookName) {
		logger.Infof("executing %s via debug-hooks", hookName)
		err = session.RunHook(hookName, runner.paths.GetCharmDir(), env)
	} else {
		err = runner.runCharmHook(hookName, env, charmLocation)
	}
	return runner.context.Flush(hookName, err)
}

func (runner *runner) runCharmHook(hookName string, env []string, charmLocation string) error {
	charmDir := runner.paths.GetCharmDir()
	hook, err := searchHook(charmDir, filepath.Join(charmLocation, hookName))
	if err != nil {
		return err
	}
	hookCmd := hookCommand(hook)
	ps := exec.Command(hookCmd[0], hookCmd[1:]...)
	ps.Env = env
	ps.Dir = charmDir
	outReader, outWriter, err := os.Pipe()
	if err != nil {
		return errors.Errorf("cannot make logging pipe: %v", err)
	}
	ps.Stdout = outWriter
	ps.Stderr = outWriter
	hookLogger := &hookLogger{
		r:      outReader,
		done:   make(chan struct{}),
		logger: runner.getLogger(hookName),
	}
	go hookLogger.run()
	err = ps.Start()
	outWriter.Close()
	if err == nil {
		// Record the *os.Process of the hook
		runner.context.SetProcess(hookProcess{ps.Process})
		// Block until execution finishes
		err = ps.Wait()
	}
	hookLogger.stop()
	return errors.Trace(err)
}

func (runner *runner) startJujucServer() (*jujuc.Server, error) {
	// Prepare server.
	getCmd := func(ctxId, cmdName string) (cmd.Command, error) {
		if ctxId != runner.context.Id() {
			return nil, errors.Errorf("expected context id %q, got %q", runner.context.Id(), ctxId)
		}
		return jujuc.NewCommand(runner.context, cmdName)
	}
	srv, err := jujuc.NewServer(getCmd, runner.paths.GetJujucSocket())
	if err != nil {
		return nil, err
	}
	go srv.Run()
	return srv, nil
}

func (runner *runner) getLogger(hookName string) loggo.Logger {
	return loggo.GetLogger(fmt.Sprintf("unit.%s.%s", runner.context.UnitName(), hookName))
}

type hookProcess struct {
	*os.Process
}

func (p hookProcess) Pid() int {
	return p.Process.Pid
}
<|endoftext|>