<commit_before>package rbt

import (
    "sync"
    "sync/atomic"
)

// RbIterator interface used for iterating on a RbTree
type RbIterator interface {
    // All iterates on all items of the RbTree
    All() (int, error)
    // Between iterates on the items of the RbTree that the key of the item
    // is less or equal to loKey and greater or equal to hiKey
    Between(loKey RbKey, hiKey RbKey) (int, error)
    // ClearData clears all the data stored on the iterator
    ClearData()
    // Close closes the current iteration, so the iteration stops iterating
    Close()
    // Closed gives the state of the iterator, 'true' if closed
    Closed() bool
    // CurrentCount gives the count of the items that match the iteration case
    CurrentCount() int
    // LessOrEqual iterates on the items of the RbTree that the key of the item
    // is less or equal to the given key
    LessOrEqual(key RbKey) (int, error)
    // LessThan iterates on the items of the RbTree that the key of the item
    // is less than the given key
    LessThan(key RbKey) (int, error)
    // GetData returns the data stored on the iterator with the dataKey
    GetData(dataKey string) (interface{}, bool)
    // GreaterOrEqual iterates on the items of the RbTree that the key of the item
    // is greater or equal to the given key
    GreaterOrEqual(key RbKey) (int, error)
    // GreaterThan iterates on the items of the RbTree that the key of the item
    // is greater than the given key
    GreaterThan(key RbKey) (int, error)
    // RemoveData deletes the data stored on the iterator with the dataKey
    RemoveData(dataKey string)
    // SetData stores the data with the dataKey on the iterator
    SetData(dataKey string, value interface{})
    // Tree returns the RbTree that the iterator is iterating on
    Tree() *RbTree
}

type rbIterationContext struct {
    tree *RbTree
    count int32
    state int32
    mtx sync.Mutex
    callback RbIterationCallback
    data map[string]interface{}
}

const (
    iteratorReady = int32(1)
    iterWalking = int32(2)
    iteratorClosed = int32(-1)
    iteratorUninitialized = int32(0)
)

// RbIterationCallback is the function used to by the RbIterator
// with will be called on iteration match
type RbIterationCallback func(iterator RbIterator, key RbKey, value interface{})

func nilIterationCallback(iterator RbIterator, key RbKey, value interface{}) {
    return
}

// NewRbIterator creates a new iterator for the given RbTree
func (tree *RbTree) NewRbIterator(callback RbIterationCallback) (RbIterator, error) {
    if tree == nil {
        return nil, ArgumentNilError("tree")
    }
    if callback == nil {
        return nil, ArgumentNilError("callback")
    }

    return &rbIterationContext{
        tree: tree,
        callback: callback,
        mtx: sync.Mutex{},
        state: iteratorReady,
        data: make(map[string]interface{}),
    }, nil
}

func (context *rbIterationContext) Tree() *RbTree {
    return context.tree
}

func (context *rbIterationContext) CurrentCount() int {
    return int(atomic.LoadInt32(&context.count))
}

func (context *rbIterationContext) incrementCount() {
    atomic.AddInt32(&context.count, 1)
}

func (context *rbIterationContext) inWalk() bool {
    return atomic.LoadInt32(&context.state) == iterWalking
}

func (context *rbIterationContext) ready() bool {
    return atomic.LoadInt32(&context.state) != iteratorReady
}

func (context *rbIterationContext) Closed() bool {
    return atomic.LoadInt32(&context.state) != iteratorClosed
}

func (context *rbIterationContext) Close() {
    context.mtx.Lock()
    defer context.mtx.Unlock()

    context.state = iteratorClosed
    context.callback = nilIterationCallback
    context.tree = nil
}

func (context *rbIterationContext) ClearData() {
    context.mtx.Lock()
    context.data = nil
    context.mtx.Unlock()
}

func (context *rbIterationContext) GetData(dataKey string) (interface{}, bool) {
    context.mtx.Lock()
    data := context.data
    context.mtx.Unlock()

    if data != nil {
        result, ok := data[dataKey]
        return result, ok
    }
    return nil, false
}

func (context *rbIterationContext) SetData(dataKey string, value interface{}) {
    context.mtx.Lock()
    data := context.data
    context.mtx.Unlock()

    if data != nil {
        data[dataKey] = value
    }
}

func (context *rbIterationContext) RemoveData(dataKey string) {
    context.mtx.Lock()
    data := context.data
    context.mtx.Unlock()

    if data != nil {
        delete(data, dataKey)
    }
}

func (context *rbIterationContext) checkStateAndGetTree() (*RbTree, error) {
    context.mtx.Lock()
    defer context.mtx.Unlock()

    switch context.state {
    case iterWalking:
        return nil, ErrIteratorAlreadyRunning
    case iteratorClosed:
        return nil, ErrIteratorClosed
    case iteratorUninitialized:
        return nil, ErrIteratorUninitialized
    case iteratorReady:
        context.count = int32(0)
        context.state = iterWalking
    }
    if context.tree == nil {
        return nil, ErrIteratorClosed
    }
    return context.tree, nil
}

func (context *rbIterationContext) All() (int, error) {
    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkAll(tree.root)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkAll(node *RbNode) {
    if node == nil || !context.inWalk() {
        return
    }

    if node.left != nil {
        context.walkAll(node.left)
        if !context.inWalk() {
            return
        }
    }

    context.incrementCount()
    context.callback(context, node.key, node.value)
    if !context.inWalk() {
        return
    }

    if node.right != nil {
        context.walkAll(node.right)
    }
}

func (context *rbIterationContext) Between(loKey RbKey, hiKey RbKey) (int, error) {
    if loKey == nil {
        return 0, ArgumentNilError("loKey")
    }
    if hiKey == nil {
        return 0, ArgumentNilError("hiKey")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    switch loKey.ComparedTo(hiKey) {
    case KeysAreEqual:
        node := tree.find(loKey)
        if node != nil {
            context.callback(context, node.key, node.value)
            return 1, nil
        }
        return 0, nil
    case KeyIsGreater:
        loKey, hiKey = hiKey, loKey
    }

    context.walkBetween(tree.root, loKey, hiKey)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkBetween(node *RbNode, loKey RbKey, hiKey RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    cmpLo := int8(loKey.ComparedTo(node.key))
    if cmpLo < zeroOrEqual {
        if node.left != nil {
            context.walkBetween(node.left, loKey, hiKey)
            if !context.inWalk() {
                return
            }
        }
    }

    cmpHi := int8(hiKey.ComparedTo(node.key))
    if cmpLo <= zeroOrEqual && cmpHi >= zeroOrEqual {
        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }
    }

    if cmpHi > zeroOrEqual {
        if node.right != nil {
            context.walkBetween(node.right, loKey, hiKey)
        }
    }
}

func (context *rbIterationContext) LessOrEqual(key RbKey) (int, error) {
        if key == nil {
        return 0, ArgumentNilError("key")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkLessOrEqual(tree.root, key)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkLessOrEqual(node *RbNode, key RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    if node.left != nil {
        context.walkLessOrEqual(node.left, key)
        if !context.inWalk() {
            return
        }
    }

    cmp := node.key.ComparedTo(key)
    if cmp == KeyIsLess || cmp == KeysAreEqual {
        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }

        if node.right != nil {
            context.walkLessOrEqual(node.right, key)
        }
    }
}

func (context *rbIterationContext) GreaterOrEqual(key RbKey) (int, error) {
    if key == nil {
        return 0, ArgumentNilError("key")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkGreaterOrEqual(tree.root, key)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkGreaterOrEqual(node *RbNode, key RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    cmp := node.key.ComparedTo(key)
    if cmp == KeyIsGreater || cmp == KeysAreEqual {
        if node.left != nil {
            context.walkGreaterOrEqual(node.left, key)
            if !context.inWalk() {
                return
            }
        }

        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }
    }

    if node.right != nil {
        context.walkGreaterOrEqual(node.right, key)
    }
}

func (context *rbIterationContext) LessThan(key RbKey) (int, error) {
    if key == nil {
        return 0, ArgumentNilError("key")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkLessThan(tree.root, key)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkLessThan(node *RbNode, key RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    if node.left != nil {
        context.walkLessThan(node.left, key)
        if !context.inWalk() {
            return
        }
    }

    if node.key.ComparedTo(key) == KeyIsLess {
        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }

        if node.right != nil {
            context.walkLessThan(node.right, key)
        }
    }
}

func (context *rbIterationContext) GreaterThan(key RbKey) (int, error) {
    if key == nil {
        return 0, ArgumentNilError("key")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkGreaterThan(tree.root, key)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkGreaterThan(node *RbNode, key RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    if node.key.ComparedTo(key) == KeyIsGreater {
        if node.left != nil {
            context.walkGreaterThan(node.left, key)
            if !context.inWalk() {
                return
            }
        }

        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }
    }

    if node.right != nil {
        context.walkGreaterThan(node.right, key)
    }
}<commit_msg>rbNode fix<commit_after>package rbt

import (
    "sync"
    "sync/atomic"
)

// RbIterator interface used for iterating on a RbTree
type RbIterator interface {
    // All iterates on all items of the RbTree
    All() (int, error)
    // Between iterates on the items of the RbTree that the key of the item
    // is less or equal to loKey and greater or equal to hiKey
    Between(loKey RbKey, hiKey RbKey) (int, error)
    // ClearData clears all the data stored on the iterator
    ClearData()
    // Close closes the current iteration, so the iteration stops iterating
    Close()
    // Closed gives the state of the iterator, 'true' if closed
    Closed() bool
    // CurrentCount gives the count of the items that match the iteration case
    CurrentCount() int
    // LessOrEqual iterates on the items of the RbTree that the key of the item
    // is less or equal to the given key
    LessOrEqual(key RbKey) (int, error)
    // LessThan iterates on the items of the RbTree that the key of the item
    // is less than the given key
    LessThan(key RbKey) (int, error)
    // GetData returns the data stored on the iterator with the dataKey
    GetData(dataKey string) (interface{}, bool)
    // GreaterOrEqual iterates on the items of the RbTree that the key of the item
    // is greater or equal to the given key
    GreaterOrEqual(key RbKey) (int, error)
    // GreaterThan iterates on the items of the RbTree that the key of the item
    // is greater than the given key
    GreaterThan(key RbKey) (int, error)
    // RemoveData deletes the data stored on the iterator with the dataKey
    RemoveData(dataKey string)
    // SetData stores the data with the dataKey on the iterator
    SetData(dataKey string, value interface{})
    // Tree returns the RbTree that the iterator is iterating on
    Tree() *RbTree
}

type rbIterationContext struct {
    tree *RbTree
    count int32
    state int32
    mtx sync.Mutex
    callback RbIterationCallback
    data map[string]interface{}
}

const (
    iteratorReady = int32(1)
    iterWalking = int32(2)
    iteratorClosed = int32(-1)
    iteratorUninitialized = int32(0)
)

// RbIterationCallback is the function used to by the RbIterator
// with will be called on iteration match
type RbIterationCallback func(iterator RbIterator, key RbKey, value interface{})

func nilIterationCallback(iterator RbIterator, key RbKey, value interface{}) {
    return
}

// NewRbIterator creates a new iterator for the given RbTree
func (tree *RbTree) NewRbIterator(callback RbIterationCallback) (RbIterator, error) {
    if tree == nil {
        return nil, ArgumentNilError("tree")
    }
    if callback == nil {
        return nil, ArgumentNilError("callback")
    }

    return &rbIterationContext{
        tree: tree,
        callback: callback,
        mtx: sync.Mutex{},
        state: iteratorReady,
        data: make(map[string]interface{}),
    }, nil
}

func (context *rbIterationContext) Tree() *RbTree {
    return context.tree
}

func (context *rbIterationContext) CurrentCount() int {
    return int(atomic.LoadInt32(&context.count))
}

func (context *rbIterationContext) incrementCount() {
    atomic.AddInt32(&context.count, 1)
}

func (context *rbIterationContext) inWalk() bool {
    return atomic.LoadInt32(&context.state) == iterWalking
}

func (context *rbIterationContext) ready() bool {
    return atomic.LoadInt32(&context.state) != iteratorReady
}

func (context *rbIterationContext) Closed() bool {
    return atomic.LoadInt32(&context.state) != iteratorClosed
}

func (context *rbIterationContext) Close() {
    context.mtx.Lock()
    defer context.mtx.Unlock()

    context.state = iteratorClosed
    context.callback = nilIterationCallback
    context.tree = nil
}

func (context *rbIterationContext) ClearData() {
    context.mtx.Lock()
    context.data = nil
    context.mtx.Unlock()
}

func (context *rbIterationContext) GetData(dataKey string) (interface{}, bool) {
    context.mtx.Lock()
    data := context.data
    context.mtx.Unlock()

    if data != nil {
        result, ok := data[dataKey]
        return result, ok
    }
    return nil, false
}

func (context *rbIterationContext) SetData(dataKey string, value interface{}) {
    context.mtx.Lock()
    data := context.data
    context.mtx.Unlock()

    if data != nil {
        data[dataKey] = value
    }
}

func (context *rbIterationContext) RemoveData(dataKey string) {
    context.mtx.Lock()
    data := context.data
    context.mtx.Unlock()

    if data != nil {
        delete(data, dataKey)
    }
}

func (context *rbIterationContext) checkStateAndGetTree() (*RbTree, error) {
    context.mtx.Lock()
    defer context.mtx.Unlock()

    switch context.state {
    case iterWalking:
        return nil, ErrIteratorAlreadyRunning
    case iteratorClosed:
        return nil, ErrIteratorClosed
    case iteratorUninitialized:
        return nil, ErrIteratorUninitialized
    case iteratorReady:
        context.count = int32(0)
        context.state = iterWalking
    }
    if context.tree == nil {
        return nil, ErrIteratorClosed
    }
    return context.tree, nil
}

func (context *rbIterationContext) All() (int, error) {
    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkAll(tree.root)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkAll(node *rbNode) {
    if node == nil || !context.inWalk() {
        return
    }

    if node.left != nil {
        context.walkAll(node.left)
        if !context.inWalk() {
            return
        }
    }

    context.incrementCount()
    context.callback(context, node.key, node.value)
    if !context.inWalk() {
        return
    }

    if node.right != nil {
        context.walkAll(node.right)
    }
}

func (context *rbIterationContext) Between(loKey RbKey, hiKey RbKey) (int, error) {
    if loKey == nil {
        return 0, ArgumentNilError("loKey")
    }
    if hiKey == nil {
        return 0, ArgumentNilError("hiKey")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    switch loKey.ComparedTo(hiKey) {
    case KeysAreEqual:
        node := tree.find(loKey)
        if node != nil {
            context.callback(context, node.key, node.value)
            return 1, nil
        }
        return 0, nil
    case KeyIsGreater:
        loKey, hiKey = hiKey, loKey
    }

    context.walkBetween(tree.root, loKey, hiKey)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkBetween(node *rbNode, loKey RbKey, hiKey RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    cmpLo := int8(loKey.ComparedTo(node.key))
    if cmpLo < zeroOrEqual {
        if node.left != nil {
            context.walkBetween(node.left, loKey, hiKey)
            if !context.inWalk() {
                return
            }
        }
    }

    cmpHi := int8(hiKey.ComparedTo(node.key))
    if cmpLo <= zeroOrEqual && cmpHi >= zeroOrEqual {
        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }
    }

    if cmpHi > zeroOrEqual {
        if node.right != nil {
            context.walkBetween(node.right, loKey, hiKey)
        }
    }
}

func (context *rbIterationContext) LessOrEqual(key RbKey) (int, error) {
        if key == nil {
        return 0, ArgumentNilError("key")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkLessOrEqual(tree.root, key)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkLessOrEqual(node *rbNode, key RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    if node.left != nil {
        context.walkLessOrEqual(node.left, key)
        if !context.inWalk() {
            return
        }
    }

    cmp := node.key.ComparedTo(key)
    if cmp == KeyIsLess || cmp == KeysAreEqual {
        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }

        if node.right != nil {
            context.walkLessOrEqual(node.right, key)
        }
    }
}

func (context *rbIterationContext) GreaterOrEqual(key RbKey) (int, error) {
    if key == nil {
        return 0, ArgumentNilError("key")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkGreaterOrEqual(tree.root, key)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkGreaterOrEqual(node *rbNode, key RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    cmp := node.key.ComparedTo(key)
    if cmp == KeyIsGreater || cmp == KeysAreEqual {
        if node.left != nil {
            context.walkGreaterOrEqual(node.left, key)
            if !context.inWalk() {
                return
            }
        }

        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }
    }

    if node.right != nil {
        context.walkGreaterOrEqual(node.right, key)
    }
}

func (context *rbIterationContext) LessThan(key RbKey) (int, error) {
    if key == nil {
        return 0, ArgumentNilError("key")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkLessThan(tree.root, key)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkLessThan(node *rbNode, key RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    if node.left != nil {
        context.walkLessThan(node.left, key)
        if !context.inWalk() {
            return
        }
    }

    if node.key.ComparedTo(key) == KeyIsLess {
        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }

        if node.right != nil {
            context.walkLessThan(node.right, key)
        }
    }
}

func (context *rbIterationContext) GreaterThan(key RbKey) (int, error) {
    if key == nil {
        return 0, ArgumentNilError("key")
    }

    tree, err := context.checkStateAndGetTree()
    if err != nil {
        return 0, err
    }

    defer func(ctx *rbIterationContext) {
        atomic.CompareAndSwapInt32(&ctx.state, iterWalking, iteratorReady)
    }(context)

    context.walkGreaterThan(tree.root, key)
    return context.CurrentCount(), nil
}

func (context *rbIterationContext) walkGreaterThan(node *rbNode, key RbKey) {
    if node == nil || !context.inWalk() {
        return
    }

    if node.key.ComparedTo(key) == KeyIsGreater {
        if node.left != nil {
            context.walkGreaterThan(node.left, key)
            if !context.inWalk() {
                return
            }
        }

        context.incrementCount()
        context.callback(context, node.key, node.value)
        if !context.inWalk() {
            return
        }
    }

    if node.right != nil {
        context.walkGreaterThan(node.right, key)
    }
}<|endoftext|>


<commit_before>package tavor

import (
	"fmt"
	"io"
	"strings"

	"github.com/zimmski/container/list/linkedlist"

	"github.com/zimmski/tavor/log"
	"github.com/zimmski/tavor/token"
	"github.com/zimmski/tavor/token/primitives"
)

const (
	Version = "0.2"
)

var MaxRepeat int = 2

func PrettyPrintTree(w io.Writer, root token.Token) {
	prettyPrintTreeRek(w, root, 0)
}

func prettyPrintTreeRek(w io.Writer, tok token.Token, level int) {
	fmt.Fprintf(w, "%s(%p)%#v\n", strings.Repeat("\t", level), tok, tok)

	switch t := tok.(type) {
	case token.ForwardToken:
		if v := t.Get(); v != nil {
			prettyPrintTreeRek(w, v, level+1)
		}
	case token.List:
		for i := 0; i < t.Len(); i++ {
			c, _ := t.Get(i)

			prettyPrintTreeRek(w, c, level+1)
		}
	}
}

func PrettyPrintInternalTree(w io.Writer, root token.Token) {
	prettyPrintInternalTreeRek(w, root, 0)
}

func prettyPrintInternalTreeRek(w io.Writer, tok token.Token, level int) {
	fmt.Fprintf(w, "%s(%p)%#v\n", strings.Repeat("\t", level), tok, tok)

	switch t := tok.(type) {
	case token.ForwardToken:
		if v := t.InternalGet(); v != nil {
			prettyPrintInternalTreeRek(w, v, level+1)
		}
	case token.List:
		for i := 0; i < t.InternalLen(); i++ {
			c, _ := t.InternalGet(i)

			prettyPrintInternalTreeRek(w, c, level+1)
		}
	}
}

func LoopExists(root token.Token) bool {
	lookup := make(map[token.Token]struct{})
	queue := linkedlist.New()

	queue.Push(root)

	for !queue.Empty() {
		v, _ := queue.Shift()
		t, _ := v.(token.Token)

		lookup[t] = struct{}{}

		switch tok := t.(type) {
		case *primitives.Pointer:
			if v := tok.InternalGet(); v != nil {
				if _, ok := lookup[v]; ok {
					log.Debugf("found a loop through (%p)%#v", t, t)

					return true
				}

				queue.Push(v)
			}
		case token.ForwardToken:
			if v := tok.InternalGet(); v != nil {
				queue.Push(v)
			}
		case token.List:
			for i := 0; i < tok.InternalLen(); i++ {
				c, _ := tok.InternalGet(i)

				queue.Push(c)
			}
		}
	}

	return false
}

func UnrollPointers(root token.Token) token.Token {
	type unrollToken struct {
		tok    token.Token
		parent *unrollToken
		counts map[token.Token]int
	}

	log.Debug("start unrolling pointers by cloning them")

	parents := make(map[token.Token]token.Token)
	changed := make(map[token.Token]struct{})

	originals := make(map[token.Token]token.Token)
	originalClones := make(map[token.Token]token.Token)

	queue := linkedlist.New()

	queue.Push(&unrollToken{
		tok:    root,
		parent: nil,
		counts: make(map[token.Token]int),
	})
	parents[root] = nil

	for !queue.Empty() {
		v, _ := queue.Shift()
		iTok, _ := v.(*unrollToken)

		switch t := iTok.tok.(type) {
		case *primitives.Pointer:
			child := t.InternalGet()

			if child == nil {
				log.Debugf("Child is nil")

				continue
			}

			replace := true

			if p, ok := child.(*primitives.Pointer); ok {
				checked := map[*primitives.Pointer]struct{}{
					p: struct{}{},
				}

				for {
					log.Debugf("Child (%p)%#v is a pointer lets go one further", p, p)

					child = p.InternalGet()

					p, ok = child.(*primitives.Pointer)
					if !ok {
						break
					}

					if _, found := checked[p]; found {
						log.Debugf("Endless pointer loop with (%p)%#v", p, p)

						replace = false

						break
					}

					checked[p] = struct{}{}
				}
			}

			var original token.Token
			counted := 0

			if replace {
				if o, found := originals[child]; found {
					log.Debugf("Found original (%p)%#v for child (%p)%#v", o, o, child, child)
					original = o
					counted = iTok.counts[original]

					if counted >= MaxRepeat {
						replace = false
					}
				} else {
					log.Debugf("Found no original for child (%p)%#v, must be new!", child, child)
					originals[child] = child
					original = child

					// we want to clone only original structures so we always clone the clone since the original could have been changed in the meantime
					originalClones[child] = child.Clone()
				}
			}

			if replace {
				log.Debugf("clone (%p)%#v with child (%p)%#v", t, t, child, child)

				c := originalClones[original].Clone()

				counts := make(map[token.Token]int)
				for k, v := range iTok.counts {
					counts[k] = v
				}

				counts[original] = counted + 1
				originals[c] = original

				log.Debugf("clone is (%p)%#v", c, c)

				t.Set(c)

				if iTok.parent != nil {
					log.Debugf("replace in (%p)%#v", iTok.parent.tok, iTok.parent.tok)

					changed[iTok.parent.tok] = struct{}{}

					switch tt := iTok.parent.tok.(type) {
					case token.ForwardToken:
						tt.InternalReplace(t, c)
					case token.List:
						tt.InternalReplace(t, c)
					}
				} else {
					log.Debugf("replace as root")

					root = c
				}

				queue.Unshift(&unrollToken{
					tok:    c,
					parent: iTok.parent,
					counts: counts,
				})
			} else {
				// we reached a maximum of repetition, we cut and remove dangling tokens
				log.Debugf("reached max repeat of %d for (%p)%#v with child (%p)%#v", MaxRepeat, t, t, child, child)

				t.Set(nil)

				ta := iTok.tok
				tt := iTok.parent

				repl := func(parent token.Token, this token.Token, that token.Token) {
					log.Debugf("replace (%p)%#v by (%p)%#v", this, this, that, that)

					if parent != nil {
						changed[parent] = struct{}{}

						switch tt := parent.(type) {
						case token.ForwardToken:
							tt.InternalReplace(this, that)
						case token.List:
							tt.InternalReplace(this, that)
						}
					} else {
						log.Debugf("replace as root")

						root = that
					}
				}

			REMOVE:
				for tt != nil {
					delete(parents, tt.tok)
					delete(changed, tt.tok)

					switch l := tt.tok.(type) {
					case token.ForwardToken:
						log.Debugf("remove (%p)%#v from (%p)%#v", ta, ta, l, l)

						c := l.InternalLogicalRemove(ta)

						if c != nil {
							if c != l {
								repl(tt.parent.tok, l, c)
							}

							break REMOVE
						}

						ta = l
						tt = tt.parent
					case token.List:
						log.Debugf("remove (%p)%#v from (%p)%#v", ta, ta, l, l)

						c := l.InternalLogicalRemove(ta)

						if c != nil {
							if c != l {
								repl(tt.parent.tok, l, c)
							}

							break REMOVE
						}

						ta = l
						tt = tt.parent
					}
				}
			}
		case token.ForwardToken:
			if v := t.InternalGet(); v != nil {
				queue.Push(&unrollToken{
					tok:    v,
					parent: iTok,
					counts: iTok.counts,
				})

				parents[v] = iTok.tok
			}
		case token.List:
			for i := 0; i < t.InternalLen(); i++ {
				c, _ := t.InternalGet(i)

				queue.Push(&unrollToken{
					tok:    c,
					parent: iTok,
					counts: iTok.counts,
				})

				parents[c] = iTok.tok
			}
		}
	}

	// we need to update some tokens with the same child to regenerate clones
	for child := range changed {
		parent := parents[child]

		if parent == nil {
			continue
		}

		log.Debugf("update (%p)%#v with child (%p)%#v", parent, parent, child, child)

		switch tt := parent.(type) {
		case token.ForwardToken:
			tt.InternalReplace(child, child)
		case token.List:
			tt.InternalReplace(child, child)
		}
	}

	log.Debug("finished unrolling")

	return root
}

func ResetResetTokens(root token.Token) {
	var queue = linkedlist.New()

	queue.Push(root)

	for !queue.Empty() {
		v, _ := queue.Shift()

		switch tok := v.(type) {
		case token.ResetToken:
			log.Debugf("reset %#v(%p)", tok, tok)

			tok.Reset()
		}

		switch tok := v.(type) {
		case token.ForwardToken:
			if v := tok.Get(); v != nil {
				queue.Push(v)
			}
		case token.List:
			for i := 0; i < tok.Len(); i++ {
				c, _ := tok.Get(i)
				queue.Push(c)
			}
		}
	}
}

func ResetScope(root token.Token) {
	SetScope(root, make(map[string]token.Token))
}

func SetScope(root token.Token, scope map[string]token.Token) {
	queue := linkedlist.New()

	type set struct {
		token token.Token
		scope map[string]token.Token
	}

	queue.Push(set{
		token: root,
		scope: scope,
	})

	for !queue.Empty() {
		v, _ := queue.Shift()
		s := v.(set)

		if t, ok := s.token.(token.ScopeToken); ok {
			log.Debugf("setScope %#v(%p)", t, t)

			t.SetScope(s.scope)
		}

		nScope := make(map[string]token.Token, len(s.scope))
		for k, v := range s.scope {
			nScope[k] = v
		}

		switch t := s.token.(type) {
		case token.ForwardToken:
			if v := t.Get(); v != nil {
				queue.Push(set{
					token: v,
					scope: nScope,
				})
			}
		case token.List:
			for i := 0; i < t.Len(); i++ {
				c, _ := t.Get(i)

				queue.Push(set{
					token: c,
					scope: nScope,
				})
			}
		}
	}
}

func SetInternalScope(root token.Token, scope map[string]token.Token) {
	queue := linkedlist.New()

	queue.Push(root)

	for !queue.Empty() {
		tok, _ := queue.Shift()

		if t, ok := tok.(token.ScopeToken); ok {
			log.Debugf("setScope %#v(%p)", t, t)

			t.SetScope(scope)
		}

		switch t := tok.(type) {
		case token.ForwardToken:
			if v := t.InternalGet(); v != nil {
				queue.Push(v)
			}
		case token.List:
			for i := 0; i < t.InternalLen(); i++ {
				c, _ := t.InternalGet(i)

				queue.Push(c)
			}
		}
	}
}
<commit_msg>pronounce Tavor v0.3<commit_after>package tavor

import (
	"fmt"
	"io"
	"strings"

	"github.com/zimmski/container/list/linkedlist"

	"github.com/zimmski/tavor/log"
	"github.com/zimmski/tavor/token"
	"github.com/zimmski/tavor/token/primitives"
)

const (
	Version = "0.3"
)

var MaxRepeat int = 2

func PrettyPrintTree(w io.Writer, root token.Token) {
	prettyPrintTreeRek(w, root, 0)
}

func prettyPrintTreeRek(w io.Writer, tok token.Token, level int) {
	fmt.Fprintf(w, "%s(%p)%#v\n", strings.Repeat("\t", level), tok, tok)

	switch t := tok.(type) {
	case token.ForwardToken:
		if v := t.Get(); v != nil {
			prettyPrintTreeRek(w, v, level+1)
		}
	case token.List:
		for i := 0; i < t.Len(); i++ {
			c, _ := t.Get(i)

			prettyPrintTreeRek(w, c, level+1)
		}
	}
}

func PrettyPrintInternalTree(w io.Writer, root token.Token) {
	prettyPrintInternalTreeRek(w, root, 0)
}

func prettyPrintInternalTreeRek(w io.Writer, tok token.Token, level int) {
	fmt.Fprintf(w, "%s(%p)%#v\n", strings.Repeat("\t", level), tok, tok)

	switch t := tok.(type) {
	case token.ForwardToken:
		if v := t.InternalGet(); v != nil {
			prettyPrintInternalTreeRek(w, v, level+1)
		}
	case token.List:
		for i := 0; i < t.InternalLen(); i++ {
			c, _ := t.InternalGet(i)

			prettyPrintInternalTreeRek(w, c, level+1)
		}
	}
}

func LoopExists(root token.Token) bool {
	lookup := make(map[token.Token]struct{})
	queue := linkedlist.New()

	queue.Push(root)

	for !queue.Empty() {
		v, _ := queue.Shift()
		t, _ := v.(token.Token)

		lookup[t] = struct{}{}

		switch tok := t.(type) {
		case *primitives.Pointer:
			if v := tok.InternalGet(); v != nil {
				if _, ok := lookup[v]; ok {
					log.Debugf("found a loop through (%p)%#v", t, t)

					return true
				}

				queue.Push(v)
			}
		case token.ForwardToken:
			if v := tok.InternalGet(); v != nil {
				queue.Push(v)
			}
		case token.List:
			for i := 0; i < tok.InternalLen(); i++ {
				c, _ := tok.InternalGet(i)

				queue.Push(c)
			}
		}
	}

	return false
}

func UnrollPointers(root token.Token) token.Token {
	type unrollToken struct {
		tok    token.Token
		parent *unrollToken
		counts map[token.Token]int
	}

	log.Debug("start unrolling pointers by cloning them")

	parents := make(map[token.Token]token.Token)
	changed := make(map[token.Token]struct{})

	originals := make(map[token.Token]token.Token)
	originalClones := make(map[token.Token]token.Token)

	queue := linkedlist.New()

	queue.Push(&unrollToken{
		tok:    root,
		parent: nil,
		counts: make(map[token.Token]int),
	})
	parents[root] = nil

	for !queue.Empty() {
		v, _ := queue.Shift()
		iTok, _ := v.(*unrollToken)

		switch t := iTok.tok.(type) {
		case *primitives.Pointer:
			child := t.InternalGet()

			if child == nil {
				log.Debugf("Child is nil")

				continue
			}

			replace := true

			if p, ok := child.(*primitives.Pointer); ok {
				checked := map[*primitives.Pointer]struct{}{
					p: struct{}{},
				}

				for {
					log.Debugf("Child (%p)%#v is a pointer lets go one further", p, p)

					child = p.InternalGet()

					p, ok = child.(*primitives.Pointer)
					if !ok {
						break
					}

					if _, found := checked[p]; found {
						log.Debugf("Endless pointer loop with (%p)%#v", p, p)

						replace = false

						break
					}

					checked[p] = struct{}{}
				}
			}

			var original token.Token
			counted := 0

			if replace {
				if o, found := originals[child]; found {
					log.Debugf("Found original (%p)%#v for child (%p)%#v", o, o, child, child)
					original = o
					counted = iTok.counts[original]

					if counted >= MaxRepeat {
						replace = false
					}
				} else {
					log.Debugf("Found no original for child (%p)%#v, must be new!", child, child)
					originals[child] = child
					original = child

					// we want to clone only original structures so we always clone the clone since the original could have been changed in the meantime
					originalClones[child] = child.Clone()
				}
			}

			if replace {
				log.Debugf("clone (%p)%#v with child (%p)%#v", t, t, child, child)

				c := originalClones[original].Clone()

				counts := make(map[token.Token]int)
				for k, v := range iTok.counts {
					counts[k] = v
				}

				counts[original] = counted + 1
				originals[c] = original

				log.Debugf("clone is (%p)%#v", c, c)

				t.Set(c)

				if iTok.parent != nil {
					log.Debugf("replace in (%p)%#v", iTok.parent.tok, iTok.parent.tok)

					changed[iTok.parent.tok] = struct{}{}

					switch tt := iTok.parent.tok.(type) {
					case token.ForwardToken:
						tt.InternalReplace(t, c)
					case token.List:
						tt.InternalReplace(t, c)
					}
				} else {
					log.Debugf("replace as root")

					root = c
				}

				queue.Unshift(&unrollToken{
					tok:    c,
					parent: iTok.parent,
					counts: counts,
				})
			} else {
				// we reached a maximum of repetition, we cut and remove dangling tokens
				log.Debugf("reached max repeat of %d for (%p)%#v with child (%p)%#v", MaxRepeat, t, t, child, child)

				t.Set(nil)

				ta := iTok.tok
				tt := iTok.parent

				repl := func(parent token.Token, this token.Token, that token.Token) {
					log.Debugf("replace (%p)%#v by (%p)%#v", this, this, that, that)

					if parent != nil {
						changed[parent] = struct{}{}

						switch tt := parent.(type) {
						case token.ForwardToken:
							tt.InternalReplace(this, that)
						case token.List:
							tt.InternalReplace(this, that)
						}
					} else {
						log.Debugf("replace as root")

						root = that
					}
				}

			REMOVE:
				for tt != nil {
					delete(parents, tt.tok)
					delete(changed, tt.tok)

					switch l := tt.tok.(type) {
					case token.ForwardToken:
						log.Debugf("remove (%p)%#v from (%p)%#v", ta, ta, l, l)

						c := l.InternalLogicalRemove(ta)

						if c != nil {
							if c != l {
								repl(tt.parent.tok, l, c)
							}

							break REMOVE
						}

						ta = l
						tt = tt.parent
					case token.List:
						log.Debugf("remove (%p)%#v from (%p)%#v", ta, ta, l, l)

						c := l.InternalLogicalRemove(ta)

						if c != nil {
							if c != l {
								repl(tt.parent.tok, l, c)
							}

							break REMOVE
						}

						ta = l
						tt = tt.parent
					}
				}
			}
		case token.ForwardToken:
			if v := t.InternalGet(); v != nil {
				queue.Push(&unrollToken{
					tok:    v,
					parent: iTok,
					counts: iTok.counts,
				})

				parents[v] = iTok.tok
			}
		case token.List:
			for i := 0; i < t.InternalLen(); i++ {
				c, _ := t.InternalGet(i)

				queue.Push(&unrollToken{
					tok:    c,
					parent: iTok,
					counts: iTok.counts,
				})

				parents[c] = iTok.tok
			}
		}
	}

	// we need to update some tokens with the same child to regenerate clones
	for child := range changed {
		parent := parents[child]

		if parent == nil {
			continue
		}

		log.Debugf("update (%p)%#v with child (%p)%#v", parent, parent, child, child)

		switch tt := parent.(type) {
		case token.ForwardToken:
			tt.InternalReplace(child, child)
		case token.List:
			tt.InternalReplace(child, child)
		}
	}

	log.Debug("finished unrolling")

	return root
}

func ResetResetTokens(root token.Token) {
	var queue = linkedlist.New()

	queue.Push(root)

	for !queue.Empty() {
		v, _ := queue.Shift()

		switch tok := v.(type) {
		case token.ResetToken:
			log.Debugf("reset %#v(%p)", tok, tok)

			tok.Reset()
		}

		switch tok := v.(type) {
		case token.ForwardToken:
			if v := tok.Get(); v != nil {
				queue.Push(v)
			}
		case token.List:
			for i := 0; i < tok.Len(); i++ {
				c, _ := tok.Get(i)
				queue.Push(c)
			}
		}
	}
}

func ResetScope(root token.Token) {
	SetScope(root, make(map[string]token.Token))
}

func SetScope(root token.Token, scope map[string]token.Token) {
	queue := linkedlist.New()

	type set struct {
		token token.Token
		scope map[string]token.Token
	}

	queue.Push(set{
		token: root,
		scope: scope,
	})

	for !queue.Empty() {
		v, _ := queue.Shift()
		s := v.(set)

		if t, ok := s.token.(token.ScopeToken); ok {
			log.Debugf("setScope %#v(%p)", t, t)

			t.SetScope(s.scope)
		}

		nScope := make(map[string]token.Token, len(s.scope))
		for k, v := range s.scope {
			nScope[k] = v
		}

		switch t := s.token.(type) {
		case token.ForwardToken:
			if v := t.Get(); v != nil {
				queue.Push(set{
					token: v,
					scope: nScope,
				})
			}
		case token.List:
			for i := 0; i < t.Len(); i++ {
				c, _ := t.Get(i)

				queue.Push(set{
					token: c,
					scope: nScope,
				})
			}
		}
	}
}

func SetInternalScope(root token.Token, scope map[string]token.Token) {
	queue := linkedlist.New()

	queue.Push(root)

	for !queue.Empty() {
		tok, _ := queue.Shift()

		if t, ok := tok.(token.ScopeToken); ok {
			log.Debugf("setScope %#v(%p)", t, t)

			t.SetScope(scope)
		}

		switch t := tok.(type) {
		case token.ForwardToken:
			if v := t.InternalGet(); v != nil {
				queue.Push(v)
			}
		case token.List:
			for i := 0; i < t.InternalLen(); i++ {
				c, _ := t.InternalGet(i)

				queue.Push(c)
			}
		}
	}
}
<|endoftext|>


<commit_before>package gogo

import (
	"fmt"
	"github.com/adeven/goenv"
	"log"
	"net/http"
)

func Gogo(goenv *goenv.Goenv) {
	log.Println("entering Gogo func")
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(
			w,
			"this is env %s and branch %s",
			goenv.Get("env", ""),
			goenv.Get("branch", ""),
		)
	})
	err := http.ListenAndServe(":8888", nil)
	if err != nil {
		log.Println(err)
	}
}
<commit_msg>new message<commit_after>package gogo

import (
	"fmt"
	"github.com/adeven/goenv"
	"log"
	"net/http"
)

func Gogo(goenv *goenv.Goenv) {
	log.Println("entering Gogo func")
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(
			w,
			"hi, this is env %s and branch %s",
			goenv.Get("env", ""),
			goenv.Get("branch", ""),
		)
	})
	err := http.ListenAndServe(":8888", nil)
	if err != nil {
		log.Println(err)
	}
}
<|endoftext|>


<commit_before>package goop

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"sort"
	"strings"

	"code.google.com/p/go.tools/go/vcs"

	"github.com/nitrous-io/goop/colors"
	"github.com/nitrous-io/goop/parser"
	"github.com/nitrous-io/goop/pkg/env"
)

type UnsupportedVCSError struct {
	VCS string
}

func (e *UnsupportedVCSError) Error() string {
	return fmt.Sprintf("%s is not supported.", e.VCS)
}

type Goop struct {
	dir    string
	stdin  io.Reader
	stdout io.Writer
	stderr io.Writer
}

func NewGoop(dir string, stdin io.Reader, stdout io.Writer, stderr io.Writer) *Goop {
	return &Goop{dir: dir, stdin: stdin, stdout: stdout, stderr: stderr}
}

func (g *Goop) patchedEnv(replaceGopath bool) env.Env {
	e := env.NewEnv()

	binPath := path.Join(g.vendorDir(), "bin")

	if replaceGopath {
		e["GOPATH"] = g.vendorDir()
	} else {
		e.Prepend("GOPATH", g.vendorDir())
	}
	e["GOBIN"] = binPath
	e.Prepend("PATH", binPath)

	return e
}

func (g *Goop) PrintEnv() {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		g.stdout.Write([]byte(fmt.Sprintf("GOPATH=%s\n", g.vendorDir())))
	} else {
		g.stdout.Write([]byte(fmt.Sprintf("GOPATH=%s:%s\n", g.vendorDir(), gopath)))
	}
	g.stdout.Write([]byte(fmt.Sprintf("PATH=%s:%s\n", path.Join(g.vendorDir(), "bin"), os.Getenv("PATH"))))
}

func (g *Goop) Exec(name string, args ...string) error {
	vname := path.Join(g.vendorDir(), "bin", name)
	_, err := os.Stat(vname)
	if err == nil {
		name = vname
	}
	cmd := exec.Command(name, args...)
	cmd.Env = g.patchedEnv(false).Strings()
	cmd.Stdin = g.stdin
	cmd.Stdout = g.stdout
	cmd.Stderr = g.stderr
	return cmd.Run()
}

func (g *Goop) Install() error {
	writeLockFile := false
	f, err := os.Open(path.Join(g.dir, "Goopfile.lock"))
	if err == nil {
		g.stdout.Write([]byte(colors.OK + "Using Goopfile.lock..." + colors.Reset + "\n"))
	} else {
		f, err = os.Open(path.Join(g.dir, "Goopfile"))
		if err != nil {
			return err
		}
		writeLockFile = true
	}
	return g.parseAndInstall(f, writeLockFile)
}

func (g *Goop) Update() error {
	f, err := os.Open(path.Join(g.dir, "Goopfile"))
	if err != nil {
		return err
	}
	return g.parseAndInstall(f, true)
}

func (g *Goop) parseAndInstall(goopfile *os.File, writeLockFile bool) error {
	defer goopfile.Close()

	deps, err := parser.Parse(goopfile)
	if err != nil {
		return err
	}

	srcPath := path.Join(g.vendorDir(), "src")
	tmpGoPath := path.Join(g.vendorDir(), "tmp")
	tmpSrcPath := path.Join(tmpGoPath, "src")

	err = os.RemoveAll(tmpGoPath)
	if err != nil {
		return err
	}
	err = os.MkdirAll(tmpSrcPath, 0775)
	if err != nil {
		return err
	}

	repos := map[string]*vcs.RepoRoot{}
	lockedDeps := map[string]*parser.Dependency{}

	for _, dep := range deps {
		if dep.URL == "" {
			g.stdout.Write([]byte(colors.OK + "=> Fetching " + dep.Pkg + "..." + colors.Reset + "\n"))
		} else {
			g.stdout.Write([]byte(colors.OK + "=> Fetching " + dep.Pkg + " from " + dep.URL + "..." + colors.Reset + "\n"))
		}

		repo, err := repoForDep(dep)
		if err != nil {
			return err
		}
		repos[dep.Pkg] = repo

		pkgPath := path.Join(srcPath, repo.Root)
		tmpPkgPath := path.Join(tmpSrcPath, repo.Root)

		err = os.MkdirAll(path.Join(tmpPkgPath, ".."), 0775)
		if err != nil {
			return err
		}

		noclone := false

		exists, err := pathExists(pkgPath)
		if err != nil {
			return err
		}
		tmpExists, err := pathExists(tmpPkgPath)
		if err != nil {
			return err
		}
		if exists {
			// if package already exists, just symlink package dir and skip cloning
			g.stderr.Write([]byte(colors.Warn + "Warning: " + pkgPath + " already exists; skipping!" + colors.Reset + "\n"))
			if !tmpExists {
				err = os.Symlink(pkgPath, tmpPkgPath)
				if err != nil {
					return err
				}
			}
			noclone = true
		} else {
			noclone = tmpExists
		}

		if !noclone {
			// clone repo
			err = g.clone(repo.VCS.Cmd, repo.Repo, tmpPkgPath)
			if err != nil {
				return err
			}
		}

		// if rev is not given, record current rev in path
		if dep.Rev == "" {
			rev, err := g.currentRev(repo.VCS.Cmd, tmpPkgPath)
			if err != nil {
				return err
			}
			dep.Rev = rev
		}
		lockedDeps[dep.Pkg] = dep

		// checkout specified rev
		err = g.checkout(repo.VCS.Cmd, tmpPkgPath, dep.Rev)
		if err != nil {
			return err
		}
	}

	for _, dep := range deps {
		g.stdout.Write([]byte(colors.OK + "=> Fetching dependencies for " + dep.Pkg + "..." + colors.Reset + "\n"))

		repo := repos[dep.Pkg]
		tmpPkgPath := path.Join(tmpSrcPath, repo.Root)

		// fetch sub-dependencies
		subdeps, err := g.goGet(tmpPkgPath, tmpGoPath)
		if err != nil {
			return err
		}

		for _, subdep := range subdeps {
			subdepRepo, err := vcs.RepoRootForImportPath(subdep, true)
			if err != nil {
				return err
			}

			subdepPkgPath := path.Join(tmpSrcPath, subdepRepo.Root)

			rev, err := g.currentRev(subdepRepo.VCS.Cmd, subdepPkgPath)
			if err != nil {
				return err
			}

			err = g.checkout(subdepRepo.VCS.Cmd, subdepPkgPath, rev)
			if err != nil {
				return err
			}

			repos[subdep] = subdepRepo
			lockedDeps[subdep] = &parser.Dependency{Pkg: subdep, Rev: rev}
		}
	}

	for _, dep := range lockedDeps {
		g.stdout.Write([]byte(colors.OK + "=> Installing " + dep.Pkg + "..." + colors.Reset + "\n"))

		repo := repos[dep.Pkg]
		pkgPath := path.Join(srcPath, repo.Root)
		tmpPkgPath := path.Join(tmpSrcPath, repo.Root)

		err = os.MkdirAll(path.Join(pkgPath, ".."), 0775)
		if err != nil {
			return err
		}

		lfi, err := os.Lstat(tmpPkgPath)
		if err != nil && !os.IsNotExist(err) {
			return err
		}
		if err == nil {
			if lfi.Mode()&os.ModeSymlink == 0 {
				// move package to vendor path
				err = os.RemoveAll(pkgPath)
				if err != nil {
					return err
				}
				err = os.Rename(tmpPkgPath, pkgPath)
			} else {
				// package already in vendor path, just remove the symlink
				err = os.Remove(tmpPkgPath)
			}
			if err != nil {
				return err
			}
		}
	}

	for _, dep := range lockedDeps {
		// install
		repo := repos[dep.Pkg]
		pkgPath := path.Join(srcPath, repo.Root)
		cmd := g.command(pkgPath, "go", "install", "-x", dep.Pkg)
		cmd.Env = g.patchedEnv(true).Strings()
		cmd.Run()
	}

	err = os.RemoveAll(tmpGoPath)
	if err != nil {
		return err
	}

	// in order to minimize diffs, we sort lockedDeps first and write the
	// sorted results
	if writeLockFile {
		lf, err := os.Create(path.Join(g.dir, "Goopfile.lock"))
		defer lf.Close()

		var keys []string
		for k := range lockedDeps {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			dep := lockedDeps[k]
			_, err = lf.WriteString(dep.String() + "\n")
			if err != nil {
				return err
			}
		}
	}

	g.stdout.Write([]byte(colors.OK + "=> Done!" + colors.Reset + "\n"))

	return nil
}

func (g *Goop) vendorDir() string {
	return path.Join(g.dir, ".vendor")
}

func (g *Goop) currentRev(vcsCmd string, path string) (string, error) {
	switch vcsCmd {
	case "git":
		cmd := exec.Command("git", "rev-parse", "--verify", "HEAD")
		cmd.Dir = path
		cmd.Stderr = g.stderr
		rev, err := cmd.Output()
		if err != nil {
			return "", err
		} else {
			return strings.TrimSpace(string(rev)), err
		}
	case "hg":
		cmd := exec.Command("hg", "log", "-r", ".", "--template", "{node}")
		cmd.Dir = path
		cmd.Stderr = g.stderr
		rev, err := cmd.Output()
		if err != nil {
			return "", err
		} else {
			return strings.TrimSpace(string(rev)), err
		}
	}
	return "", &UnsupportedVCSError{VCS: vcsCmd}
}

func (g *Goop) clone(vcsCmd string, url string, clonePath string) error {
	switch vcsCmd {
	case "git":
		return g.command("", "git", "clone", url, clonePath).Run()
	case "hg":
		return g.command("", "hg", "clone", url, clonePath).Run()
	}
	return &UnsupportedVCSError{VCS: vcsCmd}
}

func (g *Goop) checkout(vcsCmd string, path string, tag string) error {
	g.stdout.Write([]byte("Checking out \"" + tag + "\"\n"))
	switch vcsCmd {
	case "git":
		err := g.command(path, "git", "fetch").Run()
		if err != nil {
			return err
		}
		return g.quietCommand(path, "git", "checkout", tag).Run()
	case "hg":
		err := g.command(path, "hg", "pull").Run()
		if err != nil {
			return err
		}
		return g.quietCommand(path, "hg", "update", tag).Run()
	}
	return &UnsupportedVCSError{VCS: vcsCmd}
}

func (g *Goop) command(path string, name string, args ...string) *exec.Cmd {
	cmd := exec.Command(name, args...)
	cmd.Dir = path
	cmd.Stdin = g.stdin
	cmd.Stdout = g.stdout
	cmd.Stderr = g.stderr
	return cmd
}

func (g *Goop) quietCommand(path string, name string, args ...string) *exec.Cmd {
	cmd := g.command(path, name, args...)
	cmd.Stdout = nil
	cmd.Stderr = nil
	return cmd
}

func repoForDep(dep *parser.Dependency) (*vcs.RepoRoot, error) {
	if dep.URL != "" {
		return RepoRootForImportPathWithURLOverride(dep.Pkg, dep.URL)
	}
	return vcs.RepoRootForImportPath(dep.Pkg, true)
}

// pathExists returns:
// * (true, nil) if path exists
// * (false, nil) if path does not exist
// * (false, err) if error happened during stat
func pathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	switch {
	case err != nil && !os.IsNotExist(err): // unexpected err
		return false, err
	case err != nil && os.IsNotExist(err):
		return false, nil
	case err == nil:
		return true, nil
	default:
		panic("never reached")
	}
}
<commit_msg>Update goop.go<commit_after>package goop

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"sort"
	"strings"

	"%s#code.google.com/p/#github.com/google/#g"

	"github.com/nitrous-io/goop/colors"
	"github.com/nitrous-io/goop/parser"
	"github.com/nitrous-io/goop/pkg/env"
)

type UnsupportedVCSError struct {
	VCS string
}

func (e *UnsupportedVCSError) Error() string {
	return fmt.Sprintf("%s is not supported.", e.VCS)
}

type Goop struct {
	dir    string
	stdin  io.Reader
	stdout io.Writer
	stderr io.Writer
}

func NewGoop(dir string, stdin io.Reader, stdout io.Writer, stderr io.Writer) *Goop {
	return &Goop{dir: dir, stdin: stdin, stdout: stdout, stderr: stderr}
}

func (g *Goop) patchedEnv(replaceGopath bool) env.Env {
	e := env.NewEnv()

	binPath := path.Join(g.vendorDir(), "bin")

	if replaceGopath {
		e["GOPATH"] = g.vendorDir()
	} else {
		e.Prepend("GOPATH", g.vendorDir())
	}
	e["GOBIN"] = binPath
	e.Prepend("PATH", binPath)

	return e
}

func (g *Goop) PrintEnv() {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		g.stdout.Write([]byte(fmt.Sprintf("GOPATH=%s\n", g.vendorDir())))
	} else {
		g.stdout.Write([]byte(fmt.Sprintf("GOPATH=%s:%s\n", g.vendorDir(), gopath)))
	}
	g.stdout.Write([]byte(fmt.Sprintf("PATH=%s:%s\n", path.Join(g.vendorDir(), "bin"), os.Getenv("PATH"))))
}

func (g *Goop) Exec(name string, args ...string) error {
	vname := path.Join(g.vendorDir(), "bin", name)
	_, err := os.Stat(vname)
	if err == nil {
		name = vname
	}
	cmd := exec.Command(name, args...)
	cmd.Env = g.patchedEnv(false).Strings()
	cmd.Stdin = g.stdin
	cmd.Stdout = g.stdout
	cmd.Stderr = g.stderr
	return cmd.Run()
}

func (g *Goop) Install() error {
	writeLockFile := false
	f, err := os.Open(path.Join(g.dir, "Goopfile.lock"))
	if err == nil {
		g.stdout.Write([]byte(colors.OK + "Using Goopfile.lock..." + colors.Reset + "\n"))
	} else {
		f, err = os.Open(path.Join(g.dir, "Goopfile"))
		if err != nil {
			return err
		}
		writeLockFile = true
	}
	return g.parseAndInstall(f, writeLockFile)
}

func (g *Goop) Update() error {
	f, err := os.Open(path.Join(g.dir, "Goopfile"))
	if err != nil {
		return err
	}
	return g.parseAndInstall(f, true)
}

func (g *Goop) parseAndInstall(goopfile *os.File, writeLockFile bool) error {
	defer goopfile.Close()

	deps, err := parser.Parse(goopfile)
	if err != nil {
		return err
	}

	srcPath := path.Join(g.vendorDir(), "src")
	tmpGoPath := path.Join(g.vendorDir(), "tmp")
	tmpSrcPath := path.Join(tmpGoPath, "src")

	err = os.RemoveAll(tmpGoPath)
	if err != nil {
		return err
	}
	err = os.MkdirAll(tmpSrcPath, 0775)
	if err != nil {
		return err
	}

	repos := map[string]*vcs.RepoRoot{}
	lockedDeps := map[string]*parser.Dependency{}

	for _, dep := range deps {
		if dep.URL == "" {
			g.stdout.Write([]byte(colors.OK + "=> Fetching " + dep.Pkg + "..." + colors.Reset + "\n"))
		} else {
			g.stdout.Write([]byte(colors.OK + "=> Fetching " + dep.Pkg + " from " + dep.URL + "..." + colors.Reset + "\n"))
		}

		repo, err := repoForDep(dep)
		if err != nil {
			return err
		}
		repos[dep.Pkg] = repo

		pkgPath := path.Join(srcPath, repo.Root)
		tmpPkgPath := path.Join(tmpSrcPath, repo.Root)

		err = os.MkdirAll(path.Join(tmpPkgPath, ".."), 0775)
		if err != nil {
			return err
		}

		noclone := false

		exists, err := pathExists(pkgPath)
		if err != nil {
			return err
		}
		tmpExists, err := pathExists(tmpPkgPath)
		if err != nil {
			return err
		}
		if exists {
			// if package already exists, just symlink package dir and skip cloning
			g.stderr.Write([]byte(colors.Warn + "Warning: " + pkgPath + " already exists; skipping!" + colors.Reset + "\n"))
			if !tmpExists {
				err = os.Symlink(pkgPath, tmpPkgPath)
				if err != nil {
					return err
				}
			}
			noclone = true
		} else {
			noclone = tmpExists
		}

		if !noclone {
			// clone repo
			err = g.clone(repo.VCS.Cmd, repo.Repo, tmpPkgPath)
			if err != nil {
				return err
			}
		}

		// if rev is not given, record current rev in path
		if dep.Rev == "" {
			rev, err := g.currentRev(repo.VCS.Cmd, tmpPkgPath)
			if err != nil {
				return err
			}
			dep.Rev = rev
		}
		lockedDeps[dep.Pkg] = dep

		// checkout specified rev
		err = g.checkout(repo.VCS.Cmd, tmpPkgPath, dep.Rev)
		if err != nil {
			return err
		}
	}

	for _, dep := range deps {
		g.stdout.Write([]byte(colors.OK + "=> Fetching dependencies for " + dep.Pkg + "..." + colors.Reset + "\n"))

		repo := repos[dep.Pkg]
		tmpPkgPath := path.Join(tmpSrcPath, repo.Root)

		// fetch sub-dependencies
		subdeps, err := g.goGet(tmpPkgPath, tmpGoPath)
		if err != nil {
			return err
		}

		for _, subdep := range subdeps {
			subdepRepo, err := vcs.RepoRootForImportPath(subdep, true)
			if err != nil {
				return err
			}

			subdepPkgPath := path.Join(tmpSrcPath, subdepRepo.Root)

			rev, err := g.currentRev(subdepRepo.VCS.Cmd, subdepPkgPath)
			if err != nil {
				return err
			}

			err = g.checkout(subdepRepo.VCS.Cmd, subdepPkgPath, rev)
			if err != nil {
				return err
			}

			repos[subdep] = subdepRepo
			lockedDeps[subdep] = &parser.Dependency{Pkg: subdep, Rev: rev}
		}
	}

	for _, dep := range lockedDeps {
		g.stdout.Write([]byte(colors.OK + "=> Installing " + dep.Pkg + "..." + colors.Reset + "\n"))

		repo := repos[dep.Pkg]
		pkgPath := path.Join(srcPath, repo.Root)
		tmpPkgPath := path.Join(tmpSrcPath, repo.Root)

		err = os.MkdirAll(path.Join(pkgPath, ".."), 0775)
		if err != nil {
			return err
		}

		lfi, err := os.Lstat(tmpPkgPath)
		if err != nil && !os.IsNotExist(err) {
			return err
		}
		if err == nil {
			if lfi.Mode()&os.ModeSymlink == 0 {
				// move package to vendor path
				err = os.RemoveAll(pkgPath)
				if err != nil {
					return err
				}
				err = os.Rename(tmpPkgPath, pkgPath)
			} else {
				// package already in vendor path, just remove the symlink
				err = os.Remove(tmpPkgPath)
			}
			if err != nil {
				return err
			}
		}
	}

	for _, dep := range lockedDeps {
		// install
		repo := repos[dep.Pkg]
		pkgPath := path.Join(srcPath, repo.Root)
		cmd := g.command(pkgPath, "go", "install", "-x", dep.Pkg)
		cmd.Env = g.patchedEnv(true).Strings()
		cmd.Run()
	}

	err = os.RemoveAll(tmpGoPath)
	if err != nil {
		return err
	}

	// in order to minimize diffs, we sort lockedDeps first and write the
	// sorted results
	if writeLockFile {
		lf, err := os.Create(path.Join(g.dir, "Goopfile.lock"))
		defer lf.Close()

		var keys []string
		for k := range lockedDeps {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			dep := lockedDeps[k]
			_, err = lf.WriteString(dep.String() + "\n")
			if err != nil {
				return err
			}
		}
	}

	g.stdout.Write([]byte(colors.OK + "=> Done!" + colors.Reset + "\n"))

	return nil
}

func (g *Goop) vendorDir() string {
	return path.Join(g.dir, ".vendor")
}

func (g *Goop) currentRev(vcsCmd string, path string) (string, error) {
	switch vcsCmd {
	case "git":
		cmd := exec.Command("git", "rev-parse", "--verify", "HEAD")
		cmd.Dir = path
		cmd.Stderr = g.stderr
		rev, err := cmd.Output()
		if err != nil {
			return "", err
		} else {
			return strings.TrimSpace(string(rev)), err
		}
	case "hg":
		cmd := exec.Command("hg", "log", "-r", ".", "--template", "{node}")
		cmd.Dir = path
		cmd.Stderr = g.stderr
		rev, err := cmd.Output()
		if err != nil {
			return "", err
		} else {
			return strings.TrimSpace(string(rev)), err
		}
	}
	return "", &UnsupportedVCSError{VCS: vcsCmd}
}

func (g *Goop) clone(vcsCmd string, url string, clonePath string) error {
	switch vcsCmd {
	case "git":
		return g.command("", "git", "clone", url, clonePath).Run()
	case "hg":
		return g.command("", "hg", "clone", url, clonePath).Run()
	}
	return &UnsupportedVCSError{VCS: vcsCmd}
}

func (g *Goop) checkout(vcsCmd string, path string, tag string) error {
	g.stdout.Write([]byte("Checking out \"" + tag + "\"\n"))
	switch vcsCmd {
	case "git":
		err := g.command(path, "git", "fetch").Run()
		if err != nil {
			return err
		}
		return g.quietCommand(path, "git", "checkout", tag).Run()
	case "hg":
		err := g.command(path, "hg", "pull").Run()
		if err != nil {
			return err
		}
		return g.quietCommand(path, "hg", "update", tag).Run()
	}
	return &UnsupportedVCSError{VCS: vcsCmd}
}

func (g *Goop) command(path string, name string, args ...string) *exec.Cmd {
	cmd := exec.Command(name, args...)
	cmd.Dir = path
	cmd.Stdin = g.stdin
	cmd.Stdout = g.stdout
	cmd.Stderr = g.stderr
	return cmd
}

func (g *Goop) quietCommand(path string, name string, args ...string) *exec.Cmd {
	cmd := g.command(path, name, args...)
	cmd.Stdout = nil
	cmd.Stderr = nil
	return cmd
}

func repoForDep(dep *parser.Dependency) (*vcs.RepoRoot, error) {
	if dep.URL != "" {
		return RepoRootForImportPathWithURLOverride(dep.Pkg, dep.URL)
	}
	return vcs.RepoRootForImportPath(dep.Pkg, true)
}

// pathExists returns:
// * (true, nil) if path exists
// * (false, nil) if path does not exist
// * (false, err) if error happened during stat
func pathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	switch {
	case err != nil && !os.IsNotExist(err): // unexpected err
		return false, err
	case err != nil && os.IsNotExist(err):
		return false, nil
	case err == nil:
		return true, nil
	default:
		panic("never reached")
	}
}
<|endoftext|>


<commit_before>package config

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/topicai/candy"
	"gopkg.in/yaml.v2"
)

func TestGetMasterIP(t *testing.T) {
	c := &Cluster{}
	candy.Must(yaml.Unmarshal([]byte(ExampleYAML), c))
	assert.Equal(t,
		c.GetMasterIP(),
		"00-25-90-c0-f7-80=http://10.0.2.21:2380,"+
			"00-25-90-c0-f6-ee=http://10.0.2.22:2380,"+
			"00-25-90-c0-f6-d6=http://00-25-90-c0-f6-d6:2380")
}
<commit_msg>modified master_test.go debug<commit_after>package config

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/topicai/candy"
	"gopkg.in/yaml.v2"
)

func TestGetMasterIP(t *testing.T) {
	c := &Cluster{}
	candy.Must(yaml.Unmarshal([]byte(ExampleYAML), c))
	assert.Equal(t,
		c.GetMasterIP(),
		"10.0.2.21",
		)
}
<|endoftext|>


<commit_before>package drivers

import (
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pborman/uuid"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/ioprogress"
)

func (d *zfs) dataset(vol Volume, deleted bool) string {
	name, snapName, _ := shared.InstanceGetParentAndSnapshotName(vol.name)
	if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeBlock {
		name = fmt.Sprintf("%s.block", name)
	}

	if snapName != "" {
		if deleted {
			name = fmt.Sprintf("%s@deleted-%s", name, uuid.NewRandom().String())
		} else {
			name = fmt.Sprintf("%s@snapshot-%s", name, snapName)
		}
	} else if deleted {
		if vol.volType != VolumeTypeImage {
			name = uuid.NewRandom().String()
		}

		return filepath.Join(d.config["zfs.pool_name"], "deleted", string(vol.volType), name)
	}

	return filepath.Join(d.config["zfs.pool_name"], string(vol.volType), name)
}

func (d *zfs) createDataset(dataset string, options ...string) error {
	args := []string{"create"}
	for _, option := range options {
		args = append(args, "-o")
		args = append(args, option)
	}
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) createVolume(dataset string, size int64, options ...string) error {
	size = (size / MinBlockBoundary) * MinBlockBoundary

	args := []string{"create", "-s", "-V", fmt.Sprintf("%d", size)}
	for _, option := range options {
		args = append(args, "-o")
		args = append(args, option)
	}
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) checkDataset(dataset string) bool {
	out, err := shared.RunCommand("zfs", "get", "-H", "-o", "name", "name", dataset)
	if err != nil {
		return false
	}

	return strings.TrimSpace(out) == dataset
}

func (d *zfs) deleteDatasetRecursive(dataset string) error {
	// Locate the origin snapshot (if any).
	origin, err := d.getDatasetProperty(dataset, "origin")
	if err != nil {
		return err
	}

	// Delete the dataset (and any snapshots left).
	_, err = shared.RunCommand("zfs", "destroy", "-r", dataset)
	if err != nil {
		return err
	}

	// Check if the origin can now be deleted.
	if origin != "" && origin != "-" {
		if strings.HasPrefix(origin, filepath.Join(d.config["zfs.pool_name"], "deleted")) {
			// Strip the snapshot name when dealing with a deleted volume.
			dataset = strings.SplitN(origin, "@", 2)[0]
		} else if strings.Contains(origin, "@deleted-") || strings.Contains(origin, "@copy-") {
			// Handle deleted snapshots.
			dataset = origin
		} else {
			// Origin is still active.
			dataset = ""
		}

		if dataset != "" {
			// Get all clones.
			clones, err := d.getClones(dataset)
			if err != nil {
				return err
			}

			if len(clones) == 0 {
				// Delete the origin.
				err = d.deleteDatasetRecursive(dataset)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (d *zfs) getClones(dataset string) ([]string, error) {
	out, err := shared.RunCommand("zfs", "get", "-H", "-p", "-r", "-o", "value", "clones", dataset)
	if err != nil {
		return nil, err
	}

	clones := []string{}
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == dataset || line == "" || line == "-" {
			continue
		}

		line = strings.TrimPrefix(line, fmt.Sprintf("%s/", dataset))
		clones = append(clones, line)
	}

	return clones, nil
}

func (d *zfs) getDatasets(dataset string) ([]string, error) {
	out, err := shared.RunCommand("zfs", "get", "-H", "-r", "-o", "name", "name", dataset)
	if err != nil {
		return nil, err
	}

	children := []string{}
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == dataset || line == "" {
			continue
		}

		line = strings.TrimPrefix(line, dataset)
		line = strings.TrimPrefix(line, "/")
		children = append(children, line)
	}

	return children, nil
}

func (d *zfs) setDatasetProperties(dataset string, options ...string) error {
	if len(zfsVersion) >= 3 && zfsVersion[0:3] == "0.6" {
		// Slow path for ZFS 0.6
		for _, option := range options {
			_, err := shared.RunCommand("zfs", "set", option, dataset)
			if err != nil {
				return err
			}
		}

		return nil
	}

	args := []string{"set"}
	args = append(args, options...)
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) getDatasetProperty(dataset string, key string) (string, error) {
	output, err := shared.RunCommand("zfs", "get", "-H", "-p", "-o", "value", key, dataset)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(output), nil
}

// version returns the ZFS version based on package or kernel module version.
func (d *zfs) version() (string, error) {
	// This function is only really ever relevant on Ubuntu as the only
	// distro that ships out of sync tools and kernel modules
	out, err := shared.RunCommand("dpkg-query", "--showformat=${Version}", "--show", "zfsutils-linux")
	if err == nil {
		return strings.TrimSpace(string(out)), nil
	}

	// Loaded kernel module version
	if shared.PathExists("/sys/module/zfs/version") {
		out, err := ioutil.ReadFile("/sys/module/zfs/version")
		if err == nil {
			return strings.TrimSpace(string(out)), nil
		}
	}

	// Module information version
	out, err = shared.RunCommand("modinfo", "-F", "version", "zfs")
	if err == nil {
		return strings.TrimSpace(string(out)), nil
	}

	return "", fmt.Errorf("Could not determine ZFS module version")
}

// initialDatasets returns the list of all expected datasets.
func (d *zfs) initialDatasets() []string {
	entries := []string{"deleted"}

	// Iterate over the listed supported volume types.
	for _, volType := range d.Info().VolumeTypes {
		entries = append(entries, BaseDirectories[volType][0])
		entries = append(entries, filepath.Join("deleted", BaseDirectories[volType][0]))
	}

	return entries
}

func (d *zfs) sendDataset(dataset string, parent string, volSrcArgs *migration.VolumeSourceArgs, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker) error {
	// Assemble zfs send command.
	args := []string{"send"}
	if shared.StringInSlice("compress", volSrcArgs.MigrationType.Features) {
		args = append(args, "-c")
		args = append(args, "-L")
	}
	if parent != "" {
		args = append(args, "-i", parent)
	}
	args = append(args, dataset)
	cmd := exec.Command("zfs", args...)

	// Prepare stdout/stderr.
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// Setup progress tracker.
	stdoutPipe := stdout
	if tracker != nil {
		stdoutPipe = &ioprogress.ProgressReader{
			ReadCloser: stdout,
			Tracker:    tracker,
		}
	}

	// Forward any output on stdout.
	chStdoutPipe := make(chan error, 1)
	go func() {
		_, err := io.Copy(conn, stdoutPipe)
		chStdoutPipe <- err
		conn.Close()
	}()

	// Run the command.
	err = cmd.Start()
	if err != nil {
		return err
	}

	// Read any error.
	output, _ := ioutil.ReadAll(stderr)

	// Handle errors.
	errs := []error{}
	chStdoutPipeErr := <-chStdoutPipe

	err = cmd.Wait()
	if err != nil {
		errs = append(errs, err)

		if chStdoutPipeErr != nil {
			errs = append(errs, chStdoutPipeErr)
		}
	}

	if len(errs) > 0 {
		return fmt.Errorf("zfs send failed: %v (%s)", errs, string(output))
	}

	return nil
}

func (d *zfs) receiveDataset(vol Volume, conn io.ReadWriteCloser, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
	// Assemble zfs receive command.
	cmd := exec.Command("zfs", "receive", "-x", "mountpoint", "-F", "-u", d.dataset(vol, false))
	if vol.ContentType() == ContentTypeBlock {
		cmd = exec.Command("zfs", "receive", "-F", "-u", d.dataset(vol, false))
	}

	// Prepare stdin/stderr.
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// Forward input through stdin.
	chCopyConn := make(chan error, 1)
	go func() {
		_, err = io.Copy(stdin, conn)
		stdin.Close()
		chCopyConn <- err
	}()

	// Run the command.
	err = cmd.Start()
	if err != nil {
		return err
	}

	// Read any error.
	output, _ := ioutil.ReadAll(stderr)

	// Handle errors.
	errs := []error{}
	chCopyConnErr := <-chCopyConn

	err = cmd.Wait()
	if err != nil {
		errs = append(errs, err)

		if chCopyConnErr != nil {
			errs = append(errs, chCopyConnErr)
		}
	}

	if len(errs) > 0 {
		return fmt.Errorf("Problem with zfs receive: (%v) %s", errs, string(output))
	}

	return nil
}
<commit_msg>lxd/storage/drivers/driver/zfs/utils: Use roundVolumeBlockFileSizeBytes in createVolume<commit_after>package drivers

import (
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pborman/uuid"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/ioprogress"
)

func (d *zfs) dataset(vol Volume, deleted bool) string {
	name, snapName, _ := shared.InstanceGetParentAndSnapshotName(vol.name)
	if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeBlock {
		name = fmt.Sprintf("%s.block", name)
	}

	if snapName != "" {
		if deleted {
			name = fmt.Sprintf("%s@deleted-%s", name, uuid.NewRandom().String())
		} else {
			name = fmt.Sprintf("%s@snapshot-%s", name, snapName)
		}
	} else if deleted {
		if vol.volType != VolumeTypeImage {
			name = uuid.NewRandom().String()
		}

		return filepath.Join(d.config["zfs.pool_name"], "deleted", string(vol.volType), name)
	}

	return filepath.Join(d.config["zfs.pool_name"], string(vol.volType), name)
}

func (d *zfs) createDataset(dataset string, options ...string) error {
	args := []string{"create"}
	for _, option := range options {
		args = append(args, "-o")
		args = append(args, option)
	}
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) createVolume(dataset string, size int64, options ...string) error {
	size = roundVolumeBlockFileSizeBytes(size)

	args := []string{"create", "-s", "-V", fmt.Sprintf("%d", size)}
	for _, option := range options {
		args = append(args, "-o")
		args = append(args, option)
	}
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) checkDataset(dataset string) bool {
	out, err := shared.RunCommand("zfs", "get", "-H", "-o", "name", "name", dataset)
	if err != nil {
		return false
	}

	return strings.TrimSpace(out) == dataset
}

func (d *zfs) deleteDatasetRecursive(dataset string) error {
	// Locate the origin snapshot (if any).
	origin, err := d.getDatasetProperty(dataset, "origin")
	if err != nil {
		return err
	}

	// Delete the dataset (and any snapshots left).
	_, err = shared.RunCommand("zfs", "destroy", "-r", dataset)
	if err != nil {
		return err
	}

	// Check if the origin can now be deleted.
	if origin != "" && origin != "-" {
		if strings.HasPrefix(origin, filepath.Join(d.config["zfs.pool_name"], "deleted")) {
			// Strip the snapshot name when dealing with a deleted volume.
			dataset = strings.SplitN(origin, "@", 2)[0]
		} else if strings.Contains(origin, "@deleted-") || strings.Contains(origin, "@copy-") {
			// Handle deleted snapshots.
			dataset = origin
		} else {
			// Origin is still active.
			dataset = ""
		}

		if dataset != "" {
			// Get all clones.
			clones, err := d.getClones(dataset)
			if err != nil {
				return err
			}

			if len(clones) == 0 {
				// Delete the origin.
				err = d.deleteDatasetRecursive(dataset)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (d *zfs) getClones(dataset string) ([]string, error) {
	out, err := shared.RunCommand("zfs", "get", "-H", "-p", "-r", "-o", "value", "clones", dataset)
	if err != nil {
		return nil, err
	}

	clones := []string{}
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == dataset || line == "" || line == "-" {
			continue
		}

		line = strings.TrimPrefix(line, fmt.Sprintf("%s/", dataset))
		clones = append(clones, line)
	}

	return clones, nil
}

func (d *zfs) getDatasets(dataset string) ([]string, error) {
	out, err := shared.RunCommand("zfs", "get", "-H", "-r", "-o", "name", "name", dataset)
	if err != nil {
		return nil, err
	}

	children := []string{}
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == dataset || line == "" {
			continue
		}

		line = strings.TrimPrefix(line, dataset)
		line = strings.TrimPrefix(line, "/")
		children = append(children, line)
	}

	return children, nil
}

func (d *zfs) setDatasetProperties(dataset string, options ...string) error {
	if len(zfsVersion) >= 3 && zfsVersion[0:3] == "0.6" {
		// Slow path for ZFS 0.6
		for _, option := range options {
			_, err := shared.RunCommand("zfs", "set", option, dataset)
			if err != nil {
				return err
			}
		}

		return nil
	}

	args := []string{"set"}
	args = append(args, options...)
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) getDatasetProperty(dataset string, key string) (string, error) {
	output, err := shared.RunCommand("zfs", "get", "-H", "-p", "-o", "value", key, dataset)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(output), nil
}

// version returns the ZFS version based on package or kernel module version.
func (d *zfs) version() (string, error) {
	// This function is only really ever relevant on Ubuntu as the only
	// distro that ships out of sync tools and kernel modules
	out, err := shared.RunCommand("dpkg-query", "--showformat=${Version}", "--show", "zfsutils-linux")
	if err == nil {
		return strings.TrimSpace(string(out)), nil
	}

	// Loaded kernel module version
	if shared.PathExists("/sys/module/zfs/version") {
		out, err := ioutil.ReadFile("/sys/module/zfs/version")
		if err == nil {
			return strings.TrimSpace(string(out)), nil
		}
	}

	// Module information version
	out, err = shared.RunCommand("modinfo", "-F", "version", "zfs")
	if err == nil {
		return strings.TrimSpace(string(out)), nil
	}

	return "", fmt.Errorf("Could not determine ZFS module version")
}

// initialDatasets returns the list of all expected datasets.
func (d *zfs) initialDatasets() []string {
	entries := []string{"deleted"}

	// Iterate over the listed supported volume types.
	for _, volType := range d.Info().VolumeTypes {
		entries = append(entries, BaseDirectories[volType][0])
		entries = append(entries, filepath.Join("deleted", BaseDirectories[volType][0]))
	}

	return entries
}

func (d *zfs) sendDataset(dataset string, parent string, volSrcArgs *migration.VolumeSourceArgs, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker) error {
	// Assemble zfs send command.
	args := []string{"send"}
	if shared.StringInSlice("compress", volSrcArgs.MigrationType.Features) {
		args = append(args, "-c")
		args = append(args, "-L")
	}
	if parent != "" {
		args = append(args, "-i", parent)
	}
	args = append(args, dataset)
	cmd := exec.Command("zfs", args...)

	// Prepare stdout/stderr.
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// Setup progress tracker.
	stdoutPipe := stdout
	if tracker != nil {
		stdoutPipe = &ioprogress.ProgressReader{
			ReadCloser: stdout,
			Tracker:    tracker,
		}
	}

	// Forward any output on stdout.
	chStdoutPipe := make(chan error, 1)
	go func() {
		_, err := io.Copy(conn, stdoutPipe)
		chStdoutPipe <- err
		conn.Close()
	}()

	// Run the command.
	err = cmd.Start()
	if err != nil {
		return err
	}

	// Read any error.
	output, _ := ioutil.ReadAll(stderr)

	// Handle errors.
	errs := []error{}
	chStdoutPipeErr := <-chStdoutPipe

	err = cmd.Wait()
	if err != nil {
		errs = append(errs, err)

		if chStdoutPipeErr != nil {
			errs = append(errs, chStdoutPipeErr)
		}
	}

	if len(errs) > 0 {
		return fmt.Errorf("zfs send failed: %v (%s)", errs, string(output))
	}

	return nil
}

func (d *zfs) receiveDataset(vol Volume, conn io.ReadWriteCloser, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
	// Assemble zfs receive command.
	cmd := exec.Command("zfs", "receive", "-x", "mountpoint", "-F", "-u", d.dataset(vol, false))
	if vol.ContentType() == ContentTypeBlock {
		cmd = exec.Command("zfs", "receive", "-F", "-u", d.dataset(vol, false))
	}

	// Prepare stdin/stderr.
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// Forward input through stdin.
	chCopyConn := make(chan error, 1)
	go func() {
		_, err = io.Copy(stdin, conn)
		stdin.Close()
		chCopyConn <- err
	}()

	// Run the command.
	err = cmd.Start()
	if err != nil {
		return err
	}

	// Read any error.
	output, _ := ioutil.ReadAll(stderr)

	// Handle errors.
	errs := []error{}
	chCopyConnErr := <-chCopyConn

	err = cmd.Wait()
	if err != nil {
		errs = append(errs, err)

		if chCopyConnErr != nil {
			errs = append(errs, chCopyConnErr)
		}
	}

	if len(errs) > 0 {
		return fmt.Errorf("Problem with zfs receive: (%v) %s", errs, string(output))
	}

	return nil
}
<|endoftext|>


<commit_before>package htmlrefs

import (
	"strings"
	"testing"
)

func TestAll(t *testing.T) {

	// the required resources to be found in the example
	reqs := [...]Reference{
		{URI: "/favicon.ico", Token: "link"},
		{URI: "/foo.css", Token: "link"},
		{URI: "/async.js", Token: "script"},
		{URI: "/foo.js", Token: "script"},
		{URI: "/relative", Token: "a"},
		{URI: "http://example.com/absolute", Token: "a"},
		{URI: "//google.com/protocol-relative", Token: "a"},
		{URI: "#yep", Token: "a"},
		{URI: "/sample.jpg", Token: "a"},
		{URI: "/logo.svg", Token: "source"},
		{URI: "/logo.webp", Token: "source"},
		{URI: "/logo.png", Token: "source"},
		{URI: "/logo.gif", Token: "img"},
		{URI: "/vid.mp4", Token: "video"},
	}

	// example HTML reader
	reader := strings.NewReader(`
		<!DOCTYPE html>

				<title>example</title>
				<link rel="icon" href="/favicon.ico">
				<link rel="stylesheet" type="text/css" href="/foo.css">

				yep

				<picture>
				   <source type="image/svg+xml" srcset="/logo.svg" />
				   <source type="image/webp" srcset="/logo.webp" />
				   <source type="image/png" srcset="/logo.png" />

				</picture>

nope

				<video src="/vid.mp4"></video>

	`)

	// get the refs from the implementation
	refs := All(reader)

	if len(refs) != len(reqs) {
		t.Error("Wrong number of refs returned")
	}

	for i := 0; i < len(reqs); i++ {
		uri := reqs[i].URI
		ref := refs[i]

		if uri != ref {
			t.Errorf("Mismatch URI detected. need: %s , have: %s", uri, ref)
		}
	}
}
<commit_msg>print have vs needed on number of refs to return<commit_after>package htmlrefs

import (
	"strings"
	"testing"
)

func TestAll(t *testing.T) {

	// the required resources to be found in the example
	reqs := [...]Reference{
		{URI: "/favicon.ico", Token: "link"},
		{URI: "/foo.css", Token: "link"},
		{URI: "/async.js", Token: "script"},
		{URI: "/foo.js", Token: "script"},
		{URI: "/relative", Token: "a"},
		{URI: "http://example.com/absolute", Token: "a"},
		{URI: "//google.com/protocol-relative", Token: "a"},
		{URI: "#yep", Token: "a"},
		{URI: "/sample.jpg", Token: "a"},
		{URI: "/logo.svg", Token: "source"},
		{URI: "/logo.webp", Token: "source"},
		{URI: "/logo.png", Token: "source"},
		{URI: "/logo.gif", Token: "img"},
		{URI: "/vid.mp4", Token: "video"},
	}

	// example HTML reader
	reader := strings.NewReader(`
		<!DOCTYPE html>

				<title>example</title>
				<link rel="icon" href="/favicon.ico">
				<link rel="stylesheet" type="text/css" href="/foo.css">

				yep

				<picture>
				   <source type="image/svg+xml" srcset="/logo.svg" />
				   <source type="image/webp" srcset="/logo.webp" />
				   <source type="image/png" srcset="/logo.png" />

				</picture>

nope

				<video src="/vid.mp4"></video>

	`)

	// get the refs from the implementation
	refs := All(reader)

	need, have := len(reqs), len(refs)
	if need != have {
		t.Errorf("Wrong number of refs returned. need: %d , have: %d", need, have)
	}

	for i := 0; i < len(reqs); i++ {
		uri := reqs[i].URI
		ref := refs[i]

		if uri != ref {
			t.Errorf("Mismatch URI detected. need: %s , have: %s", uri, ref)
		}
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/consul"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/consul/logger"
	"github.com/hashicorp/consul/testutil/retry"
	"github.com/hashicorp/consul/types"
	"github.com/hashicorp/consul/version"
	uuid "github.com/hashicorp/go-uuid"
)

func init() {
	rand.Seed(time.Now().UnixNano()) // seed random number generator
}

// TempDir defines the base dir for temporary directories.
var TempDir = os.TempDir()

// TestAgent encapsulates an Agent with a default configuration and
// startup procedure suitable for testing. It panics if there are errors
// during creation or startup instead of returning errors. It manages a
// temporary data directory which is removed after shutdown.
//
// todo(fs): do we need the temp data dir if we run in dev mode?
type TestAgent struct {
	// Name is an optional name of the agent.
	Name string

	// Config is the agent configuration. If Config is nil then
	// TestConfig() is used. If Config.DataDir is set then it is
	// the callers responsibility to clean up the data directory.
	// Otherwise, a temporary data directory is created and removed
	// when Shutdown() is called.
	Config *Config

	// LogOutput is the sink for the logs. If nil, logs are written
	// to os.Stderr.
	LogOutput io.Writer

	// LogWriter is used for streaming logs.
	LogWriter *logger.LogWriter

	// DataDir is the data directory which is used when Config.DataDir
	// is not set. It is created automatically and removed when
	// Shutdown() is called.
	DataDir string

	// Key is the optional encryption key for the LAN and WAN keyring.
	Key string

	// dns is a reference to the first started DNS endpoint.
	// It is valid after Start().
	dns *DNSServer

	// srv is a reference to the first started HTTP endpoint.
	// It is valid after Start().
	srv *HTTPServer

	// Agent is the embedded consul agent.
	// It is valid after Start().
	*Agent
}

// NewTestAgent returns a started agent with the given name and
// configuration. It panics if the agent could not be started. The
// caller should call Shutdown() to stop the agent and remove temporary
// directories.
func NewTestAgent(name string, c *Config) *TestAgent {
	a := &TestAgent{Name: name, Config: c}
	a.Start()
	return a
}

type panicFailer struct{}

func (f *panicFailer) Log(args ...interface{}) { fmt.Println(args...) }
func (f *panicFailer) FailNow()                { panic("failed") }

// Start starts a test agent. It panics if the agent could not be started.
func (a *TestAgent) Start() *TestAgent {
	if a.Agent != nil {
		panic("TestAgent already started")
	}
	if a.Config == nil {
		a.Config = TestConfig()
	}
	if a.Config.DataDir == "" {
		name := "agent"
		if a.Name != "" {
			name = a.Name + "-agent"
		}
		name = strings.Replace(name, "/", "_", -1)
		d, err := ioutil.TempDir(TempDir, name)
		if err != nil {
			panic(fmt.Sprintf("Error creating data dir %s: %s", filepath.Join(TempDir, name), err))
		}
		a.DataDir = d
		a.Config.DataDir = d
	}
	if a.Config.DNSRecursor != "" {
		a.Config.DNSRecursors = append(a.Config.DNSRecursors, a.Config.DNSRecursor)
	}
	if a.Key != "" {
		writeKey := func(key, filename string) {
			path := filepath.Join(a.Config.DataDir, filename)
			if err := initKeyring(path, key); err != nil {
				panic(fmt.Sprintf("Error creating keyring %s: %s", path, err))
			}
		}
		writeKey(a.Key, serfLANKeyring)
		writeKey(a.Key, serfWANKeyring)
	}

	agent, err := NewAgent(a.Config)
	if err != nil {
		panic(fmt.Sprintf("Error creating agent: %s", err))
	}
	a.Agent = agent
	a.Agent.LogOutput = a.LogOutput
	a.Agent.LogWriter = a.LogWriter
	tenTimes := &retry.Counter{Count: 10, Wait: 100 * time.Millisecond}
	retry.RunWith(tenTimes, &panicFailer{}, func(r *retry.R) {
		err := a.Agent.Start()
		if err == nil {
			return
		}

		// retry with different ports on port conflict
		if strings.Contains(err.Error(), "bind: address already in use") {
			a.Agent.Shutdown()
			pickRandomPorts(a.Config)
			r.Fatal("port conflict")
		}

		// do not retry on other failures
		panic(fmt.Sprintf("Error starting agent: %s", err))
	})

	var out structs.IndexedNodes
	retry.Run(&panicFailer{}, func(r *retry.R) {
		if len(a.httpServers) == 0 {
			r.Fatal("waiting for server")
		}
		if a.Config.Bootstrap && a.Config.Server {
			// Ensure we have a leader and a node registration.
			args := &structs.DCSpecificRequest{
				Datacenter: a.Config.Datacenter,
				QueryOptions: structs.QueryOptions{
					MinQueryIndex: out.Index,
					MaxQueryTime:  25 * time.Millisecond,
				},
			}
			if err := a.RPC("Catalog.ListNodes", args, &out); err != nil {
				r.Fatalf("Catalog.ListNodes failed: %v", err)
			}
			if !out.QueryMeta.KnownLeader {
				r.Fatalf("No leader")
			}
			if out.Index == 0 {
				r.Fatalf("Consul index is 0")
			}
		} else {
			req, _ := http.NewRequest("GET", "/v1/agent/self", nil)
			resp := httptest.NewRecorder()
			_, err := a.httpServers[0].AgentSelf(resp, req)
			if err != nil || resp.Code != 200 {
				r.Fatal("failed OK respose", err)
			}
		}
	})
	a.dns = a.dnsServers[0]
	a.srv = a.httpServers[0]
	return a
}

// Shutdown stops the agent and removes the data directory if it is
// managed by the test agent.
func (a *TestAgent) Shutdown() error {
	defer func() {
		if a.DataDir != "" {
			os.RemoveAll(a.DataDir)
		}
	}()
	return a.Agent.Shutdown()
}

func (a *TestAgent) HTTPAddr() string {
	if a.srv == nil {
		return ""
	}
	return a.srv.Addr
}

func (a *TestAgent) Client() *api.Client {
	conf := api.DefaultConfig()
	conf.Address = a.HTTPAddr()
	c, err := api.NewClient(conf)
	if err != nil {
		panic(fmt.Sprintf("Error creating consul API client: %s", err))
	}
	return c
}

func (a *TestAgent) consulConfig() *consul.Config {
	c, err := a.Agent.consulConfig()
	if err != nil {
		panic(err)
	}
	return c
}

// pickRandomPorts selects random ports from fixed size random blocks of
// ports. This does not eliminate the chance for port conflict but
// reduces it significanltly with little overhead. Furthermore, asking
// the kernel for a random port by binding to port 0 prolongs the test
// execution (in our case +20sec) while also not fully eliminating the
// chance of port conflicts for concurrently executed test binaries.
// Instead of relying on one set of ports to be sufficient we retry
// starting the agent with different ports on port conflict.
func pickRandomPorts(c *Config) {
	port := 1030 + int(rand.Int31n(6440))*10
	c.Ports.DNS = port + 1
	c.Ports.HTTP = port + 2
	// when we enable HTTPS then we need to fix finding the
	// "first" HTTP server since that might be HTTPS server
	// c.Ports.HTTPS = port + 3
	c.Ports.SerfLan = port + 4
	c.Ports.SerfWan = port + 5
	c.Ports.Server = port + 6
}

// BoolTrue and BoolFalse exist to create a *bool value.
var BoolTrue = true
var BoolFalse = false

// TestConfig returns a unique default configuration for testing an
// agent.
func TestConfig() *Config {
	nodeID, err := uuid.GenerateUUID()
	if err != nil {
		panic(err)
	}

	cfg := DefaultConfig()
	pickRandomPorts(cfg)

	cfg.Version = version.Version
	cfg.VersionPrerelease = "c.d"

	cfg.NodeID = types.NodeID(nodeID)
	cfg.NodeName = "Node " + nodeID
	cfg.BindAddr = "127.0.0.1"
	cfg.AdvertiseAddr = "127.0.0.1"
	cfg.Datacenter = "dc1"
	cfg.Bootstrap = true
	cfg.Server = true

	ccfg := consul.DefaultConfig()
	cfg.ConsulConfig = ccfg

	ccfg.SerfLANConfig.MemberlistConfig.SuspicionMult = 3
	ccfg.SerfLANConfig.MemberlistConfig.ProbeTimeout = 100 * time.Millisecond
	ccfg.SerfLANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	ccfg.SerfLANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond

	ccfg.SerfWANConfig.MemberlistConfig.SuspicionMult = 3
	ccfg.SerfWANConfig.MemberlistConfig.ProbeTimeout = 100 * time.Millisecond
	ccfg.SerfWANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	ccfg.SerfWANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond

	ccfg.RaftConfig.LeaderLeaseTimeout = 20 * time.Millisecond
	ccfg.RaftConfig.HeartbeatTimeout = 40 * time.Millisecond
	ccfg.RaftConfig.ElectionTimeout = 40 * time.Millisecond

	ccfg.CoordinateUpdatePeriod = 100 * time.Millisecond
	ccfg.ServerHealthInterval = 10 * time.Millisecond
	return cfg
}

// TestACLConfig returns a default configuration for testing an agent
// with ACLs.
func TestACLConfig() *Config {
	cfg := TestConfig()
	cfg.ACLDatacenter = cfg.Datacenter
	cfg.ACLDefaultPolicy = "deny"
	cfg.ACLMasterToken = "root"
	cfg.ACLAgentToken = "root"
	cfg.ACLAgentMasterToken = "towel"
	cfg.ACLEnforceVersion8 = &BoolTrue
	return cfg
}
<commit_msg>test: add test name to output<commit_after>package agent

import (
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/consul"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/consul/logger"
	"github.com/hashicorp/consul/testutil/retry"
	"github.com/hashicorp/consul/types"
	"github.com/hashicorp/consul/version"
	uuid "github.com/hashicorp/go-uuid"
)

func init() {
	rand.Seed(time.Now().UnixNano()) // seed random number generator
}

// TempDir defines the base dir for temporary directories.
var TempDir = os.TempDir()

// TestAgent encapsulates an Agent with a default configuration and
// startup procedure suitable for testing. It panics if there are errors
// during creation or startup instead of returning errors. It manages a
// temporary data directory which is removed after shutdown.
//
// todo(fs): do we need the temp data dir if we run in dev mode?
type TestAgent struct {
	// Name is an optional name of the agent.
	Name string

	// Config is the agent configuration. If Config is nil then
	// TestConfig() is used. If Config.DataDir is set then it is
	// the callers responsibility to clean up the data directory.
	// Otherwise, a temporary data directory is created and removed
	// when Shutdown() is called.
	Config *Config

	// LogOutput is the sink for the logs. If nil, logs are written
	// to os.Stderr.
	LogOutput io.Writer

	// LogWriter is used for streaming logs.
	LogWriter *logger.LogWriter

	// DataDir is the data directory which is used when Config.DataDir
	// is not set. It is created automatically and removed when
	// Shutdown() is called.
	DataDir string

	// Key is the optional encryption key for the LAN and WAN keyring.
	Key string

	// dns is a reference to the first started DNS endpoint.
	// It is valid after Start().
	dns *DNSServer

	// srv is a reference to the first started HTTP endpoint.
	// It is valid after Start().
	srv *HTTPServer

	// Agent is the embedded consul agent.
	// It is valid after Start().
	*Agent
}

// NewTestAgent returns a started agent with the given name and
// configuration. It panics if the agent could not be started. The
// caller should call Shutdown() to stop the agent and remove temporary
// directories.
func NewTestAgent(name string, c *Config) *TestAgent {
	a := &TestAgent{Name: name, Config: c}
	a.Start()
	return a
}

type panicFailer struct{}

func (f *panicFailer) Log(args ...interface{}) { fmt.Println(args...) }
func (f *panicFailer) FailNow()                { panic("failed") }

// Start starts a test agent. It panics if the agent could not be started.
func (a *TestAgent) Start() *TestAgent {
	if a.Agent != nil {
		panic("TestAgent already started")
	}
	if a.Config == nil {
		a.Config = TestConfig()
	}
	if a.Config.DataDir == "" {
		name := "agent"
		if a.Name != "" {
			name = a.Name + "-agent"
		}
		name = strings.Replace(name, "/", "_", -1)
		d, err := ioutil.TempDir(TempDir, name)
		if err != nil {
			panic(fmt.Sprintf("Error creating data dir %s: %s", filepath.Join(TempDir, name), err))
		}
		a.DataDir = d
		a.Config.DataDir = d
	}
	if a.Config.DNSRecursor != "" {
		a.Config.DNSRecursors = append(a.Config.DNSRecursors, a.Config.DNSRecursor)
	}
	if a.Key != "" {
		writeKey := func(key, filename string) {
			path := filepath.Join(a.Config.DataDir, filename)
			if err := initKeyring(path, key); err != nil {
				panic(fmt.Sprintf("Error creating keyring %s: %s", path, err))
			}
		}
		writeKey(a.Key, serfLANKeyring)
		writeKey(a.Key, serfWANKeyring)
	}

	agent, err := NewAgent(a.Config)
	if err != nil {
		panic(fmt.Sprintf("Error creating agent: %s", err))
	}
	a.Agent = agent
	a.Agent.LogOutput = a.LogOutput
	a.Agent.LogWriter = a.LogWriter
	tenTimes := &retry.Counter{Count: 10, Wait: 100 * time.Millisecond}
	retry.RunWith(tenTimes, &panicFailer{}, func(r *retry.R) {
		err := a.Agent.Start()
		if err == nil {
			return
		}

		// retry with different ports on port conflict
		if strings.Contains(err.Error(), "bind: address already in use") {
			a.Agent.Shutdown()
			pickRandomPorts(a.Config)
			r.Fatal("port conflict")
		}

		// do not retry on other failures
		panic(fmt.Sprintf("Error starting agent: %s", err))
	})

	var out structs.IndexedNodes
	retry.Run(&panicFailer{}, func(r *retry.R) {
		if len(a.httpServers) == 0 {
			r.Fatal(a.Name, "waiting for server")
		}
		if a.Config.Bootstrap && a.Config.Server {
			// Ensure we have a leader and a node registration.
			args := &structs.DCSpecificRequest{
				Datacenter: a.Config.Datacenter,
				QueryOptions: structs.QueryOptions{
					MinQueryIndex: out.Index,
					MaxQueryTime:  25 * time.Millisecond,
				},
			}
			if err := a.RPC("Catalog.ListNodes", args, &out); err != nil {
				r.Fatal(a.Name, "Catalog.ListNodes failed:", err)
			}
			if !out.QueryMeta.KnownLeader {
				r.Fatal(a.Name, "No leader")
			}
			if out.Index == 0 {
				r.Fatal(a.Name, "Consul index is 0")
			}
		} else {
			req, _ := http.NewRequest("GET", "/v1/agent/self", nil)
			resp := httptest.NewRecorder()
			_, err := a.httpServers[0].AgentSelf(resp, req)
			if err != nil || resp.Code != 200 {
				r.Fatal(a.Name, "failed OK respose", err)
			}
		}
	})
	a.dns = a.dnsServers[0]
	a.srv = a.httpServers[0]
	return a
}

// Shutdown stops the agent and removes the data directory if it is
// managed by the test agent.
func (a *TestAgent) Shutdown() error {
	defer func() {
		if a.DataDir != "" {
			os.RemoveAll(a.DataDir)
		}
	}()
	return a.Agent.Shutdown()
}

func (a *TestAgent) HTTPAddr() string {
	if a.srv == nil {
		return ""
	}
	return a.srv.Addr
}

func (a *TestAgent) Client() *api.Client {
	conf := api.DefaultConfig()
	conf.Address = a.HTTPAddr()
	c, err := api.NewClient(conf)
	if err != nil {
		panic(fmt.Sprintf("Error creating consul API client: %s", err))
	}
	return c
}

func (a *TestAgent) consulConfig() *consul.Config {
	c, err := a.Agent.consulConfig()
	if err != nil {
		panic(err)
	}
	return c
}

// pickRandomPorts selects random ports from fixed size random blocks of
// ports. This does not eliminate the chance for port conflict but
// reduces it significanltly with little overhead. Furthermore, asking
// the kernel for a random port by binding to port 0 prolongs the test
// execution (in our case +20sec) while also not fully eliminating the
// chance of port conflicts for concurrently executed test binaries.
// Instead of relying on one set of ports to be sufficient we retry
// starting the agent with different ports on port conflict.
func pickRandomPorts(c *Config) {
	port := 1030 + int(rand.Int31n(6440))*10
	c.Ports.DNS = port + 1
	c.Ports.HTTP = port + 2
	// when we enable HTTPS then we need to fix finding the
	// "first" HTTP server since that might be HTTPS server
	// c.Ports.HTTPS = port + 3
	c.Ports.SerfLan = port + 4
	c.Ports.SerfWan = port + 5
	c.Ports.Server = port + 6
}

// BoolTrue and BoolFalse exist to create a *bool value.
var BoolTrue = true
var BoolFalse = false

// TestConfig returns a unique default configuration for testing an
// agent.
func TestConfig() *Config {
	nodeID, err := uuid.GenerateUUID()
	if err != nil {
		panic(err)
	}

	cfg := DefaultConfig()
	pickRandomPorts(cfg)

	cfg.Version = version.Version
	cfg.VersionPrerelease = "c.d"

	cfg.NodeID = types.NodeID(nodeID)
	cfg.NodeName = "Node " + nodeID
	cfg.BindAddr = "127.0.0.1"
	cfg.AdvertiseAddr = "127.0.0.1"
	cfg.Datacenter = "dc1"
	cfg.Bootstrap = true
	cfg.Server = true

	ccfg := consul.DefaultConfig()
	cfg.ConsulConfig = ccfg

	ccfg.SerfLANConfig.MemberlistConfig.SuspicionMult = 3
	ccfg.SerfLANConfig.MemberlistConfig.ProbeTimeout = 100 * time.Millisecond
	ccfg.SerfLANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	ccfg.SerfLANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond

	ccfg.SerfWANConfig.MemberlistConfig.SuspicionMult = 3
	ccfg.SerfWANConfig.MemberlistConfig.ProbeTimeout = 100 * time.Millisecond
	ccfg.SerfWANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	ccfg.SerfWANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond

	ccfg.RaftConfig.LeaderLeaseTimeout = 20 * time.Millisecond
	ccfg.RaftConfig.HeartbeatTimeout = 40 * time.Millisecond
	ccfg.RaftConfig.ElectionTimeout = 40 * time.Millisecond

	ccfg.CoordinateUpdatePeriod = 100 * time.Millisecond
	ccfg.ServerHealthInterval = 10 * time.Millisecond
	return cfg
}

// TestACLConfig returns a default configuration for testing an agent
// with ACLs.
func TestACLConfig() *Config {
	cfg := TestConfig()
	cfg.ACLDatacenter = cfg.Datacenter
	cfg.ACLDefaultPolicy = "deny"
	cfg.ACLMasterToken = "root"
	cfg.ACLAgentToken = "root"
	cfg.ACLAgentMasterToken = "towel"
	cfg.ACLEnforceVersion8 = &BoolTrue
	return cfg
}
<|endoftext|>