<commit_before>/*
Package readcache implements a read-through cache where the caller supplies
the means to read-through in the form of a function.

TODO: Add a maximum size, enforce with an LRU algorithm
*/
package readcache

import (
	"sync"
	"time"
)

// Type Cacheable is something that may be stored in a cache
type Cacheable struct {
	// The item in the cache
	Value interface{}

	// The time at which this item should expire from the cache.
	ExpiresAt time.Time
}

// Type Cache defines a read-through cache.
type Cache interface {
	// Retrieve an item from the cache if available, or from a
	// backing source if it is not.
	Get(key string) interface{}
}

// Constructs a new cache
func New(getter func(string) Cacheable) Cache {
	return &lazycache{getter, make(map[string]*Cacheable), make(map[string]*sync.Once), new(sync.RWMutex), new(sync.RWMutex)}
}

// Type lazycache implements the Cache interface
type lazycache struct {
	Getter           func(string) Cacheable
	Cache            map[string]*Cacheable
	ReadControls     map[string]*sync.Once
	CacheLock        *sync.RWMutex
	ReadControlsLock *sync.RWMutex
}

// Get an item from the cache, retrieving the item from the getter if necessary.
// This implemention is meant to be goroutine safe.  It assumes that updating a
// map while concurrently reading from it is unsafe, so it uses a read/write mutex
// to synchronize access to its internal maps.
func (c *lazycache) Get(key string) interface{} {
	cachedValue, ok := getFromCache(c, key)
	if ok {
		return cachedValue.Value
	}

	readControl, cachedValue, ok := getReadControl(c, key)
	if ok {
		return cachedValue.Value
	}

	cachedValue = doFetch(c, key, readControl)
	return cachedValue.Value
}

// Attempt to retrieve an item from the cache, if it exists and hasn't expired.
// Returns somevalue, true if exists or nil, false if it does not.
func getFromCache(c *lazycache, key string) (*Cacheable, bool) {
	c.CacheLock.RLock()
	cachedValue, ok := c.Cache[key]
	c.CacheLock.RUnlock()
	if ok {
		now := time.Now()
		if cachedValue.ExpiresAt.After(now) {
			return cachedValue, true
		}
		c.CacheLock.Lock()
		// Determine if another goroutine has updated the cache before the lock
		cachedValue, ok = c.Cache[key]
		if ok && cachedValue.ExpiresAt.After(now) {
			c.CacheLock.Unlock()
			return cachedValue, true
		}
		delete(c.Cache, key)
		c.CacheLock.Unlock()
	}
	return nil, false
}

// Get a Once for controlling the read-through on a particular cached item.
// Performs a last-minute check to determine if another goroutine has populated
// the cache before a lock is acquired, so this function may return a cached
// value instead.  If so, the third return value will be true.  Otherwise, a
// read control is returned and the third value is false.
func getReadControl(c *lazycache, key string) (readControl *sync.Once, cachedItem *Cacheable, gotCachedItem bool) {
	gotCachedItem = false

	c.ReadControlsLock.RLock()
	readControl, ok := c.ReadControls[key]
	c.ReadControlsLock.RUnlock()
	if !ok {
		c.ReadControlsLock.Lock()

		// Another goroutine may have created a read control, fetched an item, updated the
		// cache and cleaned up its read control by the time we reach this point.
		// Therefore, we verify that the cache still does not contain anything for the
		// given key.
		// Warning: possibility of deadlock when dealing with multiple locks.  Make sure
		//          they are always acquired in the same order.
		c.CacheLock.RLock()
		cachedItem, ok = c.Cache[key]
		c.CacheLock.RUnlock()

		if ok {
			c.ReadControlsLock.Unlock()
			gotCachedItem = true
			return
		}

		readControl, ok = c.ReadControls[key]
		if !ok {
			readControl = new(sync.Once)
			c.ReadControls[key] = readControl
		}
		c.ReadControlsLock.Unlock()
	}

	return
}

// Use the given read control to fetch a value and store it in the cache.
// The read control may prevent this goroutine from fetching the value if
// some other routine gets to it first.  In either case, the resulting
// fetched value is returned.
func doFetch(c *lazycache, key string, readControl *sync.Once) (cachedValue *Cacheable) {
	fetchedValue := false
	readControl.Do(func() {
		defer func() {
			c.ReadControlsLock.Lock()
			delete(c.ReadControls, key)
			c.ReadControlsLock.Unlock()
		}()

		valueFromGetter := c.Getter(key)
		cachedValue = &valueFromGetter
		fetchedValue = true
		c.CacheLock.Lock()
		c.Cache[key] = cachedValue
		c.CacheLock.Unlock()
	})

	if !fetchedValue {
		c.CacheLock.RLock()
		cachedValue, _ = c.Cache[key]
		c.CacheLock.RUnlock()
	}

	return
}
<commit_msg>Added comments<commit_after>/*
Package readcache implements a read-through cache where the caller supplies
the means to read-through in the form of a function.

TODO: Add a maximum size, enforce with an LRU algorithm
*/
package readcache

import (
	"sync"
	"time"
)

// Type Cacheable is something that may be stored in a cache
type Cacheable struct {
	// The item in the cache
	Value interface{}

	// The time at which this item should expire from the cache.
	ExpiresAt time.Time
}

// Type Cache defines a read-through cache.
type Cache interface {
	// Retrieve an item from the cache if available, or from a
	// backing source if it is not.
	Get(key string) interface{}
}

// Constructs a new cache
func New(getter func(string) Cacheable) Cache {
	return &lazycache{getter, make(map[string]*Cacheable), make(map[string]*sync.Once), new(sync.RWMutex), new(sync.RWMutex)}
}

// Type lazycache implements the Cache interface
type lazycache struct {
	// The fetcher of items
	Getter func(string) Cacheable

	// The cache of items
	Cache map[string]*Cacheable

	// Controls read of items from the getter; prevents multiple concurrent reads of the same item.
	ReadControls map[string]*sync.Once

	// Locks the entire cache for reads or writes
	CacheLock *sync.RWMutex

	// Locks the read control manifest for reads or writes
	ReadControlsLock *sync.RWMutex
}

// Get an item from the cache, retrieving the item from the getter if necessary.
// This implemention is meant to be goroutine safe.  It assumes that updating a
// map while concurrently reading from it is unsafe, so it uses a read/write mutex
// to synchronize access to its internal maps.
func (c *lazycache) Get(key string) interface{} {
	cachedValue, ok := getFromCache(c, key)
	if ok {
		return cachedValue.Value
	}

	readControl, cachedValue, ok := getReadControl(c, key)
	if ok {
		return cachedValue.Value
	}

	cachedValue = doFetch(c, key, readControl)
	return cachedValue.Value
}

// Attempt to retrieve an item from the cache, if it exists and hasn't expired.
// Returns somevalue, true if exists or nil, false if it does not.
func getFromCache(c *lazycache, key string) (*Cacheable, bool) {
	c.CacheLock.RLock()
	cachedValue, ok := c.Cache[key]
	c.CacheLock.RUnlock()
	if ok {
		now := time.Now()
		if cachedValue.ExpiresAt.After(now) {
			return cachedValue, true
		}
		c.CacheLock.Lock()
		// Determine if another goroutine has updated the cache before the lock
		cachedValue, ok = c.Cache[key]
		if ok && cachedValue.ExpiresAt.After(now) {
			c.CacheLock.Unlock()
			return cachedValue, true
		}
		delete(c.Cache, key)
		c.CacheLock.Unlock()
	}
	return nil, false
}

// Get a Once for controlling the read-through on a particular cached item.
// Performs a last-minute check to determine if another goroutine has populated
// the cache before a lock is acquired, so this function may return a cached
// value instead.  If so, the third return value will be true.  Otherwise, a
// read control is returned and the third value is false.
func getReadControl(c *lazycache, key string) (readControl *sync.Once, cachedItem *Cacheable, gotCachedItem bool) {
	gotCachedItem = false

	c.ReadControlsLock.RLock()
	readControl, ok := c.ReadControls[key]
	c.ReadControlsLock.RUnlock()
	if !ok {
		c.ReadControlsLock.Lock()

		// Another goroutine may have created a read control, fetched an item, updated the
		// cache and cleaned up its read control by the time we reach this point.
		// Therefore, we verify that the cache still does not contain anything for the
		// given key.
		// Warning: possibility of deadlock when dealing with multiple locks.  Make sure
		//          they are always acquired in the same order.
		c.CacheLock.RLock()
		cachedItem, ok = c.Cache[key]
		c.CacheLock.RUnlock()

		if ok {
			c.ReadControlsLock.Unlock()
			gotCachedItem = true
			return
		}

		readControl, ok = c.ReadControls[key]
		if !ok {
			readControl = new(sync.Once)
			c.ReadControls[key] = readControl
		}
		c.ReadControlsLock.Unlock()
	}

	return
}

// Use the given read control to fetch a value and store it in the cache.
// The read control may prevent this goroutine from fetching the value if
// some other routine gets to it first.  In either case, the resulting
// fetched value is returned.
func doFetch(c *lazycache, key string, readControl *sync.Once) (cachedValue *Cacheable) {
	fetchedValue := false
	readControl.Do(func() {
		defer func() {
			c.ReadControlsLock.Lock()
			delete(c.ReadControls, key)
			c.ReadControlsLock.Unlock()
		}()

		valueFromGetter := c.Getter(key)
		cachedValue = &valueFromGetter
		fetchedValue = true
		c.CacheLock.Lock()
		c.Cache[key] = cachedValue
		c.CacheLock.Unlock()
	})

	if !fetchedValue {
		c.CacheLock.RLock()
		cachedValue, _ = c.Cache[key]
		c.CacheLock.RUnlock()
	}

	return
}
<|endoftext|>


<commit_before>package obj

import (
	. "github.com/polydawn/go-xlate/tok"
)

/*
	Returns a `TokenSource` that will walk over structures in memory,
	emitting tokens representing values and fields as it visits them.
*/
func NewMarshaler(s *Suite, v interface{}) *MarshalDriver {
	d := &MarshalDriver{
		suite: s,
		stack: make([]MarshalMachine, 0, 10),
		step:  s.pickMarshalMachine(v),
	}
	d.step.Reset(s, v)
	return d
}

type MarshalDriver struct {
	suite *Suite
	stack []MarshalMachine
	step  MarshalMachine
}

type MarshalMachine interface {
	Reset(*Suite, interface{}) error
	Step(*MarshalDriver, *Suite, *Token) (done bool, err error)
}

// for convenience in declaring fields of state machines with internal step funcs
type marshalMachineStep func(*MarshalDriver, *Token) (done bool, err error)

func (d *MarshalDriver) Step(tok *Token) (bool, error) {
	//	fmt.Printf("> next step is %#v\n", d.step)
	done, err := d.step.Step(d, d.suite, tok)
	//	fmt.Printf(">> yield is %#v\n", TokenToString(*tok))
	// If the step errored: out, entirely.
	if err != nil {
		return true, err
	}
	// If the step wasn't done, return same status.
	if !done {
		return false, nil
	}
	// If it WAS done, pop next, or if stack empty, we're entirely done.
	nSteps := len(d.stack) - 1
	if nSteps == -1 {
		return true, nil // that's all folks
	}
	//	fmt.Printf(">> popping up from %#v\n", d.stack)
	d.step = d.stack[nSteps]
	d.stack = d.stack[0:nSteps]
	return false, nil
}

/*
	Traverses `target`,
	first looking up the machine for that type just like it's a new top-level object,
	then pushing the first step with `tok` (the upstream tends to have peeked at it
	in order to decide what to do, but if recursing, it belongs to the next obj),
	then saving this new machine: the driver will then continuing stepping the
	new machine it returns a done status, at which point we'll finally
	"return" by popping back to the last machine on the stack.

	In other words, your MarshalMachine calls this when it wants to deal
	with an object, and by the time we call back to your machine again,
	that object will be traversed and the stream ready for you to continue.
*/
func (d *MarshalDriver) Recurse(tok *Token, target interface{}, nextMach MarshalMachine) error {
	//	fmt.Printf(">>> pushing into recursion with %#v\n", nextMach)
	// Push the current machine onto the stack (we'll resume it when the new one is done),
	d.stack = append(d.stack, d.step)
	// Initialize the machine for this new target value.
	nextMach.Reset(d.suite, target)
	d.step = nextMach
	// Immediately make a step (we're still the delegate in charge of someone else's step).
	_, err := d.Step(tok)
	return err
}
<commit_msg>Check error during marshal Recurse.<commit_after>package obj

import (
	. "github.com/polydawn/go-xlate/tok"
)

/*
	Returns a `TokenSource` that will walk over structures in memory,
	emitting tokens representing values and fields as it visits them.
*/
func NewMarshaler(s *Suite, v interface{}) *MarshalDriver {
	d := &MarshalDriver{
		suite: s,
		stack: make([]MarshalMachine, 0, 10),
		step:  s.pickMarshalMachine(v),
	}
	d.step.Reset(s, v)
	return d
}

type MarshalDriver struct {
	suite *Suite
	stack []MarshalMachine
	step  MarshalMachine
}

type MarshalMachine interface {
	Reset(*Suite, interface{}) error
	Step(*MarshalDriver, *Suite, *Token) (done bool, err error)
}

// for convenience in declaring fields of state machines with internal step funcs
type marshalMachineStep func(*MarshalDriver, *Token) (done bool, err error)

func (d *MarshalDriver) Step(tok *Token) (bool, error) {
	//	fmt.Printf("> next step is %#v\n", d.step)
	done, err := d.step.Step(d, d.suite, tok)
	//	fmt.Printf(">> yield is %#v\n", TokenToString(*tok))
	// If the step errored: out, entirely.
	if err != nil {
		return true, err
	}
	// If the step wasn't done, return same status.
	if !done {
		return false, nil
	}
	// If it WAS done, pop next, or if stack empty, we're entirely done.
	nSteps := len(d.stack) - 1
	if nSteps == -1 {
		return true, nil // that's all folks
	}
	//	fmt.Printf(">> popping up from %#v\n", d.stack)
	d.step = d.stack[nSteps]
	d.stack = d.stack[0:nSteps]
	return false, nil
}

/*
	Traverses `target`,
	first looking up the machine for that type just like it's a new top-level object,
	then pushing the first step with `tok` (the upstream tends to have peeked at it
	in order to decide what to do, but if recursing, it belongs to the next obj),
	then saving this new machine: the driver will then continuing stepping the
	new machine it returns a done status, at which point we'll finally
	"return" by popping back to the last machine on the stack.

	In other words, your MarshalMachine calls this when it wants to deal
	with an object, and by the time we call back to your machine again,
	that object will be traversed and the stream ready for you to continue.
*/
func (d *MarshalDriver) Recurse(tok *Token, target interface{}, nextMach MarshalMachine) (err error) {
	//	fmt.Printf(">>> pushing into recursion with %#v\n", nextMach)
	// Push the current machine onto the stack (we'll resume it when the new one is done),
	d.stack = append(d.stack, d.step)
	// Initialize the machine for this new target value.
	err = nextMach.Reset(d.suite, target)
	if err != nil {
		return
	}
	d.step = nextMach
	// Immediately make a step (we're still the delegate in charge of someone else's step).
	_, err = d.Step(tok)
	return
}
<|endoftext|>


<commit_before>package eval

import (
	"encoding/json"
	"errors"
	"os"
	"testing"
	"time"

	"github.com/gansoi/gansoi/boltdb"
	"github.com/gansoi/gansoi/checks"
	"github.com/gansoi/gansoi/database"
	"github.com/gansoi/gansoi/plugins"
)

type (
	mockAgent struct {
		ReturnError bool `json:"return_error"`
	}

	peerStore struct {
		peers []string
	}
)

var (
	check = checks.Check{
		ID:        "test",
		AgentID:   "mock",
		Interval:  time.Second,
		Arguments: json.RawMessage("{}"),
	}
)

func (m *mockAgent) Check(result plugins.AgentResult) error {
	if m.ReturnError {
		return errors.New("error")
	}

	result.AddValue("ran", true)

	return nil
}

func init() {
	plugins.RegisterAgent("mock", mockAgent{})
}

func (p *peerStore) SetPeers(peers []string) error {
	p.peers = peers

	return nil
}

func (p *peerStore) Peers() ([]string, error) {
	return p.peers, nil
}

func newE(t *testing.T, nodes []string) (*boltdb.BoltStore, *Evaluator) {
	path := "/dev/shm/mockdb"
	peers := &peerStore{}
	peers.SetPeers(nodes)

	db, err := boltdb.NewBoltStore(path)
	if err != nil {
		t.Fatalf("Failed to create mock database: %s", err.Error())
	}

	err = os.Remove(path)
	if err != nil {
		t.Fatalf("os.remove() failed: %s", err.Error())
	}

	e := NewEvaluator(db, peers)
	if e == nil {
		t.Fatalf("NewEvaluator() returned nil")
	}

	return db, e
}

func TestEvaluatorEvaluate1Basics(t *testing.T) {
	db, e := newE(t, []string{"justone"})
	defer db.Close()

	result := &checks.CheckResult{
		TimeStamp: time.Now(),
		CheckID:   "test",
		Node:      "justone",
	}

	_, err := e.evaluate1(result)
	if err != nil {
		t.Fatalf("evaluate1() failed: %s", err.Error())
	}

	pe := []PartialEvaluation{}
	err = db.All(&pe, -1, 0, false)
	if err != nil {
		t.Fatalf("db.All() failed: %s", err.Error())
	}

	if len(pe) != 1 {
		t.Fatalf("Got wrong number of evaluations, got %d (%v)", len(pe), pe)
	}

	// Move one minute into the future.
	result.TimeStamp = result.TimeStamp.Add(time.Minute)

	_, err = e.evaluate1(result)
	if err != nil {
		t.Fatalf("evaluate1() failed: %s", err.Error())
	}

	err = db.All(&pe, -1, 0, false)
	if err != nil {
		t.Fatalf("db.All() failed: %s", err.Error())
	}

	if len(pe) != 1 {
		t.Fatalf("Got wrong number of evaluations.")
	}
}

func TestEvaluatorEvaluate1(t *testing.T) {
	db, e := newE(t, []string{"justone"})
	defer db.Close()

	cases := []struct {
		in    checks.CheckResult
		state State
	}{
		{checks.CheckResult{}, StateUp},
		{checks.CheckResult{}, StateUp},
		{checks.CheckResult{Error: "error"}, StateDown},
		{checks.CheckResult{Error: "error"}, StateDown},
		{checks.CheckResult{Error: "error"}, StateDown},
		{checks.CheckResult{}, StateUp},
		{checks.CheckResult{Error: "error"}, StateDown},
		{checks.CheckResult{}, StateUp},
	}

	for _, c := range cases {
		e, _ := e.evaluate1(&c.in)
		if e.State != c.state {
			t.Fatalf("evaluate1() concluded wrong state. Got %s, expected %s", e.State.ColorString(), c.state.ColorString())
		}
	}
}

func TestEvaluatorEvaluate2(t *testing.T) {
	db, e := newE(t, []string{"justone"})
	defer db.Close()

	err := db.Save(&check)
	if err != nil {
		t.Fatalf("db.Save() failed: %s", err.Error())
	}

	cases := []struct {
		in    checks.CheckResult
		state State
	}{
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
	}

	for i, c := range cases {
		pe, _ := e.evaluate1(&c.in)
		eval, err := e.evaluate2(pe)
		if err != nil {
			t.Fatalf("evaluate2() returned error: %s", err.Error())
		}

		evalState := eval.History[0]
		if evalState != c.state {
			t.Fatalf("evaluate2(%d) concluded wrong state. Got %s, expected %s", i, evalState.ColorString(), c.state.ColorString())
		}
	}
}

func TestEvaluatorEvaluate2Cluster(t *testing.T) {
	db, e := newE(t, []string{"one", "two"})
	defer db.Close()

	err := db.Save(&check)
	if err != nil {
		t.Fatalf("db.Save() failed: %s", err.Error())
	}

	cases := []struct {
		in    checks.CheckResult
		state State
	}{
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now(), Error: "error"}, StateDegraded},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateDegraded},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now(), Error: "error"}, StateDegraded},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUp},

		// Old results should result in unknown. Something is wrong.
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now().Add(-time.Hour)}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now().Add(-time.Hour)}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now()}, StateUp},
	}

	for i, c := range cases {
		pe, _ := e.evaluate1(&c.in)
		eval, err := e.evaluate2(pe)
		if err != nil {
			t.Fatalf("evaluate2() returned error: %s", err.Error())
		}

		evalState := eval.History[0]
		if evalState != c.state {
			t.Fatalf("evaluate2(%d) concluded wrong state. Got %s, expected %s", i, evalState.ColorString(), c.state.ColorString())
		}
	}
}

var _ database.Listener = (*Evaluator)(nil)
<commit_msg>Fix tests.<commit_after>package eval

import (
	"encoding/json"
	"errors"
	"os"
	"testing"
	"time"

	"github.com/gansoi/gansoi/boltdb"
	"github.com/gansoi/gansoi/checks"
	"github.com/gansoi/gansoi/database"
	"github.com/gansoi/gansoi/plugins"
)

type (
	mockAgent struct {
		ReturnError bool `json:"return_error"`
	}

	peerStore struct {
		peers []string
	}
)

var (
	check = checks.Check{
		AgentID:   "mock",
		Interval:  time.Second,
		Arguments: json.RawMessage("{}"),
	}
)

func (m *mockAgent) Check(result plugins.AgentResult) error {
	if m.ReturnError {
		return errors.New("error")
	}

	result.AddValue("ran", true)

	return nil
}

func init() {
	plugins.RegisterAgent("mock", mockAgent{})

	check.ID = "test"
}

func (p *peerStore) SetPeers(peers []string) error {
	p.peers = peers

	return nil
}

func (p *peerStore) Peers() ([]string, error) {
	return p.peers, nil
}

func newE(t *testing.T, nodes []string) (*boltdb.BoltStore, *Evaluator) {
	path := "/dev/shm/mockdb"
	peers := &peerStore{}
	peers.SetPeers(nodes)

	db, err := boltdb.NewBoltStore(path)
	if err != nil {
		t.Fatalf("Failed to create mock database: %s", err.Error())
	}

	err = os.Remove(path)
	if err != nil {
		t.Fatalf("os.remove() failed: %s", err.Error())
	}

	e := NewEvaluator(db, peers)
	if e == nil {
		t.Fatalf("NewEvaluator() returned nil")
	}

	return db, e
}

func TestEvaluatorEvaluate1Basics(t *testing.T) {
	db, e := newE(t, []string{"justone"})
	defer db.Close()

	result := &checks.CheckResult{
		TimeStamp: time.Now(),
		CheckID:   "test",
		Node:      "justone",
	}

	_, err := e.evaluate1(result)
	if err != nil {
		t.Fatalf("evaluate1() failed: %s", err.Error())
	}

	pe := []PartialEvaluation{}
	err = db.All(&pe, -1, 0, false)
	if err != nil {
		t.Fatalf("db.All() failed: %s", err.Error())
	}

	if len(pe) != 1 {
		t.Fatalf("Got wrong number of evaluations, got %d (%v)", len(pe), pe)
	}

	// Move one minute into the future.
	result.TimeStamp = result.TimeStamp.Add(time.Minute)

	_, err = e.evaluate1(result)
	if err != nil {
		t.Fatalf("evaluate1() failed: %s", err.Error())
	}

	err = db.All(&pe, -1, 0, false)
	if err != nil {
		t.Fatalf("db.All() failed: %s", err.Error())
	}

	if len(pe) != 1 {
		t.Fatalf("Got wrong number of evaluations.")
	}
}

func TestEvaluatorEvaluate1(t *testing.T) {
	db, e := newE(t, []string{"justone"})
	defer db.Close()

	cases := []struct {
		in    checks.CheckResult
		state State
	}{
		{checks.CheckResult{}, StateUp},
		{checks.CheckResult{}, StateUp},
		{checks.CheckResult{Error: "error"}, StateDown},
		{checks.CheckResult{Error: "error"}, StateDown},
		{checks.CheckResult{Error: "error"}, StateDown},
		{checks.CheckResult{}, StateUp},
		{checks.CheckResult{Error: "error"}, StateDown},
		{checks.CheckResult{}, StateUp},
	}

	for _, c := range cases {
		e, _ := e.evaluate1(&c.in)
		if e.State != c.state {
			t.Fatalf("evaluate1() concluded wrong state. Got %s, expected %s", e.State.ColorString(), c.state.ColorString())
		}
	}
}

func TestEvaluatorEvaluate2(t *testing.T) {
	db, e := newE(t, []string{"justone"})
	defer db.Close()

	err := db.Save(&check)
	if err != nil {
		t.Fatalf("db.Save() failed: %s", err.Error())
	}

	cases := []struct {
		in    checks.CheckResult
		state State
	}{
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "justone", TimeStamp: time.Now()}, StateUp},
	}

	for i, c := range cases {
		pe, _ := e.evaluate1(&c.in)
		eval, err := e.evaluate2(pe)
		if err != nil {
			t.Fatalf("evaluate2() returned error: %s", err.Error())
		}

		evalState := eval.History[0]
		if evalState != c.state {
			t.Fatalf("evaluate2(%d) concluded wrong state. Got %s, expected %s", i, evalState.ColorString(), c.state.ColorString())
		}
	}
}

func TestEvaluatorEvaluate2Cluster(t *testing.T) {
	db, e := newE(t, []string{"one", "two"})
	defer db.Close()

	err := db.Save(&check)
	if err != nil {
		t.Fatalf("db.Save() failed: %s", err.Error())
	}

	cases := []struct {
		in    checks.CheckResult
		state State
	}{
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now(), Error: "error"}, StateDegraded},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now(), Error: "error"}, StateDown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateDegraded},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now(), Error: "error"}, StateDegraded},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUp},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUp},

		// Old results should result in unknown. Something is wrong.
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now().Add(-time.Hour)}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now().Add(-time.Hour)}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "one", TimeStamp: time.Now()}, StateUnknown},
		{checks.CheckResult{CheckID: "test", Node: "two", TimeStamp: time.Now()}, StateUp},
	}

	for i, c := range cases {
		pe, _ := e.evaluate1(&c.in)
		eval, err := e.evaluate2(pe)
		if err != nil {
			t.Fatalf("evaluate2() returned error: %s", err.Error())
		}

		evalState := eval.History[0]
		if evalState != c.state {
			t.Fatalf("evaluate2(%d) concluded wrong state. Got %s, expected %s", i, evalState.ColorString(), c.state.ColorString())
		}
	}
}

var _ database.Listener = (*Evaluator)(nil)
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccAwsSsmParameterDataSource_basic(t *testing.T) {
	name := "test.parameter"

	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAwsSsmParameterDataSourceConfig(name, "false"),
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestCheckResourceAttrSet("data.aws_ssm_parameter.test", "arn"),
					resource.TestCheckResourceAttr("data.aws_ssm_parameter.test", "name", name),
					resource.TestCheckResourceAttr("data.aws_ssm_parameter.test", "type", "String"),
					resource.TestCheckResourceAttr("data.aws_ssm_parameter.test", "value", "TestValue"),
					resource.TestCheckResourceAttr("data.aws_ssm_parameter.test", "with_decryption", "false"),
				),
			},
			{
				Config: testAccCheckAwsSsmParameterDataSourceConfig(name, "true"),
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestCheckResourceAttrSet("data.aws_ssm_parameter.test", "arn"),
					resource.TestCheckResourceAttr("data.aws_ssm_parameter.test", "name", name),
					resource.TestCheckResourceAttr("data.aws_ssm_parameter.test", "type", "String"),
					resource.TestCheckResourceAttr("data.aws_ssm_parameter.test", "value", "TestValue"),
					resource.TestCheckResourceAttr("data.aws_ssm_parameter.test", "with_decryption", "true"),
				),
			},
		},
	})
}

func testAccCheckAwsSsmParameterDataSourceConfig(name string, withDecryption string) string {
	return fmt.Sprintf(`
resource "aws_ssm_parameter" "test" {
	name = "%s"
	type = "String"
	value = "TestValue"
}

data "aws_ssm_parameter" "test" {
	name = "${aws_ssm_parameter.test.name}"
	with_decryption = %s
}
`, name, withDecryption)
}
<commit_msg>test/data-source/aws_ssm_parameter: Add full path parameter name test for ARN checking<commit_after>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccAwsSsmParameterDataSource_basic(t *testing.T) {
	resourceName := "data.aws_ssm_parameter.test"
	name := "test.parameter"

	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAwsSsmParameterDataSourceConfig(name, "false"),
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestMatchResourceAttr(resourceName, "arn",
						regexp.MustCompile(fmt.Sprintf("^arn:aws:ssm:[a-z0-9-]+:[0-9]{12}:parameter/%s$", name))),
					resource.TestCheckResourceAttr(resourceName, "name", name),
					resource.TestCheckResourceAttr(resourceName, "type", "String"),
					resource.TestCheckResourceAttr(resourceName, "value", "TestValue"),
					resource.TestCheckResourceAttr(resourceName, "with_decryption", "false"),
				),
			},
			{
				Config: testAccCheckAwsSsmParameterDataSourceConfig(name, "true"),
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestMatchResourceAttr(resourceName, "arn",
						regexp.MustCompile(fmt.Sprintf("^arn:aws:ssm:[a-z0-9-]+:[0-9]{12}:parameter/%s$", name))),
					resource.TestCheckResourceAttr(resourceName, "name", name),
					resource.TestCheckResourceAttr(resourceName, "type", "String"),
					resource.TestCheckResourceAttr(resourceName, "value", "TestValue"),
					resource.TestCheckResourceAttr(resourceName, "with_decryption", "true"),
				),
			},
		},
	})
}

func TestAccAwsSsmParameterDataSource_fullPath(t *testing.T) {
	resourceName := "data.aws_ssm_parameter.test"
	name := "/path/parameter"

	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAwsSsmParameterDataSourceConfig(name, "false"),
				Check: resource.ComposeAggregateTestCheckFunc(
					resource.TestMatchResourceAttr(resourceName, "arn",
						regexp.MustCompile(fmt.Sprintf("^arn:aws:ssm:[a-z0-9-]+:[0-9]{12}:parameter%s$", name))),
					resource.TestCheckResourceAttr(resourceName, "name", name),
					resource.TestCheckResourceAttr(resourceName, "type", "String"),
					resource.TestCheckResourceAttr(resourceName, "value", "TestValue"),
					resource.TestCheckResourceAttr(resourceName, "with_decryption", "false"),
				),
			},
		},
	})
}

func testAccCheckAwsSsmParameterDataSourceConfig(name string, withDecryption string) string {
	return fmt.Sprintf(`
resource "aws_ssm_parameter" "test" {
	name = "%s"
	type = "String"
	value = "TestValue"
}

data "aws_ssm_parameter" "test" {
	name = "${aws_ssm_parameter.test.name}"
	with_decryption = %s
}
`, name, withDecryption)
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package grpclb

import (
	"context"
	"fmt"
	"io"
	"net"
	"reflect"
	"time"

	timestamppb "github.com/golang/protobuf/ptypes/timestamp"
	"google.golang.org/grpc"
	"google.golang.org/grpc/balancer"
	lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/internal"
	"google.golang.org/grpc/internal/channelz"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/resolver"
)

// processServerList updates balaner's internal state, create/remove SubConns
// and regenerates picker using the received serverList.
func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
	grpclog.Infof("lbBalancer: processing server list: %+v", l)
	lb.mu.Lock()
	defer lb.mu.Unlock()

	// Set serverListReceived to true so fallback will not take effect if it has
	// not hit timeout.
	lb.serverListReceived = true

	// If the new server list == old server list, do nothing.
	if reflect.DeepEqual(lb.fullServerList, l.Servers) {
		grpclog.Infof("lbBalancer: new serverlist same as the previous one, ignoring")
		return
	}
	lb.fullServerList = l.Servers

	var backendAddrs []resolver.Address
	for i, s := range l.Servers {
		if s.Drop {
			continue
		}

		md := metadata.Pairs(lbTokeyKey, s.LoadBalanceToken)
		ip := net.IP(s.IpAddress)
		ipStr := ip.String()
		if ip.To4() == nil {
			// Add square brackets to ipv6 addresses, otherwise net.Dial() and
			// net.SplitHostPort() will return too many colons error.
			ipStr = fmt.Sprintf("[%s]", ipStr)
		}
		addr := resolver.Address{
			Addr:     fmt.Sprintf("%s:%d", ipStr, s.Port),
			Metadata: &md,
		}
		grpclog.Infof("lbBalancer: server list entry[%d]: ipStr:|%s|, port:|%d|, load balancer token:|%v|",
			i, ipStr, s.Port, s.LoadBalanceToken)
		backendAddrs = append(backendAddrs, addr)
	}

	// Call refreshSubConns to create/remove SubConns.
	lb.refreshSubConns(backendAddrs, true)
	// Regenerate and update picker no matter if there's update on backends (if
	// any SubConn will be newed/removed). Because since the full serverList was
	// different, there might be updates in drops or pick weights(different
	// number of duplicates). We need to update picker with the fulllist.
	//
	// Now with cache, even if SubConn was newed/removed, there might be no
	// state changes.
	lb.regeneratePicker()
	lb.cc.UpdateBalancerState(lb.state, lb.picker)
}

// refreshSubConns creates/removes SubConns with backendAddrs. It returns a bool
// indicating whether the backendAddrs are different from the cached
// backendAddrs (whether any SubConn was newed/removed).
// Caller must hold lb.mu.
func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fromGRPCLBServer bool) bool {
	opts := balancer.NewSubConnOptions{}
	if fromGRPCLBServer {
		opts.CredsBundle = lb.grpclbBackendCreds
	}

	lb.backendAddrs = nil
	var backendsUpdated bool
	// addrsSet is the set converted from backendAddrs, it's used to quick
	// lookup for an address.
	addrsSet := make(map[resolver.Address]struct{})
	// Create new SubConns.
	for _, addr := range backendAddrs {
		addrWithoutMD := addr
		addrWithoutMD.Metadata = nil
		addrsSet[addrWithoutMD] = struct{}{}
		lb.backendAddrs = append(lb.backendAddrs, addrWithoutMD)

		if _, ok := lb.subConns[addrWithoutMD]; !ok {
			backendsUpdated = true

			// Use addrWithMD to create the SubConn.
			sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, opts)
			if err != nil {
				grpclog.Warningf("roundrobinBalancer: failed to create new SubConn: %v", err)
				continue
			}
			lb.subConns[addrWithoutMD] = sc // Use the addr without MD as key for the map.
			if _, ok := lb.scStates[sc]; !ok {
				// Only set state of new sc to IDLE. The state could already be
				// READY for cached SubConns.
				lb.scStates[sc] = connectivity.Idle
			}
			sc.Connect()
		}
	}

	for a, sc := range lb.subConns {
		// a was removed by resolver.
		if _, ok := addrsSet[a]; !ok {
			backendsUpdated = true

			lb.cc.RemoveSubConn(sc)
			delete(lb.subConns, a)
			// Keep the state of this sc in b.scStates until sc's state becomes Shutdown.
			// The entry will be deleted in HandleSubConnStateChange.
		}
	}

	return backendsUpdated
}

func (lb *lbBalancer) readServerList(s *balanceLoadClientStream) error {
	for {
		reply, err := s.Recv()
		if err != nil {
			if err == io.EOF {
				return errServerTerminatedConnection
			}
			return fmt.Errorf("grpclb: failed to recv server list: %v", err)
		}
		if serverList := reply.GetServerList(); serverList != nil {
			lb.processServerList(serverList)
		}
	}
}

func (lb *lbBalancer) sendLoadReport(s *balanceLoadClientStream, interval time.Duration) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
		case <-s.Context().Done():
			return
		}
		stats := lb.clientStats.toClientStats()
		t := time.Now()
		stats.Timestamp = &timestamppb.Timestamp{
			Seconds: t.Unix(),
			Nanos:   int32(t.Nanosecond()),
		}
		if err := s.Send(&lbpb.LoadBalanceRequest{
			LoadBalanceRequestType: &lbpb.LoadBalanceRequest_ClientStats{
				ClientStats: stats,
			},
		}); err != nil {
			return
		}
	}
}

func (lb *lbBalancer) callRemoteBalancer() (backoff bool, _ error) {
	lbClient := &loadBalancerClient{cc: lb.ccRemoteLB}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	stream, err := lbClient.BalanceLoad(ctx, grpc.FailFast(false))
	if err != nil {
		return true, fmt.Errorf("grpclb: failed to perform RPC to the remote balancer %v", err)
	}

	// grpclb handshake on the stream.
	initReq := &lbpb.LoadBalanceRequest{
		LoadBalanceRequestType: &lbpb.LoadBalanceRequest_InitialRequest{
			InitialRequest: &lbpb.InitialLoadBalanceRequest{
				Name: lb.target,
			},
		},
	}
	if err := stream.Send(initReq); err != nil {
		return true, fmt.Errorf("grpclb: failed to send init request: %v", err)
	}
	reply, err := stream.Recv()
	if err != nil {
		return true, fmt.Errorf("grpclb: failed to recv init response: %v", err)
	}
	initResp := reply.GetInitialResponse()
	if initResp == nil {
		return true, fmt.Errorf("grpclb: reply from remote balancer did not include initial response")
	}
	if initResp.LoadBalancerDelegate != "" {
		return true, fmt.Errorf("grpclb: Delegation is not supported")
	}

	go func() {
		if d := convertDuration(initResp.ClientStatsReportInterval); d > 0 {
			lb.sendLoadReport(stream, d)
		}
	}()
	// No backoff if init req/resp handshake was successful.
	return false, lb.readServerList(stream)
}

func (lb *lbBalancer) watchRemoteBalancer() {
	var retryCount int
	for {
		doBackoff, err := lb.callRemoteBalancer()
		select {
		case <-lb.doneCh:
			return
		default:
			if err != nil {
				if err == errServerTerminatedConnection {
					grpclog.Info(err)
				} else {
					grpclog.Warning(err)
				}
			}
		}

		if !doBackoff {
			retryCount = 0
			continue
		}

		timer := time.NewTimer(lb.backoff.Backoff(retryCount))
		select {
		case <-timer.C:
		case <-lb.doneCh:
			timer.Stop()
			return
		}
		retryCount++
	}
}

func (lb *lbBalancer) dialRemoteLB(remoteLBName string) {
	var dopts []grpc.DialOption
	if creds := lb.opt.DialCreds; creds != nil {
		if err := creds.OverrideServerName(remoteLBName); err == nil {
			dopts = append(dopts, grpc.WithTransportCredentials(creds))
		} else {
			grpclog.Warningf("grpclb: failed to override the server name in the credentials: %v, using Insecure", err)
			dopts = append(dopts, grpc.WithInsecure())
		}
	} else if bundle := lb.grpclbClientConnCreds; bundle != nil {
		dopts = append(dopts, grpc.WithCredentialsBundle(bundle))
	} else {
		dopts = append(dopts, grpc.WithInsecure())
	}
	if lb.opt.Dialer != nil {
		// WithDialer takes a different type of function, so we instead use a
		// special DialOption here.
		wcd := internal.WithContextDialer.(func(func(context.Context, string) (net.Conn, error)) grpc.DialOption)
		dopts = append(dopts, wcd(lb.opt.Dialer))
	}
	// Explicitly set pickfirst as the balancer.
	dopts = append(dopts, grpc.WithBalancerName(grpc.PickFirstBalancerName))
	wrb := internal.WithResolverBuilder.(func(resolver.Builder) grpc.DialOption)
	dopts = append(dopts, wrb(lb.manualResolver))
	if channelz.IsOn() {
		dopts = append(dopts, grpc.WithChannelzParentID(lb.opt.ChannelzParentID))
	}

	// DialContext using manualResolver.Scheme, which is a random scheme
	// generated when init grpclb. The target scheme here is not important.
	//
	// The grpc dial target will be used by the creds (ALTS) as the authority,
	// so it has to be set to remoteLBName that comes from resolver.
	cc, err := grpc.DialContext(context.Background(), remoteLBName, dopts...)
	if err != nil {
		grpclog.Fatalf("failed to dial: %v", err)
	}
	lb.ccRemoteLB = cc
	go lb.watchRemoteBalancer()
}
<commit_msg>grpclb: re-resolve when the stream to GRPCLB balancer fails (#2502)<commit_after>/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package grpclb

import (
	"context"
	"fmt"
	"io"
	"net"
	"reflect"
	"time"

	timestamppb "github.com/golang/protobuf/ptypes/timestamp"
	"google.golang.org/grpc"
	"google.golang.org/grpc/balancer"
	lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/internal"
	"google.golang.org/grpc/internal/channelz"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/resolver"
)

// processServerList updates balaner's internal state, create/remove SubConns
// and regenerates picker using the received serverList.
func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
	grpclog.Infof("lbBalancer: processing server list: %+v", l)
	lb.mu.Lock()
	defer lb.mu.Unlock()

	// Set serverListReceived to true so fallback will not take effect if it has
	// not hit timeout.
	lb.serverListReceived = true

	// If the new server list == old server list, do nothing.
	if reflect.DeepEqual(lb.fullServerList, l.Servers) {
		grpclog.Infof("lbBalancer: new serverlist same as the previous one, ignoring")
		return
	}
	lb.fullServerList = l.Servers

	var backendAddrs []resolver.Address
	for i, s := range l.Servers {
		if s.Drop {
			continue
		}

		md := metadata.Pairs(lbTokeyKey, s.LoadBalanceToken)
		ip := net.IP(s.IpAddress)
		ipStr := ip.String()
		if ip.To4() == nil {
			// Add square brackets to ipv6 addresses, otherwise net.Dial() and
			// net.SplitHostPort() will return too many colons error.
			ipStr = fmt.Sprintf("[%s]", ipStr)
		}
		addr := resolver.Address{
			Addr:     fmt.Sprintf("%s:%d", ipStr, s.Port),
			Metadata: &md,
		}
		grpclog.Infof("lbBalancer: server list entry[%d]: ipStr:|%s|, port:|%d|, load balancer token:|%v|",
			i, ipStr, s.Port, s.LoadBalanceToken)
		backendAddrs = append(backendAddrs, addr)
	}

	// Call refreshSubConns to create/remove SubConns.
	lb.refreshSubConns(backendAddrs, true)
	// Regenerate and update picker no matter if there's update on backends (if
	// any SubConn will be newed/removed). Because since the full serverList was
	// different, there might be updates in drops or pick weights(different
	// number of duplicates). We need to update picker with the fulllist.
	//
	// Now with cache, even if SubConn was newed/removed, there might be no
	// state changes.
	lb.regeneratePicker()
	lb.cc.UpdateBalancerState(lb.state, lb.picker)
}

// refreshSubConns creates/removes SubConns with backendAddrs. It returns a bool
// indicating whether the backendAddrs are different from the cached
// backendAddrs (whether any SubConn was newed/removed).
// Caller must hold lb.mu.
func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fromGRPCLBServer bool) bool {
	opts := balancer.NewSubConnOptions{}
	if fromGRPCLBServer {
		opts.CredsBundle = lb.grpclbBackendCreds
	}

	lb.backendAddrs = nil
	var backendsUpdated bool
	// addrsSet is the set converted from backendAddrs, it's used to quick
	// lookup for an address.
	addrsSet := make(map[resolver.Address]struct{})
	// Create new SubConns.
	for _, addr := range backendAddrs {
		addrWithoutMD := addr
		addrWithoutMD.Metadata = nil
		addrsSet[addrWithoutMD] = struct{}{}
		lb.backendAddrs = append(lb.backendAddrs, addrWithoutMD)

		if _, ok := lb.subConns[addrWithoutMD]; !ok {
			backendsUpdated = true

			// Use addrWithMD to create the SubConn.
			sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, opts)
			if err != nil {
				grpclog.Warningf("roundrobinBalancer: failed to create new SubConn: %v", err)
				continue
			}
			lb.subConns[addrWithoutMD] = sc // Use the addr without MD as key for the map.
			if _, ok := lb.scStates[sc]; !ok {
				// Only set state of new sc to IDLE. The state could already be
				// READY for cached SubConns.
				lb.scStates[sc] = connectivity.Idle
			}
			sc.Connect()
		}
	}

	for a, sc := range lb.subConns {
		// a was removed by resolver.
		if _, ok := addrsSet[a]; !ok {
			backendsUpdated = true

			lb.cc.RemoveSubConn(sc)
			delete(lb.subConns, a)
			// Keep the state of this sc in b.scStates until sc's state becomes Shutdown.
			// The entry will be deleted in HandleSubConnStateChange.
		}
	}

	return backendsUpdated
}

func (lb *lbBalancer) readServerList(s *balanceLoadClientStream) error {
	for {
		reply, err := s.Recv()
		if err != nil {
			if err == io.EOF {
				return errServerTerminatedConnection
			}
			return fmt.Errorf("grpclb: failed to recv server list: %v", err)
		}
		if serverList := reply.GetServerList(); serverList != nil {
			lb.processServerList(serverList)
		}
	}
}

func (lb *lbBalancer) sendLoadReport(s *balanceLoadClientStream, interval time.Duration) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
		case <-s.Context().Done():
			return
		}
		stats := lb.clientStats.toClientStats()
		t := time.Now()
		stats.Timestamp = &timestamppb.Timestamp{
			Seconds: t.Unix(),
			Nanos:   int32(t.Nanosecond()),
		}
		if err := s.Send(&lbpb.LoadBalanceRequest{
			LoadBalanceRequestType: &lbpb.LoadBalanceRequest_ClientStats{
				ClientStats: stats,
			},
		}); err != nil {
			return
		}
	}
}

func (lb *lbBalancer) callRemoteBalancer() (backoff bool, _ error) {
	lbClient := &loadBalancerClient{cc: lb.ccRemoteLB}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	stream, err := lbClient.BalanceLoad(ctx, grpc.FailFast(false))
	if err != nil {
		return true, fmt.Errorf("grpclb: failed to perform RPC to the remote balancer %v", err)
	}

	// grpclb handshake on the stream.
	initReq := &lbpb.LoadBalanceRequest{
		LoadBalanceRequestType: &lbpb.LoadBalanceRequest_InitialRequest{
			InitialRequest: &lbpb.InitialLoadBalanceRequest{
				Name: lb.target,
			},
		},
	}
	if err := stream.Send(initReq); err != nil {
		return true, fmt.Errorf("grpclb: failed to send init request: %v", err)
	}
	reply, err := stream.Recv()
	if err != nil {
		return true, fmt.Errorf("grpclb: failed to recv init response: %v", err)
	}
	initResp := reply.GetInitialResponse()
	if initResp == nil {
		return true, fmt.Errorf("grpclb: reply from remote balancer did not include initial response")
	}
	if initResp.LoadBalancerDelegate != "" {
		return true, fmt.Errorf("grpclb: Delegation is not supported")
	}

	go func() {
		if d := convertDuration(initResp.ClientStatsReportInterval); d > 0 {
			lb.sendLoadReport(stream, d)
		}
	}()
	// No backoff if init req/resp handshake was successful.
	return false, lb.readServerList(stream)
}

func (lb *lbBalancer) watchRemoteBalancer() {
	var retryCount int
	for {
		doBackoff, err := lb.callRemoteBalancer()
		select {
		case <-lb.doneCh:
			return
		default:
			if err != nil {
				if err == errServerTerminatedConnection {
					grpclog.Info(err)
				} else {
					grpclog.Warning(err)
				}
			}
		}
		// Trigger a re-resolve when the stream errors.
		lb.cc.cc.ResolveNow(resolver.ResolveNowOption{})

		if !doBackoff {
			retryCount = 0
			continue
		}

		timer := time.NewTimer(lb.backoff.Backoff(retryCount))
		select {
		case <-timer.C:
		case <-lb.doneCh:
			timer.Stop()
			return
		}
		retryCount++
	}
}

func (lb *lbBalancer) dialRemoteLB(remoteLBName string) {
	var dopts []grpc.DialOption
	if creds := lb.opt.DialCreds; creds != nil {
		if err := creds.OverrideServerName(remoteLBName); err == nil {
			dopts = append(dopts, grpc.WithTransportCredentials(creds))
		} else {
			grpclog.Warningf("grpclb: failed to override the server name in the credentials: %v, using Insecure", err)
			dopts = append(dopts, grpc.WithInsecure())
		}
	} else if bundle := lb.grpclbClientConnCreds; bundle != nil {
		dopts = append(dopts, grpc.WithCredentialsBundle(bundle))
	} else {
		dopts = append(dopts, grpc.WithInsecure())
	}
	if lb.opt.Dialer != nil {
		// WithDialer takes a different type of function, so we instead use a
		// special DialOption here.
		wcd := internal.WithContextDialer.(func(func(context.Context, string) (net.Conn, error)) grpc.DialOption)
		dopts = append(dopts, wcd(lb.opt.Dialer))
	}
	// Explicitly set pickfirst as the balancer.
	dopts = append(dopts, grpc.WithBalancerName(grpc.PickFirstBalancerName))
	wrb := internal.WithResolverBuilder.(func(resolver.Builder) grpc.DialOption)
	dopts = append(dopts, wrb(lb.manualResolver))
	if channelz.IsOn() {
		dopts = append(dopts, grpc.WithChannelzParentID(lb.opt.ChannelzParentID))
	}

	// DialContext using manualResolver.Scheme, which is a random scheme
	// generated when init grpclb. The target scheme here is not important.
	//
	// The grpc dial target will be used by the creds (ALTS) as the authority,
	// so it has to be set to remoteLBName that comes from resolver.
	cc, err := grpc.DialContext(context.Background(), remoteLBName, dopts...)
	if err != nil {
		grpclog.Fatalf("failed to dial: %v", err)
	}
	lb.ccRemoteLB = cc
	go lb.watchRemoteBalancer()
}
<|endoftext|>


<commit_before>package processor_test

import (
	"fmt"
	"os"
	"testing"

	"github.com/nanobox-io/nanobox/processor"
	_ "github.com/nanobox-io/nanobox/processor/code"
	_ "github.com/nanobox-io/nanobox/processor/platform"
	_ "github.com/nanobox-io/nanobox/processor/provider"
	_ "github.com/nanobox-io/nanobox/processor/service"
)

type testProcessor struct {
	run bool
}

func TestMain(m *testing.M) {
	err := os.Chdir("../testing/")
	if err != nil {
		fmt.Println(err)
		return
	}
	processor.DefaultConfig.Force = true
	processor.DefaultConfig.Quiet = true
	// for testing we dont want to drop into a console
	// or hang on mist logging
	processor.Register("dev_console", testProcessBuilder)
	processor.Register("mist_log", testProcessBuilder)
	os.Exit(m.Run())
}

func (self testProcessor) Process() error {
	self.run = true
	return nil
}

func (self testProcessor) Results() processor.ProcessControl {
	return processor.ProcessControl{}
}

func testProcessBuilder(p processor.ProcessControl) (processor.Processor, error) {
	return testProcessor{}, nil
}

func TestRegister(t *testing.T) {
	processor.Register("test", testProcessBuilder)
	err := processor.Run("test", processor.DefaultConfig)
	if err != nil {
		t.Errorf("error from processor run", err)
	}
}

func TestBuild(t *testing.T) {
	err := processor.Run("build", processor.DefaultConfig)
	if err != nil {
		panic(err)
		t.Errorf("error from build run", err)
	}
}

func TestDevDeploy(t *testing.T) {
	err := processor.Run("dev", processor.DefaultConfig)
	if err != nil {
		panic(err)
		t.Errorf("error from dev run", err)
	}
}

// func TestDevDestroy(t *testing.T) {
// 	err := processor.Run("dev_destroy", processor.DefaultConfig)
// 	if err != nil {
// 		t.Errorf("error from build run", err)
// 	}
// }
<commit_msg>remove panics<commit_after>package processor_test

import (
	"fmt"
	"os"
	"testing"

	"github.com/nanobox-io/nanobox/processor"
	_ "github.com/nanobox-io/nanobox/processor/code"
	_ "github.com/nanobox-io/nanobox/processor/platform"
	_ "github.com/nanobox-io/nanobox/processor/provider"
	_ "github.com/nanobox-io/nanobox/processor/service"
)

type testProcessor struct {
	run bool
}

func TestMain(m *testing.M) {
	err := os.Chdir("../testing/")
	if err != nil {
		fmt.Println(err)
		return
	}
	processor.DefaultConfig.Force = true
	processor.DefaultConfig.Quiet = true
	// for testing we dont want to drop into a console
	// or hang on mist logging
	processor.Register("dev_console", testProcessBuilder)
	processor.Register("mist_log", testProcessBuilder)
	os.Exit(m.Run())
}

func (self testProcessor) Process() error {
	self.run = true
	return nil
}

func (self testProcessor) Results() processor.ProcessControl {
	return processor.ProcessControl{}
}

func testProcessBuilder(p processor.ProcessControl) (processor.Processor, error) {
	return testProcessor{}, nil
}

func TestRegister(t *testing.T) {
	processor.Register("test", testProcessBuilder)
	err := processor.Run("test", processor.DefaultConfig)
	if err != nil {
		t.Errorf("error from processor run", err)
	}
}

func TestBuild(t *testing.T) {
	err := processor.Run("build", processor.DefaultConfig)
	if err != nil {
		t.Errorf("error from build run", err)
	}
}

func TestDevDeploy(t *testing.T) {
	err := processor.Run("dev", processor.DefaultConfig)
	if err != nil {
		t.Errorf("error from dev run", err)
	}
}

// func TestDevDestroy(t *testing.T) {
// 	err := processor.Run("dev_destroy", processor.DefaultConfig)
// 	if err != nil {
// 		t.Errorf("error from build run", err)
// 	}
// }
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package profiler

import (
	"bytes"
	"reflect"
	"testing"

	"github.com/google/pprof/profile"
)

type fakeFunc struct {
	name   string
	file   string
	lineno int
}

func (f *fakeFunc) Name() string {
	return f.name
}
func (f *fakeFunc) FileLine(_ uintptr) (string, int) {
	return f.file, f.lineno
}

// TestRuntimeFunctionTrimming tests if symbolize trims runtime functions as intended.
func TestRuntimeFunctionTrimming(t *testing.T) {
	fakeFuncMap := map[uintptr]*fakeFunc{
		0x10: &fakeFunc{"runtime.goexit", "runtime.go", 10},
		0x20: &fakeFunc{"runtime.other", "runtime.go", 20},
		0x30: &fakeFunc{"foo", "foo.go", 30},
		0x40: &fakeFunc{"bar", "bar.go", 40},
	}
	backupFuncForPC := funcForPC
	funcForPC = func(pc uintptr) function {
		return fakeFuncMap[pc]
	}
	defer func() {
		funcForPC = backupFuncForPC
	}()
	testLoc := []*profile.Location{
		{ID: 1, Address: 0x10},
		{ID: 2, Address: 0x20},
		{ID: 3, Address: 0x30},
		{ID: 4, Address: 0x40},
	}
	testProfile := &profile.Profile{
		Sample: []*profile.Sample{
			{Location: []*profile.Location{testLoc[0], testLoc[1], testLoc[3], testLoc[2]}},
			{Location: []*profile.Location{testLoc[1], testLoc[3], testLoc[2]}},
			{Location: []*profile.Location{testLoc[3], testLoc[2], testLoc[1]}},
			{Location: []*profile.Location{testLoc[3], testLoc[2], testLoc[0]}},
			{Location: []*profile.Location{testLoc[0], testLoc[1], testLoc[3], testLoc[0]}},
			{Location: []*profile.Location{testLoc[1], testLoc[0]}},
		},
		Location: testLoc,
	}
	testProfiles := make([]*profile.Profile, 2)
	testProfiles[0] = testProfile.Copy()
	testProfiles[1] = testProfile.Copy()
	// Test case for CPU profile.
	testProfiles[0].PeriodType = &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}
	// Test case for heap profile.
	testProfiles[1].PeriodType = &profile.ValueType{Type: "space", Unit: "bytes"}
	wantFunc := []*profile.Function{
		{ID: 1, Name: "runtime.goexit", SystemName: "runtime.goexit", Filename: "runtime.go"},
		{ID: 2, Name: "runtime.other", SystemName: "runtime.other", Filename: "runtime.go"},
		{ID: 3, Name: "foo", SystemName: "foo", Filename: "foo.go"},
		{ID: 4, Name: "bar", SystemName: "bar", Filename: "bar.go"},
	}
	wantLoc := []*profile.Location{
		{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
		{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
		{ID: 3, Address: 0x30, Line: []profile.Line{{Function: wantFunc[2], Line: 30}}},
		{ID: 4, Address: 0x40, Line: []profile.Line{{Function: wantFunc[3], Line: 40}}},
	}
	wantProfiles := []*profile.Profile{
		{
			PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
			Sample: []*profile.Sample{
				{Location: []*profile.Location{wantLoc[1], wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[1], wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2], wantLoc[1]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[1], wantLoc[3]}},
				{Location: []*profile.Location{wantLoc[1]}},
			},
			Location: wantLoc,
			Function: wantFunc,
		},
		{
			PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"},
			Sample: []*profile.Sample{
				{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2], wantLoc[1]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[3]}},
				{Location: []*profile.Location{wantLoc[0]}},
			},
			Location: wantLoc,
			Function: wantFunc,
		},
	}
	for i := 0; i < 2; i++ {
		symbolize(testProfiles[i])
		if !reflect.DeepEqual(testProfiles[i], wantProfiles[i]) {
			t.Errorf("incorrect trimming (testcase = %d): got {%v}, want {%v}", i, testProfiles[i], wantProfiles[i])
		}
	}
}

// TestParseAndSymbolize tests if parseAndSymbolize parses and symbolizes
// profiles as intended.
func TestParseAndSymbolize(t *testing.T) {
	fakeFuncMap := map[uintptr]*fakeFunc{
		0x10: &fakeFunc{"foo", "foo.go", 10},
		0x20: &fakeFunc{"bar", "bar.go", 20},
	}
	backupFuncForPC := funcForPC
	funcForPC = func(pc uintptr) function {
		return fakeFuncMap[pc]
	}
	defer func() {
		funcForPC = backupFuncForPC
	}()

	testLoc := []*profile.Location{
		{ID: 1, Address: 0x10},
		{ID: 2, Address: 0x20},
	}
	testProfile := &profile.Profile{
		SampleType: []*profile.ValueType{
			&profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
		},
		PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
		Sample: []*profile.Sample{
			{Location: []*profile.Location{testLoc[0], testLoc[1]}, Value: []int64{1}},
			{Location: []*profile.Location{testLoc[1]}, Value: []int64{1}},
		},
		Location: testLoc,
	}
	testProfiles := make([]*profile.Profile, 2)
	testProfiles[0] = testProfile.Copy()
	testProfiles[1] = testProfile.Copy()

	wantFunc := []*profile.Function{
		{ID: 1, Name: "foo", SystemName: "foo", Filename: "foo.go"},
		{ID: 2, Name: "bar", SystemName: "bar", Filename: "bar.go"},
	}
	wantLoc := []*profile.Location{
		{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
		{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
	}
	wantProfile := &profile.Profile{
		SampleType: []*profile.ValueType{
			&profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
		},
		PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
		Sample: []*profile.Sample{
			{Location: []*profile.Location{wantLoc[0], wantLoc[1]}, Value: []int64{1}},
			{Location: []*profile.Location{wantLoc[1]}, Value: []int64{1}},
		},
		Location: wantLoc,
		Function: wantFunc,
	}

	// Profile already symbolized.
	testProfiles[1].Location = []*profile.Location{
		{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
		{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
	}
	testProfiles[1].Function = []*profile.Function{
		{ID: 1, Name: "foo", SystemName: "foo", Filename: "foo.go"},
		{ID: 2, Name: "bar", SystemName: "bar", Filename: "bar.go"},
	}
	for i := 0; i < 2; i++ {
		var prof bytes.Buffer
		testProfiles[i].Write(&prof)

		parseAndSymbolize(&prof)
		gotProfile, err := profile.ParseData(prof.Bytes())
		if err != nil {
			t.Errorf("parsing symbolized profile (testcase = %d) got err: %v, want no error", i, err)
		}
		if !reflect.DeepEqual(gotProfile, wantProfile) {
			t.Errorf("incorrect symbolization (testcase = %d): got {%v}, want {%v}", i, gotProfile, wantProfile)
		}
	}
}

func TestIsSymbolizedGoVersion(t *testing.T) {
	for _, tc := range []struct {
		input string
		want  bool
	}{
		{"go1.9beta2", true},
		{"go1.9", true},
		{"go1.9.1", true},
		{"go1.10", true},
		{"go1.10.1", true},
		{"go2.0", true},
		{"go3.1", true},
		{"go1.8", false},
		{"go1.8.1", false},
		{"go1.7", false},
		{"devel ", false},
	} {
		if got := isSymbolizedGoVersion(tc.input); got != tc.want {
			t.Errorf("isSymbolizedGoVersion(%v) got %v, want %v", tc.input, got, tc.want)
		}
	}
}
<commit_msg>profiler: replace reflect.DeepEqual with cmp<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package profiler

import (
	"bytes"
	"testing"

	"cloud.google.com/go/internal/testutil"
	"github.com/google/go-cmp/cmp/cmpopts"
	"github.com/google/pprof/profile"
)

type fakeFunc struct {
	name   string
	file   string
	lineno int
}

func (f *fakeFunc) Name() string {
	return f.name
}
func (f *fakeFunc) FileLine(_ uintptr) (string, int) {
	return f.file, f.lineno
}

var cmpOpt = cmpopts.IgnoreUnexported(profile.Profile{}, profile.Function{},
	profile.Line{}, profile.Location{}, profile.Sample{}, profile.ValueType{})

// TestRuntimeFunctionTrimming tests if symbolize trims runtime functions as intended.
func TestRuntimeFunctionTrimming(t *testing.T) {
	fakeFuncMap := map[uintptr]*fakeFunc{
		0x10: &fakeFunc{"runtime.goexit", "runtime.go", 10},
		0x20: &fakeFunc{"runtime.other", "runtime.go", 20},
		0x30: &fakeFunc{"foo", "foo.go", 30},
		0x40: &fakeFunc{"bar", "bar.go", 40},
	}
	backupFuncForPC := funcForPC
	funcForPC = func(pc uintptr) function {
		return fakeFuncMap[pc]
	}
	defer func() {
		funcForPC = backupFuncForPC
	}()
	testLoc := []*profile.Location{
		{ID: 1, Address: 0x10},
		{ID: 2, Address: 0x20},
		{ID: 3, Address: 0x30},
		{ID: 4, Address: 0x40},
	}
	testProfile := &profile.Profile{
		Sample: []*profile.Sample{
			{Location: []*profile.Location{testLoc[0], testLoc[1], testLoc[3], testLoc[2]}},
			{Location: []*profile.Location{testLoc[1], testLoc[3], testLoc[2]}},
			{Location: []*profile.Location{testLoc[3], testLoc[2], testLoc[1]}},
			{Location: []*profile.Location{testLoc[3], testLoc[2], testLoc[0]}},
			{Location: []*profile.Location{testLoc[0], testLoc[1], testLoc[3], testLoc[0]}},
			{Location: []*profile.Location{testLoc[1], testLoc[0]}},
		},
		Location: testLoc,
	}
	testProfiles := make([]*profile.Profile, 2)
	testProfiles[0] = testProfile.Copy()
	testProfiles[1] = testProfile.Copy()
	// Test case for CPU profile.
	testProfiles[0].PeriodType = &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}
	// Test case for heap profile.
	testProfiles[1].PeriodType = &profile.ValueType{Type: "space", Unit: "bytes"}
	wantFunc := []*profile.Function{
		{ID: 1, Name: "runtime.goexit", SystemName: "runtime.goexit", Filename: "runtime.go"},
		{ID: 2, Name: "runtime.other", SystemName: "runtime.other", Filename: "runtime.go"},
		{ID: 3, Name: "foo", SystemName: "foo", Filename: "foo.go"},
		{ID: 4, Name: "bar", SystemName: "bar", Filename: "bar.go"},
	}
	wantLoc := []*profile.Location{
		{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
		{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
		{ID: 3, Address: 0x30, Line: []profile.Line{{Function: wantFunc[2], Line: 30}}},
		{ID: 4, Address: 0x40, Line: []profile.Line{{Function: wantFunc[3], Line: 40}}},
	}
	wantProfiles := []*profile.Profile{
		{
			PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
			Sample: []*profile.Sample{
				{Location: []*profile.Location{wantLoc[1], wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[1], wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2], wantLoc[1]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[1], wantLoc[3]}},
				{Location: []*profile.Location{wantLoc[1]}},
			},
			Location: wantLoc,
			Function: wantFunc,
		},
		{
			PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"},
			Sample: []*profile.Sample{
				{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2], wantLoc[1]}},
				{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
				{Location: []*profile.Location{wantLoc[3]}},
				{Location: []*profile.Location{wantLoc[0]}},
			},
			Location: wantLoc,
			Function: wantFunc,
		},
	}
	for i := 0; i < 2; i++ {
		symbolize(testProfiles[i])
		if !testutil.Equal(testProfiles[i], wantProfiles[i], cmpOpt) {
			t.Errorf("incorrect trimming (testcase = %d): got {%v}, want {%v}", i, testProfiles[i], wantProfiles[i])
		}
	}
}

// TestParseAndSymbolize tests if parseAndSymbolize parses and symbolizes
// profiles as intended.
func TestParseAndSymbolize(t *testing.T) {
	fakeFuncMap := map[uintptr]*fakeFunc{
		0x10: &fakeFunc{"foo", "foo.go", 10},
		0x20: &fakeFunc{"bar", "bar.go", 20},
	}
	backupFuncForPC := funcForPC
	funcForPC = func(pc uintptr) function {
		return fakeFuncMap[pc]
	}
	defer func() {
		funcForPC = backupFuncForPC
	}()

	testLoc := []*profile.Location{
		{ID: 1, Address: 0x10},
		{ID: 2, Address: 0x20},
	}
	testProfile := &profile.Profile{
		SampleType: []*profile.ValueType{
			&profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
		},
		PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
		Sample: []*profile.Sample{
			{Location: []*profile.Location{testLoc[0], testLoc[1]}, Value: []int64{1}},
			{Location: []*profile.Location{testLoc[1]}, Value: []int64{1}},
		},
		Location: testLoc,
	}
	testProfiles := make([]*profile.Profile, 2)
	testProfiles[0] = testProfile.Copy()
	testProfiles[1] = testProfile.Copy()

	wantFunc := []*profile.Function{
		{ID: 1, Name: "foo", SystemName: "foo", Filename: "foo.go"},
		{ID: 2, Name: "bar", SystemName: "bar", Filename: "bar.go"},
	}
	wantLoc := []*profile.Location{
		{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
		{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
	}
	wantProfile := &profile.Profile{
		SampleType: []*profile.ValueType{
			&profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
		},
		PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
		Sample: []*profile.Sample{
			{Location: []*profile.Location{wantLoc[0], wantLoc[1]}, Value: []int64{1}},
			{Location: []*profile.Location{wantLoc[1]}, Value: []int64{1}},
		},
		Location: wantLoc,
		Function: wantFunc,
	}

	// Profile already symbolized.
	testProfiles[1].Location = []*profile.Location{
		{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
		{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
	}
	testProfiles[1].Function = []*profile.Function{
		{ID: 1, Name: "foo", SystemName: "foo", Filename: "foo.go"},
		{ID: 2, Name: "bar", SystemName: "bar", Filename: "bar.go"},
	}
	for i := 0; i < 2; i++ {
		var prof bytes.Buffer
		testProfiles[i].Write(&prof)

		parseAndSymbolize(&prof)
		gotProfile, err := profile.ParseData(prof.Bytes())
		if err != nil {
			t.Errorf("parsing symbolized profile (testcase = %d) got err: %v, want no error", i, err)
		}
		if !testutil.Equal(gotProfile, wantProfile, cmpOpt) {
			t.Errorf("incorrect symbolization (testcase = %d): got {%v}, want {%v}", i, gotProfile, wantProfile)
		}
	}
}

func TestIsSymbolizedGoVersion(t *testing.T) {
	for _, tc := range []struct {
		input string
		want  bool
	}{
		{"go1.9beta2", true},
		{"go1.9", true},
		{"go1.9.1", true},
		{"go1.10", true},
		{"go1.10.1", true},
		{"go2.0", true},
		{"go3.1", true},
		{"go1.8", false},
		{"go1.8.1", false},
		{"go1.7", false},
		{"devel ", false},
	} {
		if got := isSymbolizedGoVersion(tc.input); got != tc.want {
			t.Errorf("isSymbolizedGoVersion(%v) got %v, want %v", tc.input, got, tc.want)
		}
	}
}
<|endoftext|>


<commit_before>package network_test

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/APTrust/exchange/testdata"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"
	"time"
)

func TestNewPharosClient(t *testing.T) {
	_, err := network.NewPharosClient("http://example.com", "v1", "user", "key")
	if err != nil {
		t.Error(err)
	}
}

func TestInstitutionGet(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(institutionGetHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.InstitutionGet("college.edu")

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/institutions/college.edu", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)
	assert.EqualValues(t, "Institution", response.ObjectType())
	if response.Institution() == nil {
		t.Errorf("Institution should not be nil")
	}
	assert.NotEqual(t, "", response.Institution().Identifier)
}

func TestInstitutionList(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(institutionListHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.InstitutionList()

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/institutions/", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)
	assert.EqualValues(t, "Institution", response.ObjectType())

	list := response.Institutions()
	if list == nil {
		t.Errorf("Institution list should not be nil")
		return
	}
	if len(list) != 4 {
		t.Errorf("Institutions list should have four items. Found %d.", len(list))
		return
	}
	for _, inst := range list {
		assert.NotEqual(t, "", len(inst.Identifier))
	}
}

func TestIntellectualObjectGet(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(intellectualObjectGetHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.IntellectualObjectGet("college.edu/object")

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/objects/college.edu%2Fobject", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)

	obj := response.IntellectualObject()
	assert.EqualValues(t, "IntellectualObject", response.ObjectType())
	if obj == nil {
		t.Errorf("IntellectualObject should not be nil")
	}
	assert.NotEqual(t, "", obj.Identifier)

	// Check that child objects were parsed correctly
	assert.Equal(t, 2, len(obj.GenericFiles))
	assert.Equal(t, 3, len(obj.PremisEvents))
	assert.Equal(t, 4, len(obj.GenericFiles[0].Checksums))
	assert.Equal(t, 5, len(obj.IngestTags))
}

func TestIntellectualObjectList(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(intellectualObjectListHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.IntellectualObjectList(nil)

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/objects/?", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)
	assert.EqualValues(t, "IntellectualObject", response.ObjectType())

	list := response.IntellectualObjects()
	if list == nil {
		t.Errorf("IntellectualObject list should not be nil")
		return
	}
	if len(list) != 4 {
		t.Errorf("IntellectualObjects list should have four items. Found %d.", len(list))
		return
	}
	for _, inst := range list {
		assert.NotEqual(t, "", len(inst.Identifier))
	}
}

func TestIntellectualObjectSave(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(intellectualObjectSaveHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	// ---------------------------------------------
	// First, test create...
	// ---------------------------------------------
	obj := testdata.MakeIntellectualObject(0,0,0,0)
	obj.Id = 0
	response := client.IntellectualObjectSave(obj)

	// Check the request URL and method
	assert.Equal(t, "POST", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/objects/", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)

	obj = response.IntellectualObject()
	assert.EqualValues(t, "IntellectualObject", response.ObjectType())
	if obj == nil {
		t.Errorf("IntellectualObject should not be nil")
	}
	assert.NotEqual(t, "", obj.Identifier)

	// Make sure the client returns the SAVED object,
	// not the unsaved one we sent.
	assert.NotEqual(t, 0, obj.Id)

	// ---------------------------------------------
	// Now test with an update...
	// ---------------------------------------------
	obj = testdata.MakeIntellectualObject(0,0,0,0)
	origModTime := obj.UpdatedAt
	response = client.IntellectualObjectSave(obj)

	// Check the request URL and method
	expectedUrl := fmt.Sprintf("/api/v1/objects/%s", strings.Replace(obj.Identifier, "/", "%2F", -1))
	assert.Equal(t, "PUT", response.Response.Request.Method)
	assert.Equal(t, expectedUrl, response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)

	obj = response.IntellectualObject()
	assert.EqualValues(t, "IntellectualObject", response.ObjectType())
	if obj == nil {
		t.Errorf("IntellectualObject should not be nil")
	}
	assert.NotEqual(t, "", obj.Identifier)
	assert.Equal(t, 1000, obj.Id)
	assert.NotEqual(t, origModTime, obj.UpdatedAt)
}

// -------------------------------------------------------------------------

// Build a simple struct that mimics the structure of a Pharos
// JSON list response. That includes keys count, next, previous,
// and results. The caller will add ["results"] with a list of
// objects of the appropriate type.
func listResponseData() (map[string]interface{}) {
	data := make(map[string]interface{})
	data["count"] = 100
	data["next"] = "http://example.com/?page=11"
	data["previous"] = "http://example.com/?page=9"
	return data
}

func institutionGetHander(w http.ResponseWriter, r *http.Request) {
	obj := testdata.MakeInstitution()
	objJson, _ := json.Marshal(obj)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(objJson))
}

func institutionListHander(w http.ResponseWriter, r *http.Request) {
	list := make([]*models.Institution, 4)
	for i := 0; i < 4; i++ {
		list[i] = testdata.MakeInstitution()
	}
	data := listResponseData()
	data["results"] = list
	listJson, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(listJson))
}

func intellectualObjectGetHander(w http.ResponseWriter, r *http.Request) {
	obj := testdata.MakeIntellectualObject(2,3,4,5)
	objJson, _ := json.Marshal(obj)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(objJson))
}

func intellectualObjectListHander(w http.ResponseWriter, r *http.Request) {
	list := make([]*models.IntellectualObject, 4)
	for i := 0; i < 4; i++ {
		list[i] = testdata.MakeIntellectualObject(2,3,4,5)
	}
	data := listResponseData()
	data["results"] = list
	listJson, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(listJson))
}

func intellectualObjectSaveHander(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
    data := make(map[string]interface{})
    err := decoder.Decode(&data)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error decoding JSON data: %v", err)
		fmt.Fprintln(w, "")
		return
	}
	// Assign ID and timestamps, as if the object has been saved.
	data["id"] = 1000
	data["created_at"] = time.Now().UTC()
	data["updated_at"] = time.Now().UTC()
	objJson, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(objJson))
}
<commit_msg>Test GenericFileGet()<commit_after>package network_test

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/APTrust/exchange/testdata"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"
	"time"
)

func TestNewPharosClient(t *testing.T) {
	_, err := network.NewPharosClient("http://example.com", "v1", "user", "key")
	if err != nil {
		t.Error(err)
	}
}

func TestInstitutionGet(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(institutionGetHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.InstitutionGet("college.edu")

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/institutions/college.edu", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)
	assert.EqualValues(t, "Institution", response.ObjectType())
	if response.Institution() == nil {
		t.Errorf("Institution should not be nil")
	}
	assert.NotEqual(t, "", response.Institution().Identifier)
}

func TestInstitutionList(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(institutionListHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.InstitutionList()

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/institutions/", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)
	assert.EqualValues(t, "Institution", response.ObjectType())

	list := response.Institutions()
	if list == nil {
		t.Errorf("Institution list should not be nil")
		return
	}
	if len(list) != 4 {
		t.Errorf("Institutions list should have four items. Found %d.", len(list))
		return
	}
	for _, inst := range list {
		assert.NotEqual(t, "", len(inst.Identifier))
	}
}

func TestIntellectualObjectGet(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(intellectualObjectGetHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.IntellectualObjectGet("college.edu/object")

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/objects/college.edu%2Fobject", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)

	obj := response.IntellectualObject()
	assert.EqualValues(t, "IntellectualObject", response.ObjectType())
	if obj == nil {
		t.Errorf("IntellectualObject should not be nil")
	}
	assert.NotEqual(t, "", obj.Identifier)

	// Check that child objects were parsed correctly
	assert.Equal(t, 2, len(obj.GenericFiles))
	assert.Equal(t, 3, len(obj.PremisEvents))
	assert.Equal(t, 4, len(obj.GenericFiles[0].Checksums))
	assert.Equal(t, 5, len(obj.IngestTags))
}

func TestIntellectualObjectList(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(intellectualObjectListHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.IntellectualObjectList(nil)

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/objects/?", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)
	assert.EqualValues(t, "IntellectualObject", response.ObjectType())

	list := response.IntellectualObjects()
	if list == nil {
		t.Errorf("IntellectualObject list should not be nil")
		return
	}
	if len(list) != 4 {
		t.Errorf("IntellectualObjects list should have four items. Found %d.", len(list))
		return
	}
	for _, inst := range list {
		assert.NotEqual(t, "", len(inst.Identifier))
	}
}

func TestIntellectualObjectSave(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(intellectualObjectSaveHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	// ---------------------------------------------
	// First, test create...
	// ---------------------------------------------
	obj := testdata.MakeIntellectualObject(0,0,0,0)
	obj.Id = 0
	response := client.IntellectualObjectSave(obj)

	// Check the request URL and method
	assert.Equal(t, "POST", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/objects/", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)

	obj = response.IntellectualObject()
	assert.EqualValues(t, "IntellectualObject", response.ObjectType())
	if obj == nil {
		t.Errorf("IntellectualObject should not be nil")
	}
	assert.NotEqual(t, "", obj.Identifier)

	// Make sure the client returns the SAVED object,
	// not the unsaved one we sent.
	assert.NotEqual(t, 0, obj.Id)

	// ---------------------------------------------
	// Now test with an update...
	// ---------------------------------------------
	obj = testdata.MakeIntellectualObject(0,0,0,0)
	origModTime := obj.UpdatedAt
	response = client.IntellectualObjectSave(obj)

	// Check the request URL and method
	expectedUrl := fmt.Sprintf("/api/v1/objects/%s", strings.Replace(obj.Identifier, "/", "%2F", -1))
	assert.Equal(t, "PUT", response.Response.Request.Method)
	assert.Equal(t, expectedUrl, response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)

	obj = response.IntellectualObject()
	assert.EqualValues(t, "IntellectualObject", response.ObjectType())
	if obj == nil {
		t.Errorf("IntellectualObject should not be nil")
	}
	assert.NotEqual(t, "", obj.Identifier)
	assert.Equal(t, 1000, obj.Id)
	assert.NotEqual(t, origModTime, obj.UpdatedAt)
}

func TestGenericFileGet(t *testing.T) {
	testServer := httptest.NewServer(http.HandlerFunc(genericFileGetHander))
	defer testServer.Close()

	client, err := network.NewPharosClient(testServer.URL, "v1", "user", "key")
	if err != nil {
		t.Error(err)
		return
	}

	response := client.GenericFileGet("college.edu/object/file.xml")

	// Check the request URL and method
	assert.Equal(t, "GET", response.Response.Request.Method)
	assert.Equal(t, "/api/v1/files/college.edu%2Fobject%2Ffile.xml", response.Request.URL.Opaque)

	// Basic sanity check on response values
	assert.Nil(t, response.Error)

	obj := response.GenericFile()
	assert.EqualValues(t, "GenericFile", response.ObjectType())
	if obj == nil {
		t.Errorf("GenericFile should not be nil")
	}
	assert.True(t, strings.HasPrefix(obj.Identifier, "kollege.kom/objekt"))

	// Check that child objects were parsed correctly
	assert.Equal(t, 2, len(obj.PremisEvents))
	assert.Equal(t, 3, len(obj.Checksums))
}

// -------------------------------------------------------------------------
// -------------------------------------------------------------------------
// -------------------------------------------------------------------------

// Build a simple struct that mimics the structure of a Pharos
// JSON list response. That includes keys count, next, previous,
// and results. The caller will add ["results"] with a list of
// objects of the appropriate type.
func listResponseData() (map[string]interface{}) {
	data := make(map[string]interface{})
	data["count"] = 100
	data["next"] = "http://example.com/?page=11"
	data["previous"] = "http://example.com/?page=9"
	return data
}

func institutionGetHander(w http.ResponseWriter, r *http.Request) {
	obj := testdata.MakeInstitution()
	objJson, _ := json.Marshal(obj)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(objJson))
}

func institutionListHander(w http.ResponseWriter, r *http.Request) {
	list := make([]*models.Institution, 4)
	for i := 0; i < 4; i++ {
		list[i] = testdata.MakeInstitution()
	}
	data := listResponseData()
	data["results"] = list
	listJson, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(listJson))
}

func intellectualObjectGetHander(w http.ResponseWriter, r *http.Request) {
	obj := testdata.MakeIntellectualObject(2,3,4,5)
	objJson, _ := json.Marshal(obj)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(objJson))
}

func intellectualObjectListHander(w http.ResponseWriter, r *http.Request) {
	list := make([]*models.IntellectualObject, 4)
	for i := 0; i < 4; i++ {
		list[i] = testdata.MakeIntellectualObject(2,3,4,5)
	}
	data := listResponseData()
	data["results"] = list
	listJson, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(listJson))
}

func intellectualObjectSaveHander(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
    data := make(map[string]interface{})
    err := decoder.Decode(&data)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error decoding JSON data: %v", err)
		fmt.Fprintln(w, "")
		return
	}
	// Assign ID and timestamps, as if the object has been saved.
	data["id"] = 1000
	data["created_at"] = time.Now().UTC()
	data["updated_at"] = time.Now().UTC()
	objJson, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(objJson))
}

func genericFileGetHander(w http.ResponseWriter, r *http.Request) {
	obj := testdata.MakeGenericFile(2,3, "kollege.kom/objekt")
	objJson, _ := json.Marshal(obj)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintln(w, string(objJson))
}
<|endoftext|>


<commit_before>package lumberjack

// lumberjack.go defines the lumberjack plugin as being registered with all
// output plugins

import (
	"crypto/tls"
	"errors"
	"time"

	"github.com/elastic/libbeat/common"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/outputs"
	"github.com/elastic/libbeat/outputs/mode"
)

var debug = logp.MakeDebug("lumberjack")

func init() {
	outputs.RegisterOutputPlugin("lumberjack", lumberjackOutputPlugin{})
}

type lumberjackOutputPlugin struct{}

func (p lumberjackOutputPlugin) NewOutput(
	beat string,
	config *outputs.MothershipConfig,
	topologyExpire int,
) (outputs.Outputer, error) {
	output := &lumberjack{}
	err := output.init(beat, *config, topologyExpire)
	if err != nil {
		return nil, err
	}
	return output, nil
}

type lumberjack struct {
	mode mode.ConnectionMode
}

const (
	lumberjackDefaultTimeout = 5 * time.Second
	defaultSendRetries       = 3
)

// ErrNoHostsConfigured indicates missing host or hosts configuration
var ErrNoHostsConfigured = errors.New("no host configuration found")

var waitRetry = time.Duration(1) * time.Second

func (lj *lumberjack) init(
	beat string,
	config outputs.MothershipConfig,
	topologyExpire int,
) error {
	useTLS := false
	if config.TLS != nil {
		useTLS = !config.TLS.Disabled
	}

	timeout := lumberjackDefaultTimeout
	if config.Timeout != 0 {
		timeout = time.Duration(config.Timeout) * time.Second
	}

	var clients []mode.ProtocolClient
	var err error
	if useTLS {
		var tlsConfig *tls.Config
		tlsConfig, err = outputs.LoadTLSConfig(config.TLS)
		if err != nil {
			return err
		}

		clients, err = makeClients(config, timeout,
			func(host string) (TransportClient, error) {
				return newTLSClient(host, tlsConfig)
			})
	} else {
		clients, err = makeClients(config, timeout,
			func(host string) (TransportClient, error) {
				return newTCPClient(host)
			})
	}
	if err != nil {
		return err
	}

	sendRetries := defaultSendRetries
	if config.Max_retries != nil {
		sendRetries = *config.Max_retries
	}

	var m mode.ConnectionMode
	if len(clients) == 1 {
		m, err = mode.NewSingleConnectionMode(clients[0],
			sendRetries, waitRetry, timeout)
	} else {
		loadBalance := config.LoadBalance == nil || *config.LoadBalance
		if loadBalance {
			m, err = mode.NewLoadBalancerMode(clients, sendRetries, waitRetry, timeout)
		} else {
			m, err = mode.NewFailOverConnectionMode(clients, sendRetries, waitRetry, timeout)
		}
	}
	if err != nil {
		return err
	}

	lj.mode = m
	return nil
}

func makeClients(
	config outputs.MothershipConfig,
	timeout time.Duration,
	newTransp func(string) (TransportClient, error),
) ([]mode.ProtocolClient, error) {
	switch {
	case len(config.Hosts) > 0:
		var clients []mode.ProtocolClient
		for _, host := range config.Hosts {
			transp, err := newTransp(host)
			if err != nil {
				for _, client := range clients {
					_ = client.Close() // ignore error
				}
				return nil, err
			}
			client := newLumberjackClient(transp, timeout)
			clients = append(clients, client)
		}
		return clients, nil
	case config.Host != "":
		transp, err := newTransp(config.Host)
		if err != nil {
			return nil, err
		}
		return []mode.ProtocolClient{newLumberjackClient(transp, timeout)}, nil
	default:
		return nil, ErrNoHostsConfigured
	}
}

// TODO: update Outputer interface to support multiple events for batch-like
//       processing (e.g. for filebeat). Batch like processing might reduce
//       send/receive overhead per event for other implementors too.
func (lj *lumberjack) PublishEvent(
	signaler outputs.Signaler,
	ts time.Time,
	event common.MapStr,
) error {
	return lj.mode.PublishEvent(signaler, event)
}

// BulkPublish implements the BulkOutputer interface pushing a bulk of events
// via lumberjack.
func (lj *lumberjack) BulkPublish(
	trans outputs.Signaler,
	ts time.Time,
	events []common.MapStr,
) error {
	return lj.mode.PublishEvents(trans, events)
}
<commit_msg>lumberjack: field compat for logstash codec<commit_after>package lumberjack

// lumberjack.go defines the lumberjack plugin as being registered with all
// output plugins

import (
	"crypto/tls"
	"errors"
	"time"

	"github.com/elastic/libbeat/common"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/outputs"
	"github.com/elastic/libbeat/outputs/mode"
)

var debug = logp.MakeDebug("lumberjack")

func init() {
	outputs.RegisterOutputPlugin("lumberjack", lumberjackOutputPlugin{})
}

type lumberjackOutputPlugin struct{}

func (p lumberjackOutputPlugin) NewOutput(
	beat string,
	config *outputs.MothershipConfig,
	topologyExpire int,
) (outputs.Outputer, error) {
	output := &lumberjack{}
	err := output.init(beat, *config, topologyExpire)
	if err != nil {
		return nil, err
	}
	return output, nil
}

type lumberjack struct {
	mode mode.ConnectionMode
}

const (
	lumberjackDefaultTimeout = 5 * time.Second
	defaultSendRetries       = 3
)

// ErrNoHostsConfigured indicates missing host or hosts configuration
var ErrNoHostsConfigured = errors.New("no host configuration found")

var waitRetry = time.Duration(1) * time.Second

func (lj *lumberjack) init(
	beat string,
	config outputs.MothershipConfig,
	topologyExpire int,
) error {
	useTLS := false
	if config.TLS != nil {
		useTLS = !config.TLS.Disabled
	}

	timeout := lumberjackDefaultTimeout
	if config.Timeout != 0 {
		timeout = time.Duration(config.Timeout) * time.Second
	}

	var clients []mode.ProtocolClient
	var err error
	if useTLS {
		var tlsConfig *tls.Config
		tlsConfig, err = outputs.LoadTLSConfig(config.TLS)
		if err != nil {
			return err
		}

		clients, err = makeClients(config, timeout,
			func(host string) (TransportClient, error) {
				return newTLSClient(host, tlsConfig)
			})
	} else {
		clients, err = makeClients(config, timeout,
			func(host string) (TransportClient, error) {
				return newTCPClient(host)
			})
	}
	if err != nil {
		return err
	}

	sendRetries := defaultSendRetries
	if config.Max_retries != nil {
		sendRetries = *config.Max_retries
	}

	var m mode.ConnectionMode
	if len(clients) == 1 {
		m, err = mode.NewSingleConnectionMode(clients[0],
			sendRetries, waitRetry, timeout)
	} else {
		loadBalance := config.LoadBalance == nil || *config.LoadBalance
		if loadBalance {
			m, err = mode.NewLoadBalancerMode(clients, sendRetries, waitRetry, timeout)
		} else {
			m, err = mode.NewFailOverConnectionMode(clients, sendRetries, waitRetry, timeout)
		}
	}
	if err != nil {
		return err
	}

	lj.mode = m
	return nil
}

func makeClients(
	config outputs.MothershipConfig,
	timeout time.Duration,
	newTransp func(string) (TransportClient, error),
) ([]mode.ProtocolClient, error) {
	switch {
	case len(config.Hosts) > 0:
		var clients []mode.ProtocolClient
		for _, host := range config.Hosts {
			transp, err := newTransp(host)
			if err != nil {
				for _, client := range clients {
					_ = client.Close() // ignore error
				}
				return nil, err
			}
			client := newLumberjackClient(transp, timeout)
			clients = append(clients, client)
		}
		return clients, nil
	case config.Host != "":
		transp, err := newTransp(config.Host)
		if err != nil {
			return nil, err
		}
		return []mode.ProtocolClient{newLumberjackClient(transp, timeout)}, nil
	default:
		return nil, ErrNoHostsConfigured
	}
}

// TODO: update Outputer interface to support multiple events for batch-like
//       processing (e.g. for filebeat). Batch like processing might reduce
//       send/receive overhead per event for other implementors too.
func (lj *lumberjack) PublishEvent(
	signaler outputs.Signaler,
	ts time.Time,
	event common.MapStr,
) error {
	compatFix(event)
	return lj.mode.PublishEvent(signaler, event)
}

// BulkPublish implements the BulkOutputer interface pushing a bulk of events
// via lumberjack.
func (lj *lumberjack) BulkPublish(
	trans outputs.Signaler,
	ts time.Time,
	events []common.MapStr,
) error {
	for _, event := range events {
		compatFix(event)
	}
	return lj.mode.PublishEvents(trans, events)
}

// compatFix adapts events to be compatible with logstash forwarer messages by renaming
// the "message" field to "line". The lumberjack server in logstash will
// decode/rename the "line" field into "message".
func compatFix(event common.MapStr) {
	if msg, hasMessage := event["message"]; hasMessage {
		if _, hasLine := event["line"]; !hasLine {
			event["line"] = msg
			delete(event, "message")
		}
	}
}
<|endoftext|>