<commit_before>package contractcourt

import (
	"bytes"
	"io"
	"io/ioutil"
	"testing"

	"github.com/lightningnetwork/lnd/channeldb"
	"github.com/lightningnetwork/lnd/htlcswitch/hop"
	"github.com/lightningnetwork/lnd/invoices"
	"github.com/lightningnetwork/lnd/lnwallet"

	"github.com/lightningnetwork/lnd/chainntnfs"
	"github.com/lightningnetwork/lnd/lntypes"
)

const (
	testInitialBlockHeight = 100
	testHtlcExpiry         = 150
)

var (
	testResPreimage   = lntypes.Preimage{1, 2, 3}
	testResHash       = testResPreimage.Hash()
	testResCircuitKey = channeldb.CircuitKey{}
	testOnionBlob     = []byte{4, 5, 6}
)

// TestHtlcIncomingResolverFwdPreimageKnown tests resolution of a forwarded htlc
// for which the preimage is already known initially.
func TestHtlcIncomingResolverFwdPreimageKnown(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyErr = channeldb.ErrInvoiceNotFound
	ctx.witnessBeacon.lookupPreimage[testResHash] = testResPreimage
	ctx.resolve()
	ctx.waitForResult(true)
}

// TestHtlcIncomingResolverFwdContestedSuccess tests resolution of a forwarded
// htlc for which the preimage becomes known after the resolver has been
// started.
func TestHtlcIncomingResolverFwdContestedSuccess(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyErr = channeldb.ErrInvoiceNotFound
	ctx.resolve()

	// Simulate a new block coming in. HTLC is not yet expired.
	ctx.notifyEpoch(testInitialBlockHeight + 1)

	ctx.witnessBeacon.preImageUpdates <- testResPreimage
	ctx.waitForResult(true)
}

// TestHtlcIncomingResolverFwdContestedTimeout tests resolution of a forwarded
// htlc that times out after the resolver has been started.
func TestHtlcIncomingResolverFwdContestedTimeout(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyErr = channeldb.ErrInvoiceNotFound
	ctx.resolve()

	// Simulate a new block coming in. HTLC expires.
	ctx.notifyEpoch(testHtlcExpiry)

	ctx.waitForResult(false)
}

// TestHtlcIncomingResolverFwdTimeout tests resolution of a forwarded htlc that
// has already expired when the resolver starts.
func TestHtlcIncomingResolverFwdTimeout(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)

	ctx.registry.notifyErr = channeldb.ErrInvoiceNotFound
	ctx.witnessBeacon.lookupPreimage[testResHash] = testResPreimage
	ctx.resolver.htlcExpiry = 90
	ctx.resolve()
	ctx.waitForResult(false)
}

// TestHtlcIncomingResolverExitSettle tests resolution of an exit hop htlc for
// which the invoice has already been settled when the resolver starts.
func TestHtlcIncomingResolverExitSettle(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyEvent = &invoices.HodlEvent{
		CircuitKey: testResCircuitKey,
		Preimage:   &testResPreimage,
	}
	ctx.resolve()

	data := <-ctx.registry.notifyChan
	if data.expiry != testHtlcExpiry {
		t.Fatal("incorrect expiry")
	}
	if data.currentHeight != testInitialBlockHeight {
		t.Fatal("incorrect block height")
	}

	ctx.waitForResult(true)

	if !bytes.Equal(
		ctx.onionProcessor.offeredOnionBlob, testOnionBlob,
	) {
		t.Fatal("unexpected onion blob")
	}
}

// TestHtlcIncomingResolverExitCancel tests resolution of an exit hop htlc for
// an invoice that is already canceled when the resolver starts.
func TestHtlcIncomingResolverExitCancel(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyEvent = &invoices.HodlEvent{
		CircuitKey: testResCircuitKey,
	}
	ctx.resolve()
	ctx.waitForResult(false)
}

// TestHtlcIncomingResolverExitSettleHodl tests resolution of an exit hop htlc
// for a hodl invoice that is settled after the resolver has started.
func TestHtlcIncomingResolverExitSettleHodl(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.resolve()

	notifyData := <-ctx.registry.notifyChan
	notifyData.hodlChan <- invoices.HodlEvent{
		CircuitKey: testResCircuitKey,
		Preimage:   &testResPreimage,
	}

	ctx.waitForResult(true)
}

// TestHtlcIncomingResolverExitTimeoutHodl tests resolution of an exit hop htlc
// for a hodl invoice that times out.
func TestHtlcIncomingResolverExitTimeoutHodl(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.resolve()
	ctx.notifyEpoch(testHtlcExpiry)
	ctx.waitForResult(false)
}

// TestHtlcIncomingResolverExitCancelHodl tests resolution of an exit hop htlc
// for a hodl invoice that is canceled after the resolver has started.
func TestHtlcIncomingResolverExitCancelHodl(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.resolve()
	notifyData := <-ctx.registry.notifyChan
	notifyData.hodlChan <- invoices.HodlEvent{
		CircuitKey: testResCircuitKey,
	}
	ctx.waitForResult(false)
}

type mockHopIterator struct {
	hop.Iterator
}

func (h *mockHopIterator) HopPayload() (*hop.Payload, error) {
	return nil, nil
}

type mockOnionProcessor struct {
	offeredOnionBlob []byte
}

func (o *mockOnionProcessor) ReconstructHopIterator(r io.Reader, rHash []byte) (
	hop.Iterator, error) {

	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	o.offeredOnionBlob = data

	return &mockHopIterator{}, nil
}

type incomingResolverTestContext struct {
	registry       *mockRegistry
	witnessBeacon  *mockWitnessBeacon
	resolver       *htlcIncomingContestResolver
	notifier       *mockNotifier
	onionProcessor *mockOnionProcessor
	resolveErr     chan error
	nextResolver   ContractResolver
	t              *testing.T
}

func newIncomingResolverTestContext(t *testing.T) *incomingResolverTestContext {
	notifier := &mockNotifier{
		epochChan: make(chan *chainntnfs.BlockEpoch),
		spendChan: make(chan *chainntnfs.SpendDetail),
		confChan:  make(chan *chainntnfs.TxConfirmation),
	}
	witnessBeacon := newMockWitnessBeacon()
	registry := &mockRegistry{
		notifyChan: make(chan notifyExitHopData, 1),
	}

	onionProcessor := &mockOnionProcessor{}

	checkPointChan := make(chan struct{}, 1)

	chainCfg := ChannelArbitratorConfig{
		ChainArbitratorConfig: ChainArbitratorConfig{
			Notifier:       notifier,
			PreimageDB:     witnessBeacon,
			Registry:       registry,
			OnionProcessor: onionProcessor,
		},
	}

	cfg := ResolverConfig{
		ChannelArbitratorConfig: chainCfg,
		Checkpoint: func(_ ContractResolver) error {
			checkPointChan <- struct{}{}
			return nil
		},
	}
	resolver := &htlcIncomingContestResolver{
		htlcSuccessResolver: htlcSuccessResolver{
			contractResolverKit: *newContractResolverKit(cfg),
			htlcResolution:      lnwallet.IncomingHtlcResolution{},
			htlc: channeldb.HTLC{
				RHash:     testResHash,
				OnionBlob: testOnionBlob,
			},
		},
		htlcExpiry: testHtlcExpiry,
	}

	return &incomingResolverTestContext{
		registry:       registry,
		witnessBeacon:  witnessBeacon,
		resolver:       resolver,
		notifier:       notifier,
		onionProcessor: onionProcessor,
		t:              t,
	}
}

func (i *incomingResolverTestContext) resolve() {
	// Start resolver.
	i.resolveErr = make(chan error, 1)
	go func() {
		var err error
		i.nextResolver, err = i.resolver.Resolve()
		i.resolveErr <- err
	}()

	// Notify initial block height.
	i.notifyEpoch(testInitialBlockHeight)
}

func (i *incomingResolverTestContext) notifyEpoch(height int32) {
	i.notifier.epochChan <- &chainntnfs.BlockEpoch{
		Height: height,
	}
}

func (i *incomingResolverTestContext) waitForResult(expectSuccessRes bool) {
	i.t.Helper()

	err := <-i.resolveErr
	if err != nil {
		i.t.Fatal(err)
	}

	if !expectSuccessRes {
		if err != nil {
			i.t.Fatal("expected no next resolver")
		}
		return
	}

	successResolver, ok := i.nextResolver.(*htlcSuccessResolver)
	if !ok {
		i.t.Fatal("expected htlcSuccessResolver")
	}

	if successResolver.htlcResolution.Preimage != testResPreimage {
		i.t.Fatal("invalid preimage")
	}

	successTx := successResolver.htlcResolution.SignedSuccessTx
	if successTx != nil &&
		!bytes.Equal(successTx.TxIn[0].Witness[3], testResPreimage[:]) {

		i.t.Fatal("invalid preimage")
	}
}
<commit_msg>cnct: fix incoming resolver assertion<commit_after>package contractcourt

import (
	"bytes"
	"io"
	"io/ioutil"
	"testing"

	"github.com/lightningnetwork/lnd/channeldb"
	"github.com/lightningnetwork/lnd/htlcswitch/hop"
	"github.com/lightningnetwork/lnd/invoices"
	"github.com/lightningnetwork/lnd/lnwallet"

	"github.com/lightningnetwork/lnd/chainntnfs"
	"github.com/lightningnetwork/lnd/lntypes"
)

const (
	testInitialBlockHeight = 100
	testHtlcExpiry         = 150
)

var (
	testResPreimage   = lntypes.Preimage{1, 2, 3}
	testResHash       = testResPreimage.Hash()
	testResCircuitKey = channeldb.CircuitKey{}
	testOnionBlob     = []byte{4, 5, 6}
)

// TestHtlcIncomingResolverFwdPreimageKnown tests resolution of a forwarded htlc
// for which the preimage is already known initially.
func TestHtlcIncomingResolverFwdPreimageKnown(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyErr = channeldb.ErrInvoiceNotFound
	ctx.witnessBeacon.lookupPreimage[testResHash] = testResPreimage
	ctx.resolve()
	ctx.waitForResult(true)
}

// TestHtlcIncomingResolverFwdContestedSuccess tests resolution of a forwarded
// htlc for which the preimage becomes known after the resolver has been
// started.
func TestHtlcIncomingResolverFwdContestedSuccess(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyErr = channeldb.ErrInvoiceNotFound
	ctx.resolve()

	// Simulate a new block coming in. HTLC is not yet expired.
	ctx.notifyEpoch(testInitialBlockHeight + 1)

	ctx.witnessBeacon.preImageUpdates <- testResPreimage
	ctx.waitForResult(true)
}

// TestHtlcIncomingResolverFwdContestedTimeout tests resolution of a forwarded
// htlc that times out after the resolver has been started.
func TestHtlcIncomingResolverFwdContestedTimeout(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyErr = channeldb.ErrInvoiceNotFound
	ctx.resolve()

	// Simulate a new block coming in. HTLC expires.
	ctx.notifyEpoch(testHtlcExpiry)

	ctx.waitForResult(false)
}

// TestHtlcIncomingResolverFwdTimeout tests resolution of a forwarded htlc that
// has already expired when the resolver starts.
func TestHtlcIncomingResolverFwdTimeout(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)

	ctx.registry.notifyErr = channeldb.ErrInvoiceNotFound
	ctx.witnessBeacon.lookupPreimage[testResHash] = testResPreimage
	ctx.resolver.htlcExpiry = 90
	ctx.resolve()
	ctx.waitForResult(false)
}

// TestHtlcIncomingResolverExitSettle tests resolution of an exit hop htlc for
// which the invoice has already been settled when the resolver starts.
func TestHtlcIncomingResolverExitSettle(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyEvent = &invoices.HodlEvent{
		CircuitKey: testResCircuitKey,
		Preimage:   &testResPreimage,
	}
	ctx.resolve()

	data := <-ctx.registry.notifyChan
	if data.expiry != testHtlcExpiry {
		t.Fatal("incorrect expiry")
	}
	if data.currentHeight != testInitialBlockHeight {
		t.Fatal("incorrect block height")
	}

	ctx.waitForResult(true)

	if !bytes.Equal(
		ctx.onionProcessor.offeredOnionBlob, testOnionBlob,
	) {
		t.Fatal("unexpected onion blob")
	}
}

// TestHtlcIncomingResolverExitCancel tests resolution of an exit hop htlc for
// an invoice that is already canceled when the resolver starts.
func TestHtlcIncomingResolverExitCancel(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.registry.notifyEvent = &invoices.HodlEvent{
		CircuitKey: testResCircuitKey,
	}
	ctx.resolve()
	ctx.waitForResult(false)
}

// TestHtlcIncomingResolverExitSettleHodl tests resolution of an exit hop htlc
// for a hodl invoice that is settled after the resolver has started.
func TestHtlcIncomingResolverExitSettleHodl(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.resolve()

	notifyData := <-ctx.registry.notifyChan
	notifyData.hodlChan <- invoices.HodlEvent{
		CircuitKey: testResCircuitKey,
		Preimage:   &testResPreimage,
	}

	ctx.waitForResult(true)
}

// TestHtlcIncomingResolverExitTimeoutHodl tests resolution of an exit hop htlc
// for a hodl invoice that times out.
func TestHtlcIncomingResolverExitTimeoutHodl(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.resolve()
	ctx.notifyEpoch(testHtlcExpiry)
	ctx.waitForResult(false)
}

// TestHtlcIncomingResolverExitCancelHodl tests resolution of an exit hop htlc
// for a hodl invoice that is canceled after the resolver has started.
func TestHtlcIncomingResolverExitCancelHodl(t *testing.T) {
	t.Parallel()
	defer timeout(t)()

	ctx := newIncomingResolverTestContext(t)
	ctx.resolve()
	notifyData := <-ctx.registry.notifyChan
	notifyData.hodlChan <- invoices.HodlEvent{
		CircuitKey: testResCircuitKey,
	}
	ctx.waitForResult(false)
}

type mockHopIterator struct {
	hop.Iterator
}

func (h *mockHopIterator) HopPayload() (*hop.Payload, error) {
	return nil, nil
}

type mockOnionProcessor struct {
	offeredOnionBlob []byte
}

func (o *mockOnionProcessor) ReconstructHopIterator(r io.Reader, rHash []byte) (
	hop.Iterator, error) {

	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	o.offeredOnionBlob = data

	return &mockHopIterator{}, nil
}

type incomingResolverTestContext struct {
	registry       *mockRegistry
	witnessBeacon  *mockWitnessBeacon
	resolver       *htlcIncomingContestResolver
	notifier       *mockNotifier
	onionProcessor *mockOnionProcessor
	resolveErr     chan error
	nextResolver   ContractResolver
	t              *testing.T
}

func newIncomingResolverTestContext(t *testing.T) *incomingResolverTestContext {
	notifier := &mockNotifier{
		epochChan: make(chan *chainntnfs.BlockEpoch),
		spendChan: make(chan *chainntnfs.SpendDetail),
		confChan:  make(chan *chainntnfs.TxConfirmation),
	}
	witnessBeacon := newMockWitnessBeacon()
	registry := &mockRegistry{
		notifyChan: make(chan notifyExitHopData, 1),
	}

	onionProcessor := &mockOnionProcessor{}

	checkPointChan := make(chan struct{}, 1)

	chainCfg := ChannelArbitratorConfig{
		ChainArbitratorConfig: ChainArbitratorConfig{
			Notifier:       notifier,
			PreimageDB:     witnessBeacon,
			Registry:       registry,
			OnionProcessor: onionProcessor,
		},
	}

	cfg := ResolverConfig{
		ChannelArbitratorConfig: chainCfg,
		Checkpoint: func(_ ContractResolver) error {
			checkPointChan <- struct{}{}
			return nil
		},
	}
	resolver := &htlcIncomingContestResolver{
		htlcSuccessResolver: htlcSuccessResolver{
			contractResolverKit: *newContractResolverKit(cfg),
			htlcResolution:      lnwallet.IncomingHtlcResolution{},
			htlc: channeldb.HTLC{
				RHash:     testResHash,
				OnionBlob: testOnionBlob,
			},
		},
		htlcExpiry: testHtlcExpiry,
	}

	return &incomingResolverTestContext{
		registry:       registry,
		witnessBeacon:  witnessBeacon,
		resolver:       resolver,
		notifier:       notifier,
		onionProcessor: onionProcessor,
		t:              t,
	}
}

func (i *incomingResolverTestContext) resolve() {
	// Start resolver.
	i.resolveErr = make(chan error, 1)
	go func() {
		var err error
		i.nextResolver, err = i.resolver.Resolve()
		i.resolveErr <- err
	}()

	// Notify initial block height.
	i.notifyEpoch(testInitialBlockHeight)
}

func (i *incomingResolverTestContext) notifyEpoch(height int32) {
	i.notifier.epochChan <- &chainntnfs.BlockEpoch{
		Height: height,
	}
}

func (i *incomingResolverTestContext) waitForResult(expectSuccessRes bool) {
	i.t.Helper()

	err := <-i.resolveErr
	if err != nil {
		i.t.Fatal(err)
	}

	if !expectSuccessRes {
		if i.nextResolver != nil {
			i.t.Fatal("expected no next resolver")
		}
		return
	}

	successResolver, ok := i.nextResolver.(*htlcSuccessResolver)
	if !ok {
		i.t.Fatal("expected htlcSuccessResolver")
	}

	if successResolver.htlcResolution.Preimage != testResPreimage {
		i.t.Fatal("invalid preimage")
	}

	successTx := successResolver.htlcResolution.SignedSuccessTx
	if successTx != nil &&
		!bytes.Equal(successTx.TxIn[0].Witness[3], testResPreimage[:]) {

		i.t.Fatal("invalid preimage")
	}
}
<|endoftext|>


<commit_before>package sched

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"html/template"
	"io"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/expr"
	"github.com/StackExchange/bosun/expr/parse"
)

type Context struct {
	*State
	Alert *conf.Alert

	schedule    *Schedule
	Attachments []*conf.Attachment
}

func (s *Schedule) Data(st *State, a *conf.Alert, isEmail bool) *Context {
	c := Context{
		State:    st,
		Alert:    a,
		schedule: s,
	}
	if isEmail {
		c.Attachments = make([]*conf.Attachment, 0)
	}
	return &c
}

type unknownContext struct {
	Time  time.Time
	Name  string
	Group expr.AlertKeys

	schedule *Schedule
}

func (s *Schedule) unknownData(t time.Time, name string, group expr.AlertKeys) *unknownContext {
	return &unknownContext{
		Time:     t,
		Group:    group,
		Name:     name,
		schedule: s,
	}
}

// URL returns a prepopulated URL for external access, with path and query empty.
func (s *Schedule) URL() *url.URL {
	u := url.URL{
		Scheme: "http",
		Host:   s.Conf.HttpListen,
	}
	if strings.HasPrefix(s.Conf.HttpListen, ":") {
		h, err := os.Hostname()
		if err != nil {
			u.Host = "localhost" + u.Host
		} else {
			u.Host = h + u.Host
		}
	}
	return &u
}

// Ack returns the URL to acknowledge an alert.
func (c *Context) Ack() string {
	u := c.schedule.URL()
	u.Path = "/action"
	u.RawQuery = url.Values{
		"type": []string{"ack"},
		"key":  []string{c.Alert.Name + c.State.Group.String()},
	}.Encode()
	return u.String()
}

// HostView returns the URL to the host view page.
func (c *Context) HostView(host string) string {
	u := c.schedule.URL()
	u.Path = "/host"
	u.RawQuery = fmt.Sprintf("time=1d-ago&host=%s", host)
	return u.String()
}

func (c *Context) makeLink(path string, v *url.Values) (string, error) {
	u := url.URL{
		Scheme:   "http",
		Host:     c.schedule.Conf.HttpListen,
		Path:     path,
		RawQuery: v.Encode(),
	}
	if strings.HasPrefix(c.schedule.Conf.HttpListen, ":") {
		h, err := os.Hostname()
		if err != nil {
			return "", err
		}
		u.Host = h + u.Host
	}
	return u.String(), nil
}

func (c *Context) Expr(v string) (string, error) {
	p := url.Values{}
	p.Add("expr", base64.StdEncoding.EncodeToString([]byte(opentsdb.ReplaceTags(v, c.Group))))
	return c.makeLink("/expr", &p)
}

func (c *Context) Rule() (string, error) {
	t, err := c.schedule.Conf.AlertTemplateStrings()
	if err != nil {
		return "", err
	}
	p := url.Values{}
	adef := base64.StdEncoding.EncodeToString([]byte(t.Alerts[c.Alert.Name]))
	tdef := base64.StdEncoding.EncodeToString([]byte(t.Templates[c.Alert.Template.Name]))
	p.Add("alert", adef)
	p.Add("template", tdef)
	return c.makeLink("/rule", &p)
}

func (s *Schedule) ExecuteBody(w io.Writer, a *conf.Alert, st *State, isEmail bool) ([]*conf.Attachment, error) {
	t := a.Template
	if t == nil || t.Body == nil {
		return nil, nil
	}
	c := s.Data(st, a, isEmail)
	return c.Attachments, t.Body.Execute(w, c)
}

func (s *Schedule) ExecuteSubject(w io.Writer, a *conf.Alert, st *State) error {
	t := a.Template
	if t == nil || t.Subject == nil {
		return nil
	}
	return t.Subject.Execute(w, s.Data(st, a, false))
}

func (c *Context) eval(v interface{}, filter bool, series bool, autods int) ([]*expr.Result, error) {
	var e *expr.Expr
	var err error
	switch v := v.(type) {
	case string:
		e, err = expr.New(v)
	case *expr.Expr:
		e = v
	default:
		return nil, fmt.Errorf("expected string or expression, got %T (%v)", v, v)
	}
	if err != nil {
		return nil, fmt.Errorf("%v: %v", v, err)
	}
	if series && e.Root.Return() != parse.TYPE_SERIES {
		return nil, fmt.Errorf("egraph: requires an expression that returns a series")
	}
	res, _, err := e.Execute(c.schedule.cache, nil, c.schedule.CheckStart, autods, c.Alert.UnjoinedOK, c.schedule.Search, c.schedule.Lookups, c.schedule.Conf.AlertSquelched(c.Alert))
	if err != nil {
		return nil, fmt.Errorf("%s: %v", v, err)
	}
	if !filter {
		return res.Results, nil
	}
	for _, r := range res.Results {
		if r.Group.Equal(c.State.Group) {
			return []*expr.Result{r}, nil
		}
	}
	for _, r := range res.Results {
		if c.State.Group.Subset(r.Group) {
			return []*expr.Result{r}, nil
		}
	}
	for _, r := range res.Results {
		if r.Group == nil {
			return []*expr.Result{r}, nil
		}
	}
	return nil, nil
}

// Lookup returns the value for a key in the lookup table for the context's tagset.
func (c *Context) Lookup(table, key string) (string, error) {
	l, ok := c.schedule.Lookups[table]
	if !ok {
		return "", fmt.Errorf("unknown lookup table %v", table)
	}
	if v, ok := l.Get(key, c.Group); ok {
		return v, nil
	} else {
		return "", fmt.Errorf("no entry for key %v in table %v for tagset %v", key, table, c.Group)
	}
}

// Eval executes the given expression and returns a value with corresponding
// tags to the context's tags. If no such result is found, the first result with
// nil tags is returned. If no such result is found, nil is returned.
func (c *Context) Eval(v interface{}) (interface{}, error) {
	res, err := c.eval(v, true, false, 0)
	if err != nil {
		return nil, err
	}
	if len(res) != 1 {
		return nil, fmt.Errorf("expected 1 result, got %v", len(res))
	}
	return res[0].Value, nil
}

// EvalAll returns the executed expression.
func (c *Context) EvalAll(v interface{}) (interface{}, error) {
	return c.eval(v, false, false, 0)
}

func (c *Context) IsEmail() bool {
	return c.Attachments != nil
}

func (c *Context) graph(v interface{}, filter bool) (interface{}, error) {
	res, err := c.eval(v, filter, true, 1000)
	if err != nil {
		return nil, err
	}
	var buf bytes.Buffer
	if err := c.schedule.ExprGraph(nil, &buf, res, fmt.Sprint(v), time.Now().UTC()); err != nil {
		return nil, err
	}
	if c.IsEmail() {
		name := fmt.Sprintf("%d.svg", len(c.Attachments)+1)
		c.Attachments = append(c.Attachments, &conf.Attachment{
			Data:        buf.Bytes(),
			Filename:    name,
			ContentType: "image/svg+xml",
		})
		return template.HTML(fmt.Sprintf(`<img alt="%s" src="cid:%s" />`,
			template.HTMLEscapeString(fmt.Sprint(v)),
			name,
		)), nil
	}
	return template.HTML(buf.String()), nil
}

func (c *Context) Graph(v interface{}) (interface{}, error) {
	return c.graph(v, true)
}

func (c *Context) GraphAll(v interface{}) (interface{}, error) {
	return c.graph(v, false)
}

func (c *Context) GetMeta(metric, name string, v interface{}) (interface{}, error) {
	var t opentsdb.TagSet
	switch v := v.(type) {
	case string:
		var err error
		t, err = opentsdb.ParseTags(v)
		if err != nil {
			return t, err
		}
	case opentsdb.TagSet:
		t = v
	}
	meta := c.schedule.GetMetadata(metric, t)
	if name == "" {
		return meta, nil
	}
	fm := make([]metadata.Metasend, 0)
	for _, m := range meta {
		if m.Name == name {
			fm = append(fm, m)
		}
	}
	return fm, nil
}

func (c *Context) LeftJoin(q ...interface{}) (interface{}, error) {
	if len(q) < 2 {
		return nil, fmt.Errorf("need at least two expressions, got %v", len(q))
	}
	matrix := make([][]*expr.Result, 0)
	results := make([][]*expr.Result, len(q))
	for col, v := range q {
		res, err := c.eval(v, false, false, 0)
		if err != nil {
			return nil, err
		}
		results[col] = res
	}
	for row, first := range results[0] {
		matrix = append(matrix, make([]*expr.Result, len(q)))
		matrix[row][0] = first
		for col, res := range results[1:] {
			for _, r := range res {
				if first.Group.Subset(r.Group) {
					matrix[row][col+1] = r
				}
			}
		}
	}
	return matrix, nil
}
<commit_msg>Set empty rows to NaN on leftjoin<commit_after>package sched

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"html/template"
	"io"
	"math"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/expr"
	"github.com/StackExchange/bosun/expr/parse"
)

type Context struct {
	*State
	Alert *conf.Alert

	schedule    *Schedule
	Attachments []*conf.Attachment
}

func (s *Schedule) Data(st *State, a *conf.Alert, isEmail bool) *Context {
	c := Context{
		State:    st,
		Alert:    a,
		schedule: s,
	}
	if isEmail {
		c.Attachments = make([]*conf.Attachment, 0)
	}
	return &c
}

type unknownContext struct {
	Time  time.Time
	Name  string
	Group expr.AlertKeys

	schedule *Schedule
}

func (s *Schedule) unknownData(t time.Time, name string, group expr.AlertKeys) *unknownContext {
	return &unknownContext{
		Time:     t,
		Group:    group,
		Name:     name,
		schedule: s,
	}
}

// URL returns a prepopulated URL for external access, with path and query empty.
func (s *Schedule) URL() *url.URL {
	u := url.URL{
		Scheme: "http",
		Host:   s.Conf.HttpListen,
	}
	if strings.HasPrefix(s.Conf.HttpListen, ":") {
		h, err := os.Hostname()
		if err != nil {
			u.Host = "localhost" + u.Host
		} else {
			u.Host = h + u.Host
		}
	}
	return &u
}

// Ack returns the URL to acknowledge an alert.
func (c *Context) Ack() string {
	u := c.schedule.URL()
	u.Path = "/action"
	u.RawQuery = url.Values{
		"type": []string{"ack"},
		"key":  []string{c.Alert.Name + c.State.Group.String()},
	}.Encode()
	return u.String()
}

// HostView returns the URL to the host view page.
func (c *Context) HostView(host string) string {
	u := c.schedule.URL()
	u.Path = "/host"
	u.RawQuery = fmt.Sprintf("time=1d-ago&host=%s", host)
	return u.String()
}

func (c *Context) makeLink(path string, v *url.Values) (string, error) {
	u := url.URL{
		Scheme:   "http",
		Host:     c.schedule.Conf.HttpListen,
		Path:     path,
		RawQuery: v.Encode(),
	}
	if strings.HasPrefix(c.schedule.Conf.HttpListen, ":") {
		h, err := os.Hostname()
		if err != nil {
			return "", err
		}
		u.Host = h + u.Host
	}
	return u.String(), nil
}

func (c *Context) Expr(v string) (string, error) {
	p := url.Values{}
	p.Add("expr", base64.StdEncoding.EncodeToString([]byte(opentsdb.ReplaceTags(v, c.Group))))
	return c.makeLink("/expr", &p)
}

func (c *Context) Rule() (string, error) {
	t, err := c.schedule.Conf.AlertTemplateStrings()
	if err != nil {
		return "", err
	}
	p := url.Values{}
	adef := base64.StdEncoding.EncodeToString([]byte(t.Alerts[c.Alert.Name]))
	tdef := base64.StdEncoding.EncodeToString([]byte(t.Templates[c.Alert.Template.Name]))
	p.Add("alert", adef)
	p.Add("template", tdef)
	return c.makeLink("/rule", &p)
}

func (s *Schedule) ExecuteBody(w io.Writer, a *conf.Alert, st *State, isEmail bool) ([]*conf.Attachment, error) {
	t := a.Template
	if t == nil || t.Body == nil {
		return nil, nil
	}
	c := s.Data(st, a, isEmail)
	return c.Attachments, t.Body.Execute(w, c)
}

func (s *Schedule) ExecuteSubject(w io.Writer, a *conf.Alert, st *State) error {
	t := a.Template
	if t == nil || t.Subject == nil {
		return nil
	}
	return t.Subject.Execute(w, s.Data(st, a, false))
}

func (c *Context) eval(v interface{}, filter bool, series bool, autods int) ([]*expr.Result, error) {
	var e *expr.Expr
	var err error
	switch v := v.(type) {
	case string:
		e, err = expr.New(v)
	case *expr.Expr:
		e = v
	default:
		return nil, fmt.Errorf("expected string or expression, got %T (%v)", v, v)
	}
	if err != nil {
		return nil, fmt.Errorf("%v: %v", v, err)
	}
	if series && e.Root.Return() != parse.TYPE_SERIES {
		return nil, fmt.Errorf("egraph: requires an expression that returns a series")
	}
	res, _, err := e.Execute(c.schedule.cache, nil, c.schedule.CheckStart, autods, c.Alert.UnjoinedOK, c.schedule.Search, c.schedule.Lookups, c.schedule.Conf.AlertSquelched(c.Alert))
	if err != nil {
		return nil, fmt.Errorf("%s: %v", v, err)
	}
	if !filter {
		return res.Results, nil
	}
	for _, r := range res.Results {
		if r.Group.Equal(c.State.Group) {
			return []*expr.Result{r}, nil
		}
	}
	for _, r := range res.Results {
		if c.State.Group.Subset(r.Group) {
			return []*expr.Result{r}, nil
		}
	}
	for _, r := range res.Results {
		if r.Group == nil {
			return []*expr.Result{r}, nil
		}
	}
	return nil, nil
}

// Lookup returns the value for a key in the lookup table for the context's tagset.
func (c *Context) Lookup(table, key string) (string, error) {
	l, ok := c.schedule.Lookups[table]
	if !ok {
		return "", fmt.Errorf("unknown lookup table %v", table)
	}
	if v, ok := l.Get(key, c.Group); ok {
		return v, nil
	} else {
		return "", fmt.Errorf("no entry for key %v in table %v for tagset %v", key, table, c.Group)
	}
}

// Eval executes the given expression and returns a value with corresponding
// tags to the context's tags. If no such result is found, the first result with
// nil tags is returned. If no such result is found, nil is returned.
func (c *Context) Eval(v interface{}) (interface{}, error) {
	res, err := c.eval(v, true, false, 0)
	if err != nil {
		return nil, err
	}
	if len(res) != 1 {
		return nil, fmt.Errorf("expected 1 result, got %v", len(res))
	}
	return res[0].Value, nil
}

// EvalAll returns the executed expression.
func (c *Context) EvalAll(v interface{}) (interface{}, error) {
	return c.eval(v, false, false, 0)
}

func (c *Context) IsEmail() bool {
	return c.Attachments != nil
}

func (c *Context) graph(v interface{}, filter bool) (interface{}, error) {
	res, err := c.eval(v, filter, true, 1000)
	if err != nil {
		return nil, err
	}
	var buf bytes.Buffer
	if err := c.schedule.ExprGraph(nil, &buf, res, fmt.Sprint(v), time.Now().UTC()); err != nil {
		return nil, err
	}
	if c.IsEmail() {
		name := fmt.Sprintf("%d.svg", len(c.Attachments)+1)
		c.Attachments = append(c.Attachments, &conf.Attachment{
			Data:        buf.Bytes(),
			Filename:    name,
			ContentType: "image/svg+xml",
		})
		return template.HTML(fmt.Sprintf(`<img alt="%s" src="cid:%s" />`,
			template.HTMLEscapeString(fmt.Sprint(v)),
			name,
		)), nil
	}
	return template.HTML(buf.String()), nil
}

func (c *Context) Graph(v interface{}) (interface{}, error) {
	return c.graph(v, true)
}

func (c *Context) GraphAll(v interface{}) (interface{}, error) {
	return c.graph(v, false)
}

func (c *Context) GetMeta(metric, name string, v interface{}) (interface{}, error) {
	var t opentsdb.TagSet
	switch v := v.(type) {
	case string:
		var err error
		t, err = opentsdb.ParseTags(v)
		if err != nil {
			return t, err
		}
	case opentsdb.TagSet:
		t = v
	}
	meta := c.schedule.GetMetadata(metric, t)
	if name == "" {
		return meta, nil
	}
	fm := make([]metadata.Metasend, 0)
	for _, m := range meta {
		if m.Name == name {
			fm = append(fm, m)
		}
	}
	return fm, nil
}

func (c *Context) LeftJoin(q ...interface{}) (interface{}, error) {
	if len(q) < 2 {
		return nil, fmt.Errorf("need at least two expressions, got %v", len(q))
	}
	matrix := make([][]*expr.Result, 0)
	results := make([][]*expr.Result, len(q))
	for col, v := range q {
		res, err := c.eval(v, false, false, 0)
		if err != nil {
			return nil, err
		}
		results[col] = res
	}
	for row, first := range results[0] {
		matrix = append(matrix, make([]*expr.Result, len(q)))
		matrix[row][0] = first
		for col, res := range results[1:] {
			for _, r := range res {
				if first.Group.Subset(r.Group) {
					matrix[row][col+1] = r
					break
				}
				// Fill emtpy cells with NaN Value, so calling .Valie is not a nil pointer dereference
				matrix[row][col+1] = &expr.Result{Value: expr.Number(math.NaN())}
			}
		}
	}
	return matrix, nil
}
<|endoftext|>


<commit_before>// +build windows

package main

import (
	"log"
	"testing"
)

func TestCreateOpenDeleteKey(t *testing.T) {

	var registry = realRegistry{}

	// create
	err := registry.CreateKey(PATH_TIMERS)
	if err != nil {
		log.Fatalln("Error in CreateKey", err)
	}

	// store value
	expected := uint64(1234)
	err = registry.SetQword(PATH_TIMERS, "t1", expected)
	if err != nil {
		log.Fatalln("Error in SetQword", err)
	}

	// list values
	timers1 := registry.EnumValues(PATH_TIMERS)
	if len(timers1) == 0 {
		t.Errorf("No timers found")
	}

	// read value
	actual, err1 := registry.GetQword(PATH_TIMERS, "t1")
	if err1 != nil {
		log.Fatalln("Error in GetQword", err1)
	}
	if actual != expected {
		t.Errorf("Expected: %q, was: %q", expected, actual)
	}

	// delete value
	err = registry.DeleteValue(PATH_TIMERS, "t1")
	if err != nil {
		t.Errorf("Error deleting value t1, %s", err)
	}
	timers2 := registry.EnumValues(PATH_TIMERS)
	if len(timers2) != len(timers1)-1 {
		t.Errorf("Timers should have been deleted")
	}

	// delete keys
	err = registry.DeleteKey(PATH_TIMERS)
	if err != nil {
		t.Errorf("Error deleting %s, %s", PATH_TIMERS.lpSubKey, err)
	}
	err = registry.DeleteKey(PATH_SOFTWARE)
	if err != nil {
		t.Errorf("Error deleting %s, %s", PATH_SOFTWARE.lpSubKey, err)
	}
}
<commit_msg>log errors instead of failing<commit_after>// +build windows

package main

import (
	"testing"
)

func TestCreateOpenDeleteKey(t *testing.T) {

	var registry = realRegistry{}

	// create
	err := registry.CreateKey(PATH_TIMERS)
	if err != nil {
		t.Errorf("Error in CreateKey", err)
	}

	// store value
	expected := uint64(1234)
	err = registry.SetQword(PATH_TIMERS, "t1", expected)
	if err != nil {
		t.Errorf("Error in SetQword", err)
	}

	// list values
	timers1 := registry.EnumValues(PATH_TIMERS)
	if len(timers1) == 0 {
		t.Errorf("No timers found")
	}

	// read value
	actual, err1 := registry.GetQword(PATH_TIMERS, "t1")
	if err1 != nil {
		t.Errorf("Error in GetQword", err1)
	}
	if actual != expected {
		t.Errorf("Expected: %q, was: %q", expected, actual)
	}

	// delete value
	err = registry.DeleteValue(PATH_TIMERS, "t1")
	if err != nil {
		t.Errorf("Error deleting value t1, %s", err)
	}
	timers2 := registry.EnumValues(PATH_TIMERS)
	if len(timers2) != len(timers1)-1 {
		t.Errorf("Timers should have been deleted")
	}

	// delete keys
	err = registry.DeleteKey(PATH_TIMERS)
	if err != nil {
		t.Errorf("Error deleting %s, %s", PATH_TIMERS.lpSubKey, err)
	}
	err = registry.DeleteKey(PATH_SOFTWARE)
	if err != nil {
		t.Errorf("Error deleting %s, %s", PATH_SOFTWARE.lpSubKey, err)
	}
}
<|endoftext|>


<commit_before>package gearcmd

import (
	"bufio"
	"bytes"
	"container/ring"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/Clever/baseworker-go"
	"github.com/Clever/gearcmd/argsparser"
	kayvee "gopkg.in/Clever/kayvee-go.v1"
)

// TaskConfig defines the configuration for the task.
type TaskConfig struct {
	FunctionName, FunctionCmd string
	WarningLines              int
	ParseArgs                 bool
	CmdTimeout                time.Duration
}

// Process runs the Gearman job by running the configured task.
// We need to implement the Task interface so we return (byte[], error)
// though the byte[] is always nil.
func (conf TaskConfig) Process(job baseworker.Job) ([]byte, error) {
	// This wraps the actual processing to do some logging
	log.Printf(kayvee.FormatLog("gearcmd", "info", "START",
		map[string]interface{}{"function_name": conf.FunctionName, "job_id": getJobId(job), "job_data": string(job.Data())}))
	start := time.Now()
	err := conf.doProcess(job)
	end := time.Now()
	durationStr := fmt.Sprintf("%d", int32(end.Sub(start).Seconds()*1000))
	if err != nil {
		log.Printf(kayvee.FormatLog("gearcmd", "error", "END_WITH_ERROR",
			map[string]interface{}{"function_name": conf.FunctionName, "job_id": getJobId(job),
				"error_message": err.Error(), "job_data": string(job.Data()), "duration": durationStr}))
	} else {
		log.Printf(kayvee.FormatLog("gearcmd", "info", "END",
			map[string]interface{}{"function_name": conf.FunctionName, "job_id": getJobId(job),
				"job_data": string(job.Data()), "duration": durationStr}))
	}
	return nil, err
}

// getJobId returns the jobId from the job handle
func getJobId(job baseworker.Job) string {
	splits := strings.Split(job.Handle(), ":")
	return splits[len(splits)-1]
}

func (conf TaskConfig) doProcess(job baseworker.Job) error {

	defer func() {
		// If we panicked then set the panic message as a warning. Gearman-go will
		// handle marking this job as failed.
		if r := recover(); r != nil {
			err := r.(error)
			job.SendWarning([]byte(err.Error()))
		}
	}()
	var args []string
	var err error
	if conf.ParseArgs {
		args, err = argsparser.ParseArgs(string(job.Data()))
		if err != nil {
			return err
		}
	} else {
		args = []string{string(job.Data())}

	}
	cmd := exec.Command(conf.FunctionCmd, args...)

	// Write the stdout and stderr of the process to both this process' stdout and stderr
	// and also write it to a byte buffer so that we can return it with the Gearman job
	// data as necessary.
	var stderrbuf bytes.Buffer
	cmd.Stderr = io.MultiWriter(os.Stderr, &stderrbuf)
	defer sendStderrWarnings(&stderrbuf, job, conf.WarningLines)

	stdoutReader, stdoutWriter := io.Pipe()
	cmd.Stdout = io.MultiWriter(os.Stdout, stdoutWriter)

	done := make(chan error)
	go func() {
		defer close(done)

		finishedProcessingStdout := make(chan error)
		go func() {
			finishedProcessingStdout <- streamToGearman(stdoutReader, job)
		}()

		// Save the cmdErr. We want to process stdout and stderr before we return it
		cmdErr := cmd.Run()
		stdoutWriter.Close()

		stdoutErr := <-finishedProcessingStdout
		if cmdErr != nil {
			done <- cmdErr
		} else if stdoutErr != nil {
			done <- stdoutErr
		}
	}()
	// No timeout
	if conf.CmdTimeout == 0 {
		// Will be nil if the channel was closed without any errors
		return <-done
	}
	select {
	case err := <-done:
		// Will be nil if the channel was closed without any errors
		return err
	case <-time.After(conf.CmdTimeout):
		return fmt.Errorf("process timed out after %s", conf.CmdTimeout.String())
	}
}

// This function streams the reader to the Gearman job (through job.SendData())
func streamToGearman(reader io.Reader, job baseworker.Job) error {
	buffer := make([]byte, 1024)
	for {
		n, err := reader.Read(buffer)
		// Process the data before processing the error (as per the io.Reader docs)
		if n > 0 {
			job.SendData(buffer[:n])
		}
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// sendStderrWarnings sends the last X lines in the stderr output and to the job's warnings
// field
func sendStderrWarnings(buffer io.Reader, job baseworker.Job, warningLines int) error {
	scanner := bufio.NewScanner(buffer)
	// Create a circular buffer for the last X lines
	lastStderrLines := ring.New(warningLines)
	for scanner.Scan() {
		lastStderrLines = lastStderrLines.Next()
		lastStderrLines.Value = scanner.Bytes()
	}
	// Walk forward through the buffer to get all the last X entries. Note that we call next first
	// so that we start at the oldest entry.
	for i := 0; i < lastStderrLines.Len(); i++ {
		if lastStderrLines = lastStderrLines.Next(); lastStderrLines.Value != nil {
			job.SendWarning(append(lastStderrLines.Value.([]byte), byte('\n')))
		}
	}
	return scanner.Err()
}
<commit_msg>add metric type to duration<commit_after>package gearcmd

import (
	"bufio"
	"bytes"
	"container/ring"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/Clever/baseworker-go"
	"github.com/Clever/gearcmd/argsparser"
	"gopkg.in/Clever/kayvee-go.v1"
)

// TaskConfig defines the configuration for the task.
type TaskConfig struct {
	FunctionName, FunctionCmd string
	WarningLines              int
	ParseArgs                 bool
	CmdTimeout                time.Duration
}

// Process runs the Gearman job by running the configured task.
// We need to implement the Task interface so we return (byte[], error)
// though the byte[] is always nil.
func (conf TaskConfig) Process(job baseworker.Job) ([]byte, error) {
	// This wraps the actual processing to do some logging
	log.Printf(kayvee.FormatLog("gearcmd", "info", "START",
		map[string]interface{}{"function_name": conf.FunctionName, "job_id": getJobId(job), "job_data": string(job.Data())}))
	start := time.Now()
	err := conf.doProcess(job)
	end := time.Now()
	data := map[string]interface{}{
		"function": conf.FunctionName, "job_id": getJobId(job), "job_data": string(job.Data()),
	}
	if err != nil {
		data["error_message"] = err.Error()
		log.Printf(kayvee.FormatLog("gearcmd", "error", "END_WITH_ERROR", data))
	} else {
		log.Printf(kayvee.FormatLog("gearcmd", "info", "END", data))
		// Hopefully none of our jobs last long enough for a uint64...
		log.Printf(kayvee.FormatLog("gearcmd", "info", "duration",
			map[string]interface{}{
				"value": uint64(end.Sub(start).Seconds() * 1000),
				"type":  "gauge", "function": conf.FunctionName,
			},
		))
	}
	return nil, err
}

// getJobId returns the jobId from the job handle
func getJobId(job baseworker.Job) string {
	splits := strings.Split(job.Handle(), ":")
	return splits[len(splits)-1]
}

func (conf TaskConfig) doProcess(job baseworker.Job) error {

	defer func() {
		// If we panicked then set the panic message as a warning. Gearman-go will
		// handle marking this job as failed.
		if r := recover(); r != nil {
			err := r.(error)
			job.SendWarning([]byte(err.Error()))
		}
	}()
	var args []string
	var err error
	if conf.ParseArgs {
		args, err = argsparser.ParseArgs(string(job.Data()))
		if err != nil {
			return err
		}
	} else {
		args = []string{string(job.Data())}

	}
	cmd := exec.Command(conf.FunctionCmd, args...)

	// Write the stdout and stderr of the process to both this process' stdout and stderr
	// and also write it to a byte buffer so that we can return it with the Gearman job
	// data as necessary.
	var stderrbuf bytes.Buffer
	cmd.Stderr = io.MultiWriter(os.Stderr, &stderrbuf)
	defer sendStderrWarnings(&stderrbuf, job, conf.WarningLines)

	stdoutReader, stdoutWriter := io.Pipe()
	cmd.Stdout = io.MultiWriter(os.Stdout, stdoutWriter)

	done := make(chan error)
	go func() {
		defer close(done)

		finishedProcessingStdout := make(chan error)
		go func() {
			finishedProcessingStdout <- streamToGearman(stdoutReader, job)
		}()

		// Save the cmdErr. We want to process stdout and stderr before we return it
		cmdErr := cmd.Run()
		stdoutWriter.Close()

		stdoutErr := <-finishedProcessingStdout
		if cmdErr != nil {
			done <- cmdErr
		} else if stdoutErr != nil {
			done <- stdoutErr
		}
	}()
	// No timeout
	if conf.CmdTimeout == 0 {
		// Will be nil if the channel was closed without any errors
		return <-done
	}
	select {
	case err := <-done:
		// Will be nil if the channel was closed without any errors
		return err
	case <-time.After(conf.CmdTimeout):
		return fmt.Errorf("process timed out after %s", conf.CmdTimeout.String())
	}
}

// This function streams the reader to the Gearman job (through job.SendData())
func streamToGearman(reader io.Reader, job baseworker.Job) error {
	buffer := make([]byte, 1024)
	for {
		n, err := reader.Read(buffer)
		// Process the data before processing the error (as per the io.Reader docs)
		if n > 0 {
			job.SendData(buffer[:n])
		}
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// sendStderrWarnings sends the last X lines in the stderr output and to the job's warnings
// field
func sendStderrWarnings(buffer io.Reader, job baseworker.Job, warningLines int) error {
	scanner := bufio.NewScanner(buffer)
	// Create a circular buffer for the last X lines
	lastStderrLines := ring.New(warningLines)
	for scanner.Scan() {
		lastStderrLines = lastStderrLines.Next()
		lastStderrLines.Value = scanner.Bytes()
	}
	// Walk forward through the buffer to get all the last X entries. Note that we call next first
	// so that we start at the oldest entry.
	for i := 0; i < lastStderrLines.Len(); i++ {
		if lastStderrLines = lastStderrLines.Next(); lastStderrLines.Value != nil {
			job.SendWarning(append(lastStderrLines.Value.([]byte), byte('\n')))
		}
	}
	return scanner.Err()
}
<|endoftext|>


<commit_before>package security_groups_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
)

var _ = Describe("Security Groups", func() {

	type AppResource struct {
		Metadata struct {
			Url string
		}
	}
	type AppsResponse struct {
		Resources []AppResource
	}

	type Stat struct {
		Stats struct {
			Host string
			Port int
		}
	}
	type StatsResponse map[string]Stat

	type DoraCurlResponse struct {
		Stdout     string
		Stderr     string
		ReturnCode int `json:"return_code"`
	}

	var serverAppName, securityGroupName, privateHost string
	var privatePort int

	BeforeEach(func() {
		serverAppName = generator.PrefixedRandomName("CATS-APP-")
		Expect(cf.Cf("push", serverAppName, "-m", "128M", "-p", assets.NewAssets().Dora, "-d", config.AppsDomain).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

		// gather app url
		var appsResponse AppsResponse
		cfResponse := cf.Cf("curl", fmt.Sprintf("/v2/apps?q=name:%s", serverAppName)).Wait(DEFAULT_TIMEOUT).Out.Contents()
		json.Unmarshal(cfResponse, &appsResponse)
		serverAppUrl := appsResponse.Resources[0].Metadata.Url

		// gather app stats for dea ip and app port
		var statsResponse StatsResponse
		cfResponse = cf.Cf("curl", fmt.Sprintf("%s/stats", serverAppUrl)).Wait(DEFAULT_TIMEOUT).Out.Contents()
		json.Unmarshal(cfResponse, &statsResponse)

		privateHost = statsResponse["0"].Stats.Host
		privatePort = statsResponse["0"].Stats.Port
	})

	AfterEach(func() {
		Expect(cf.Cf("delete", serverAppName, "-f").Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
	})

	// this test assumes the default running security groups block access to the DEAs
	// the test takes advantage of the fact that the DEA ip address and internal container ip address
	//  are discoverable via the cc api and dora's myip endpoint
	It("allows previously-blocked ip traffic after applying a security group, and re-blocks it when the group is removed", func() {
		clientAppName := generator.PrefixedRandomName("CATS-APP-")
		Expect(cf.Cf("push", clientAppName, "-m", "128M", "-p", assets.NewAssets().Dora, "-d", config.AppsDomain).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
		defer func() { cf.Cf("delete", clientAppName, "-f").Wait(CF_PUSH_TIMEOUT) }()

		// gather container ip
		curlResponse := helpers.CurlApp(serverAppName, "/myip")
		containerIp := strings.TrimSpace(curlResponse)

		// test app egress rules
		var doraCurlResponse DoraCurlResponse
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).ToNot(Equal(0))

		// apply security group
		rules := fmt.Sprintf(
			`[{"destination":"%s","ports":"%d","protocol":"tcp"},
        {"destination":"%s","ports":"%d","protocol":"tcp"}]`,
			privateHost, privatePort, containerIp, privatePort)

		file, _ := ioutil.TempFile(os.TempDir(), "CATS-sg-rules")
		defer os.Remove(file.Name())
		file.WriteString(rules)

		rulesPath := file.Name()
		securityGroupName = fmt.Sprintf("CATS-SG-%s", generator.RandomName())

		cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
			Expect(cf.Cf("create-security-group", securityGroupName, rulesPath).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			Expect(
				cf.Cf("bind-security-group",
					securityGroupName,
					context.RegularUserContext().Org,
					context.RegularUserContext().Space).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		})
		defer func() {
			cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
				Expect(cf.Cf("delete-security-group", securityGroupName, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			})
		}()

		Expect(cf.Cf("restart", clientAppName).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

		// test app egress rules
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).To(Equal(0))

		// unapply security group
		cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
			Expect(cf.Cf("unbind-security-group", securityGroupName, context.RegularUserContext().Org, context.RegularUserContext().Space).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		})
		Expect(cf.Cf("restart", clientAppName).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

		// test app egress rules
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).ToNot(Equal(0))
	})

	It("allows external and denies internal traffic during staging based on default staging security rules", func() {
		buildpack := fmt.Sprintf("CATS-SGBP-%s", generator.RandomName())
		testAppName := generator.PrefixedRandomName("CATS-APP-")
		privateUri := fmt.Sprintf("%s:%d", privateHost, privatePort)

		buildpackZip := assets.NewAssets().SecurityGroupBuildpack

		cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
			Expect(cf.Cf("create-buildpack", buildpack, buildpackZip, "999").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		})
		defer func() {
			cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
				Expect(cf.Cf("delete-buildpack", buildpack, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			})
		}()

		Expect(cf.Cf("push", testAppName, "-m", "128M", "-b", buildpack, "-p", assets.NewAssets().HelloWorld, "--no-start", "-d", config.AppsDomain).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
		defer func() { cf.Cf("delete", testAppName, "-f").Wait(CF_PUSH_TIMEOUT) }()

		Expect(cf.Cf("set-env", testAppName, "TESTURI", "www.google.com").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		Expect(cf.Cf("start", testAppName).Wait(CF_PUSH_TIMEOUT)).To(Exit(1))
		Eventually(func() *Session {
			appLogsSession := cf.Cf("logs", "--recent", testAppName)
			Expect(appLogsSession.Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			return appLogsSession
		}, 5).Should(Say("CURL_EXIT=0"))

		Expect(cf.Cf("set-env", testAppName, "TESTURI", privateUri).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		Expect(cf.Cf("restart", testAppName).Wait(CF_PUSH_TIMEOUT)).To(Exit(1))
		Eventually(func() *Session {
			appLogsSession := cf.Cf("logs", "--recent", testAppName)
			Expect(appLogsSession.Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			return appLogsSession
		}, 5).Should(Say("CURL_EXIT=[^0]"))
	})
})
<commit_msg>Convert comments to By() statements in long running tests<commit_after>package security_groups_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
)

var _ = Describe("Security Groups", func() {

	type AppResource struct {
		Metadata struct {
			Url string
		}
	}
	type AppsResponse struct {
		Resources []AppResource
	}

	type Stat struct {
		Stats struct {
			Host string
			Port int
		}
	}
	type StatsResponse map[string]Stat

	type DoraCurlResponse struct {
		Stdout     string
		Stderr     string
		ReturnCode int `json:"return_code"`
	}

	var serverAppName, securityGroupName, privateHost string
	var privatePort int

	BeforeEach(func() {
		serverAppName = generator.PrefixedRandomName("CATS-APP-")
		Expect(cf.Cf("push", serverAppName, "-m", "128M", "-p", assets.NewAssets().Dora, "-d", config.AppsDomain).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

		// gather app url
		var appsResponse AppsResponse
		cfResponse := cf.Cf("curl", fmt.Sprintf("/v2/apps?q=name:%s", serverAppName)).Wait(DEFAULT_TIMEOUT).Out.Contents()
		json.Unmarshal(cfResponse, &appsResponse)
		serverAppUrl := appsResponse.Resources[0].Metadata.Url

		// gather app stats for dea ip and app port
		var statsResponse StatsResponse
		cfResponse = cf.Cf("curl", fmt.Sprintf("%s/stats", serverAppUrl)).Wait(DEFAULT_TIMEOUT).Out.Contents()
		json.Unmarshal(cfResponse, &statsResponse)

		privateHost = statsResponse["0"].Stats.Host
		privatePort = statsResponse["0"].Stats.Port
	})

	AfterEach(func() {
		Expect(cf.Cf("delete", serverAppName, "-f").Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
	})

	// this test assumes the default running security groups block access to the DEAs
	// the test takes advantage of the fact that the DEA ip address and internal container ip address
	// are discoverable via the cc api and dora's myip endpoint
	It("allows previously-blocked ip traffic after applying a security group, and re-blocks it when the group is removed", func() {

		clientAppName := generator.PrefixedRandomName("CATS-APP-")
		Expect(cf.Cf("push", clientAppName, "-m", "128M", "-p", assets.NewAssets().Dora, "-d", config.AppsDomain).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
		defer func() { cf.Cf("delete", clientAppName, "-f").Wait(CF_PUSH_TIMEOUT) }()

		By("Gathering container ip")
		curlResponse := helpers.CurlApp(serverAppName, "/myip")
		containerIp := strings.TrimSpace(curlResponse)

		By("Testing app egress rules")
		var doraCurlResponse DoraCurlResponse
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).ToNot(Equal(0))

		By("Applying security group")
		rules := fmt.Sprintf(
			`[{"destination":"%s","ports":"%d","protocol":"tcp"},
        {"destination":"%s","ports":"%d","protocol":"tcp"}]`,
			privateHost, privatePort, containerIp, privatePort)

		file, _ := ioutil.TempFile(os.TempDir(), "CATS-sg-rules")
		defer os.Remove(file.Name())
		file.WriteString(rules)

		rulesPath := file.Name()
		securityGroupName = fmt.Sprintf("CATS-SG-%s", generator.RandomName())

		cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
			Expect(cf.Cf("create-security-group", securityGroupName, rulesPath).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			Expect(
				cf.Cf("bind-security-group",
					securityGroupName,
					context.RegularUserContext().Org,
					context.RegularUserContext().Space).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		})
		defer func() {
			cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
				Expect(cf.Cf("delete-security-group", securityGroupName, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			})
		}()

		Expect(cf.Cf("restart", clientAppName).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

		By("Testing app egress rules")
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).To(Equal(0))

		By("Unapplying security group")
		cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
			Expect(cf.Cf("unbind-security-group", securityGroupName, context.RegularUserContext().Org, context.RegularUserContext().Space).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		})
		Expect(cf.Cf("restart", clientAppName).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

		By("Testing app egress rules")
		curlResponse = helpers.CurlApp(clientAppName, fmt.Sprintf("/curl/%s/%d", privateHost, privatePort))
		json.Unmarshal([]byte(curlResponse), &doraCurlResponse)
		Expect(doraCurlResponse.ReturnCode).ToNot(Equal(0))
	})

	It("allows external and denies internal traffic during staging based on default staging security rules", func() {
		buildpack := fmt.Sprintf("CATS-SGBP-%s", generator.RandomName())
		testAppName := generator.PrefixedRandomName("CATS-APP-")
		privateUri := fmt.Sprintf("%s:%d", privateHost, privatePort)

		buildpackZip := assets.NewAssets().SecurityGroupBuildpack

		cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
			Expect(cf.Cf("create-buildpack", buildpack, buildpackZip, "999").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		})
		defer func() {
			cf.AsUser(context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
				Expect(cf.Cf("delete-buildpack", buildpack, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			})
		}()

		Expect(cf.Cf("push", testAppName, "-m", "128M", "-b", buildpack, "-p", assets.NewAssets().HelloWorld, "--no-start", "-d", config.AppsDomain).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
		defer func() { cf.Cf("delete", testAppName, "-f").Wait(CF_PUSH_TIMEOUT) }()

		Expect(cf.Cf("set-env", testAppName, "TESTURI", "www.google.com").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		Expect(cf.Cf("start", testAppName).Wait(CF_PUSH_TIMEOUT)).To(Exit(1))
		Eventually(func() *Session {
			appLogsSession := cf.Cf("logs", "--recent", testAppName)
			Expect(appLogsSession.Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			return appLogsSession
		}, 5).Should(Say("CURL_EXIT=0"))

		Expect(cf.Cf("set-env", testAppName, "TESTURI", privateUri).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		Expect(cf.Cf("restart", testAppName).Wait(CF_PUSH_TIMEOUT)).To(Exit(1))
		Eventually(func() *Session {
			appLogsSession := cf.Cf("logs", "--recent", testAppName)
			Expect(appLogsSession.Wait(DEFAULT_TIMEOUT)).To(Exit(0))
			return appLogsSession
		}, 5).Should(Say("CURL_EXIT=[^0]"))
	})
})
<|endoftext|>


<commit_before>package coprhd

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"net/url"

	log "github.com/Sirupsen/logrus"

	"github.com/portworx/kvdb"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"

	napping "gopkg.in/jmcvetta/napping.v3"
)

const (
	Name = "coprhd"
	Type = api.Block

	// URI_LOGIN path to create a authentication token
	URI_LOGIN = "login.json"
	// URI_LOGIN path to create volume
	URI_CREATE_VOL = "block/volumes.json"
	// URI_EXPORT_VOL path to export a volume
	URI_EXPORT_VOL = "block/export.json"
	// URI_TPL_DEL_VOL template path to delete/deactivate a volume
	URI_TPL_DEL_VOL = "block/volumes/%s/deactivate.json"
	// URL_TPL_NEW_SNAP path to create a volume snapshot
	URL_TPL_NEW_SNAP = "block/volumes/%s/protections/snapshots.json"
	// URI_TPL_UNEXP_VOL path template to remove a volume export
	URI_TPL_UNEXP_VOL = "block/export/%s/deactivate.json"
)

type (
	driver struct {
		*volume.IoNotSupported
		*volume.DefaultEnumerator
		consistency_group string
		project           string
		varray            string
		vpool             string
		url               string
		httpClient        *http.Client
		creds             *url.Userinfo
	}

	// ApiError represents the default api error code
	ApiError struct {
		Code        string `json:"code"`
		Retryable   string `json:"retryable"`
		Description string `json:"description"`
		Details     string `json:"details"`
	}

	// CreateVolumeArgs represents the json parameters for the create volume REST call
	CreateVolumeArgs struct {
		ConsistencyGroup string `json:"consistency_group"`
		Count            int    `json:"count"`
		Name             string `json:"name"`
		Project          string `json:"project"`
		Size             string `json:"size"`
		VArray           string `json:"varray"`
		VPool            string `json:"vpool"`
	}

	// CreateVolumeReply is the reply from the create volume REST call
	CreateVolumeReply struct {
		Task []struct {
			Resource struct {
				Name string       `json:"name"`
				Id   api.VolumeID `json:"id"`
			} `json:"resource"`
		} `json:"task"`
	}
)

func init() {
	volume.Register(Name, Init)
}

func Init(params volume.DriverParams) (volume.VolumeDriver, error) {

	restUrl, ok := params["restUrl"]
	if !ok {
		return nil, fmt.Errorf("rest api 'url' configuration parameter must be set")
	}

	user, ok := params["user"]
	if !ok {
		return nil, fmt.Errorf("rest auth 'user' must be set")
	}

	pass, ok := params["password"]
	if !ok {
		return nil, fmt.Errorf("rest auth 'password' must be set")
	}

	consistency_group, ok := params["consistency_group"]
	if !ok {
		return nil, fmt.Errorf("'consistency_group' configuration parameter must be set")
	}

	project, ok := params["project"]
	if !ok {
		return nil, fmt.Errorf("'project' configuration parameter must be set")
	}

	varray, ok := params["varray"]
	if !ok {
		return nil, fmt.Errorf("'varray' configuration parameter must be set")
	}

	vpool, ok := params["vpool"]
	if !ok {
		return nil, fmt.Errorf("'vpool' configuration parameter must be set")
	}

	d := &driver{
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
		consistency_group: consistency_group,
		project:           project,
		varray:            varray,
		vpool:             vpool,
		url:               restUrl,
		creds:             url.UserPassword(user, pass),
		httpClient: &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			},
		},
	}

	return d, nil
}

func (d *driver) String() string {
	return Name
}

func (d *driver) Type() api.DriverType {
	return Type
}

func (d *driver) Create(
	locator api.VolumeLocator,
	source *api.Source,
	spec *api.VolumeSpec) (api.VolumeID, error) {

	s, err := d.getAuthSession()

	if err != nil {
		log.Errorf("Failed to create session: %s", err.Error())
		return api.BadVolumeID, err
	}

	e := ApiError{}

	res := &CreateVolumeReply{}

	sz := int64(spec.Size / (1024 * 1024 * 1000))

	payload := CreateVolumeArgs{
		"Default",                 // ConsistencyGroup
		1,                         // Count
		locator.Name,              // Name
		d.project,                 // Project
		fmt.Sprintf("%.6fGB", sz), // Volume Size
		d.varray,                  // Virtual Block Array
		d.vpool,                   // Virtual Block Pool
	}

	url := d.url + URI_CREATE_VOL

	resp, err := s.Post(url, &payload, res, &e)

	if resp.Status() != http.StatusAccepted {

		return api.BadVolumeID, fmt.Errorf("Failed to create volume: %s", resp.Status())
	}

	return res.Task[0].Resource.Id, err
}

func (d *driver) Delete(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Stats(volumeID api.VolumeID) (api.Stats, error) {
	return api.Stats{}, volume.ErrNotSupported
}

func (d *driver) Alerts(volumeID api.VolumeID) (api.Alerts, error) {
	return api.Alerts{}, volume.ErrNotSupported
}

func (d *driver) Attach(volumeID api.VolumeID) (path string, err error) {
	return "", nil
}

func (d *driver) Detach(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Mount(volumeID api.VolumeID, mountpath string) error {
	return nil
}

func (d *driver) Unmount(volumeID api.VolumeID, mountpath string) error {

	return nil
}

func (d *driver) Set(volumeID api.VolumeID, locator *api.VolumeLocator, spec *api.VolumeSpec) error {
	return volume.ErrNotSupported
}

func (d *driver) Shutdown() {
	log.Infof("%s Shutting down", Name)
}

func (d *driver) Snapshot(volumeID api.VolumeID, readonly bool, locator api.VolumeLocator) (api.VolumeID, error) {
	return "", nil
}

func (v *driver) Status() [][2]string {
	return [][2]string{}
}

// getAuthSession returns an authenticated API Session
func (d *driver) getAuthSession() (session *napping.Session, err error) {

	e := ApiError{}

	s := napping.Session{
		Userinfo: d.creds,
		Client:   d.httpClient,
	}

	url := d.url + URI_LOGIN

	resp, err := s.Get(url, nil, nil, &e)

	if err != nil {
		return
	}

	token := resp.HttpResponse().Header.Get("X-SDS-AUTH-TOKEN")

	h := http.Header{}

	h.Set("X-SDS-AUTH-TOKEN", token)

	session = &napping.Session{
		Client: d.httpClient,
		Header: &h,
	}

	return
}
<commit_msg>fixing the consistency group driver setting in Create<commit_after>package coprhd

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"net/url"

	log "github.com/Sirupsen/logrus"

	"github.com/portworx/kvdb"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"

	napping "gopkg.in/jmcvetta/napping.v3"
)

const (
	Name = "coprhd"
	Type = api.Block

	// URI_LOGIN path to create a authentication token
	URI_LOGIN = "login.json"
	// URI_LOGIN path to create volume
	URI_CREATE_VOL = "block/volumes.json"
	// URI_EXPORT_VOL path to export a volume
	URI_EXPORT_VOL = "block/export.json"
	// URI_TPL_DEL_VOL template path to delete/deactivate a volume
	URI_TPL_DEL_VOL = "block/volumes/%s/deactivate.json"
	// URL_TPL_NEW_SNAP path to create a volume snapshot
	URL_TPL_NEW_SNAP = "block/volumes/%s/protections/snapshots.json"
	// URI_TPL_UNEXP_VOL path template to remove a volume export
	URI_TPL_UNEXP_VOL = "block/export/%s/deactivate.json"
)

type (
	driver struct {
		*volume.IoNotSupported
		*volume.DefaultEnumerator
		consistency_group string
		project           string
		varray            string
		vpool             string
		url               string
		httpClient        *http.Client
		creds             *url.Userinfo
	}

	// ApiError represents the default api error code
	ApiError struct {
		Code        string `json:"code"`
		Retryable   string `json:"retryable"`
		Description string `json:"description"`
		Details     string `json:"details"`
	}

	// CreateVolumeArgs represents the json parameters for the create volume REST call
	CreateVolumeArgs struct {
		ConsistencyGroup string `json:"consistency_group"`
		Count            int    `json:"count"`
		Name             string `json:"name"`
		Project          string `json:"project"`
		Size             string `json:"size"`
		VArray           string `json:"varray"`
		VPool            string `json:"vpool"`
	}

	// CreateVolumeReply is the reply from the create volume REST call
	CreateVolumeReply struct {
		Task []struct {
			Resource struct {
				Name string       `json:"name"`
				Id   api.VolumeID `json:"id"`
			} `json:"resource"`
		} `json:"task"`
	}
)

func init() {
	volume.Register(Name, Init)
}

func Init(params volume.DriverParams) (volume.VolumeDriver, error) {

	restUrl, ok := params["restUrl"]
	if !ok {
		return nil, fmt.Errorf("rest api 'url' configuration parameter must be set")
	}

	user, ok := params["user"]
	if !ok {
		return nil, fmt.Errorf("rest auth 'user' must be set")
	}

	pass, ok := params["password"]
	if !ok {
		return nil, fmt.Errorf("rest auth 'password' must be set")
	}

	consistency_group, ok := params["consistency_group"]
	if !ok {
		return nil, fmt.Errorf("'consistency_group' configuration parameter must be set")
	}

	project, ok := params["project"]
	if !ok {
		return nil, fmt.Errorf("'project' configuration parameter must be set")
	}

	varray, ok := params["varray"]
	if !ok {
		return nil, fmt.Errorf("'varray' configuration parameter must be set")
	}

	vpool, ok := params["vpool"]
	if !ok {
		return nil, fmt.Errorf("'vpool' configuration parameter must be set")
	}

	d := &driver{
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
		consistency_group: consistency_group,
		project:           project,
		varray:            varray,
		vpool:             vpool,
		url:               restUrl,
		creds:             url.UserPassword(user, pass),
		httpClient: &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			},
		},
	}

	return d, nil
}

func (d *driver) String() string {
	return Name
}

func (d *driver) Type() api.DriverType {
	return Type
}

func (d *driver) Create(
	locator api.VolumeLocator,
	source *api.Source,
	spec *api.VolumeSpec) (api.VolumeID, error) {

	s, err := d.getAuthSession()

	if err != nil {
		log.Errorf("Failed to create session: %s", err.Error())
		return api.BadVolumeID, err
	}

	e := ApiError{}

	res := &CreateVolumeReply{}

	sz := int64(spec.Size / (1024 * 1024 * 1000))

	payload := CreateVolumeArgs{
		d.consistency_group,       // ConsistencyGroup
		1,                         // Count
		locator.Name,              // Name
		d.project,                 // Project
		fmt.Sprintf("%.6fGB", sz), // Volume Size
		d.varray,                  // Virtual Block Array
		d.vpool,                   // Virtual Block Pool
	}

	url := d.url + URI_CREATE_VOL

	resp, err := s.Post(url, &payload, res, &e)

	if resp.Status() != http.StatusAccepted {

		return api.BadVolumeID, fmt.Errorf("Failed to create volume: %s", resp.Status())
	}

	return res.Task[0].Resource.Id, err
}

func (d *driver) Delete(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Stats(volumeID api.VolumeID) (api.Stats, error) {
	return api.Stats{}, volume.ErrNotSupported
}

func (d *driver) Alerts(volumeID api.VolumeID) (api.Alerts, error) {
	return api.Alerts{}, volume.ErrNotSupported
}

func (d *driver) Attach(volumeID api.VolumeID) (path string, err error) {
	return "", nil
}

func (d *driver) Detach(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Mount(volumeID api.VolumeID, mountpath string) error {
	return nil
}

func (d *driver) Unmount(volumeID api.VolumeID, mountpath string) error {

	return nil
}

func (d *driver) Set(volumeID api.VolumeID, locator *api.VolumeLocator, spec *api.VolumeSpec) error {
	return volume.ErrNotSupported
}

func (d *driver) Shutdown() {
	log.Infof("%s Shutting down", Name)
}

func (d *driver) Snapshot(volumeID api.VolumeID, readonly bool, locator api.VolumeLocator) (api.VolumeID, error) {
	return "", nil
}

func (v *driver) Status() [][2]string {
	return [][2]string{}
}

// getAuthSession returns an authenticated API Session
func (d *driver) getAuthSession() (session *napping.Session, err error) {

	e := ApiError{}

	s := napping.Session{
		Userinfo: d.creds,
		Client:   d.httpClient,
	}

	url := d.url + URI_LOGIN

	resp, err := s.Get(url, nil, nil, &e)

	if err != nil {
		return
	}

	token := resp.HttpResponse().Header.Get("X-SDS-AUTH-TOKEN")

	h := http.Header{}

	h.Set("X-SDS-AUTH-TOKEN", token)

	session = &napping.Session{
		Client: d.httpClient,
		Header: &h,
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"
	"runtime"
	"time"
)

func main() {
	SetupConfig()
	runtime.GOMAXPROCS(runtime.NumCPU())
	SetupNumerics()
	SetupSytemUser()
	var listeners []net.Listener
	// Listen for incoming connections.
	for _, LISTENING_IP := range config.ListenIPs {
		for _, LISTENING_PORT := range config.ListenPorts {
			l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", LISTENING_IP, LISTENING_PORT))
			if err != nil {
				log.Println("Error listening:", err.Error())
				os.Exit(1)
			} else {
				listeners = append(listeners, l)
				log.Printf("Listening on %s:%d", LISTENING_IP, LISTENING_PORT)
			}
		}
	}
	// Close the listener when the application closes.
	for _, l := range listeners {
		defer l.Close()
	}
	for _, l := range listeners {
		go listenerthing(l)
	}
	periodicStatusUpdate()
}

func listenerthing(l net.Listener) {
	for {
		conn, err := l.Accept()
		if err != nil {
			log.Println("Error accepting: ", err.Error())
		} else {
			user := NewUser()
			user.SetConn(conn)
			go user.HandleRequests()
		}
	}
}

func checkMaxUsers() {
	if len(userlist) > maxUsers {
		maxUsers = len(userlist)
	}
}

//periodicStatusUpdate shows information about the ircd every 5 seconds or so,
//as well as updating the max users, and goroutines numbers. Since these are
//only ran every 5 seconds or so, it may not be 100% accurate, but who cares
func periodicStatusUpdate() {
	for {
		checkMaxUsers()
		gor := runtime.NumGoroutine()
		if gor > maxRoutines {
			maxRoutines = gor
		}
		log.Printf("Status: %d current users", len(userlist))
		log.Printf("Status: %d current channels", len(chanlist))
		if config.Debug {
			log.Printf("Status: %d current Goroutines", gor)
			log.Printf("Status: %d max Goroutines", maxRoutines)
		}
		time.Sleep(config.StatTime * time.Second)
	}
}
<commit_msg>make some stats make more sense<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"runtime"
	"time"
)

func main() {
	SetupConfig()
	runtime.GOMAXPROCS(runtime.NumCPU())
	SetupNumerics()
	SetupSytemUser()
	var listeners []net.Listener
	// Listen for incoming connections.
	for _, LISTENING_IP := range config.ListenIPs {
		for _, LISTENING_PORT := range config.ListenPorts {
			l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", LISTENING_IP, LISTENING_PORT))
			if err != nil {
				log.Println("Error listening:", err.Error())
				os.Exit(1)
			} else {
				listeners = append(listeners, l)
				log.Printf("Listening on %s:%d", LISTENING_IP, LISTENING_PORT)
			}
		}
	}
	// Close the listener when the application closes.
	for _, l := range listeners {
		defer l.Close()
	}
	for _, l := range listeners {
		go listenerthing(l)
	}
	periodicStatusUpdate()
}

func listenerthing(l net.Listener) {
	for {
		conn, err := l.Accept()
		if err != nil {
			log.Println("Error accepting: ", err.Error())
		} else {
			user := NewUser()
			user.SetConn(conn)
			checkMaxUsers()
			go user.HandleRequests()
		}
	}
}

func checkMaxUsers() {
	if len(userlist) > maxUsers {
		maxUsers = len(userlist)
	}
}

//periodicStatusUpdate shows information about the ircd every 5 seconds or so,
//as well as updating the max users, and goroutines numbers. Since these are
//only ran every 5 seconds or so, it may not be 100% accurate, but who cares
func periodicStatusUpdate() {
	for {
		log.Printf("Status: %d current users", len(userlist))
		log.Printf("Status: %d current channels", len(chanlist))
		if config.Debug {
			log.Printf("Status: %d current Goroutines", runtime.NumGoroutine())
		}
		time.Sleep(config.StatTime * time.Second)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"time"

	"golang.org/x/net/context"

	"github.com/coreos/agro"
	"github.com/coreos/agro/models"
	"github.com/coreos/agro/ring"
	"github.com/coreos/pkg/capnslog"
)

type RebalanceStrategy int32

type Rebalancer interface {
	Leader(chans [2]chan *models.RebalanceStatus)
	AdvanceState(s *models.RebalanceStatus) (*models.RebalanceStatus, bool, error)
	OnError(error) *models.RebalanceStatus
	RebalanceMessage(context.Context, *models.RebalanceRequest) (*models.RebalanceResponse, error)
	Timeout()
}

const (
	Error   RebalanceStrategy = iota
	Replace                   = 1
	Full                      = 2
)

type makeRebalanceFunc func(d *distributor, newring agro.Ring) Rebalancer

var (
	rebalanceTimeout   = 30 * time.Second
	rebalancerRegistry = make(map[RebalanceStrategy]makeRebalanceFunc)
	rlog               = capnslog.NewPackageLogger("github.com/coreos/agro", "rebalancer")
)

// Goroutine which watches for new rings and kicks off
// the rebalance dance.
func (d *distributor) rebalanceWatcher(closer chan struct{}) {
	ch := make(chan agro.Ring)
	d.srv.mds.SubscribeNewRings(ch)
exit:
	for {
		select {
		case <-closer:
			d.srv.mds.UnsubscribeNewRings(ch)
			close(ch)
			break exit
		case newring, ok := <-ch:
			if ok {
				if newring.Version() == d.ring.Version() {
					// No problem. We're seeing the same ring.
					continue
				}
				if newring.Version() != d.ring.Version()+1 {
					panic("replacing old ring with ring in the far future!")
				}
				d.Rebalance(newring)
			} else {
				break exit
			}
		}
	}
}

func (d *distributor) Rebalance(newring agro.Ring) {
	d.srv.updatePeerMap()
	isMember := d.ring.Members().Union(newring.Members()).Has(d.UUID())
	if !isMember {
		clog.Infof("rebalance detected, but not a member")
	}
	// TODO(barakmich): Rebalancing is tricky. But here's the entry point.
	clog.Infof("rebalancing beginning: new ring version %d for %s", newring.Version(), d.UUID())
	chans, leader, err := d.srv.mds.OpenRebalanceChannels()
	if err != nil {
		clog.Error(err)
		return
	}
	if leader {
		clog.Infof("elected as leader")
		d.rebalanceLeader(chans, newring)
	} else {
		clog.Infof("elected to follow")
		d.rebalanceFollower(chans, newring)
	}
	d.mut.Lock()
	defer d.mut.Unlock()
	d.rebalancer = nil
}

func (d *distributor) rebalanceLeader(chans [2]chan *models.RebalanceStatus, newring agro.Ring) {
	var re Rebalancer
	switch d.ring.Type() {
	case ring.Empty:
		// We can always replace the empty ring.
		clog.Infof("replacing empty ring")
		re = rebalancerRegistry[Replace](d, newring)
	default:
		re = rebalancerRegistry[Full](d, newring)
	}
	d.mut.Lock()
	d.rebalancer = re
	d.mut.Unlock()
	re.Leader(chans)
	d.srv.mut.Lock()
	defer d.srv.mut.Unlock()
	clog.Info("leader: success, setting new ring")
	d.ring = newring
	d.srv.mds.SetRing(newring, true)
	close(chans[1])
}

func (d *distributor) rebalanceFollower(inOut [2]chan *models.RebalanceStatus, newring agro.Ring) {
	in, out := inOut[0], inOut[1]
	for {
		select {
		case s := <-in:
			if !s.FromLeader {
				panic("got a message not from leader")
			}
			if d.rebalancer == nil {
				d.mut.Lock()
				rlog.Debugf("creating rebalancer %d", s.RebalanceType)
				d.rebalancer = rebalancerRegistry[RebalanceStrategy(s.RebalanceType)](d, newring)
				d.mut.Unlock()
			}
			news, done, err := d.rebalancer.AdvanceState(s)
			if err != nil {
				clog.Error(err)
				stat := d.rebalancer.OnError(err)
				if stat != nil {
					out <- stat
				}
				close(out)
				return
			}
			news.UUID = d.UUID()
			out <- news
			if done {
				close(out)
				d.srv.mut.Lock()
				defer d.srv.mut.Unlock()
				clog.Info("follower: success, setting new ring")
				d.ring = newring
				return
			}
		case <-time.After(rebalanceTimeout):
			close(out)
			d.rebalancer.Timeout()
			// Re-elect
			d.Rebalance(newring)
		}
	}
}

func waitAll(c chan *models.RebalanceStatus, newring agro.Ring, phase int32) error {
	member := newring.Members()
	for len(member) > 0 {

		// TODO(barakmich) Check if the status is an error, such as the TTL of
		// the key being lost in etcd (thus a machine has timed out and we're in
		// trouble). LEASES.
		stat, ok := <-c
		if !ok {
			clog.Error("close before end of rebalance")
			return agro.ErrClosed
		}
		if stat.Phase == phase {
			for i, m := range member {
				if m == stat.UUID {
					clog.Debugf("got response from %s", stat.UUID)
					member = append(member[:i], member[i+1:]...)
					break
				}
			}
		}
	}
	clog.Debugf("finished waiting for members")
	return nil
}
<commit_msg>remove wrong timeout (the timeout should come from lease TTL on etcd)<commit_after>package server

import (
	"time"

	"golang.org/x/net/context"

	"github.com/coreos/agro"
	"github.com/coreos/agro/models"
	"github.com/coreos/agro/ring"
	"github.com/coreos/pkg/capnslog"
)

type RebalanceStrategy int32

type Rebalancer interface {
	Leader(chans [2]chan *models.RebalanceStatus)
	AdvanceState(s *models.RebalanceStatus) (*models.RebalanceStatus, bool, error)
	OnError(error) *models.RebalanceStatus
	RebalanceMessage(context.Context, *models.RebalanceRequest) (*models.RebalanceResponse, error)
	Timeout()
}

const (
	Error   RebalanceStrategy = iota
	Replace                   = 1
	Full                      = 2
)

type makeRebalanceFunc func(d *distributor, newring agro.Ring) Rebalancer

var (
	rebalanceTimeout   = 30 * time.Second
	rebalancerRegistry = make(map[RebalanceStrategy]makeRebalanceFunc)
	rlog               = capnslog.NewPackageLogger("github.com/coreos/agro", "rebalancer")
)

// Goroutine which watches for new rings and kicks off
// the rebalance dance.
func (d *distributor) rebalanceWatcher(closer chan struct{}) {
	ch := make(chan agro.Ring)
	d.srv.mds.SubscribeNewRings(ch)
exit:
	for {
		select {
		case <-closer:
			d.srv.mds.UnsubscribeNewRings(ch)
			close(ch)
			break exit
		case newring, ok := <-ch:
			if ok {
				if newring.Version() == d.ring.Version() {
					// No problem. We're seeing the same ring.
					continue
				}
				if newring.Version() != d.ring.Version()+1 {
					panic("replacing old ring with ring in the far future!")
				}
				d.Rebalance(newring)
			} else {
				break exit
			}
		}
	}
}

func (d *distributor) Rebalance(newring agro.Ring) {
	d.srv.updatePeerMap()
	isMember := d.ring.Members().Union(newring.Members()).Has(d.UUID())
	if !isMember {
		clog.Infof("rebalance detected, but not a member")
	}
	// TODO(barakmich): Rebalancing is tricky. But here's the entry point.
	clog.Infof("rebalancing beginning: new ring version %d for %s", newring.Version(), d.UUID())
	chans, leader, err := d.srv.mds.OpenRebalanceChannels()
	if err != nil {
		clog.Error(err)
		return
	}
	if leader {
		clog.Infof("elected as leader")
		d.rebalanceLeader(chans, newring)
	} else {
		clog.Infof("elected to follow")
		d.rebalanceFollower(chans, newring)
	}
	d.mut.Lock()
	defer d.mut.Unlock()
	d.rebalancer = nil
}

func (d *distributor) rebalanceLeader(chans [2]chan *models.RebalanceStatus, newring agro.Ring) {
	var re Rebalancer
	switch d.ring.Type() {
	case ring.Empty:
		// We can always replace the empty ring.
		clog.Infof("replacing empty ring")
		re = rebalancerRegistry[Replace](d, newring)
	default:
		re = rebalancerRegistry[Full](d, newring)
	}
	d.mut.Lock()
	d.rebalancer = re
	d.mut.Unlock()
	re.Leader(chans)
	d.srv.mut.Lock()
	defer d.srv.mut.Unlock()
	clog.Info("leader: success, setting new ring")
	d.ring = newring
	d.srv.mds.SetRing(newring, true)
	close(chans[1])
}

func (d *distributor) rebalanceFollower(inOut [2]chan *models.RebalanceStatus, newring agro.Ring) {
	in, out := inOut[0], inOut[1]
	for {
		s := <-in
		if !s.FromLeader {
			panic("got a message not from leader")
		}
		if d.rebalancer == nil {
			d.mut.Lock()
			rlog.Debugf("creating rebalancer %d", s.RebalanceType)
			d.rebalancer = rebalancerRegistry[RebalanceStrategy(s.RebalanceType)](d, newring)
			d.mut.Unlock()
		}
		news, done, err := d.rebalancer.AdvanceState(s)
		if err != nil {
			clog.Error(err)
			stat := d.rebalancer.OnError(err)
			if stat != nil {
				out <- stat
			}
			close(out)
			return
		}
		news.UUID = d.UUID()
		out <- news
		if done {
			close(out)
			d.srv.mut.Lock()
			defer d.srv.mut.Unlock()
			clog.Info("follower: success, setting new ring")
			d.ring = newring
			return
		}
	}
}

func waitAll(c chan *models.RebalanceStatus, newring agro.Ring, phase int32) error {
	member := newring.Members()
	for len(member) > 0 {

		// TODO(barakmich) Check if the status is an error, such as the TTL of
		// the key being lost in etcd (thus a machine has timed out and we're in
		// trouble). LEASES.
		stat, ok := <-c
		if !ok {
			clog.Error("close before end of rebalance")
			return agro.ErrClosed
		}
		if stat.Phase == phase {
			for i, m := range member {
				if m == stat.UUID {
					clog.Debugf("got response from %s", stat.UUID)
					member = append(member[:i], member[i+1:]...)
					break
				}
			}
		}
	}
	clog.Debugf("finished waiting for members")
	return nil
}
<|endoftext|>