<commit_before>package speedbump

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/facebookgo/clock"
	"github.com/stretchr/testify/assert"

	"gopkg.in/redis.v3"
)

func createClient() *redis.Client {
	if os.Getenv("WERCKER_REDIS_HOST") != "" {
		return redis.NewClient(&redis.Options{
			Addr:     os.Getenv("WERCKER_REDIS_HOST") + ":6379",
			Password: "",
			DB:       0,
		})
	}

	return redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	})
}

func Test_NewLimiter(t *testing.T) {
	client := createClient()
	hasher := PerSecondHasher{}

	NewLimiter(client, hasher, 10)
}

func ExampleNewLimiter() {
	// Create a Redis client.
	client := createClient()

	// Create a new hasher.
	hasher := PerSecondHasher{}

	// Create a new limiter that will only allow 10 requests per second.
	limiter := NewLimiter(client, hasher, 10)

	fmt.Println(limiter.Attempt("127.0.0.1"))
	// Output: true <nil>
}

func Test_Attempts(t *testing.T) {
	client := createClient()
	mock := clock.NewMock()
	hasher := PerMinuteHasher{
		Clock: mock,
	}

	limiter := NewLimiter(client, hasher, 5)

	has, err := limiter.Has("127.0.0.1")
	assert.Nil(t, err)
	assert.False(t, has)

	ok, err := limiter.Attempt("127.0.0.1")
	assert.Nil(t, err)
	assert.True(t, ok)

	limiter.Attempt("127.0.0.1")
	limiter.Attempt("127.0.0.1")
	limiter.Attempt("127.0.0.1")
	limiter.Attempt("127.0.0.1")
	limiter.Attempt("127.0.0.1")
	limiter.Attempt("127.0.0.1")
	limiter.Attempt("127.0.0.2")
	limiter.Attempt("127.0.0.1")
	limiter.Attempt("127.0.0.2")
	ok, err = limiter.Attempt("127.0.0.1")

	assert.Nil(t, err)
	assert.False(t, ok)

	left, err := limiter.Left("127.0.0.1")
	assert.Nil(t, err)
	assert.True(t, 0 == left)

	left, err = limiter.Left("127.0.0.2")
	assert.Nil(t, err)
	assert.True(t, left > 0)

	ok, err = limiter.Attempt("127.0.0.2")
	assert.Nil(t, err)
	assert.True(t, ok)

	mock.Add(time.Minute)

	left, err = limiter.Left("127.0.0.1")
	assert.Nil(t, err)
	assert.True(t, 5 == left)
}
<commit_msg>expand and clean up tests<commit_after>package speedbump

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/facebookgo/clock"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"gopkg.in/redis.v3"
)

func createClient() *redis.Client {
	if os.Getenv("WERCKER_REDIS_HOST") != "" {
		return redis.NewClient(&redis.Options{
			Addr:     os.Getenv("WERCKER_REDIS_HOST") + ":6379",
			Password: "",
			DB:       0,
		})
	}

	return redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	})
}

func teardown(t *testing.T, client *redis.Client) {
	// Flush Redis.
	require.NoError(t, client.FlushAll().Err())
}

func TestNewLimiter(t *testing.T) {
	client := createClient()
	hasher := PerSecondHasher{}
	max := int64(10)
	actual := NewLimiter(client, hasher, max)

	assert.Exactly(t, RateLimiter{
		redisClient: client,
		hasher:      hasher,
		max:         max,
	}, *actual)
}

func ExampleNewLimiter() {
	// Create a Redis client.
	client := createClient()

	// Create a new hasher.
	hasher := PerSecondHasher{}

	// Create a new limiter that will only allow 10 requests per second.
	limiter := NewLimiter(client, hasher, 10)

	fmt.Println(limiter.Attempt("127.0.0.1"))
	// Output: true <nil>
}

func TestHas(t *testing.T) {
	// Create Redis client and defer DB teardown.
	client := createClient()
	defer teardown(t, client)
	// Create limiter of 5 requests/min.
	limiter := NewLimiter(client, PerMinuteHasher{}, 5)
	// Choose an arbitrary id.
	testID := "test_id"

	// Ensure testID returns false initially.
	has, err := limiter.Has(testID)
	require.NoError(t, err)
	assert.False(t, has)

	// Make request attempts, including attempt for testID.
	_, err = limiter.Attempt("some_id")
	require.NoError(t, err)
	_, err = limiter.Attempt(testID)
	require.NoError(t, err)
	_, err = limiter.Attempt("some_other_id")
	require.NoError(t, err)

	// Ensure testID returns true.
	has, err = limiter.Has(testID)
	require.NoError(t, err)
	assert.True(t, has)

	// Ensure other ids return false.
	has, err = limiter.Has("some_unseen_id")
	require.NoError(t, err)
	assert.False(t, has)
}

type attempt struct {
	id          string
	expectedOK  bool
	expectedErr string
}

func TestAttempt(t *testing.T) {
	// Create Redis client and defer DB teardown.
	client := createClient()
	defer teardown(t, client)
	// Create PerMinuteHasher with mock clock.
	mock := clock.NewMock()
	hasher := PerMinuteHasher{
		Clock: mock,
	}
	// Create limiter of 5 requests/min.
	limiter := NewLimiter(client, hasher, 5)
	// Choose an arbitrary id.
	testID := "test_id"
	// Ensure no key exists before first request for testID.
	has, err := limiter.Has(testID)
	require.NoError(t, err)
	assert.False(t, has)

	// Set up series of requests that include > max requests for testID.
	attempts := []attempt{
		// Attempt #1 for testID.
		{
			id:          testID,
			expectedOK:  true,
			expectedErr: "",
		},
		// Attempt #2 for testID.
		{
			id:          testID,
			expectedOK:  true,
			expectedErr: "",
		},
		{
			id:          "some_id",
			expectedOK:  true,
			expectedErr: "",
		},
		{
			id:          "some_id",
			expectedOK:  true,
			expectedErr: "",
		},
		// Attempt #3 for testID.
		{
			id:          testID,
			expectedOK:  true,
			expectedErr: "",
		},
		// Attempt #4 for testID.
		{
			id:          testID,
			expectedOK:  true,
			expectedErr: "",
		},
		{
			id:          "some_other_id",
			expectedOK:  true,
			expectedErr: "",
		},
		// Attempt #5 for testID.
		{
			id:          testID,
			expectedOK:  true,
			expectedErr: "",
		},
		// Attempt #6 for testID -- exceeded max.
		{
			id:          testID,
			expectedOK:  false,
			expectedErr: "",
		},
		// Attempt #7 for testID -- exceeded max.
		{
			id:          testID,
			expectedOK:  false,
			expectedErr: "",
		},
		{
			id:          "some_id",
			expectedOK:  true,
			expectedErr: "",
		},
		// Attempt #8 for testID -- exceeded max.
		{
			id:          testID,
			expectedOK:  false,
			expectedErr: "",
		},
	}

	// Make attempts and keep track of results. Avoid asserting in loop to ensure
	// execution takes < 1 minute.
	resultsOK := []bool{}
	resultsErr := []error{}
	for i := 0; i < len(attempts); i++ {
		ok, err := limiter.Attempt(attempts[i].id)
		resultsOK = append(resultsOK, ok)
		resultsErr = append(resultsErr, err)
	}
	// Assert attempt results.
	for i := 0; i < len(resultsErr); i++ {
		if attempts[i].expectedErr == "" {
			require.NoError(t, resultsErr[i], "failed case #%d: %+v", i, attempts[i])
			assert.Exactly(t, attempts[i].expectedOK, resultsOK[i], "failed case #%d: %+v", i, attempts[i])
		} else {
			require.Error(t, resultsErr[i], "failed case #%d: %+v", i, attempts[i])
			assert.Contains(t, resultsErr[i].Error(), attempts[i].expectedErr, "failed case #%d: %+v", i, attempts[i])
		}
	}

	// Mock add 1 minute to simulate waiting 1 minute, expect true for testID.
	mock.Add(time.Minute)
	ok, err := limiter.Attempt(attempts[0].id)
	require.NoError(t, err)
	assert.True(t, ok, "Attempts returned false after waiting for interval")
}

func makeNAttempts(t *testing.T, limiter *RateLimiter, id string, n int64) {
	var i int64
	for i = 0; i < n; i++ {
		_, err := limiter.Attempt(id)
		require.NoError(t, err, "got error during request attempt")
	}
}

func TestAttemptedLeft(t *testing.T) {
	// Create Redis client and defer DB teardown.
	client := createClient()
	defer teardown(t, client)
	// Create PerMinuteHasher with mock clock.
	mock := clock.NewMock()
	hasher := PerMinuteHasher{
		Clock: mock,
	}
	max := int64(5)
	// Create limiter of 5 requests/min.
	limiter := NewLimiter(client, hasher, max)
	// Choose an arbitrary id.
	testID := "test_id"

	// Check we have max left, 0 attempted initially.
	{
		left, err := limiter.Left(testID)
		require.NoError(t, err)
		assert.Exactly(t, max, left)
		attempted, err := limiter.Attempted(testID)
		require.NoError(t, err)
		assert.Exactly(t, int64(0), attempted)
	}

	// Make 2 attempts.
	makeNAttempts(t, limiter, testID, 2)
	{
		// Expect max-2 left for testID.
		left, err := limiter.Left(testID)
		require.NoError(t, err)
		assert.Exactly(t, max-2, left)
		// Expect max left for any other id.
		leftOther, err := limiter.Left("some_id")
		require.NoError(t, err)
		assert.Exactly(t, max, leftOther)
		// Expect 2 attempted for testID.
		attempted, err := limiter.Attempted(testID)
		require.NoError(t, err)
		assert.Exactly(t, int64(2), attempted)
		// Expect 0 attempted for any other id.
		attemptedOther, err := limiter.Attempted("some_id")
		require.NoError(t, err)
		assert.Exactly(t, int64(0), attemptedOther)
	}

	// Make max-2 more attempts.
	makeNAttempts(t, limiter, testID, max-2)
	{
		// Expect 0 left for testID.
		left, err := limiter.Left(testID)
		require.NoError(t, err)
		assert.Exactly(t, int64(0), left)
		// Expect max left for any other id.
		leftOther, err := limiter.Left("some_id")
		require.NoError(t, err)
		assert.Exactly(t, max, leftOther)
		// Expect max attempted for testID.
		attempted, err := limiter.Attempted(testID)
		require.NoError(t, err)
		assert.Exactly(t, max, attempted)
		// Expect 0 attempted for any other id.
		attemptedOther, err := limiter.Attempted("some_id")
		require.NoError(t, err)
		assert.Exactly(t, int64(0), attemptedOther)

	}

	// Make 10 more attempts.
	makeNAttempts(t, limiter, testID, 10)
	{
		// Expect 0 left for testID.
		left, err := limiter.Left(testID)
		require.NoError(t, err)
		assert.Exactly(t, int64(0), left)
		// Expect max left for any other id.
		leftOther, err := limiter.Left("some_id")
		require.NoError(t, err)
		assert.Exactly(t, max, leftOther)
		// Expect max attempted for testID.
		attempted, err := limiter.Attempted(testID)
		require.NoError(t, err)
		assert.Exactly(t, max, attempted)
		// Expect 0 attempted for any other id.
		attemptedOther, err := limiter.Attempted("some_id")
		require.NoError(t, err)
		assert.Exactly(t, int64(0), attemptedOther)
	}

	// Mock add 1 minute to simulate waiting 1 minute.
	mock.Add(time.Minute)
	{
		// Expect max left for testID.
		left, err := limiter.Left(testID)
		require.NoError(t, err)
		assert.Exactly(t, max, left)
		// Expect max left for any other id.
		leftOther, err := limiter.Left("some_id")
		require.NoError(t, err)
		assert.Exactly(t, max, leftOther)
		// Expect 0 attempted for testID.
		attempted, err := limiter.Attempted(testID)
		require.NoError(t, err)
		assert.Exactly(t, int64(0), attempted)
		// Expect 0 attempted for any other id.
		attemptedOther, err := limiter.Attempted("some_id")
		require.NoError(t, err)
		assert.Exactly(t, int64(0), attemptedOther)

	}
}
<|endoftext|>


<commit_before>package mpb

import (
	"fmt"
	"io"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/vbauerster/uilive"
)

type opType uint

const (
	opBarAdd opType = iota
	opBarRemove
)

type SortType uint

const (
	SortNone SortType = iota
	SortTop
	SortBottom
)

const refreshRate = 60

// Progress represents the container that renders Progress bars
type Progress struct {
	out     io.Writer
	width   int
	sort    SortType
	stopped bool

	op            chan *operation
	rrChangeReqCh chan time.Duration

	wg *sync.WaitGroup
}

type operation struct {
	kind   opType
	bar    *Bar
	result chan bool
}

// New returns a new progress bar with defaults
func New() *Progress {
	p := &Progress{
		out:           os.Stdout,
		width:         70,
		op:            make(chan *operation),
		rrChangeReqCh: make(chan time.Duration),
		wg:            new(sync.WaitGroup),
	}
	go p.server()
	return p
}

func (p *Progress) SetWidth(n int) *Progress {
	if n <= 0 {
		return p
	}
	p.width = n
	return p
}

// SetOut sets underlying writer of progress
// default is os.Stdout
func (p *Progress) SetOut(w io.Writer) *Progress {
	if w == nil {
		return p
	}
	p.out = w
	return p
}

// RefreshRate overrides default (30ms) refreshRate value
func (p *Progress) RefreshRate(d time.Duration) *Progress {
	p.rrChangeReqCh <- d
	return p
}

func (p *Progress) WithSort(sort SortType) *Progress {
	p.sort = sort
	return p
}

// AddBar creates a new progress bar and adds to the container
func (p *Progress) AddBar(total int) *Bar {
	p.wg.Add(1)
	bar := newBar(total, p.width, p.wg)
	p.op <- &operation{opBarAdd, bar, nil}
	return bar
}

func (p *Progress) RemoveBar(b *Bar) bool {
	result := make(chan bool)
	p.op <- &operation{opBarRemove, b, result}
	return <-result
}

// WaitAndStop stops listening
func (p *Progress) WaitAndStop() {
	if !p.stopped {
		// fmt.Fprintln(os.Stderr, "p.WaitAndStop")
		p.stopped = true
		p.wg.Wait()
		close(p.op)
	}
}

// server monitors underlying channels and renders any progress bars
func (p *Progress) server() {
	t := time.NewTicker(refreshRate * time.Millisecond)
	bars := make([]*Bar, 0, 4)
	lw := uilive.New(p.out)
	for {
		select {
		case op, ok := <-p.op:
			if !ok {
				// fmt.Fprintln(os.Stderr, "Sopping bars")
				for _, b := range bars {
					b.Stop()
				}
				t.Stop()
				return
			}
			switch op.kind {
			case opBarAdd:
				bars = append(bars, op.bar)
			case opBarRemove:
				var ok bool
				for i, b := range bars {
					if b == op.bar {
						bars = append(bars[:i], bars[i+1:]...)
						ok = true
						b.Stop()
						break
					}
				}
				op.result <- ok
			}
		case <-t.C:
			switch p.sort {
			case SortTop:
				sort.Sort(sort.Reverse(SortableBarSlice(bars)))
			case SortBottom:
				sort.Sort(SortableBarSlice(bars))
			}
			for _, b := range bars {
				// cannot parallel this, because order matters
				fmt.Fprintln(lw, b)
			}
			lw.Flush()
			for _, b := range bars {
				go func(b *Bar) {
					b.flushedCh <- struct{}{}
				}(b)
			}
		case d := <-p.rrChangeReqCh:
			t.Stop()
			t = time.NewTicker(d)
		}
	}
}
<commit_msg>default refreshRate = 100 ms<commit_after>package mpb

import (
	"fmt"
	"io"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/vbauerster/uilive"
)

type opType uint

const (
	opBarAdd opType = iota
	opBarRemove
)

type SortType uint

const (
	SortNone SortType = iota
	SortTop
	SortBottom
)

const refreshRate = 100

// Progress represents the container that renders Progress bars
type Progress struct {
	out     io.Writer
	width   int
	sort    SortType
	stopped bool

	op            chan *operation
	rrChangeReqCh chan time.Duration

	wg *sync.WaitGroup
}

type operation struct {
	kind   opType
	bar    *Bar
	result chan bool
}

// New returns a new progress bar with defaults
func New() *Progress {
	p := &Progress{
		out:           os.Stdout,
		width:         70,
		op:            make(chan *operation),
		rrChangeReqCh: make(chan time.Duration),
		wg:            new(sync.WaitGroup),
	}
	go p.server()
	return p
}

func (p *Progress) SetWidth(n int) *Progress {
	if n <= 0 {
		return p
	}
	p.width = n
	return p
}

// SetOut sets underlying writer of progress
// default is os.Stdout
func (p *Progress) SetOut(w io.Writer) *Progress {
	if w == nil {
		return p
	}
	p.out = w
	return p
}

// RefreshRate overrides default (30ms) refreshRate value
func (p *Progress) RefreshRate(d time.Duration) *Progress {
	p.rrChangeReqCh <- d
	return p
}

func (p *Progress) WithSort(sort SortType) *Progress {
	p.sort = sort
	return p
}

// AddBar creates a new progress bar and adds to the container
func (p *Progress) AddBar(total int) *Bar {
	p.wg.Add(1)
	bar := newBar(total, p.width, p.wg)
	p.op <- &operation{opBarAdd, bar, nil}
	return bar
}

func (p *Progress) RemoveBar(b *Bar) bool {
	result := make(chan bool)
	p.op <- &operation{opBarRemove, b, result}
	return <-result
}

// WaitAndStop stops listening
func (p *Progress) WaitAndStop() {
	if !p.stopped {
		// fmt.Fprintln(os.Stderr, "p.WaitAndStop")
		p.stopped = true
		p.wg.Wait()
		close(p.op)
	}
}

// server monitors underlying channels and renders any progress bars
func (p *Progress) server() {
	t := time.NewTicker(refreshRate * time.Millisecond)
	bars := make([]*Bar, 0, 4)
	lw := uilive.New(p.out)
	for {
		select {
		case op, ok := <-p.op:
			if !ok {
				// fmt.Fprintln(os.Stderr, "Sopping bars")
				for _, b := range bars {
					b.Stop()
				}
				t.Stop()
				return
			}
			switch op.kind {
			case opBarAdd:
				bars = append(bars, op.bar)
			case opBarRemove:
				var ok bool
				for i, b := range bars {
					if b == op.bar {
						bars = append(bars[:i], bars[i+1:]...)
						ok = true
						b.Stop()
						break
					}
				}
				op.result <- ok
			}
		case <-t.C:
			switch p.sort {
			case SortTop:
				sort.Sort(sort.Reverse(SortableBarSlice(bars)))
			case SortBottom:
				sort.Sort(SortableBarSlice(bars))
			}
			for _, b := range bars {
				// cannot parallel this, because order matters
				fmt.Fprintln(lw, b)
			}
			lw.Flush()
			for _, b := range bars {
				go func(b *Bar) {
					b.flushedCh <- struct{}{}
				}(b)
			}
		case d := <-p.rrChangeReqCh:
			t.Stop()
			t = time.NewTicker(d)
		}
	}
}
<|endoftext|>


<commit_before>package cluster

import (
	"context"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"time"

	dqlite "github.com/canonical/go-dqlite"
	client "github.com/canonical/go-dqlite/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/node"
	"github.com/pkg/errors"
)

// ListDatabaseNodes returns a list of database node names.
func ListDatabaseNodes(database *db.Node) ([]string, error) {
	nodes := []db.RaftNode{}
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		nodes, err = tx.GetRaftNodes()
		return err
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to list database nodes")
	}
	addresses := make([]string, 0)
	for _, node := range nodes {
		if node.Role != db.RaftVoter {
			continue
		}
		addresses = append(addresses, node.Address)
	}
	return addresses, nil
}

// Recover attempts data recovery on the cluster database.
func Recover(database *db.Node) error {
	// Figure out if we actually act as dqlite node.
	var info *db.RaftNode
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		info, err = node.DetermineRaftNode(tx)
		return err
	})
	if err != nil {
		return errors.Wrap(err, "Failed to determine node role")
	}

	// If we're not a database node, return an error.
	if info == nil {
		return fmt.Errorf("This LXD instance has no database role")
	}

	// If this is a standalone node not exposed to the network, return an
	// error.
	if info.Address == "" {
		return fmt.Errorf("This LXD instance is not clustered")
	}

	dir := filepath.Join(database.Dir(), "global")
	server, err := dqlite.New(
		uint64(info.ID),
		info.Address,
		dir,
	)
	if err != nil {
		return errors.Wrap(err, "Failed to create dqlite server")
	}

	cluster := []dqlite.NodeInfo{
		{ID: uint64(info.ID), Address: info.Address},
	}

	err = server.Recover(cluster)
	if err != nil {
		return errors.Wrap(err, "Failed to recover database state")
	}

	// Update the list of raft nodes.
	err = database.Transaction(func(tx *db.NodeTx) error {
		nodes := []db.RaftNode{
			{ID: info.ID, Address: info.Address},
		}
		return tx.ReplaceRaftNodes(nodes)
	})
	if err != nil {
		return errors.Wrap(err, "Failed to update database nodes")
	}

	return nil
}

// Reconfigure replaces the entire cluster configuration.
// Addresses and node roles may be updated. Node IDs are read-only.
func Reconfigure(database *db.Node, nodes []db.RaftNode) error {
	var info *db.RaftNode
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		info, err = node.DetermineRaftNode(tx)

		return err
	})
	if err != nil || info == nil {
		return errors.Wrap(err, "Failed to determine node role")
	}

	// Update cluster.https_address if changed.
	for _, n := range nodes {
		if n.ID == info.ID && n.Address != info.Address {
			err := updateLocalAddress(database, n.Address)
			if err != nil {
				return err
			}

			break
		}
	}

	dir := filepath.Join(database.Dir(), "global")
	// Replace cluster configuration in dqlite.
	err = dqlite.ReconfigureMembershipExt(dir, nodes)
	if err != nil {
		return errors.Wrap(err, "Failed to recover database state")
	}

	// Replace cluster configuration in local raft_nodes database.
	err = database.Transaction(func(tx *db.NodeTx) error {
		return tx.ReplaceRaftNodes(nodes)
	})
	if err != nil {
		return err
	}

	// Create patch file for global nodes database.
	content := ""
	for _, node := range nodes {
		content += fmt.Sprintf("UPDATE nodes SET address = %q WHERE id = %d;\n", node.Address, node.ID)
	}

	if len(content) > 0 {
		dir := filepath.Join(database.Dir(), "patch.global.sql")
		err := ioutil.WriteFile(dir, []byte(content), 0644)
		if err != nil {
			return err
		}
	}

	return nil
}

// RemoveRaftNode removes a raft node from the raft configuration.
func RemoveRaftNode(gateway *Gateway, address string) error {
	nodes, err := gateway.currentRaftNodes()
	if err != nil {
		return errors.Wrap(err, "Failed to get current raft nodes")
	}
	var id uint64
	for _, node := range nodes {
		if node.Address == address {
			id = node.ID
			break
		}
	}
	if id == 0 {
		return fmt.Errorf("No raft node with address %q", address)
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	client, err := client.FindLeader(
		ctx, gateway.NodeStore(),
		client.WithDialFunc(gateway.raftDial()),
		client.WithLogFunc(DqliteLog),
	)
	if err != nil {
		return errors.Wrap(err, "Failed to connect to cluster leader")
	}
	defer client.Close()
	err = client.Remove(ctx, id)
	if err != nil {
		return errors.Wrap(err, "Failed to remove node")
	}
	return nil
}
<commit_msg>lxd/cluster/recover: add updateLocalAddress<commit_after>package cluster

import (
	"context"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"time"

	dqlite "github.com/canonical/go-dqlite"
	client "github.com/canonical/go-dqlite/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/node"
	"github.com/pkg/errors"
)

// ListDatabaseNodes returns a list of database node names.
func ListDatabaseNodes(database *db.Node) ([]string, error) {
	nodes := []db.RaftNode{}
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		nodes, err = tx.GetRaftNodes()
		return err
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to list database nodes")
	}
	addresses := make([]string, 0)
	for _, node := range nodes {
		if node.Role != db.RaftVoter {
			continue
		}
		addresses = append(addresses, node.Address)
	}
	return addresses, nil
}

// Recover attempts data recovery on the cluster database.
func Recover(database *db.Node) error {
	// Figure out if we actually act as dqlite node.
	var info *db.RaftNode
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		info, err = node.DetermineRaftNode(tx)
		return err
	})
	if err != nil {
		return errors.Wrap(err, "Failed to determine node role")
	}

	// If we're not a database node, return an error.
	if info == nil {
		return fmt.Errorf("This LXD instance has no database role")
	}

	// If this is a standalone node not exposed to the network, return an
	// error.
	if info.Address == "" {
		return fmt.Errorf("This LXD instance is not clustered")
	}

	dir := filepath.Join(database.Dir(), "global")
	server, err := dqlite.New(
		uint64(info.ID),
		info.Address,
		dir,
	)
	if err != nil {
		return errors.Wrap(err, "Failed to create dqlite server")
	}

	cluster := []dqlite.NodeInfo{
		{ID: uint64(info.ID), Address: info.Address},
	}

	err = server.Recover(cluster)
	if err != nil {
		return errors.Wrap(err, "Failed to recover database state")
	}

	// Update the list of raft nodes.
	err = database.Transaction(func(tx *db.NodeTx) error {
		nodes := []db.RaftNode{
			{ID: info.ID, Address: info.Address},
		}
		return tx.ReplaceRaftNodes(nodes)
	})
	if err != nil {
		return errors.Wrap(err, "Failed to update database nodes")
	}

	return nil
}

// updateLocalAddress updates the cluster.https_address for this node.
func updateLocalAddress(database *db.Node, address string) error {
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		config, err := node.ConfigLoad(tx)
		if err != nil {
			return err
		}

		newConfig := map[string]interface{}{"cluster.https_address": address}
		_, err = config.Patch(newConfig)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return errors.Wrapf(err, "Failed to update node configuration")
	}

	return nil
}

// Reconfigure replaces the entire cluster configuration.
// Addresses and node roles may be updated. Node IDs are read-only.
func Reconfigure(database *db.Node, nodes []db.RaftNode) error {
	var info *db.RaftNode
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		info, err = node.DetermineRaftNode(tx)

		return err
	})
	if err != nil || info == nil {
		return errors.Wrap(err, "Failed to determine node role")
	}

	// Update cluster.https_address if changed.
	for _, n := range nodes {
		if n.ID == info.ID && n.Address != info.Address {
			err := updateLocalAddress(database, n.Address)
			if err != nil {
				return err
			}

			break
		}
	}

	dir := filepath.Join(database.Dir(), "global")
	// Replace cluster configuration in dqlite.
	err = dqlite.ReconfigureMembershipExt(dir, nodes)
	if err != nil {
		return errors.Wrap(err, "Failed to recover database state")
	}

	// Replace cluster configuration in local raft_nodes database.
	err = database.Transaction(func(tx *db.NodeTx) error {
		return tx.ReplaceRaftNodes(nodes)
	})
	if err != nil {
		return err
	}

	// Create patch file for global nodes database.
	content := ""
	for _, node := range nodes {
		content += fmt.Sprintf("UPDATE nodes SET address = %q WHERE id = %d;\n", node.Address, node.ID)
	}

	if len(content) > 0 {
		dir := filepath.Join(database.Dir(), "patch.global.sql")
		err := ioutil.WriteFile(dir, []byte(content), 0644)
		if err != nil {
			return err
		}
	}

	return nil
}

// RemoveRaftNode removes a raft node from the raft configuration.
func RemoveRaftNode(gateway *Gateway, address string) error {
	nodes, err := gateway.currentRaftNodes()
	if err != nil {
		return errors.Wrap(err, "Failed to get current raft nodes")
	}
	var id uint64
	for _, node := range nodes {
		if node.Address == address {
			id = node.ID
			break
		}
	}
	if id == 0 {
		return fmt.Errorf("No raft node with address %q", address)
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	client, err := client.FindLeader(
		ctx, gateway.NodeStore(),
		client.WithDialFunc(gateway.raftDial()),
		client.WithLogFunc(DqliteLog),
	)
	if err != nil {
		return errors.Wrap(err, "Failed to connect to cluster leader")
	}
	defer client.Close()
	err = client.Remove(ctx, id)
	if err != nil {
		return errors.Wrap(err, "Failed to remove node")
	}
	return nil
}
<|endoftext|>


<commit_before>package request

import (
	"net/http"

	"github.com/lxc/lxd/shared/api"
)

// CreateRequestor extracts the lifecycle event requestor data from an http.Request context
func CreateRequestor(r *http.Request) *api.EventLifecycleRequestor {
	ctx := r.Context()
	requestor := &api.EventLifecycleRequestor{}

	// Normal requestor.
	val, ok := ctx.Value(CtxUsername).(string)
	if ok {
		requestor.Username = val
	}

	val, ok = ctx.Value(CtxProtocol).(string)
	if ok {
		requestor.Protocol = val
	}

	// Forwarded requestor override.
	val, ok = ctx.Value(CtxForwardedUsername).(string)
	if ok {
		requestor.Username = val
	}

	val, ok = ctx.Value(CtxForwardedProtocol).(string)
	if ok {
		requestor.Protocol = val
	}
	return requestor
}
<commit_msg>lxd/request: Support for Address field<commit_after>package request

import (
	"net/http"

	"github.com/lxc/lxd/shared/api"
)

// CreateRequestor extracts the lifecycle event requestor data from an http.Request context
func CreateRequestor(r *http.Request) *api.EventLifecycleRequestor {
	ctx := r.Context()
	requestor := &api.EventLifecycleRequestor{}

	// Normal requestor.
	val, ok := ctx.Value(CtxUsername).(string)
	if ok {
		requestor.Username = val
	}

	val, ok = ctx.Value(CtxProtocol).(string)
	if ok {
		requestor.Protocol = val
	}

	requestor.Address = r.RemoteAddr

	// Forwarded requestor override.
	val, ok = ctx.Value(CtxForwardedUsername).(string)
	if ok {
		requestor.Username = val
	}

	val, ok = ctx.Value(CtxForwardedProtocol).(string)
	if ok {
		requestor.Protocol = val
	}

	val, ok = ctx.Value(CtxForwardedAddress).(string)
	if ok {
		requestor.Address = val
	}
	return requestor
}
<|endoftext|>


<commit_before>package hstspreload

import (
	"net/url"
	"sync"
	"testing"
)

func chainsEqual(actual []*url.URL, expected []string) bool {
	if len(actual) != len(expected) {
		return false
	}
	for i, u := range actual {
		if u.String() != expected[i] {
			return false
		}
	}
	return true
}

var tooManyRedirectsTests = []struct {
	description    string
	url            string
	expectedChain  []string
	expectedIssues Issues
}{
	{
		"almost too many redirects",
		"https://httpbin.org/redirect/3",
		[]string{"https://httpbin.org/relative-redirect/2", "https://httpbin.org/relative-redirect/1", "https://httpbin.org/get"},
		Issues{},
	},
	{
		"too many redirects",
		"https://httpbin.org/redirect/4",
		[]string{"https://httpbin.org/relative-redirect/3", "https://httpbin.org/relative-redirect/2", "https://httpbin.org/relative-redirect/1", "https://httpbin.org/get"},
		Issues{Errors: []Issue{{
			Code:    "redirects.too_many",
			Message: "There are more than 3 redirects starting from `https://httpbin.org/redirect/4`.",
		}}},
	},
}

func TestTooManyRedirects(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	for _, tt := range tooManyRedirectsTests {
		chain, issues := preloadableRedirects(tt.url)
		if !chainsEqual(chain, tt.expectedChain) {
			t.Errorf("[%s] Unexpected chain: %v", tt.description, chain)
		}

		if !issues.Match(tt.expectedIssues) {
			t.Errorf("[%s] "+issuesShouldMatch, tt.description, issues, tt.expectedIssues)
		}
	}
}

func TestInsecureRedirect(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "https://httpbin.org/redirect-to?url=http://httpbin.org"

	chain, issues := preloadableRedirects(u)
	if !chainsEqual(chain, []string{"http://httpbin.org"}) {
		t.Errorf("Unexpected chain: %v", chain)
	}
	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	httpsIssues := preloadableHTTPSRedirectsURL(u)
	expected := Issues{Errors: []Issue{{
		Code:    "redirects.insecure.initial",
		Message: "`https://httpbin.org/redirect-to?url=http://httpbin.org` redirects to an insecure page: `http://httpbin.org`",
	}}}
	if !httpsIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, httpsIssues, expected)
	}
}

func TestIndirectInsecureRedirect(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "https://httpbin.org/redirect-to?url=https://httpbin.org/redirect-to?url=http://httpbin.org"

	chain, issues := preloadableRedirects(u)
	if !chainsEqual(chain, []string{"https://httpbin.org/redirect-to?url=http://httpbin.org", "http://httpbin.org"}) {
		t.Errorf("Unexpected chain: %v", chain)
	}
	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	httpsIssues := preloadableHTTPSRedirectsURL(u)
	expected := Issues{Errors: []Issue{{
		Code:    "redirects.insecure.subsequent",
		Message: "`https://httpbin.org/redirect-to?url=https://httpbin.org/redirect-to?url=http://httpbin.org` redirects to an insecure page on redirect #2: `http://httpbin.org`",
	}}}
	if !httpsIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, httpsIssues, expected)
	}
}

func TestExplicitPortFirstRedirect(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "https://tls-v1-1.badssl.com"

	chain, issues := preloadableRedirects(u)
	if !chainsEqual(chain, []string{"https://tls-v1-1.badssl.com:1011/"}) {
		t.Errorf("Unexpected chain: %v", chain)
	}
	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	httpsIssues := preloadableHTTPSRedirectsURL(u)
	expected := Issues{}
	if !httpsIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, httpsIssues, expected)
	}
}

func TestHTTPUnavailable(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "http://psa.gov"
	domain := "psa.gov"

	// Test the helper
	issues, cont := checkHSTSOverHTTP(u)
	expected := Issues{Warnings: []Issue{{
		Code:    "redirects.http.does_not_exist",
		Message: "The site appears to be unavailable over plain HTTP (http://psa.gov). This can prevent users without a freshly updated modern browser from connecting to the site when they visit a URL with the http:// scheme (or with an unspecified scheme). However, this is okay if the site does not wish to support those users.",
	}}}
	if !issues.Match(expected) {
		t.Errorf(issuesShouldMatch, issues, expected)
	}
	if cont {
		t.Errorf("Should continue.")
	}

	// Mini integration test
	mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirectsURL(u, domain)
	expected = Issues{
		Warnings: []Issue{{Code: "redirects.http.does_not_exist"}},
	}
	if !mainIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, mainIssues, expected)
	}

	if !firstRedirectHSTSIssues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, firstRedirectHSTSIssues)
	}
}

func TestHSTSOverHTTP(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "http://history.google.com"
	domain := "history.google.com"

	_, issues := preloadableRedirects(u)
	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	// Test the helper
	issues, cont := checkHSTSOverHTTP(u)
	expected := Issues{Warnings: []Issue{{
		Code:    "redirects.http.useless_header",
		Message: "The HTTP page at http://history.google.com sends an HSTS header. This has no effect over HTTP, and should be removed.",
	}}}
	if !issues.Match(expected) {
		t.Errorf(issuesShouldMatch, issues, expected)
	}
	if !cont {
		t.Fatalf("Should not continue.")
	}

	// Mini integration test
	mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirectsURL(u, domain)
	expected = Issues{
		Errors:   []Issue{{Code: "redirects.http.first_redirect.insecure"}},
		Warnings: []Issue{{Code: "redirects.http.useless_header"}},
	}
	if !mainIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, mainIssues, expected)
	}

	if !firstRedirectHSTSIssues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, firstRedirectHSTSIssues)
	}
}

func TestHTTPNoRedirect(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "http://httpbin.org"
	domain := "httpbin.org"

	chain, issues := preloadableRedirects(u)
	if !chainsEqual(chain, []string{}) {
		t.Errorf("Unexpected chain: %v", chain)
	}

	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirectsURL(u, domain)
	expected := Issues{Errors: []Issue{{
		Code:    "redirects.http.no_redirect",
		Message: "`http://httpbin.org` does not redirect to `https://httpbin.org`.",
	}}}
	if !mainIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, mainIssues, expected)
	}

	if !firstRedirectHSTSIssues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, firstRedirectHSTSIssues)
	}
}

type preloadableHTTPRedirectsTest struct {
	description                     string
	domain                          string
	expectedMainIssues              Issues
	expectedFirstRedirectHSTSIssues Issues
}

var preloadableHTTPRedirectsTests = []preloadableHTTPRedirectsTest{
	{
		"different host",
		"bofa.com", // http://bofa.com redirects to https://www.bankofamerica.com
		Issues{Errors: []Issue{{
			Code:    "redirects.http.first_redirect.insecure",
			Message: "`http://bofa.com` (HTTP) redirects to `https://www.bankofamerica.com/vanity/redirect.go?src=/`. The first redirect from `http://bofa.com` should be to a secure page on the same host (`https://bofa.com`).",
		}}},
		Issues{},
	},
	{
		"same origin",
		"www.wikia.com", // http://www.wikia.com redirects to http://www.wikia.com/fandom
		Issues{Errors: []Issue{{
			Code:    "redirects.http.first_redirect.insecure",
			Message: "`http://www.wikia.com` (HTTP) redirects to `http://www.wikia.com/fandom`. The first redirect from `http://www.wikia.com` should be to a secure page on the same host (`https://www.wikia.com`).",
		}}},
		Issues{},
	},
	{
		"www first and > 3 redirects",
		"blogger.com",
		Issues{
			Errors: []Issue{
				{
					Code:    "redirects.too_many",
					Message: "There are more than 3 redirects starting from `http://blogger.com`.",
				},
				{
					Code:    "redirects.http.www_first",
					Message: "`http://blogger.com` (HTTP) should immediately redirect to `https://blogger.com` (HTTPS) before adding the www subdomain. Right now, the first redirect is to `http://www.blogger.com/`. The extra redirect is required to ensure that any browser which supports HSTS will record the HSTS entry for the top level domain, not just the subdomain.",
				},
			},
		},
		Issues{},
	},
	{
		"correct origin but not HSTS",
		"sha256.badssl.com",
		Issues{},
		Issues{Errors: []Issue{{
			Code:    "redirects.http.first_redirect.no_hsts",
			Message: "`http://sha256.badssl.com` redirects to `https://sha256.badssl.com/`, which does not serve a HSTS header that satisfies preload conditions. First error: No HSTS header",
		}}},
	},
}

func TestPreloadableHTTPRedirects(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	wg := sync.WaitGroup{}
	wg.Add(len(preloadableHTTPRedirectsTests))

	for _, tt := range preloadableHTTPRedirectsTests {
		go func(tt preloadableHTTPRedirectsTest) {
			mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirects(tt.domain)

			if !mainIssues.Match(tt.expectedMainIssues) {
				t.Errorf("[%s] main issues for %s: "+issuesShouldMatch, tt.description, tt.domain, mainIssues, tt.expectedMainIssues)
			}

			if !firstRedirectHSTSIssues.Match(tt.expectedFirstRedirectHSTSIssues) {
				t.Errorf("[%s] first redirect HSTS issues for %s: "+issuesShouldMatch, tt.description, tt.domain, firstRedirectHSTSIssues, tt.expectedFirstRedirectHSTSIssues)
			}
			wg.Done()
		}(tt)
	}

	wg.Wait()
}
<commit_msg>Update test URL for TestHTTPUnavailable.<commit_after>package hstspreload

import (
	"net/url"
	"sync"
	"testing"
)

func chainsEqual(actual []*url.URL, expected []string) bool {
	if len(actual) != len(expected) {
		return false
	}
	for i, u := range actual {
		if u.String() != expected[i] {
			return false
		}
	}
	return true
}

var tooManyRedirectsTests = []struct {
	description    string
	url            string
	expectedChain  []string
	expectedIssues Issues
}{
	{
		"almost too many redirects",
		"https://httpbin.org/redirect/3",
		[]string{"https://httpbin.org/relative-redirect/2", "https://httpbin.org/relative-redirect/1", "https://httpbin.org/get"},
		Issues{},
	},
	{
		"too many redirects",
		"https://httpbin.org/redirect/4",
		[]string{"https://httpbin.org/relative-redirect/3", "https://httpbin.org/relative-redirect/2", "https://httpbin.org/relative-redirect/1", "https://httpbin.org/get"},
		Issues{Errors: []Issue{{
			Code:    "redirects.too_many",
			Message: "There are more than 3 redirects starting from `https://httpbin.org/redirect/4`.",
		}}},
	},
}

func TestTooManyRedirects(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	for _, tt := range tooManyRedirectsTests {
		chain, issues := preloadableRedirects(tt.url)
		if !chainsEqual(chain, tt.expectedChain) {
			t.Errorf("[%s] Unexpected chain: %v", tt.description, chain)
		}

		if !issues.Match(tt.expectedIssues) {
			t.Errorf("[%s] "+issuesShouldMatch, tt.description, issues, tt.expectedIssues)
		}
	}
}

func TestInsecureRedirect(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "https://httpbin.org/redirect-to?url=http://httpbin.org"

	chain, issues := preloadableRedirects(u)
	if !chainsEqual(chain, []string{"http://httpbin.org"}) {
		t.Errorf("Unexpected chain: %v", chain)
	}
	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	httpsIssues := preloadableHTTPSRedirectsURL(u)
	expected := Issues{Errors: []Issue{{
		Code:    "redirects.insecure.initial",
		Message: "`https://httpbin.org/redirect-to?url=http://httpbin.org` redirects to an insecure page: `http://httpbin.org`",
	}}}
	if !httpsIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, httpsIssues, expected)
	}
}

func TestIndirectInsecureRedirect(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "https://httpbin.org/redirect-to?url=https://httpbin.org/redirect-to?url=http://httpbin.org"

	chain, issues := preloadableRedirects(u)
	if !chainsEqual(chain, []string{"https://httpbin.org/redirect-to?url=http://httpbin.org", "http://httpbin.org"}) {
		t.Errorf("Unexpected chain: %v", chain)
	}
	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	httpsIssues := preloadableHTTPSRedirectsURL(u)
	expected := Issues{Errors: []Issue{{
		Code:    "redirects.insecure.subsequent",
		Message: "`https://httpbin.org/redirect-to?url=https://httpbin.org/redirect-to?url=http://httpbin.org` redirects to an insecure page on redirect #2: `http://httpbin.org`",
	}}}
	if !httpsIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, httpsIssues, expected)
	}
}

func TestExplicitPortFirstRedirect(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "https://tls-v1-1.badssl.com"

	chain, issues := preloadableRedirects(u)
	if !chainsEqual(chain, []string{"https://tls-v1-1.badssl.com:1011/"}) {
		t.Errorf("Unexpected chain: %v", chain)
	}
	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	httpsIssues := preloadableHTTPSRedirectsURL(u)
	expected := Issues{}
	if !httpsIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, httpsIssues, expected)
	}
}

func TestHTTPUnavailable(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "http://oskuro.net"
	domain := "oskuro.net"

	// Test the helper
	issues, cont := checkHSTSOverHTTP(u)
	expected := Issues{Warnings: []Issue{{
		Code:    "redirects.http.does_not_exist",
		Message: "The site appears to be unavailable over plain HTTP (http://oskuro.net). This can prevent users without a freshly updated modern browser from connecting to the site when they visit a URL with the http:// scheme (or with an unspecified scheme). However, this is okay if the site does not wish to support those users.",
	}}}
	if !issues.Match(expected) {
		t.Errorf(issuesShouldMatch, issues, expected)
	}
	if cont {
		t.Errorf("Should continue.")
	}

	// Mini integration test
	mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirectsURL(u, domain)
	expected = Issues{
		Warnings: []Issue{{Code: "redirects.http.does_not_exist"}},
	}
	if !mainIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, mainIssues, expected)
	}

	if !firstRedirectHSTSIssues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, firstRedirectHSTSIssues)
	}
}

func TestHSTSOverHTTP(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "http://history.google.com"
	domain := "history.google.com"

	_, issues := preloadableRedirects(u)
	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	// Test the helper
	issues, cont := checkHSTSOverHTTP(u)
	expected := Issues{Warnings: []Issue{{
		Code:    "redirects.http.useless_header",
		Message: "The HTTP page at http://history.google.com sends an HSTS header. This has no effect over HTTP, and should be removed.",
	}}}
	if !issues.Match(expected) {
		t.Errorf(issuesShouldMatch, issues, expected)
	}
	if !cont {
		t.Fatalf("Should not continue.")
	}

	// Mini integration test
	mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirectsURL(u, domain)
	expected = Issues{
		Errors:   []Issue{{Code: "redirects.http.first_redirect.insecure"}},
		Warnings: []Issue{{Code: "redirects.http.useless_header"}},
	}
	if !mainIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, mainIssues, expected)
	}

	if !firstRedirectHSTSIssues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, firstRedirectHSTSIssues)
	}
}

func TestHTTPNoRedirect(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	u := "http://httpbin.org"
	domain := "httpbin.org"

	chain, issues := preloadableRedirects(u)
	if !chainsEqual(chain, []string{}) {
		t.Errorf("Unexpected chain: %v", chain)
	}

	if !issues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, issues)
	}

	mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirectsURL(u, domain)
	expected := Issues{Errors: []Issue{{
		Code:    "redirects.http.no_redirect",
		Message: "`http://httpbin.org` does not redirect to `https://httpbin.org`.",
	}}}
	if !mainIssues.Match(expected) {
		t.Errorf(issuesShouldMatch, mainIssues, expected)
	}

	if !firstRedirectHSTSIssues.Match(Issues{}) {
		t.Errorf(issuesShouldBeEmpty, firstRedirectHSTSIssues)
	}
}

type preloadableHTTPRedirectsTest struct {
	description                     string
	domain                          string
	expectedMainIssues              Issues
	expectedFirstRedirectHSTSIssues Issues
}

var preloadableHTTPRedirectsTests = []preloadableHTTPRedirectsTest{
	{
		"different host",
		"bofa.com", // http://bofa.com redirects to https://www.bankofamerica.com
		Issues{Errors: []Issue{{
			Code:    "redirects.http.first_redirect.insecure",
			Message: "`http://bofa.com` (HTTP) redirects to `https://www.bankofamerica.com/vanity/redirect.go?src=/`. The first redirect from `http://bofa.com` should be to a secure page on the same host (`https://bofa.com`).",
		}}},
		Issues{},
	},
	{
		"same origin",
		"www.wikia.com", // http://www.wikia.com redirects to http://www.wikia.com/fandom
		Issues{Errors: []Issue{{
			Code:    "redirects.http.first_redirect.insecure",
			Message: "`http://www.wikia.com` (HTTP) redirects to `http://www.wikia.com/fandom`. The first redirect from `http://www.wikia.com` should be to a secure page on the same host (`https://www.wikia.com`).",
		}}},
		Issues{},
	},
	{
		"www first and > 3 redirects",
		"blogger.com",
		Issues{
			Errors: []Issue{
				{
					Code:    "redirects.too_many",
					Message: "There are more than 3 redirects starting from `http://blogger.com`.",
				},
				{
					Code:    "redirects.http.www_first",
					Message: "`http://blogger.com` (HTTP) should immediately redirect to `https://blogger.com` (HTTPS) before adding the www subdomain. Right now, the first redirect is to `http://www.blogger.com/`. The extra redirect is required to ensure that any browser which supports HSTS will record the HSTS entry for the top level domain, not just the subdomain.",
				},
			},
		},
		Issues{},
	},
	{
		"correct origin but not HSTS",
		"sha256.badssl.com",
		Issues{},
		Issues{Errors: []Issue{{
			Code:    "redirects.http.first_redirect.no_hsts",
			Message: "`http://sha256.badssl.com` redirects to `https://sha256.badssl.com/`, which does not serve a HSTS header that satisfies preload conditions. First error: No HSTS header",
		}}},
	},
}

func TestPreloadableHTTPRedirects(t *testing.T) {
	skipIfShort(t)
	t.Parallel()

	wg := sync.WaitGroup{}
	wg.Add(len(preloadableHTTPRedirectsTests))

	for _, tt := range preloadableHTTPRedirectsTests {
		go func(tt preloadableHTTPRedirectsTest) {
			mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirects(tt.domain)

			if !mainIssues.Match(tt.expectedMainIssues) {
				t.Errorf("[%s] main issues for %s: "+issuesShouldMatch, tt.description, tt.domain, mainIssues, tt.expectedMainIssues)
			}

			if !firstRedirectHSTSIssues.Match(tt.expectedFirstRedirectHSTSIssues) {
				t.Errorf("[%s] first redirect HSTS issues for %s: "+issuesShouldMatch, tt.description, tt.domain, firstRedirectHSTSIssues, tt.expectedFirstRedirectHSTSIssues)
			}
			wg.Done()
		}(tt)
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package batchposter

import (
	"errors"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"
)

var (
	ErrBufferFull = errors.New("buffer full")
)

type BatchPoster struct {
	url        string        // the url to POST to when flushing the buffer
	buffersize int           // the maximum amount of posts that can be batched
	autoflush  time.Duration // max time a buffer can fillup before flushing
	log        *log.Logger   // where errors get reported to

	mu     sync.RWMutex
	buffer []string
	idx    int
}

func New(url string, maxbatch int, maxtime time.Duration, l *log.Logger) *BatchPoster {

	obj := &BatchPoster{url: url, buffersize: maxbatch, autoflush: maxtime, log: l}
	obj.idx = 0
	obj.buffer = make([]string, obj.buffersize)

	go func() {
		for {
			// TODO: Add ability to start/stop autoflushing
			time.Sleep(obj.autoflush)
			obj.flush()
		}
	}()

	return obj
}

func (b *BatchPoster) Post(payload string) error {

	err := b.add(payload)
	if err != nil {
		return err
	}

	if b.full() {
		err := b.flush()
		if err != nil {
			return err
		}
	}

	return nil
}

//////////////////////////////////////////////////////////////////////////
// private parts
//////////////////////////////////////////////////////////////////////////

func (b *BatchPoster) full() bool {

	b.mu.RLock()
	defer b.mu.RUnlock()

	return b.idx >= b.buffersize
}

func (b *BatchPoster) add(payload string) error {

	if b.full() {
		_ = b.flush()
		b.log.Println(ErrBufferFull)
		return ErrBufferFull
	}

	b.mu.Lock()
	defer b.mu.Unlock()

	b.buffer[b.idx] = payload
	b.idx++

	return nil
}

func (b *BatchPoster) flush() error {

	b.mu.Lock()
	defer b.mu.Unlock()

	if b.idx == 0 {
		return nil
	}

	batch := strings.Join(b.buffer, "\n")
	_, err := http.Post(b.url, "", strings.NewReader(batch))
	if err != nil {
		b.log.Println(err)
		return err
	}

	// reset
	for i := 0; i < b.buffersize; i++ {
		b.buffer[i] = ""
	}
	b.idx = 0

	return nil
}
<commit_msg>small refactoring, commenting<commit_after>package batchposter

import (
	"errors"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"
)

var (
	ErrBufferFull = errors.New("buffer full")
)

type BatchPoster struct {
	url        string        // the url to POST to when flushing the buffer
	buffersize int           // the maximum amount of posts that can be batched
	autoflush  time.Duration // max time a buffer can fillup before flushing
	log        *log.Logger   // where errors get reported to

	mu     sync.RWMutex
	buffer []string
	idx    int
}

func New(url string, maxbatch int, maxtime time.Duration, l *log.Logger) *BatchPoster {

	obj := &BatchPoster{url: url, buffersize: maxbatch, autoflush: maxtime, log: l}
	obj.idx = 0
	obj.buffer = make([]string, obj.buffersize)

	go func() {
		for {
			// TODO: Add ability to start/stop autoflushing
			time.Sleep(obj.autoflush)
			obj.flush()
		}
	}()

	return obj
}

func (b *BatchPoster) Post(payload string) error {

	if b.full() {

		// something is wrong, buffer should never be full. Try flushing and ignore errors.
		go b.flush()

		b.log.Println(ErrBufferFull)
		return ErrBufferFull
	}

	err := b.add(payload)
	if err != nil {
		return err
	}

	if b.full() {
		err := b.flush()
		if err != nil {
			return err
		}
	}

	return nil
}

//////////////////////////////////////////////////////////////////////////
// private parts
//////////////////////////////////////////////////////////////////////////

func (b *BatchPoster) full() bool {

	b.mu.RLock()
	defer b.mu.RUnlock()

	return b.idx >= b.buffersize
}

func (b *BatchPoster) add(payload string) error {

	b.mu.Lock()
	defer b.mu.Unlock()

	// add to buffer
	b.buffer[b.idx] = payload
	b.idx++

	return nil
}

func (b *BatchPoster) flush() error {

	b.mu.Lock()
	defer b.mu.Unlock()

	// bail on nothing to post
	if b.idx == 0 {
		return nil
	}

	// post batch
	batch := strings.Join(b.buffer, "\n")
	_, err := http.Post(b.url, "", strings.NewReader(batch))
	if err != nil {
		b.log.Println(err)
		return err
	}

	// reset
	for i := 0; i < b.buffersize; i++ {
		b.buffer[i] = ""
	}
	b.idx = 0

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"fmt"
	"strings"
	"time"

	"golang.org/x/net/context"

	"go.chromium.org/gae/service/datastore"

	"go.chromium.org/luci/common/logging"
)

// InvalidBuilderIDURL is returned if a BuilderID cannot be parsed and a URL generated.
const InvalidBuilderIDURL = "#invalid-builder-id"

// BuilderSummary holds builder state for the purpose of representing e.g. header consoles.
type BuilderSummary struct {
	// BuilderID is the global identifier for the builder that this Build belongs to, i.e.:
	//   "buildbot/<mastername>/<buildername>"
	//   "buildbucket/<bucketname>/<buildername>"
	// Matches field in BuildSummary.
	BuilderID string `gae:"$id"`

	// The LUCI project ID associated with this build. This is used for generating links.
	ProjectID string

	// LastFinishedCreated is the time the last finished build was created.
	LastFinishedCreated time.Time

	// LastFinishedStatus is the status of last finished build on builder.
	LastFinishedStatus Status

	// LastFinishedBuildID is the BuildID of the BuildSummary associated with last finished build on
	// the builder.
	LastFinishedBuildID string

	// Consoles lists consoles of which this builder is part.
	// Elements of this list should be of the form
	// <common.Console.GetProjectID()>/<common.Console.ID>.
	Consoles []string // indexed on this

	// Ignore unrecognized fields and strip in future writes.
	_ datastore.PropertyMap `gae:"-,extra"`
}

// LastFinishedBuildIDLink returns a link to the last finished build.
func (b *BuilderSummary) LastFinishedBuildIDLink() string {
	return buildIDLink(b.LastFinishedBuildID, b.ProjectID)
}

// SelfLink returns a link to the associated builder.
func (b *BuilderSummary) SelfLink() string {
	return BuilderIDLink(b.BuilderID, b.ProjectID)
}

// BuilderIDLink gets a builder link from builder ID and project.
// Depends on routes.go.
func BuilderIDLink(b, project string) string {
	parts := strings.Split(b, "/")
	if len(parts) != 3 {
		return InvalidBuilderIDURL
	}

	switch source := parts[0]; source {
	case "buildbot":
		return fmt.Sprintf("/buildbot/%s/%s", parts[1], parts[2])
	case "buildbucket":
		return fmt.Sprintf("/p/%s/builders/%s/%s", project, parts[1], parts[2])
	default:
		return InvalidBuilderIDURL
	}
}

// UpdateBuilderForBuild updates the appropriate BuilderSummary for the
// provided BuildSummary, if needed.
// In particular, a BuilderSummary is updated with a BuildSummary if the latter is marked complete
// and has a more recent creation time than the one stored in the BuilderSummary.
// If there is no existing BuilderSummary for the BuildSummary provided, one is created.
// Idempotent.
// c must have a current datastore transaction.
func UpdateBuilderForBuild(c context.Context, build *BuildSummary) error {
	if datastore.CurrentTransaction(c) == nil {
		panic("UpdateBuilderForBuild was called outside of a transaction")
	}

	if !build.Summary.Status.Terminal() {
		// this build did not complete yet.
		// There is no new info for the builder.
		// Do not even bother starting a transaction.
		return nil
	}

	// Get or create the relevant BuilderSummary.
	builder := &BuilderSummary{BuilderID: build.BuilderID}
	switch err := datastore.Get(c, builder); {
	case err == datastore.ErrNoSuchEntity:
		logging.Warningf(c, "creating new BuilderSummary for BuilderID: %s", build.BuilderID)
	case err != nil:
		return err
	case build.Created.Before(builder.LastFinishedCreated):
		logging.Warningf(c, "message for build %s is out of order", build.BuildID)
		return nil
	}

	// Overwrite previous value of ProjectID in case a builder has moved to
	// another project.
	builder.ProjectID = build.ProjectID
	builder.LastFinishedCreated = build.Created
	builder.LastFinishedStatus = build.Summary.Status
	builder.LastFinishedBuildID = build.BuildID
	return datastore.Put(c, builder)
}
<commit_msg>[milo] remove BuilderSummary.Console<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"fmt"
	"strings"
	"time"

	"golang.org/x/net/context"

	"go.chromium.org/gae/service/datastore"

	"go.chromium.org/luci/common/logging"
)

// InvalidBuilderIDURL is returned if a BuilderID cannot be parsed and a URL generated.
const InvalidBuilderIDURL = "#invalid-builder-id"

// BuilderSummary holds builder state for the purpose of representing e.g. header consoles.
type BuilderSummary struct {
	// BuilderID is the global identifier for the builder that this Build belongs to, i.e.:
	//   "buildbot/<mastername>/<buildername>"
	//   "buildbucket/<bucketname>/<buildername>"
	// Matches field in BuildSummary.
	BuilderID string `gae:"$id"`

	// The LUCI project ID associated with this build. This is used for generating links.
	ProjectID string

	// LastFinishedCreated is the time the last finished build was created.
	LastFinishedCreated time.Time

	// LastFinishedStatus is the status of last finished build on builder.
	LastFinishedStatus Status

	// LastFinishedBuildID is the BuildID of the BuildSummary associated with last finished build on
	// the builder.
	LastFinishedBuildID string

	// Ignore unrecognized fields and strip in future writes.
	_ datastore.PropertyMap `gae:"-,extra"`
}

// LastFinishedBuildIDLink returns a link to the last finished build.
func (b *BuilderSummary) LastFinishedBuildIDLink() string {
	return buildIDLink(b.LastFinishedBuildID, b.ProjectID)
}

// SelfLink returns a link to the associated builder.
func (b *BuilderSummary) SelfLink() string {
	return BuilderIDLink(b.BuilderID, b.ProjectID)
}

// BuilderIDLink gets a builder link from builder ID and project.
// Depends on routes.go.
func BuilderIDLink(b, project string) string {
	parts := strings.Split(b, "/")
	if len(parts) != 3 {
		return InvalidBuilderIDURL
	}

	switch source := parts[0]; source {
	case "buildbot":
		return fmt.Sprintf("/buildbot/%s/%s", parts[1], parts[2])
	case "buildbucket":
		return fmt.Sprintf("/p/%s/builders/%s/%s", project, parts[1], parts[2])
	default:
		return InvalidBuilderIDURL
	}
}

// UpdateBuilderForBuild updates the appropriate BuilderSummary for the
// provided BuildSummary, if needed.
// In particular, a BuilderSummary is updated with a BuildSummary if the latter is marked complete
// and has a more recent creation time than the one stored in the BuilderSummary.
// If there is no existing BuilderSummary for the BuildSummary provided, one is created.
// Idempotent.
// c must have a current datastore transaction.
func UpdateBuilderForBuild(c context.Context, build *BuildSummary) error {
	if datastore.CurrentTransaction(c) == nil {
		panic("UpdateBuilderForBuild was called outside of a transaction")
	}

	if !build.Summary.Status.Terminal() {
		// this build did not complete yet.
		// There is no new info for the builder.
		// Do not even bother starting a transaction.
		return nil
	}

	// Get or create the relevant BuilderSummary.
	builder := &BuilderSummary{BuilderID: build.BuilderID}
	switch err := datastore.Get(c, builder); {
	case err == datastore.ErrNoSuchEntity:
		logging.Warningf(c, "creating new BuilderSummary for BuilderID: %s", build.BuilderID)
	case err != nil:
		return err
	case build.Created.Before(builder.LastFinishedCreated):
		logging.Warningf(c, "message for build %s is out of order", build.BuildID)
		return nil
	}

	// Overwrite previous value of ProjectID in case a builder has moved to
	// another project.
	builder.ProjectID = build.ProjectID
	builder.LastFinishedCreated = build.Created
	builder.LastFinishedStatus = build.Summary.Status
	builder.LastFinishedBuildID = build.BuildID
	return datastore.Put(c, builder)
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire

import (
	"strconv"
	"strings"
)

const (
	MainPort               = "8333"
	TestNetPort            = "18333"
	ProtocolVersion uint32 = 70001
	TxVersion              = 1

	// MultipleAddressVersion is the protocol version which added multiple
	// addresses per message (pver >= MultipleAddressVersion).
	MultipleAddressVersion uint32 = 209

	// NetAddressTimeVersion is the protocol version which added the
	// timestamp field (pver >= NetAddressTimeVersion).
	NetAddressTimeVersion uint32 = 31402

	// BIP0031Version is the protocol version AFTER which a pong message
	// and nonce field in ping were added (pver > BIP0031Version).
	BIP0031Version uint32 = 60000

	// BIP0035Version is the protocol version which added the mempool
	// message (pver >= BIP0035Version).
	BIP0035Version uint32 = 60002

	// BIP0037Version is the protocol version which added new connection
	// bloom filtering related messages and extended the version message
	// with a relay flag (pver >= BIP0037Version).
	BIP0037Version uint32 = 70001
)

// ServiceFlag identifies services supported by a bitcoin peer.
type ServiceFlag uint64

const (
	SFNodeNetwork ServiceFlag = 1 << iota
)

// Map of service flags back to their constant names for pretty printing.
var sfStrings = map[ServiceFlag]string{
	SFNodeNetwork: "SFNodeNetwork",
}

// String returns the ServiceFlag in human-readable form.
func (f ServiceFlag) String() string {
	// No flags are set.
	if f == 0 {
		return "0x0"
	}

	// Add individual bit flags.
	s := ""
	for flag, name := range sfStrings {
		if f&flag == flag {
			s += name + "|"
			f -= flag
		}
	}

	// Add any remaining flags which aren't accounted for as hex.
	s = strings.TrimRight(s, "|")
	if f != 0 {
		s += "|0x" + strconv.FormatUint(uint64(f), 16)
	}
	s = strings.TrimLeft(s, "|")
	return s
}

// BitcoinNet represents which bitcoin network a message belongs to.
type BitcoinNet uint32

// Constants used to indicate the message bitcoin network.  They can also be
// used to seek to the next message when a stream's state is unknown, but
// this package does not provide that functionality since it's generally a
// better idea to simply disconnect clients that are misbehaving over TCP.
const (
	MainNet  BitcoinNet = 0xd9b4bef9
	TestNet  BitcoinNet = 0xdab5bffa
	TestNet3 BitcoinNet = 0x0709110b
)
<commit_msg>Add constant for regression test netowrk port.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire

import (
	"strconv"
	"strings"
)

const (
	MainPort                  = "8333"
	TestNetPort               = "18333"
	RegressionTestPort        = "18444"
	ProtocolVersion    uint32 = 70001
	TxVersion                 = 1

	// MultipleAddressVersion is the protocol version which added multiple
	// addresses per message (pver >= MultipleAddressVersion).
	MultipleAddressVersion uint32 = 209

	// NetAddressTimeVersion is the protocol version which added the
	// timestamp field (pver >= NetAddressTimeVersion).
	NetAddressTimeVersion uint32 = 31402

	// BIP0031Version is the protocol version AFTER which a pong message
	// and nonce field in ping were added (pver > BIP0031Version).
	BIP0031Version uint32 = 60000

	// BIP0035Version is the protocol version which added the mempool
	// message (pver >= BIP0035Version).
	BIP0035Version uint32 = 60002

	// BIP0037Version is the protocol version which added new connection
	// bloom filtering related messages and extended the version message
	// with a relay flag (pver >= BIP0037Version).
	BIP0037Version uint32 = 70001
)

// ServiceFlag identifies services supported by a bitcoin peer.
type ServiceFlag uint64

const (
	SFNodeNetwork ServiceFlag = 1 << iota
)

// Map of service flags back to their constant names for pretty printing.
var sfStrings = map[ServiceFlag]string{
	SFNodeNetwork: "SFNodeNetwork",
}

// String returns the ServiceFlag in human-readable form.
func (f ServiceFlag) String() string {
	// No flags are set.
	if f == 0 {
		return "0x0"
	}

	// Add individual bit flags.
	s := ""
	for flag, name := range sfStrings {
		if f&flag == flag {
			s += name + "|"
			f -= flag
		}
	}

	// Add any remaining flags which aren't accounted for as hex.
	s = strings.TrimRight(s, "|")
	if f != 0 {
		s += "|0x" + strconv.FormatUint(uint64(f), 16)
	}
	s = strings.TrimLeft(s, "|")
	return s
}

// BitcoinNet represents which bitcoin network a message belongs to.
type BitcoinNet uint32

// Constants used to indicate the message bitcoin network.  They can also be
// used to seek to the next message when a stream's state is unknown, but
// this package does not provide that functionality since it's generally a
// better idea to simply disconnect clients that are misbehaving over TCP.
const (
	MainNet  BitcoinNet = 0xd9b4bef9
	TestNet  BitcoinNet = 0xdab5bffa
	TestNet3 BitcoinNet = 0x0709110b
)
<|endoftext|>


<commit_before>package zygo
func init() { GITLASTTAG = "4.6.7"; GITLASTCOMMIT = "1f28172126b0172b30e9531b0e450ef40ef32648" }
<commit_msg>v4.6.8 with if-then-else infix parsing<commit_after>package zygo
func init() { GITLASTTAG = "v4.6.8"; GITLASTCOMMIT = "fdb9976d417226f84f49078ef5a65b97847e7846" }
<|endoftext|>


<commit_before>package zygo
func init() { GITLASTTAG = "v1.1.9"; GITLASTCOMMIT = "e0f04261735e6c6ceb2ba1eefc78ddd5e20ae4e6" }
<commit_msg>v1.2.0 zygomys<commit_after>package zygo
func init() { GITLASTTAG = "v1.2.0"; GITLASTCOMMIT = "a3209cee4630a3f8e404ec162fd3348f778bca53" }
<|endoftext|>


<commit_before>package reporting

// TODO: get this under unit test.

import (
	"bytes"
	"encoding/json"
	"github.com/smartystreets/goconvey/printing"
	"strconv"
	"strings"
)

func (self *jsonReporter) BeginStory(story *StoryReport) {}

func (self *jsonReporter) Enter(scope *ScopeReport) {
	if _, found := self.titlesById[scope.ID]; !found {
		self.registerScope(scope)
	}
	self.depth++
}
func (self *jsonReporter) registerScope(scope *ScopeReport) {
	self.titlesById[scope.ID] = scope.ID
	next := newScopeResult(scope.Title, self.depth, scope.File, scope.Line)
	self.scopes = append(self.scopes, next)
	self.stack = append(self.stack, next)
}

func (self *jsonReporter) Report(report *AssertionReport) {
	current := self.stack[len(self.stack)-1]
	current.Assertions = append(current.Assertions, newAssertionResult(report))
}

func (self *jsonReporter) Exit() {
	self.depth--
	if len(self.stack) > 0 {
		self.stack = self.stack[:len(self.stack)-1]
	}
}

func (self *jsonReporter) EndStory() {
	self.report()
	self.reset()
}
func (self *jsonReporter) report() {
	self.out.Print(OpenJson + "\n")
	scopes := []string{}
	for _, scope := range self.scopes {
		serialized, err := json.Marshal(scope)
		if err != nil {
			self.out.Println(jsonMarshalFailure)
			panic(err)
		}
		var buffer bytes.Buffer
		json.Indent(&buffer, serialized, "", "  ")
		scopes = append(scopes, buffer.String())
	}
	self.out.Print(strings.Join(scopes, ",") + ",\n")
	self.out.Print(CloseJson + "\n")
}
func (self *jsonReporter) reset() {
	self.titlesById = make(map[string]string)
	self.scopes = []*ScopeResult{}
	self.stack = []*ScopeResult{}
	self.depth = 0
}

func NewJsonReporter(out *printing.Printer) *jsonReporter {
	self := &jsonReporter{}
	self.out = out
	self.reset()
	return self
}

type jsonReporter struct {
	out        *printing.Printer
	titlesById map[string]string
	scopes     []*ScopeResult
	stack      []*ScopeResult
	depth      int
}

type ScopeResult struct {
	Title      string
	File       string
	Line       int
	Depth      int
	Assertions []AssertionResult
}

func newScopeResult(title string, depth int, file string, line int) *ScopeResult {
	self := &ScopeResult{}
	self.Title = title
	self.Depth = depth
	self.File = file
	self.Line = line
	self.Assertions = []AssertionResult{}
	return self
}

type AssertionResult struct {
	File       string
	Line       int
	Failure    string
	Error      interface{}
	Skipped    bool
	StackTrace string
}

func newAssertionResult(report *AssertionReport) AssertionResult {
	self := AssertionResult{}
	self.File = report.File
	self.Line = report.Line
	quotedFailure := strconv.Quote(report.Failure)
	if quotedFailure[1:len(quotedFailure)-1] != report.Failure {
		// TODO: test
		self.Failure = quotedFailure + " (NOTE: GoConvey used strconv.Quote on the assertion failure string so it could be safely parsed.)"
	}
	self.Error = report.Error
	self.StackTrace = report.stackTrace
	self.Skipped = report.Skipped
	return self
}

const OpenJson = ">>>>>"  // "⌦"
const CloseJson = "<<<<<" // "⌫"
const jsonMarshalFailure = `

There was an error when attempting to convert test results to JSON.
Please file a bug report and reference the code that caused this failure if possible.

Here's the panic:

`
<commit_msg>Not trying to escape invalid utf-8 (or anything else that can't be marshalled to JSON. Just panic and be done. (fixes #72 ...again)<commit_after>package reporting

// TODO: get this under unit test.

import (
	"bytes"
	"encoding/json"
	"github.com/smartystreets/goconvey/printing"
	"strings"
)

func (self *jsonReporter) BeginStory(story *StoryReport) {}

func (self *jsonReporter) Enter(scope *ScopeReport) {
	if _, found := self.titlesById[scope.ID]; !found {
		self.registerScope(scope)
	}
	self.depth++
}
func (self *jsonReporter) registerScope(scope *ScopeReport) {
	self.titlesById[scope.ID] = scope.ID
	next := newScopeResult(scope.Title, self.depth, scope.File, scope.Line)
	self.scopes = append(self.scopes, next)
	self.stack = append(self.stack, next)
}

func (self *jsonReporter) Report(report *AssertionReport) {
	current := self.stack[len(self.stack)-1]
	current.Assertions = append(current.Assertions, newAssertionResult(report))
}

func (self *jsonReporter) Exit() {
	self.depth--
	if len(self.stack) > 0 {
		self.stack = self.stack[:len(self.stack)-1]
	}
}

func (self *jsonReporter) EndStory() {
	self.report()
	self.reset()
}
func (self *jsonReporter) report() {
	self.out.Print(OpenJson + "\n")
	scopes := []string{}
	for _, scope := range self.scopes {
		serialized, err := json.Marshal(scope)
		if err != nil {
			self.out.Println(jsonMarshalFailure)
			panic(err)
		}
		var buffer bytes.Buffer
		json.Indent(&buffer, serialized, "", "  ")
		scopes = append(scopes, buffer.String())
	}
	self.out.Print(strings.Join(scopes, ",") + ",\n")
	self.out.Print(CloseJson + "\n")
}
func (self *jsonReporter) reset() {
	self.titlesById = make(map[string]string)
	self.scopes = []*ScopeResult{}
	self.stack = []*ScopeResult{}
	self.depth = 0
}

func NewJsonReporter(out *printing.Printer) *jsonReporter {
	self := &jsonReporter{}
	self.out = out
	self.reset()
	return self
}

type jsonReporter struct {
	out        *printing.Printer
	titlesById map[string]string
	scopes     []*ScopeResult
	stack      []*ScopeResult
	depth      int
}

type ScopeResult struct {
	Title      string
	File       string
	Line       int
	Depth      int
	Assertions []AssertionResult
}

func newScopeResult(title string, depth int, file string, line int) *ScopeResult {
	self := &ScopeResult{}
	self.Title = title
	self.Depth = depth
	self.File = file
	self.Line = line
	self.Assertions = []AssertionResult{}
	return self
}

type AssertionResult struct {
	File       string
	Line       int
	Failure    string
	Error      interface{}
	Skipped    bool
	StackTrace string
}

func newAssertionResult(report *AssertionReport) AssertionResult {
	self := AssertionResult{}
	self.File = report.File
	self.Line = report.Line
	self.Failure = report.Failure
	self.Error = report.Error
	self.StackTrace = report.stackTrace
	self.Skipped = report.Skipped
	return self
}

const OpenJson = ">>>>>"  // "⌦"
const CloseJson = "<<<<<" // "⌫"
const jsonMarshalFailure = `

GOCONVEY_JSON_MARSHALL_FAILURE: There was an error when attempting to convert test results to JSON.
Please file a bug report and reference the code that caused this failure if possible.

Here's the panic:

`
<|endoftext|>


<commit_before>package crypto

import (
	"fmt"
	"github.com/pkg/errors"

	secp256k1 "github.com/btcsuite/btcd/btcec"
	ledger "github.com/zondax/ledger-goclient"
)

var device *ledger.Ledger

type DerivationPath = []uint32

// getLedger gets a copy of the device, and caches it
func getLedger() (*ledger.Ledger, error) {
	var err error
	if device == nil {
		device, err = ledger.FindLedger()
	}
	return device, err
}

func pubkeyLedger(device *ledger.Ledger, path DerivationPath) (pub PubKey, err error) {
	key, err := device.GetPublicKeySECP256K1(path)
	if err != nil {
		return pub, fmt.Errorf("Error fetching public key: %v", err)
	}
	var p PubKeySecp256k1
	// Reserialize in the 33-byte compressed format
	cmp, err := secp256k1.ParsePubKey(key[:], secp256k1.S256())
	copy(p[:], cmp.SerializeCompressed())
	return p, err
}

func signLedger(device *ledger.Ledger, path DerivationPath, msg []byte) (sig Signature, err error) {
	bsig, err := device.SignSECP256K1(path, msg)
	if err != nil {
		return sig, err
	}
	sig = SignatureSecp256k1FromBytes(bsig)
	return sig, nil
}

// PrivKeyLedgerSecp256k1 implements PrivKey, calling the ledger nano
// we cache the PubKey from the first call to use it later
type PrivKeyLedgerSecp256k1 struct {
	// PubKey should be private, but we want to encode it via go-amino
	// so we can view the address later, even without having the ledger
	// attached
	CachedPubKey PubKey
	Path         DerivationPath
}

// NewPrivKeyLedgerSecp256k1 will generate a new key and store the
// public key for later use.
func NewPrivKeyLedgerSecp256k1(path DerivationPath) (PrivKey, error) {
	var pk PrivKeyLedgerSecp256k1
	pk.Path = path
	// getPubKey will cache the pubkey for later use,
	// this allows us to return an error early if the ledger
	// is not plugged in
	_, err := pk.getPubKey()
	return &pk, err
}

// ValidateKey allows us to verify the sanity of a key
// after loading it from disk
func (pk PrivKeyLedgerSecp256k1) ValidateKey() error {
	// getPubKey will return an error if the ledger is not
	// properly set up...
	pub, err := pk.forceGetPubKey()
	if err != nil {
		return err
	}
	// verify this matches cached address
	if !pub.Equals(pk.CachedPubKey) {
		return errors.New("Cached key does not match retrieved key")
	}
	return nil
}

// AssertIsPrivKeyInner fulfils PrivKey Interface
func (pk *PrivKeyLedgerSecp256k1) AssertIsPrivKeyInner() {}

// Bytes fulfils PrivKey Interface - but it stores the cached pubkey so we can verify
// the same key when we reconnect to a ledger
func (pk PrivKeyLedgerSecp256k1) Bytes() []byte {
	bin, err := cdc.MarshalBinaryBare(pk)
	if err != nil {
		panic(err)
	}
	return bin
}

// Sign calls the ledger and stores the PubKey for future use
//
// XXX/TODO: panics if there is an error communicating with the ledger.
//
// Communication is checked on NewPrivKeyLedger and PrivKeyFromBytes,
// returning an error, so this should only trigger if the privkey is held
// in memory for a while before use.
func (pk PrivKeyLedgerSecp256k1) Sign(msg []byte) Signature {
	// oh, I wish there was better error handling
	dev, err := getLedger()
	if err != nil {
		panic(err)
	}

	sig, err := signLedger(dev, pk.Path, msg)
	if err != nil {
		panic(err)
	}

	pub, err := pubkeyLedger(dev, pk.Path)
	if err != nil {
		panic(err)
	}

	// if we have no pubkey yet, store it for future queries
	if pk.CachedPubKey == nil {
		pk.CachedPubKey = pub
	} else if !pk.CachedPubKey.Equals(pub) {
		panic("Stored key does not match signing key")
	}
	return sig
}

// PubKey returns the stored PubKey
// TODO: query the ledger if not there, once it is not volatile
func (pk PrivKeyLedgerSecp256k1) PubKey() PubKey {
	key, err := pk.getPubKey()
	if err != nil {
		panic(err)
	}
	return key
}

// getPubKey reads the pubkey from cache or from the ledger itself
// since this involves IO, it may return an error, which is not exposed
// in the PubKey interface, so this function allows better error handling
func (pk PrivKeyLedgerSecp256k1) getPubKey() (key PubKey, err error) {
	// if we have no pubkey, set it
	if pk.CachedPubKey == nil {
		pk.CachedPubKey, err = pk.forceGetPubKey()
	}
	return pk.CachedPubKey, err
}

// forceGetPubKey is like getPubKey but ignores any cached key
// and ensures we get it from the ledger itself.
func (pk PrivKeyLedgerSecp256k1) forceGetPubKey() (key PubKey, err error) {
	dev, err := getLedger()
	if err != nil {
		return key, errors.New(fmt.Sprintf("Cannot connect to Ledger device - error: %v", err))
	}
	key, err = pubkeyLedger(dev, pk.Path)
	if err != nil {
		return key, errors.New(fmt.Sprintf("Please open Cosmos app on the Ledger device - error: %v", err))
	}
	return key, err
}

// Equals fulfils PrivKey Interface - makes sure both keys refer to the
// same
func (pk PrivKeyLedgerSecp256k1) Equals(other PrivKey) bool {
	if ledger, ok := other.(*PrivKeyLedgerSecp256k1); ok {
		return pk.CachedPubKey.Equals(ledger.CachedPubKey)
	}
	return false
}
<commit_msg>Clarify function names<commit_after>package crypto

import (
	"fmt"
	"github.com/pkg/errors"

	secp256k1 "github.com/btcsuite/btcd/btcec"
	ledger "github.com/zondax/ledger-goclient"
)

var device *ledger.Ledger

type DerivationPath = []uint32

// getLedger gets a copy of the device, and caches it
func getLedger() (*ledger.Ledger, error) {
	var err error
	if device == nil {
		device, err = ledger.FindLedger()
	}
	return device, err
}

func pubkeyLedgerSecp256k1(device *ledger.Ledger, path DerivationPath) (pub PubKey, err error) {
	key, err := device.GetPublicKeySECP256K1(path)
	if err != nil {
		return pub, fmt.Errorf("Error fetching public key: %v", err)
	}
	var p PubKeySecp256k1
	// Reserialize in the 33-byte compressed format
	cmp, err := secp256k1.ParsePubKey(key[:], secp256k1.S256())
	copy(p[:], cmp.SerializeCompressed())
	return p, err
}

func signLedgerSecp256k1(device *ledger.Ledger, path DerivationPath, msg []byte) (sig Signature, err error) {
	bsig, err := device.SignSECP256K1(path, msg)
	if err != nil {
		return sig, err
	}
	sig = SignatureSecp256k1FromBytes(bsig)
	return sig, nil
}

// PrivKeyLedgerSecp256k1 implements PrivKey, calling the ledger nano
// we cache the PubKey from the first call to use it later
type PrivKeyLedgerSecp256k1 struct {
	// PubKey should be private, but we want to encode it via go-amino
	// so we can view the address later, even without having the ledger
	// attached
	CachedPubKey PubKey
	Path         DerivationPath
}

// NewPrivKeyLedgerSecp256k1 will generate a new key and store the
// public key for later use.
func NewPrivKeyLedgerSecp256k1(path DerivationPath) (PrivKey, error) {
	var pk PrivKeyLedgerSecp256k1
	pk.Path = path
	// getPubKey will cache the pubkey for later use,
	// this allows us to return an error early if the ledger
	// is not plugged in
	_, err := pk.getPubKey()
	return &pk, err
}

// ValidateKey allows us to verify the sanity of a key
// after loading it from disk
func (pk PrivKeyLedgerSecp256k1) ValidateKey() error {
	// getPubKey will return an error if the ledger is not
	// properly set up...
	pub, err := pk.forceGetPubKey()
	if err != nil {
		return err
	}
	// verify this matches cached address
	if !pub.Equals(pk.CachedPubKey) {
		return errors.New("Cached key does not match retrieved key")
	}
	return nil
}

// AssertIsPrivKeyInner fulfils PrivKey Interface
func (pk *PrivKeyLedgerSecp256k1) AssertIsPrivKeyInner() {}

// Bytes fulfils PrivKey Interface - but it stores the cached pubkey so we can verify
// the same key when we reconnect to a ledger
func (pk PrivKeyLedgerSecp256k1) Bytes() []byte {
	bin, err := cdc.MarshalBinaryBare(pk)
	if err != nil {
		panic(err)
	}
	return bin
}

// Sign calls the ledger and stores the PubKey for future use
//
// XXX/TODO: panics if there is an error communicating with the ledger.
//
// Communication is checked on NewPrivKeyLedger and PrivKeyFromBytes,
// returning an error, so this should only trigger if the privkey is held
// in memory for a while before use.
func (pk PrivKeyLedgerSecp256k1) Sign(msg []byte) Signature {
	// oh, I wish there was better error handling
	dev, err := getLedger()
	if err != nil {
		panic(err)
	}

	sig, err := signLedgerSecp256k1(dev, pk.Path, msg)
	if err != nil {
		panic(err)
	}

	pub, err := pubkeyLedgerSecp256k1(dev, pk.Path)
	if err != nil {
		panic(err)
	}

	// if we have no pubkey yet, store it for future queries
	if pk.CachedPubKey == nil {
		pk.CachedPubKey = pub
	} else if !pk.CachedPubKey.Equals(pub) {
		panic("Stored key does not match signing key")
	}
	return sig
}

// PubKey returns the stored PubKey
// TODO: query the ledger if not there, once it is not volatile
func (pk PrivKeyLedgerSecp256k1) PubKey() PubKey {
	key, err := pk.getPubKey()
	if err != nil {
		panic(err)
	}
	return key
}

// getPubKey reads the pubkey from cache or from the ledger itself
// since this involves IO, it may return an error, which is not exposed
// in the PubKey interface, so this function allows better error handling
func (pk PrivKeyLedgerSecp256k1) getPubKey() (key PubKey, err error) {
	// if we have no pubkey, set it
	if pk.CachedPubKey == nil {
		pk.CachedPubKey, err = pk.forceGetPubKey()
	}
	return pk.CachedPubKey, err
}

// forceGetPubKey is like getPubKey but ignores any cached key
// and ensures we get it from the ledger itself.
func (pk PrivKeyLedgerSecp256k1) forceGetPubKey() (key PubKey, err error) {
	dev, err := getLedger()
	if err != nil {
		return key, errors.New(fmt.Sprintf("Cannot connect to Ledger device - error: %v", err))
	}
	key, err = pubkeyLedgerSecp256k1(dev, pk.Path)
	if err != nil {
		return key, errors.New(fmt.Sprintf("Please open Cosmos app on the Ledger device - error: %v", err))
	}
	return key, err
}

// Equals fulfils PrivKey Interface - makes sure both keys refer to the
// same
func (pk PrivKeyLedgerSecp256k1) Equals(other PrivKey) bool {
	if ledger, ok := other.(*PrivKeyLedgerSecp256k1); ok {
		return pk.CachedPubKey.Equals(ledger.CachedPubKey)
	}
	return false
}
<|endoftext|>