<commit_before>package json

import (
	"fmt"
	"strconv"
	"strings"
)

type DecodeState struct {
	cur string // current bytes
	err error
}

func (s *DecodeState) SkipSpaces() {
	if len(s.cur) == 0 {
		return
	}
	for {
		if s.cur[0] > '\x20' {
			return
		} else if s.cur[0] == '\x20' || s.cur[0] == '\x0A' || s.cur[0] == '\x0D' || s.cur[0] == '\x09' {
			s.cur = s.cur[1:]
			if len(s.cur) == 0 {
				return
			}
		} else {
			return
		}
	}
}

func (s *DecodeState) DecodeSlice() []interface{} {
	arr := make([]interface{}, 0, 2)

	s.SkipSpaces()
	for {
		if len(s.cur) == 0 {
			s.err = fmt.Errorf("incorrect syntax")
			return arr
		}

		if s.cur[0] == ']' {
			s.cur = s.cur[1:]
			return arr
		}

		val := s.DecodeValue()
		if s.err != nil {
			return arr
		}
		arr = append(arr, val)

		s.SkipSpaces()
		if len(s.cur) > 0 && s.cur[0] == ',' {
			s.cur = s.cur[1:]
			s.SkipSpaces()
		}
	}
}

func (s *DecodeState) DecodeObject() map[string]interface{} {
	obj := make(map[string]interface{}, 2)

	for {
		s.SkipSpaces()

		if len(s.cur) == 0 {
			s.err = fmt.Errorf("incorrect syntax")
			return obj
		}

		switch s.cur[0] {
		case '}':
			s.cur = s.cur[1:]
			return obj
		case '"':
			s.cur = s.cur[1:]
			key := s.DecodeString()
			if s.err != nil {
				return obj
			}
			s.SkipSpaces()
			if len(s.cur) > 0 && s.cur[0] == ':' {
				s.cur = s.cur[1:]
			} else {
				s.err = fmt.Errorf("incorrect syntax (expect ':' after key)")
				return obj
			}
			obj[key] = s.DecodeValue()
			s.SkipSpaces()
			if len(s.cur) > 0 && s.cur[0] == ',' {
				s.cur = s.cur[1:]
			}
		default:
			s.err = fmt.Errorf("incorrect syntax (expect object key)")
			return obj
		}
	}

	return obj
}

func (s *DecodeState) DecodeString() string {
	quotePos := strings.IndexByte(s.cur, '"')
	if quotePos < 0 {
		s.err = fmt.Errorf("incorrect syntax (expected close quote)")
		return ""
	}
	// fast path
	val := s.cur[:quotePos]
	if strings.IndexByte(val, '\\') < 0 {
		s.cur = s.cur[quotePos+1:]
		return val
	}

	return ""
}

func (s *DecodeState) DecodeNumber() float64 {
	var pos int = 0

	// sign
	if s.cur[pos] == '-' {
		pos++
	}

	// significand
	// - integer
	if len(s.cur) > pos && s.cur[pos] >= '1' && s.cur[pos] <= '9' {
		pos++
		for len(s.cur) > pos && len(s.cur) >= 1 && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
			pos++
		}
	} else if len(s.cur) > pos && s.cur[pos] == '0' {
		pos++
	} else {
		s.err = fmt.Errorf("incorrect number (expected digit)")
		return 0.0
	}

	// - fractional
	if len(s.cur) > pos && s.cur[pos] == '.' {
		pos++
		if len(s.cur) > pos && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
			pos++
			for len(s.cur) > pos && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
				pos++
			}
		} else {
			s.err = fmt.Errorf("incorrect number (expected fractional)")
			return 0.0
		}
	}

	// exponential
	if len(s.cur) > pos && (s.cur[pos] == 'e' || s.cur[pos] == 'E') {
		pos++
		if len(s.cur) > pos && (s.cur[pos] == '+' || s.cur[pos] == '-') {
			pos++
		}

		if len(s.cur) > pos && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
			pos++
			for len(s.cur) > pos && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
				pos++
			}
		} else {
			s.err = fmt.Errorf("incorrect number (expected digit)")
			return 0.0
		}
	}

	val, err := strconv.ParseFloat(string(s.cur[:pos]), 64)
	if err != nil {
		s.err = err
	}
	s.cur = s.cur[pos:]

	return val
}

func (s *DecodeState) DecodeValue() interface{} {
	s.SkipSpaces()
	if len(s.cur) == 0 {
		s.err = fmt.Errorf("incorrect syntax")
		return nil
	}
	switch s.cur[0] {
	case '"':
		s.cur = s.cur[1:]
		val := s.DecodeString()
		if s.err != nil {
			return nil
		} else {
			return val
		}
	case '{':
		s.cur = s.cur[1:]
		val := s.DecodeObject()
		if s.err != nil {
			return nil
		} else {
			return val
		}
	case '[':
		s.cur = s.cur[1:]
		val := s.DecodeSlice()
		if s.err != nil {
			return nil
		} else {
			return val
		}
	case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
		val := s.DecodeNumber()
		if s.err != nil {
			return nil
		} else {
			return val
		}
	case 't':
		if len(s.cur) >= 4 && s.cur[:4] == "true" {
			s.cur = s.cur[4:]
			return true
		} else {
			s.err = fmt.Errorf("'true' expected")
		}
	case 'f':
		if len(s.cur) >= 5 && s.cur[:5] == "false" {
			s.cur = s.cur[5:]
			return false
		} else {
			s.err = fmt.Errorf("'false' expected")
		}
	case 'n':
		if len(s.cur) >= 4 && s.cur[:4] == "null" {
			s.cur = s.cur[4:]
			return nil
		} else {
			s.err = fmt.Errorf("'null' expected")
		}
	default:
		s.err = fmt.Errorf("incorrect syntax")
	}
	return nil
}

func Decode(json string) (interface{}, error) {
	state := DecodeState{cur: json}
	ret := state.DecodeValue()
	return ret, state.err
}
<commit_msg>extract some tunables to vars<commit_after>package json

import (
	"fmt"
	"strconv"
	"strings"
)

var (
	PreallocateSliceElems  = 2
	PreallocateObjectElems = 2
)

type DecodeState struct {
	cur string // current bytes
	err error
}

func (s *DecodeState) SkipSpaces() {
	if len(s.cur) == 0 {
		return
	}
	for {
		if s.cur[0] > '\x20' {
			return
		} else if s.cur[0] == '\x20' || s.cur[0] == '\x0A' || s.cur[0] == '\x0D' || s.cur[0] == '\x09' {
			s.cur = s.cur[1:]
			if len(s.cur) == 0 {
				return
			}
		} else {
			return
		}
	}
}

func (s *DecodeState) DecodeSlice() []interface{} {
	arr := make([]interface{}, 0, PreallocateSliceElems)

	s.SkipSpaces()
	for {
		if len(s.cur) == 0 {
			s.err = fmt.Errorf("incorrect syntax")
			return arr
		}

		if s.cur[0] == ']' {
			s.cur = s.cur[1:]
			return arr
		}

		val := s.DecodeValue()
		if s.err != nil {
			return arr
		}
		arr = append(arr, val)

		s.SkipSpaces()
		if len(s.cur) > 0 && s.cur[0] == ',' {
			s.cur = s.cur[1:]
			s.SkipSpaces()
		}
	}
}

func (s *DecodeState) DecodeObject() map[string]interface{} {
	obj := make(map[string]interface{}, PreallocateObjectElems)

	for {
		s.SkipSpaces()

		if len(s.cur) == 0 {
			s.err = fmt.Errorf("incorrect syntax")
			return obj
		}

		switch s.cur[0] {
		case '}':
			s.cur = s.cur[1:]
			return obj
		case '"':
			s.cur = s.cur[1:]
			key := s.DecodeString()
			if s.err != nil {
				return obj
			}
			s.SkipSpaces()
			if len(s.cur) > 0 && s.cur[0] == ':' {
				s.cur = s.cur[1:]
			} else {
				s.err = fmt.Errorf("incorrect syntax (expect ':' after key)")
				return obj
			}
			obj[key] = s.DecodeValue()
			s.SkipSpaces()
			if len(s.cur) > 0 && s.cur[0] == ',' {
				s.cur = s.cur[1:]
			}
		default:
			s.err = fmt.Errorf("incorrect syntax (expect object key)")
			return obj
		}
	}

	return obj
}

func (s *DecodeState) DecodeString() string {
	quotePos := strings.IndexByte(s.cur, '"')
	if quotePos < 0 {
		s.err = fmt.Errorf("incorrect syntax (expected close quote)")
		return ""
	}
	// fast path
	val := s.cur[:quotePos]
	if strings.IndexByte(val, '\\') < 0 {
		s.cur = s.cur[quotePos+1:]
		return val
	}

	return ""
}

func (s *DecodeState) DecodeNumber() float64 {
	var pos int = 0

	// sign
	if s.cur[pos] == '-' {
		pos++
	}

	// significand
	// - integer
	if len(s.cur) > pos && s.cur[pos] >= '1' && s.cur[pos] <= '9' {
		pos++
		for len(s.cur) > pos && len(s.cur) >= 1 && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
			pos++
		}
	} else if len(s.cur) > pos && s.cur[pos] == '0' {
		pos++
	} else {
		s.err = fmt.Errorf("incorrect number (expected digit)")
		return 0.0
	}

	// - fractional
	if len(s.cur) > pos && s.cur[pos] == '.' {
		pos++
		if len(s.cur) > pos && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
			pos++
			for len(s.cur) > pos && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
				pos++
			}
		} else {
			s.err = fmt.Errorf("incorrect number (expected fractional)")
			return 0.0
		}
	}

	// exponential
	if len(s.cur) > pos && (s.cur[pos] == 'e' || s.cur[pos] == 'E') {
		pos++
		if len(s.cur) > pos && (s.cur[pos] == '+' || s.cur[pos] == '-') {
			pos++
		}

		if len(s.cur) > pos && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
			pos++
			for len(s.cur) > pos && s.cur[pos] >= '0' && s.cur[pos] <= '9' {
				pos++
			}
		} else {
			s.err = fmt.Errorf("incorrect number (expected digit)")
			return 0.0
		}
	}

	val, err := strconv.ParseFloat(string(s.cur[:pos]), 64)
	if err != nil {
		s.err = err
	}
	s.cur = s.cur[pos:]

	return val
}

func (s *DecodeState) DecodeValue() interface{} {
	s.SkipSpaces()
	if len(s.cur) == 0 {
		s.err = fmt.Errorf("incorrect syntax")
		return nil
	}
	switch s.cur[0] {
	case '"':
		s.cur = s.cur[1:]
		val := s.DecodeString()
		if s.err != nil {
			return nil
		} else {
			return val
		}
	case '{':
		s.cur = s.cur[1:]
		val := s.DecodeObject()
		if s.err != nil {
			return nil
		} else {
			return val
		}
	case '[':
		s.cur = s.cur[1:]
		val := s.DecodeSlice()
		if s.err != nil {
			return nil
		} else {
			return val
		}
	case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
		val := s.DecodeNumber()
		if s.err != nil {
			return nil
		} else {
			return val
		}
	case 't':
		if len(s.cur) >= 4 && s.cur[:4] == "true" {
			s.cur = s.cur[4:]
			return true
		} else {
			s.err = fmt.Errorf("'true' expected")
		}
	case 'f':
		if len(s.cur) >= 5 && s.cur[:5] == "false" {
			s.cur = s.cur[5:]
			return false
		} else {
			s.err = fmt.Errorf("'false' expected")
		}
	case 'n':
		if len(s.cur) >= 4 && s.cur[:4] == "null" {
			s.cur = s.cur[4:]
			return nil
		} else {
			s.err = fmt.Errorf("'null' expected")
		}
	default:
		s.err = fmt.Errorf("incorrect syntax")
	}
	return nil
}

func Decode(json string) (interface{}, error) {
	state := DecodeState{cur: json}
	ret := state.DecodeValue()
	return ret, state.err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"unicode/utf8"
)

type Node struct {
	tag string
	attrs map[string]string
	text string
}

type Parser struct {
	Scanner Scanner
	output string

	// node heirarchy
	stack []*Node

	// states as the Parser object figures out how to output the tokens

	// new line has started
	isNewLine bool

	isIndent bool
	isDedent bool
	dedentCount int

	node *Node
}

func (p *Parser) Output() string {
	var tokens []rune
	var text string

	p.stack = make([]*Node, 0)
	p.output = ""
	p.isNewLine = true
	p.isIndent = false
	p.isDedent = false
	p.dedentCount = 0
	p.node = new(Node)

	for {
		tokens = p.Scanner.Scan()
		for _, token := range tokens {
			text = p.Scanner.TokenText()
			p.processToken(token, text)
			fmt.Println(TokenString(token))
		}
		if tokens[0] == TokEOF {
			break
		}
	}
	return p.output
}

func (p *Parser) closeLastNode() {
	if len(p.stack) > 0 {
		oldNode := p.stack[len(p.stack) - 1]
		p.stack = p.stack[0: len(p.stack) - 1]
		if r, l := utf8.DecodeLastRuneInString(p.output); r == ' ' {
			p.output = p.output[0: len(p.output) - l]
		}
		p.output += "</" + oldNode.tag + ">"
	}
}

func (p *Parser) lastNode() *Node {
	return p.stack[len(p.stack) - 1]
}

func (p *Parser) outputNode(node *Node) {
	p.output += "<" + node.tag + ">"
	p.output += node.text
}

func (p *Parser) processToken(tok rune, text string) {
	switch tok {
	case TokWord:
		if p.node.tag == "" {
			p.node.tag = text
		} else {
			p.node.text += text
		}
	case TokWhitespace:
		p.node.text += text
	case TokIndent:
		p.isIndent = true
		p.isDedent = false
	case TokDedent:
		p.isIndent = false
		p.isDedent = true
		p.dedentCount += 1
	case TokNewLine:
		// up to this point a node has been constructed but not appended to the
		// output

		fmt.Println(p.node)
		if p.isDedent {
			// for dedents, pop off the stack and close the node
			for i := 0; i < p.dedentCount; i += 1 {
				p.closeLastNode()
			}
			// closing one more time because this new node replaces the top node in the
			// stack
			p.closeLastNode()
			p.outputNode(p.node)
			p.stack = append(p.stack, p.node)
		} else if p.isIndent {
			p.outputNode(p.node)
			p.stack = append(p.stack, p.node)
		} else {
			p.closeLastNode()
			p.outputNode(p.node)
			p.stack = append(p.stack, p.node)
		}
		p.node = new(Node)
		p.isIndent = false
		p.isDedent = false
	case TokEOF:
		if p.node.tag != "" {
			p.stack = append(p.stack, p.node)
			p.outputNode(p.node)
		}
		// close the remaining nodes in the stack
		for len(p.stack) > 0 {
			p.closeLastNode()
		}
	}
}

<commit_msg>parser remove tag text initial whitespace<commit_after>package main

import (
	//"fmt"
	"unicode/utf8"
)

type Node struct {
	tag string
	attrs map[string]string
	text string
}

type Parser struct {
	Scanner Scanner
	output string

	// node heirarchy
	stack []*Node

	// states as the Parser object figures out how to output the tokens

	// new line has started
	isNewLine bool

	isIndent bool
	isDedent bool
	dedentCount int

	node *Node
}

func (p *Parser) Output() string {
	var tokens []rune
	var text string

	p.stack = make([]*Node, 0)
	p.output = ""
	p.isNewLine = true
	p.isIndent = false
	p.isDedent = false
	p.dedentCount = 0
	p.node = new(Node)

	for {
		tokens = p.Scanner.Scan()
		for _, token := range tokens {
			text = p.Scanner.TokenText()
			p.processToken(token, text)
		}
		if tokens[0] == TokEOF {
			break
		}
	}
	return p.output
}

func (p *Parser) closeLastNode() {
	if len(p.stack) > 0 {
		oldNode := p.stack[len(p.stack) - 1]
		p.stack = p.stack[0: len(p.stack) - 1]
		if r, l := utf8.DecodeLastRuneInString(p.output); r == ' ' {
			p.output = p.output[0: len(p.output) - l]
		}
		p.output += "</" + oldNode.tag + ">"
	}
}

func (p *Parser) lastNode() *Node {
	return p.stack[len(p.stack) - 1]
}

func (p *Parser) outputNode(node *Node) {
	p.output += "<" + node.tag + ">"
	p.output += node.text
}

func (p *Parser) processToken(tok rune, text string) {
	switch tok {
	case TokWord:
		if p.node.tag == "" {
			p.node.tag = text
		} else {
			p.node.text += text
		}
	case TokWhitespace:
		// skip initial whitespace for text
		if p.node.text != "" {
			p.node.text += text
		}
	case TokIndent:
		p.isIndent = true
		p.isDedent = false
	case TokDedent:
		p.isIndent = false
		p.isDedent = true
		p.dedentCount += 1
	case TokNewLine:
		// up to this point a node has been constructed but not appended to the
		// output
		if p.isDedent {
			// for dedents, pop off the stack and close the node
			for p.dedentCount > 0 {
				p.closeLastNode()
				p.dedentCount -= 1
			}
			// closing one more time because this new node replaces the top node in the
			// stack
			p.closeLastNode()
			p.outputNode(p.node)
			p.stack = append(p.stack, p.node)
		} else if p.isIndent {
			p.outputNode(p.node)
			p.stack = append(p.stack, p.node)
		} else {
			p.closeLastNode()
			p.outputNode(p.node)
			p.stack = append(p.stack, p.node)
		}
		p.node = new(Node)
		p.isIndent = false
		p.isDedent = false
	case TokEOF:
		if p.node.tag != "" {
			p.stack = append(p.stack, p.node)
			p.outputNode(p.node)
		}
		// close the remaining nodes in the stack
		for len(p.stack) > 0 {
			p.closeLastNode()
		}
	}
}

<|endoftext|>


<commit_before>package mempool

import (
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/go-kit/kit/log/term"

	"github.com/tendermint/abci/example/dummy"
	"github.com/tendermint/tmlibs/log"

	cfg "github.com/tendermint/tendermint/config"
	"github.com/tendermint/tendermint/p2p"
	"github.com/tendermint/tendermint/proxy"
	"github.com/tendermint/tendermint/types"
)

// mempoolLogger is a TestingLogger which uses a different
// color for each validator ("validator" key must exist).
func mempoolLogger() log.Logger {
	return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor {
		for i := 0; i < len(keyvals)-1; i += 2 {
			if keyvals[i] == "validator" {
				return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))}
			}
		}
		return term.FgBgColor{}
	})
}

// connect N mempool reactors through N switches
func makeAndConnectMempoolReactors(config *cfg.Config, N int) []*MempoolReactor {
	reactors := make([]*MempoolReactor, N)
	logger := mempoolLogger()
	for i := 0; i < N; i++ {
		app := dummy.NewDummyApplication()
		cc := proxy.NewLocalClientCreator(app)
		mempool := newMempoolWithApp(cc)

		reactors[i] = NewMempoolReactor(config.Mempool, mempool) // so we dont start the consensus states
		reactors[i].SetLogger(logger.With("validator", i))
	}

	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
		s.AddReactor("MEMPOOL", reactors[i])
		return s

	}, p2p.Connect2Switches)
	return reactors
}

// wait for all txs on all reactors
func waitForTxs(t *testing.T, txs types.Txs, reactors []*MempoolReactor) {
	// wait for the txs in all mempools
	wg := new(sync.WaitGroup)
	for i := 0; i < len(reactors); i++ {
		wg.Add(1)
		go _waitForTxs(t, wg, txs, i, reactors)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	timer := time.After(TIMEOUT)
	select {
	case <-timer:
		t.Fatal("Timed out waiting for txs")
	case <-done:
	}
}

// wait for all txs on a single mempool
func _waitForTxs(t *testing.T, wg *sync.WaitGroup, txs types.Txs, reactorIdx int, reactors []*MempoolReactor) {

	mempool := reactors[reactorIdx].Mempool
	for mempool.Size() != len(txs) {
		time.Sleep(time.Millisecond * 100)
	}

	reapedTxs := mempool.Reap(len(txs))
	for i, tx := range txs {
		assert.Equal(t, tx, reapedTxs[i], fmt.Sprintf("txs at index %d on reactor %d don't match: %v vs %v", i, reactorIdx, tx, reapedTxs[i]))
	}
	wg.Done()
}

const (
	NUM_TXS = 1000
	TIMEOUT = 120 * time.Second // ridiculously high because CircleCI is slow
)

func TestReactorBroadcastTxMessage(t *testing.T) {
	config := cfg.TestConfig()
	const N = 4
	reactors := makeAndConnectMempoolReactors(config, N)
	defer func() {
		for _, r := range reactors {
			r.Stop()
		}
	}()

	// send a bunch of txs to the first reactor's mempool
	// and wait for them all to be received in the others
	txs := checkTxs(t, reactors[0].Mempool, NUM_TXS)
	waitForTxs(t, txs, reactors)
}

// func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) {
// 	if testing.Short() {
// 		t.Skip("skipping test in short mode.")
// 	}

// 	config := cfg.TestConfig()
// 	const N = 2
// 	reactors := makeAndConnectMempoolReactors(config, N)
// 	defer func() {
// 		for _, r := range reactors {
// 			r.Stop()
// 		}
// 	}()

// 	// stop peer
// 	sw := reactors[1].Switch
// 	sw.StopPeerForError(sw.Peers().List()[0], errors.New("some reason"))

// 	// check that we are not leaking any go-routines
// 	// i.e. broadcastTxRoutine finishes when peer is stopped
// 	leaktest.CheckTimeout(t, 10*time.Second)()
// }

// func TestBroadcastTxForPeerStopsWhenReactorStops(t *testing.T) {
// 	if testing.Short() {
// 		t.Skip("skipping test in short mode.")
// 	}

// 	config := cfg.TestConfig()
// 	const N = 2
// 	reactors := makeAndConnectMempoolReactors(config, N)

// 	// stop reactors
// 	for _, r := range reactors {
// 		r.Stop()
// 	}

// 	// check that we are not leaking any go-routines
// 	// i.e. broadcastTxRoutine finishes when reactor is stopped
// 	leaktest.CheckTimeout(t, 10*time.Second)()
// }
<commit_msg>restore mempool memory leak tests<commit_after>package mempool

import (
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/fortytw2/leaktest"
	"github.com/pkg/errors"
	"github.com/stretchr/testify/assert"

	"github.com/go-kit/kit/log/term"

	"github.com/tendermint/abci/example/dummy"
	"github.com/tendermint/tmlibs/log"

	cfg "github.com/tendermint/tendermint/config"
	"github.com/tendermint/tendermint/p2p"
	"github.com/tendermint/tendermint/proxy"
	"github.com/tendermint/tendermint/types"
)

// mempoolLogger is a TestingLogger which uses a different
// color for each validator ("validator" key must exist).
func mempoolLogger() log.Logger {
	return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor {
		for i := 0; i < len(keyvals)-1; i += 2 {
			if keyvals[i] == "validator" {
				return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))}
			}
		}
		return term.FgBgColor{}
	})
}

// connect N mempool reactors through N switches
func makeAndConnectMempoolReactors(config *cfg.Config, N int) []*MempoolReactor {
	reactors := make([]*MempoolReactor, N)
	logger := mempoolLogger()
	for i := 0; i < N; i++ {
		app := dummy.NewDummyApplication()
		cc := proxy.NewLocalClientCreator(app)
		mempool := newMempoolWithApp(cc)

		reactors[i] = NewMempoolReactor(config.Mempool, mempool) // so we dont start the consensus states
		reactors[i].SetLogger(logger.With("validator", i))
	}

	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
		s.AddReactor("MEMPOOL", reactors[i])
		return s

	}, p2p.Connect2Switches)
	return reactors
}

// wait for all txs on all reactors
func waitForTxs(t *testing.T, txs types.Txs, reactors []*MempoolReactor) {
	// wait for the txs in all mempools
	wg := new(sync.WaitGroup)
	for i := 0; i < len(reactors); i++ {
		wg.Add(1)
		go _waitForTxs(t, wg, txs, i, reactors)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	timer := time.After(TIMEOUT)
	select {
	case <-timer:
		t.Fatal("Timed out waiting for txs")
	case <-done:
	}
}

// wait for all txs on a single mempool
func _waitForTxs(t *testing.T, wg *sync.WaitGroup, txs types.Txs, reactorIdx int, reactors []*MempoolReactor) {

	mempool := reactors[reactorIdx].Mempool
	for mempool.Size() != len(txs) {
		time.Sleep(time.Millisecond * 100)
	}

	reapedTxs := mempool.Reap(len(txs))
	for i, tx := range txs {
		assert.Equal(t, tx, reapedTxs[i], fmt.Sprintf("txs at index %d on reactor %d don't match: %v vs %v", i, reactorIdx, tx, reapedTxs[i]))
	}
	wg.Done()
}

const (
	NUM_TXS = 1000
	TIMEOUT = 120 * time.Second // ridiculously high because CircleCI is slow
)

func TestReactorBroadcastTxMessage(t *testing.T) {
	config := cfg.TestConfig()
	const N = 4
	reactors := makeAndConnectMempoolReactors(config, N)
	defer func() {
		for _, r := range reactors {
			r.Stop()
		}
	}()

	// send a bunch of txs to the first reactor's mempool
	// and wait for them all to be received in the others
	txs := checkTxs(t, reactors[0].Mempool, NUM_TXS)
	waitForTxs(t, txs, reactors)
}

func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode.")
	}

	config := cfg.TestConfig()
	const N = 2
	reactors := makeAndConnectMempoolReactors(config, N)
	defer func() {
		for _, r := range reactors {
			r.Stop()
		}
	}()

	// stop peer
	sw := reactors[1].Switch
	sw.StopPeerForError(sw.Peers().List()[0], errors.New("some reason"))

	// check that we are not leaking any go-routines
	// i.e. broadcastTxRoutine finishes when peer is stopped
	leaktest.CheckTimeout(t, 10*time.Second)()
}

func TestBroadcastTxForPeerStopsWhenReactorStops(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode.")
	}

	config := cfg.TestConfig()
	const N = 2
	reactors := makeAndConnectMempoolReactors(config, N)

	// stop reactors
	for _, r := range reactors {
		r.Stop()
	}

	// check that we are not leaking any go-routines
	// i.e. broadcastTxRoutine finishes when reactor is stopped
	leaktest.CheckTimeout(t, 10*time.Second)()
}
<|endoftext|>


<commit_before>// A memcached binary protocol client.
package memcached

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"math"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/dustin/gomemcached"
)

const bufsize = 1024

// The Client itself.
type Client struct {
	conn    io.ReadWriteCloser
	healthy bool

	hdrBuf []byte
}

// Connect to a memcached server.
func Connect(prot, dest string) (rv *Client, err error) {
	conn, err := net.Dial(prot, dest)
	if err != nil {
		return nil, err
	}
	return Wrap(conn)
}

// Wrap an existing transport.
func Wrap(rwc io.ReadWriteCloser) (rv *Client, err error) {
	return &Client{
		conn:    rwc,
		healthy: true,
		hdrBuf:  make([]byte, gomemcached.HDR_LEN),
	}, nil
}

// Close the connection when you're done.
func (c *Client) Close() {
	c.conn.Close()
}

// Return false if this client has had issues communicating.
//
// This is useful for connection pools where we want to
// non-destructively determine that a connection may be reused.
func (c Client) IsHealthy() bool {
	return c.healthy
}

// Send a custom request and get the response.
func (client *Client) Send(req *gomemcached.MCRequest) (rv *gomemcached.MCResponse, err error) {
	err = transmitRequest(client.conn, req)
	if err != nil {
		client.healthy = false
		return
	}
	resp, err := getResponse(client.conn, client.hdrBuf)
	client.healthy = !gomemcached.IsFatal(err)
	return resp, err
}

// Send a request, but do not wait for a response.
func (client *Client) Transmit(req *gomemcached.MCRequest) error {
	err := transmitRequest(client.conn, req)
	if err != nil {
		client.healthy = false
	}
	return err
}

// Receive a response
func (client *Client) Receive() (*gomemcached.MCResponse, error) {
	resp, err := getResponse(client.conn, client.hdrBuf)
	if err != nil {
		client.healthy = false
	}
	return resp, err
}

// Get the value for a key.
func (client *Client) Get(vb uint16, key string) (*gomemcached.MCResponse, error) {
	return client.Send(&gomemcached.MCRequest{
		Opcode:  gomemcached.GET,
		VBucket: vb,
		Key:     []byte(key),
	})
}

// Delete a key.
func (client *Client) Del(vb uint16, key string) (*gomemcached.MCResponse, error) {
	return client.Send(&gomemcached.MCRequest{
		Opcode:  gomemcached.DELETE,
		VBucket: vb,
		Key:     []byte(key)})
}

// List auth mechanisms
func (client *Client) AuthList() (*gomemcached.MCResponse, error) {
	return client.Send(&gomemcached.MCRequest{
		Opcode: gomemcached.SASL_LIST_MECHS})
}

func (client *Client) Auth(user, pass string) (*gomemcached.MCResponse, error) {
	res, err := client.AuthList()

	if err != nil {
		return res, err
	}

	authMech := string(res.Body)
	if strings.Index(authMech, "PLAIN") != -1 {
		return client.Send(&gomemcached.MCRequest{
			Opcode: gomemcached.SASL_AUTH,
			Key:    []byte("PLAIN"),
			Body:   []byte(fmt.Sprintf("\x00%s\x00%s", user, pass))})
	}
	return res, fmt.Errorf("Auth mechanism PLAIN not supported")
}

func (client *Client) store(opcode gomemcached.CommandCode, vb uint16,
	key string, flags int, exp int, body []byte) (*gomemcached.MCResponse, error) {

	req := &gomemcached.MCRequest{
		Opcode:  opcode,
		VBucket: vb,
		Key:     []byte(key),
		Cas:     0,
		Opaque:  0,
		Extras:  []byte{0, 0, 0, 0, 0, 0, 0, 0},
		Body:    body}

	binary.BigEndian.PutUint64(req.Extras, uint64(flags)<<32|uint64(exp))
	return client.Send(req)
}

// Increment a value.
func (client *Client) Incr(vb uint16, key string,
	amt, def uint64, exp int) (uint64, error) {

	req := &gomemcached.MCRequest{
		Opcode:  gomemcached.INCREMENT,
		VBucket: vb,
		Key:     []byte(key),
		Extras:  make([]byte, 8+8+4),
	}
	binary.BigEndian.PutUint64(req.Extras[:8], amt)
	binary.BigEndian.PutUint64(req.Extras[8:16], def)
	binary.BigEndian.PutUint32(req.Extras[16:20], uint32(exp))

	resp, err := client.Send(req)
	if err != nil {
		return 0, err
	}

	return binary.BigEndian.Uint64(resp.Body), nil
}

// Add a value for a key (store if not exists).
func (client *Client) Add(vb uint16, key string, flags int, exp int,
	body []byte) (*gomemcached.MCResponse, error) {
	return client.store(gomemcached.ADD, vb, key, flags, exp, body)
}

// Set the value for a key.
func (client *Client) Set(vb uint16, key string, flags int, exp int,
	body []byte) (*gomemcached.MCResponse, error) {
	return client.store(gomemcached.SET, vb, key, flags, exp, body)
}

// Get keys in bulk
func (client *Client) GetBulk(vb uint16, keys []string) (map[string]*gomemcached.MCResponse, error) {
	terminalOpaque := uint32(len(keys) + 5)
	rv := map[string]*gomemcached.MCResponse{}
	wg := sync.WaitGroup{}
	going := true

	defer func() {
		going = false
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for going {
			res, err := client.Receive()
			if err != nil {
				return
			}
			if res.Opaque == terminalOpaque {
				return
			}
			if res.Opcode != gomemcached.GETQ {
				log.Panicf("Unexpected opcode in GETQ response: %+v",
					res)
			}
			rv[keys[res.Opaque]] = res
		}
	}()

	for i, k := range keys {
		err := client.Transmit(&gomemcached.MCRequest{
			Opcode:  gomemcached.GETQ,
			VBucket: vb,
			Key:     []byte(k),
			Opaque:  uint32(i),
		})
		if err != nil {
			return rv, err
		}
	}

	err := client.Transmit(&gomemcached.MCRequest{
		Opcode: gomemcached.NOOP,
		Opaque: terminalOpaque})
	if err != nil {
		return rv, err
	}

	wg.Wait()

	return rv, nil
}

// Value status reported by the Observe method
type ObservedStatus uint8

const (
	ObservedNotPersisted     = ObservedStatus(0x00) // found, not persisted
	ObservedPersisted        = ObservedStatus(0x01) // found, persisted
	ObservedNotFound         = ObservedStatus(0x80) // not found (or a persisted delete)
	ObservedLogicallyDeleted = ObservedStatus(0x81) // pending deletion (not persisted yet)
)

// Data obtained by an Observe call
type ObserveResult struct {
	Status          ObservedStatus // Whether the value has been persisted/deleted
	Cas             uint64         // Current value's CAS
	PersistenceTime time.Duration  // Node's average time to persist a value
	ReplicationTime time.Duration  // Node's average time to replicate a value
}

// Gets the persistence/replication/CAS state of a key
func (client *Client) Observe(vb uint16, key string) (result ObserveResult, err error) {
	// http://www.couchbase.com/wiki/display/couchbase/Observe
	body := make([]byte, 4+len(key))
	binary.BigEndian.PutUint16(body[0:2], vb)
	binary.BigEndian.PutUint16(body[2:4], uint16(len(key)))
	copy(body[4:4+len(key)], key)

	res, err := client.Send(&gomemcached.MCRequest{
		Opcode:  gomemcached.OBSERVE,
		VBucket: vb,
		Body:    body,
	})
	if err != nil {
		return
	}

	// Parse the response data from the body:
	if len(res.Body) < 2+2+1 {
		err = io.ErrUnexpectedEOF
		return
	}
	outVb := binary.BigEndian.Uint16(res.Body[0:2])
	keyLen := binary.BigEndian.Uint16(res.Body[2:4])
	if len(res.Body) < 2+2+int(keyLen)+1+8 {
		err = io.ErrUnexpectedEOF
		return
	}
	outKey := string(res.Body[4 : 4+keyLen])
	if outVb != vb || outKey != key {
		err = fmt.Errorf("Observe returned wrong vbucket/key: %d/%q", outVb, outKey)
		return
	}
	result.Status = ObservedStatus(res.Body[4+keyLen])
	result.Cas = binary.BigEndian.Uint64(res.Body[5+keyLen:])
	// The response reuses the Cas field to store time statistics:
	result.PersistenceTime = time.Duration(res.Cas>>32) * time.Millisecond
	result.ReplicationTime = time.Duration(res.Cas&math.MaxUint32) * time.Millisecond
	return
}

// Checks whether a stored value has been persisted to disk yet.
func (result ObserveResult) CheckPersistence(cas uint64, deletion bool) (persisted bool, overwritten bool) {
	switch {
	case result.Status == ObservedNotFound && deletion:
		persisted = true
	case result.Cas != cas:
		overwritten = true
	case result.Status == ObservedPersisted:
		persisted = true
	}
	return
}

// Operation to perform on this CAS loop.
type CasOp uint8

const (
	// Store the new value normally
	CASStore = CasOp(iota)
	// Stop attempting to CAS, leave value untouched
	CASQuit
	// Delete the current value
	CASDelete
)

// User specified termination is returned as an error.
func (c CasOp) Error() string {
	switch c {
	case CASStore:
		return "CAS store"
	case CASQuit:
		return "CAS quit"
	case CASDelete:
		return "CAS delete"
	}
	panic("Unhandled value")
}

// A function to perform a CAS transform.
// Input is the current value, or nil if no value exists.
// The function should return the new value (if any) to set, and the store/quit/delete operation.
type CasFunc func(current []byte) ([]byte, CasOp)

// Perform a CAS transform with the given function.
//
// If the value does not exist, a nil current value will be sent to f.
func (client *Client) CAS(vb uint16, k string, f CasFunc,
	initexp int) (*gomemcached.MCResponse, error) {

	flags := 0
	exp := 0

	for {
		orig, err := client.Get(vb, k)
		if err != nil && (orig == nil ||
			orig.Status != gomemcached.KEY_ENOENT) {
			return orig, err
		}

		if orig.Status == gomemcached.KEY_ENOENT {
			init, operation := f(nil)
			if operation == CASQuit || operation == CASDelete {
				return nil, operation
			}
			// If it doesn't exist, add it
			resp, err := UnwrapMemcachedError(client.Add(vb, k, 0,
				initexp, init))
			if err == nil && resp.Status != gomemcached.KEY_EEXISTS {
				if resp.Status != gomemcached.SUCCESS {
					return nil, resp
				}
				return nil, nil
			}
		} else {
			var req *gomemcached.MCRequest
			newValue, operation := f(orig.Body)

			switch operation {
			case CASQuit:
				return nil, operation
			case CASStore:
				req = &gomemcached.MCRequest{
					Opcode:  gomemcached.SET,
					VBucket: vb,
					Key:     []byte(k),
					Cas:     orig.Cas,
					Opaque:  0,
					Extras:  []byte{0, 0, 0, 0, 0, 0, 0, 0},
					Body:    newValue}

				binary.BigEndian.PutUint64(req.Extras,
					uint64(flags)<<32|uint64(exp))
			case CASDelete:
				req = &gomemcached.MCRequest{
					Opcode:  gomemcached.DELETE,
					VBucket: vb,
					Key:     []byte(k),
					Cas:     orig.Cas}
			}
			resp, err := client.Send(req)
			if err == nil {
				return resp, nil
			}
		}
	}
}

// Stats returns a slice of these.
type StatValue struct {
	// The stat key
	Key string
	// The stat value
	Val string
}

// Get stats from the server
// use "" as the stat key for toplevel stats.
func (client *Client) Stats(key string) ([]StatValue, error) {
	rv := make([]StatValue, 0, 128)

	req := &gomemcached.MCRequest{
		Opcode: gomemcached.STAT,
		Key:    []byte(key),
		Opaque: 918494,
	}

	err := transmitRequest(client.conn, req)
	if err != nil {
		return rv, err
	}

	for {
		res, err := getResponse(client.conn, client.hdrBuf)
		if err != nil {
			return rv, err
		}
		k := string(res.Key)
		if k == "" {
			break
		}
		rv = append(rv, StatValue{
			Key: k,
			Val: string(res.Body),
		})
	}

	return rv, nil
}

// Get the stats from the server as a map
func (client *Client) StatsMap(key string) (map[string]string, error) {
	rv := make(map[string]string)
	st, err := client.Stats(key)
	if err != nil {
		return rv, err
	}
	for _, sv := range st {
		rv[sv.Key] = sv.Val
	}
	return rv, nil
}
<commit_msg>Return the response if a CAS degrades to an Add.<commit_after>// A memcached binary protocol client.
package memcached

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"math"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/dustin/gomemcached"
)

const bufsize = 1024

// The Client itself.
type Client struct {
	conn    io.ReadWriteCloser
	healthy bool

	hdrBuf []byte
}

// Connect to a memcached server.
func Connect(prot, dest string) (rv *Client, err error) {
	conn, err := net.Dial(prot, dest)
	if err != nil {
		return nil, err
	}
	return Wrap(conn)
}

// Wrap an existing transport.
func Wrap(rwc io.ReadWriteCloser) (rv *Client, err error) {
	return &Client{
		conn:    rwc,
		healthy: true,
		hdrBuf:  make([]byte, gomemcached.HDR_LEN),
	}, nil
}

// Close the connection when you're done.
func (c *Client) Close() {
	c.conn.Close()
}

// Return false if this client has had issues communicating.
//
// This is useful for connection pools where we want to
// non-destructively determine that a connection may be reused.
func (c Client) IsHealthy() bool {
	return c.healthy
}

// Send a custom request and get the response.
func (client *Client) Send(req *gomemcached.MCRequest) (rv *gomemcached.MCResponse, err error) {
	err = transmitRequest(client.conn, req)
	if err != nil {
		client.healthy = false
		return
	}
	resp, err := getResponse(client.conn, client.hdrBuf)
	client.healthy = !gomemcached.IsFatal(err)
	return resp, err
}

// Send a request, but do not wait for a response.
func (client *Client) Transmit(req *gomemcached.MCRequest) error {
	err := transmitRequest(client.conn, req)
	if err != nil {
		client.healthy = false
	}
	return err
}

// Receive a response
func (client *Client) Receive() (*gomemcached.MCResponse, error) {
	resp, err := getResponse(client.conn, client.hdrBuf)
	if err != nil {
		client.healthy = false
	}
	return resp, err
}

// Get the value for a key.
func (client *Client) Get(vb uint16, key string) (*gomemcached.MCResponse, error) {
	return client.Send(&gomemcached.MCRequest{
		Opcode:  gomemcached.GET,
		VBucket: vb,
		Key:     []byte(key),
	})
}

// Delete a key.
func (client *Client) Del(vb uint16, key string) (*gomemcached.MCResponse, error) {
	return client.Send(&gomemcached.MCRequest{
		Opcode:  gomemcached.DELETE,
		VBucket: vb,
		Key:     []byte(key)})
}

// List auth mechanisms
func (client *Client) AuthList() (*gomemcached.MCResponse, error) {
	return client.Send(&gomemcached.MCRequest{
		Opcode: gomemcached.SASL_LIST_MECHS})
}

func (client *Client) Auth(user, pass string) (*gomemcached.MCResponse, error) {
	res, err := client.AuthList()

	if err != nil {
		return res, err
	}

	authMech := string(res.Body)
	if strings.Index(authMech, "PLAIN") != -1 {
		return client.Send(&gomemcached.MCRequest{
			Opcode: gomemcached.SASL_AUTH,
			Key:    []byte("PLAIN"),
			Body:   []byte(fmt.Sprintf("\x00%s\x00%s", user, pass))})
	}
	return res, fmt.Errorf("Auth mechanism PLAIN not supported")
}

func (client *Client) store(opcode gomemcached.CommandCode, vb uint16,
	key string, flags int, exp int, body []byte) (*gomemcached.MCResponse, error) {

	req := &gomemcached.MCRequest{
		Opcode:  opcode,
		VBucket: vb,
		Key:     []byte(key),
		Cas:     0,
		Opaque:  0,
		Extras:  []byte{0, 0, 0, 0, 0, 0, 0, 0},
		Body:    body}

	binary.BigEndian.PutUint64(req.Extras, uint64(flags)<<32|uint64(exp))
	return client.Send(req)
}

// Increment a value.
func (client *Client) Incr(vb uint16, key string,
	amt, def uint64, exp int) (uint64, error) {

	req := &gomemcached.MCRequest{
		Opcode:  gomemcached.INCREMENT,
		VBucket: vb,
		Key:     []byte(key),
		Extras:  make([]byte, 8+8+4),
	}
	binary.BigEndian.PutUint64(req.Extras[:8], amt)
	binary.BigEndian.PutUint64(req.Extras[8:16], def)
	binary.BigEndian.PutUint32(req.Extras[16:20], uint32(exp))

	resp, err := client.Send(req)
	if err != nil {
		return 0, err
	}

	return binary.BigEndian.Uint64(resp.Body), nil
}

// Add a value for a key (store if not exists).
func (client *Client) Add(vb uint16, key string, flags int, exp int,
	body []byte) (*gomemcached.MCResponse, error) {
	return client.store(gomemcached.ADD, vb, key, flags, exp, body)
}

// Set the value for a key.
func (client *Client) Set(vb uint16, key string, flags int, exp int,
	body []byte) (*gomemcached.MCResponse, error) {
	return client.store(gomemcached.SET, vb, key, flags, exp, body)
}

// Get keys in bulk
func (client *Client) GetBulk(vb uint16, keys []string) (map[string]*gomemcached.MCResponse, error) {
	terminalOpaque := uint32(len(keys) + 5)
	rv := map[string]*gomemcached.MCResponse{}
	wg := sync.WaitGroup{}
	going := true

	defer func() {
		going = false
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for going {
			res, err := client.Receive()
			if err != nil {
				return
			}
			if res.Opaque == terminalOpaque {
				return
			}
			if res.Opcode != gomemcached.GETQ {
				log.Panicf("Unexpected opcode in GETQ response: %+v",
					res)
			}
			rv[keys[res.Opaque]] = res
		}
	}()

	for i, k := range keys {
		err := client.Transmit(&gomemcached.MCRequest{
			Opcode:  gomemcached.GETQ,
			VBucket: vb,
			Key:     []byte(k),
			Opaque:  uint32(i),
		})
		if err != nil {
			return rv, err
		}
	}

	err := client.Transmit(&gomemcached.MCRequest{
		Opcode: gomemcached.NOOP,
		Opaque: terminalOpaque})
	if err != nil {
		return rv, err
	}

	wg.Wait()

	return rv, nil
}

// Value status reported by the Observe method
type ObservedStatus uint8

const (
	ObservedNotPersisted     = ObservedStatus(0x00) // found, not persisted
	ObservedPersisted        = ObservedStatus(0x01) // found, persisted
	ObservedNotFound         = ObservedStatus(0x80) // not found (or a persisted delete)
	ObservedLogicallyDeleted = ObservedStatus(0x81) // pending deletion (not persisted yet)
)

// Data obtained by an Observe call
type ObserveResult struct {
	Status          ObservedStatus // Whether the value has been persisted/deleted
	Cas             uint64         // Current value's CAS
	PersistenceTime time.Duration  // Node's average time to persist a value
	ReplicationTime time.Duration  // Node's average time to replicate a value
}

// Gets the persistence/replication/CAS state of a key
func (client *Client) Observe(vb uint16, key string) (result ObserveResult, err error) {
	// http://www.couchbase.com/wiki/display/couchbase/Observe
	body := make([]byte, 4+len(key))
	binary.BigEndian.PutUint16(body[0:2], vb)
	binary.BigEndian.PutUint16(body[2:4], uint16(len(key)))
	copy(body[4:4+len(key)], key)

	res, err := client.Send(&gomemcached.MCRequest{
		Opcode:  gomemcached.OBSERVE,
		VBucket: vb,
		Body:    body,
	})
	if err != nil {
		return
	}

	// Parse the response data from the body:
	if len(res.Body) < 2+2+1 {
		err = io.ErrUnexpectedEOF
		return
	}
	outVb := binary.BigEndian.Uint16(res.Body[0:2])
	keyLen := binary.BigEndian.Uint16(res.Body[2:4])
	if len(res.Body) < 2+2+int(keyLen)+1+8 {
		err = io.ErrUnexpectedEOF
		return
	}
	outKey := string(res.Body[4 : 4+keyLen])
	if outVb != vb || outKey != key {
		err = fmt.Errorf("Observe returned wrong vbucket/key: %d/%q", outVb, outKey)
		return
	}
	result.Status = ObservedStatus(res.Body[4+keyLen])
	result.Cas = binary.BigEndian.Uint64(res.Body[5+keyLen:])
	// The response reuses the Cas field to store time statistics:
	result.PersistenceTime = time.Duration(res.Cas>>32) * time.Millisecond
	result.ReplicationTime = time.Duration(res.Cas&math.MaxUint32) * time.Millisecond
	return
}

// Checks whether a stored value has been persisted to disk yet.
func (result ObserveResult) CheckPersistence(cas uint64, deletion bool) (persisted bool, overwritten bool) {
	switch {
	case result.Status == ObservedNotFound && deletion:
		persisted = true
	case result.Cas != cas:
		overwritten = true
	case result.Status == ObservedPersisted:
		persisted = true
	}
	return
}

// Operation to perform on this CAS loop.
type CasOp uint8

const (
	// Store the new value normally
	CASStore = CasOp(iota)
	// Stop attempting to CAS, leave value untouched
	CASQuit
	// Delete the current value
	CASDelete
)

// User specified termination is returned as an error.
func (c CasOp) Error() string {
	switch c {
	case CASStore:
		return "CAS store"
	case CASQuit:
		return "CAS quit"
	case CASDelete:
		return "CAS delete"
	}
	panic("Unhandled value")
}

// A function to perform a CAS transform.
// Input is the current value, or nil if no value exists.
// The function should return the new value (if any) to set, and the store/quit/delete operation.
type CasFunc func(current []byte) ([]byte, CasOp)

// Perform a CAS transform with the given function.
//
// If the value does not exist, a nil current value will be sent to f.
func (client *Client) CAS(vb uint16, k string, f CasFunc,
	initexp int) (*gomemcached.MCResponse, error) {

	flags := 0
	exp := 0

	for {
		orig, err := client.Get(vb, k)
		if err != nil && (orig == nil ||
			orig.Status != gomemcached.KEY_ENOENT) {
			return orig, err
		}

		if orig.Status == gomemcached.KEY_ENOENT {
			init, operation := f(nil)
			if operation == CASQuit || operation == CASDelete {
				return nil, operation
			}
			// If it doesn't exist, add it
			resp, err := UnwrapMemcachedError(client.Add(vb, k, 0,
				initexp, init))
			if err == nil && resp.Status != gomemcached.KEY_EEXISTS {
				if resp.Status != gomemcached.SUCCESS {
					return nil, resp
				}
				return resp, nil
			}
		} else {
			var req *gomemcached.MCRequest
			newValue, operation := f(orig.Body)

			switch operation {
			case CASQuit:
				return nil, operation
			case CASStore:
				req = &gomemcached.MCRequest{
					Opcode:  gomemcached.SET,
					VBucket: vb,
					Key:     []byte(k),
					Cas:     orig.Cas,
					Opaque:  0,
					Extras:  []byte{0, 0, 0, 0, 0, 0, 0, 0},
					Body:    newValue}

				binary.BigEndian.PutUint64(req.Extras,
					uint64(flags)<<32|uint64(exp))
			case CASDelete:
				req = &gomemcached.MCRequest{
					Opcode:  gomemcached.DELETE,
					VBucket: vb,
					Key:     []byte(k),
					Cas:     orig.Cas}
			}
			resp, err := client.Send(req)
			if err == nil {
				return resp, nil
			}
		}
	}
}

// Stats returns a slice of these.
type StatValue struct {
	// The stat key
	Key string
	// The stat value
	Val string
}

// Get stats from the server
// use "" as the stat key for toplevel stats.
func (client *Client) Stats(key string) ([]StatValue, error) {
	rv := make([]StatValue, 0, 128)

	req := &gomemcached.MCRequest{
		Opcode: gomemcached.STAT,
		Key:    []byte(key),
		Opaque: 918494,
	}

	err := transmitRequest(client.conn, req)
	if err != nil {
		return rv, err
	}

	for {
		res, err := getResponse(client.conn, client.hdrBuf)
		if err != nil {
			return rv, err
		}
		k := string(res.Key)
		if k == "" {
			break
		}
		rv = append(rv, StatValue{
			Key: k,
			Val: string(res.Body),
		})
	}

	return rv, nil
}

// Get the stats from the server as a map
func (client *Client) StatsMap(key string) (map[string]string, error) {
	rv := make(map[string]string)
	st, err := client.Stats(key)
	if err != nil {
		return rv, err
	}
	for _, sv := range st {
		rv[sv.Key] = sv.Val
	}
	return rv, nil
}
<|endoftext|>


<commit_before><commit_msg>http: improve a test<commit_after><|endoftext|>


<commit_before>package web

import (
	"context"

	"net/http"

	"github.com/go-martini/martini"
	"github.com/lexicality/vending/backend"
	"github.com/lexicality/vending/hardware"
	"github.com/martini-contrib/render"
	"github.com/op/go-logging"
)

func render404(r render.Render) {
	r.HTML(404, "404", nil)
}

// Server runs the web server (!)
func Server(
	ctx context.Context,
	addr string,
	webRoot string,
	log *logging.Logger,
	stock *backend.Stock,
	hw hardware.Hardware,
) error {
	doneC := ctx.Done()

	m := martini.Classic()
	m.Use(render.Renderer(render.Options{
		Directory:  webRoot + "/tpl",
		Extensions: []string{".tmpl", ".html"},
		Layout:     "layout",
	}))
	m.Use(martini.Static(webRoot, martini.StaticOptions{
		Prefix:  "static",
		Exclude: "/static/tpl/",
	}))

	// Tell active HTTP requests to stop when we stop
	m.Use(func(req *http.Request, c martini.Context) {
		ctx := req.Context()
		newCtx, cancel := context.WithCancel(ctx)
		c.Map(req.WithContext(newCtx))
		go func() {
			select {
			case <-doneC:
				cancel()
			case <-ctx.Done():
				// exit
			}
		}()
	})
	m.Map(stock)
	m.Map(log)
	m.Map(hw)

	m.Get("/", renderHome)
	m.Get("/items/:ID", renderItem)
	m.Get("/items/:ID/vend", renderVendItem)
	m.NotFound(render404)

	server := &http.Server{
		Handler: m,
		Addr:    addr,
	}

	serverErrC := make(chan error)
	go func() {
		serverErrC <- server.ListenAndServe()
	}()

	select {
	case err := <-serverErrC:
		return err
	case <-doneC:
		// TODO: Timeouts?
		return server.Shutdown(context.TODO())
	}
}
<commit_msg>Silence Martini<commit_after>package web

import (
	"context"

	"net/http"

	"github.com/go-martini/martini"
	"github.com/lexicality/vending/backend"
	"github.com/lexicality/vending/hardware"
	"github.com/martini-contrib/render"
	"github.com/op/go-logging"
)

func render404(r render.Render) {
	r.HTML(404, "404", nil)
}

// Server runs the web server (!)
func Server(
	ctx context.Context,
	addr string,
	webRoot string,
	log *logging.Logger,
	stock *backend.Stock,
	hw hardware.Hardware,
) error {
	doneC := ctx.Done()

	m := martini.Classic()
	m.Use(render.Renderer(render.Options{
		Directory:  webRoot + "/tpl",
		Extensions: []string{".tmpl", ".html"},
		Layout:     "layout",
	}))
	m.Use(martini.Static(webRoot, martini.StaticOptions{
		Prefix:      "static",
		Exclude:     "/static/tpl/",
		SkipLogging: true,
	}))

	// Tell active HTTP requests to stop when we stop
	m.Use(func(req *http.Request, c martini.Context) {
		ctx := req.Context()
		newCtx, cancel := context.WithCancel(ctx)
		c.Map(req.WithContext(newCtx))
		go func() {
			select {
			case <-doneC:
				cancel()
			case <-ctx.Done():
				// exit
			}
		}()
	})
	m.Map(stock)
	m.Map(log)
	m.Map(hw)

	m.Get("/", renderHome)
	m.Get("/items/:ID", renderItem)
	m.Get("/items/:ID/vend", renderVendItem)
	m.NotFound(render404)

	server := &http.Server{
		Handler: m,
		Addr:    addr,
	}

	serverErrC := make(chan error)
	go func() {
		serverErrC <- server.ListenAndServe()
	}()

	select {
	case err := <-serverErrC:
		return err
	case <-doneC:
		// TODO: Timeouts?
		return server.Shutdown(context.TODO())
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	pprof_runtime "runtime/pprof"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/web/api"
	"github.com/prometheus/prometheus/web/blob"
)

var localhostRepresentations = []string{"127.0.0.1", "localhost"}

// Commandline flags.
var (
	listenAddress  = flag.String("web.listen-address", ":9090", "Address to listen on for the web interface, API, and telemetry.")
	metricsPath    = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	useLocalAssets = flag.Bool("web.use-local-assets", false, "Read assets/templates from file instead of binary.")
	userAssetsPath = flag.String("web.user-assets", "", "Path to static asset directory, available at /user.")
	enableQuit     = flag.Bool("web.enable-remote-shutdown", false, "Enable remote service shutdown.")
)

// WebService handles the HTTP endpoints with the exception of /api.
type WebService struct {
	StatusHandler   *PrometheusStatusHandler
	MetricsHandler  *api.MetricsService
	AlertsHandler   *AlertsHandler
	ConsolesHandler *ConsolesHandler
	GraphsHandler   *GraphsHandler

	QuitChan chan struct{}
}

// ServeForever serves the HTTP endpoints and only returns upon errors.
func (ws WebService) ServeForever(pathPrefix string) {

	http.Handle("/favicon.ico", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "", 404)
	}))

	http.Handle(pathPrefix, prometheus.InstrumentHandler(
		pathPrefix, ws.StatusHandler,
	))
	http.Handle(pathPrefix+"alerts", prometheus.InstrumentHandler(
		pathPrefix+"alerts", ws.AlertsHandler,
	))
	http.Handle(pathPrefix+"consoles/", prometheus.InstrumentHandler(
		pathPrefix+"consoles/", http.StripPrefix(pathPrefix+"consoles/", ws.ConsolesHandler),
	))
	http.Handle(pathPrefix+"graph", prometheus.InstrumentHandler(
		pathPrefix+"graph", ws.GraphsHandler,
	))
	http.Handle(pathPrefix+"heap", prometheus.InstrumentHandler(
		pathPrefix+"heap", http.HandlerFunc(dumpHeap),
	))

	ws.MetricsHandler.RegisterHandler(pathPrefix)
	http.Handle(pathPrefix+strings.TrimLeft(*metricsPath, "/"), prometheus.Handler())
	if *useLocalAssets {
		http.Handle(pathPrefix+"static/", prometheus.InstrumentHandler(
			pathPrefix+"static/", http.StripPrefix(pathPrefix+"static/", http.FileServer(http.Dir("web/static"))),
		))
	} else {
		http.Handle(pathPrefix+"static/", prometheus.InstrumentHandler(
			pathPrefix+"static/", http.StripPrefix(pathPrefix+"static/", new(blob.Handler)),
		))
	}

	if *userAssetsPath != "" {
		http.Handle(pathPrefix+"user/", prometheus.InstrumentHandler(
			pathPrefix+"user/", http.StripPrefix(pathPrefix+"user/", http.FileServer(http.Dir(*userAssetsPath))),
		))
	}

	if *enableQuit {
		http.Handle(pathPrefix+"-/quit", http.HandlerFunc(ws.quitHandler))
	}

	if pathPrefix != "/" {
		http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			http.Redirect(w, r, pathPrefix, http.StatusFound)
		}))
	}

	log.Infof("Listening on %s", *listenAddress)

	// If we cannot bind to a port, retry after 30 seconds.
	for {
		err := http.ListenAndServe(*listenAddress, nil)
		if err != nil {
			log.Errorf("Could not listen on %s: %s", *listenAddress, err)
		}
		time.Sleep(30 * time.Second)
	}
}

func (ws WebService) quitHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Add("Allow", "POST")
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	fmt.Fprintf(w, "Requesting termination... Goodbye!")

	close(ws.QuitChan)
}

func getTemplateFile(name string) (string, error) {
	if *useLocalAssets {
		file, err := ioutil.ReadFile(fmt.Sprintf("web/templates/%s.html", name))
		if err != nil {
			log.Errorf("Could not read %s template: %s", name, err)
			return "", err
		}
		return string(file), nil
	}
	file, err := blob.GetFile(blob.TemplateFiles, name+".html")
	if err != nil {
		log.Errorf("Could not read %s template: %s", name, err)
		return "", err
	}
	return string(file), nil
}

func getConsoles(pathPrefix string) string {
	if _, err := os.Stat(*consoleTemplatesPath + "/index.html"); !os.IsNotExist(err) {
		return pathPrefix + "consoles/index.html"
	}
	if *userAssetsPath != "" {
		if _, err := os.Stat(*userAssetsPath + "/index.html"); !os.IsNotExist(err) {
			return pathPrefix + "user/index.html"
		}
	}
	return ""
}

func getTemplate(name string, pathPrefix string) (*template.Template, error) {
	t := template.New("_base")
	var err error

	t.Funcs(template.FuncMap{
		"since":       time.Since,
		"getConsoles": func() string { return getConsoles(pathPrefix) },
		"pathPrefix":  func() string { return pathPrefix },
		"stripLabels": func(lset clientmodel.LabelSet, labels ...clientmodel.LabelName) clientmodel.LabelSet {
			for _, ln := range labels {
				delete(lset, ln)
			}
			return lset
		},
		"globalURL": func(url string) string {
			hostname, err := os.Hostname()
			if err != nil {
				log.Warnf("Couldn't get hostname: %s, returning target.URL()", err)
				return url
			}
			for _, localhostRepresentation := range localhostRepresentations {
				url = strings.Replace(url, "//"+localhostRepresentation, "//"+hostname, 1)
			}
			return url
		},
	})

	file, err := getTemplateFile("_base")
	if err != nil {
		log.Errorln("Could not read base template:", err)
		return nil, err
	}
	t, err = t.Parse(file)
	if err != nil {
		log.Errorln("Could not parse base template:", err)
	}

	file, err = getTemplateFile(name)
	if err != nil {
		log.Error("Could not read template %s: %s", name, err)
		return nil, err
	}
	t, err = t.Parse(file)
	if err != nil {
		log.Errorf("Could not parse template %s: %s", name, err)
	}
	return t, err
}

func executeTemplate(w http.ResponseWriter, name string, data interface{}, pathPrefix string) {
	tpl, err := getTemplate(name, pathPrefix)
	if err != nil {
		log.Error("Error preparing layout template: ", err)
		return
	}
	err = tpl.Execute(w, data)
	if err != nil {
		log.Error("Error executing template: ", err)
	}
}

func dumpHeap(w http.ResponseWriter, r *http.Request) {
	target := fmt.Sprintf("/tmp/%d.heap", time.Now().Unix())
	f, err := os.Create(target)
	if err != nil {
		log.Error("Could not dump heap: ", err)
	}
	fmt.Fprintf(w, "Writing to %s...", target)
	defer f.Close()
	pprof_runtime.WriteHeapProfile(f)
	fmt.Fprintf(w, "Done")
}

// MustBuildServerURL returns the server URL and panics in case an error occurs.
func MustBuildServerURL(pathPrefix string) string {
	_, port, err := net.SplitHostPort(*listenAddress)
	if err != nil {
		panic(err)
	}
	hostname, err := os.Hostname()
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("http://%s:%s%s", hostname, port, pathPrefix)
}
<commit_msg>allow to override the default os.Hostname() when creating PrometheusURLs<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	pprof_runtime "runtime/pprof"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/web/api"
	"github.com/prometheus/prometheus/web/blob"
)

var localhostRepresentations = []string{"127.0.0.1", "localhost"}

// Commandline flags.
var (
	listenAddress  = flag.String("web.listen-address", ":9090", "Address to listen on for the web interface, API, and telemetry.")
	hostname       = flag.String("web.hostname", "", "Hostname on which the server is available.")
	metricsPath    = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	useLocalAssets = flag.Bool("web.use-local-assets", false, "Read assets/templates from file instead of binary.")
	userAssetsPath = flag.String("web.user-assets", "", "Path to static asset directory, available at /user.")
	enableQuit     = flag.Bool("web.enable-remote-shutdown", false, "Enable remote service shutdown.")
)

// WebService handles the HTTP endpoints with the exception of /api.
type WebService struct {
	StatusHandler   *PrometheusStatusHandler
	MetricsHandler  *api.MetricsService
	AlertsHandler   *AlertsHandler
	ConsolesHandler *ConsolesHandler
	GraphsHandler   *GraphsHandler

	QuitChan chan struct{}
}

// ServeForever serves the HTTP endpoints and only returns upon errors.
func (ws WebService) ServeForever(pathPrefix string) {

	http.Handle("/favicon.ico", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "", 404)
	}))

	http.Handle(pathPrefix, prometheus.InstrumentHandler(
		pathPrefix, ws.StatusHandler,
	))
	http.Handle(pathPrefix+"alerts", prometheus.InstrumentHandler(
		pathPrefix+"alerts", ws.AlertsHandler,
	))
	http.Handle(pathPrefix+"consoles/", prometheus.InstrumentHandler(
		pathPrefix+"consoles/", http.StripPrefix(pathPrefix+"consoles/", ws.ConsolesHandler),
	))
	http.Handle(pathPrefix+"graph", prometheus.InstrumentHandler(
		pathPrefix+"graph", ws.GraphsHandler,
	))
	http.Handle(pathPrefix+"heap", prometheus.InstrumentHandler(
		pathPrefix+"heap", http.HandlerFunc(dumpHeap),
	))

	ws.MetricsHandler.RegisterHandler(pathPrefix)
	http.Handle(pathPrefix+strings.TrimLeft(*metricsPath, "/"), prometheus.Handler())
	if *useLocalAssets {
		http.Handle(pathPrefix+"static/", prometheus.InstrumentHandler(
			pathPrefix+"static/", http.StripPrefix(pathPrefix+"static/", http.FileServer(http.Dir("web/static"))),
		))
	} else {
		http.Handle(pathPrefix+"static/", prometheus.InstrumentHandler(
			pathPrefix+"static/", http.StripPrefix(pathPrefix+"static/", new(blob.Handler)),
		))
	}

	if *userAssetsPath != "" {
		http.Handle(pathPrefix+"user/", prometheus.InstrumentHandler(
			pathPrefix+"user/", http.StripPrefix(pathPrefix+"user/", http.FileServer(http.Dir(*userAssetsPath))),
		))
	}

	if *enableQuit {
		http.Handle(pathPrefix+"-/quit", http.HandlerFunc(ws.quitHandler))
	}

	if pathPrefix != "/" {
		http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			http.Redirect(w, r, pathPrefix, http.StatusFound)
		}))
	}

	log.Infof("Listening on %s", *listenAddress)

	// If we cannot bind to a port, retry after 30 seconds.
	for {
		err := http.ListenAndServe(*listenAddress, nil)
		if err != nil {
			log.Errorf("Could not listen on %s: %s", *listenAddress, err)
		}
		time.Sleep(30 * time.Second)
	}
}

func (ws WebService) quitHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Add("Allow", "POST")
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	fmt.Fprintf(w, "Requesting termination... Goodbye!")

	close(ws.QuitChan)
}

func getTemplateFile(name string) (string, error) {
	if *useLocalAssets {
		file, err := ioutil.ReadFile(fmt.Sprintf("web/templates/%s.html", name))
		if err != nil {
			log.Errorf("Could not read %s template: %s", name, err)
			return "", err
		}
		return string(file), nil
	}
	file, err := blob.GetFile(blob.TemplateFiles, name+".html")
	if err != nil {
		log.Errorf("Could not read %s template: %s", name, err)
		return "", err
	}
	return string(file), nil
}

func getConsoles(pathPrefix string) string {
	if _, err := os.Stat(*consoleTemplatesPath + "/index.html"); !os.IsNotExist(err) {
		return pathPrefix + "consoles/index.html"
	}
	if *userAssetsPath != "" {
		if _, err := os.Stat(*userAssetsPath + "/index.html"); !os.IsNotExist(err) {
			return pathPrefix + "user/index.html"
		}
	}
	return ""
}

func getTemplate(name string, pathPrefix string) (*template.Template, error) {
	t := template.New("_base")
	var err error

	t.Funcs(template.FuncMap{
		"since":       time.Since,
		"getConsoles": func() string { return getConsoles(pathPrefix) },
		"pathPrefix":  func() string { return pathPrefix },
		"stripLabels": func(lset clientmodel.LabelSet, labels ...clientmodel.LabelName) clientmodel.LabelSet {
			for _, ln := range labels {
				delete(lset, ln)
			}
			return lset
		},
		"globalURL": func(url string) string {
			hostname, err := getHostname()
			if err != nil {
				log.Warnf("Couldn't get hostname: %s, returning target.URL()", err)
				return url
			}
			for _, localhostRepresentation := range localhostRepresentations {
				url = strings.Replace(url, "//"+localhostRepresentation, "//"+hostname, 1)
			}
			return url
		},
	})

	file, err := getTemplateFile("_base")
	if err != nil {
		log.Errorln("Could not read base template:", err)
		return nil, err
	}
	t, err = t.Parse(file)
	if err != nil {
		log.Errorln("Could not parse base template:", err)
	}

	file, err = getTemplateFile(name)
	if err != nil {
		log.Error("Could not read template %s: %s", name, err)
		return nil, err
	}
	t, err = t.Parse(file)
	if err != nil {
		log.Errorf("Could not parse template %s: %s", name, err)
	}
	return t, err
}

func executeTemplate(w http.ResponseWriter, name string, data interface{}, pathPrefix string) {
	tpl, err := getTemplate(name, pathPrefix)
	if err != nil {
		log.Error("Error preparing layout template: ", err)
		return
	}
	err = tpl.Execute(w, data)
	if err != nil {
		log.Error("Error executing template: ", err)
	}
}

func dumpHeap(w http.ResponseWriter, r *http.Request) {
	target := fmt.Sprintf("/tmp/%d.heap", time.Now().Unix())
	f, err := os.Create(target)
	if err != nil {
		log.Error("Could not dump heap: ", err)
	}
	fmt.Fprintf(w, "Writing to %s...", target)
	defer f.Close()
	pprof_runtime.WriteHeapProfile(f)
	fmt.Fprintf(w, "Done")
}

// MustBuildServerURL returns the server URL and panics in case an error occurs.
func MustBuildServerURL(pathPrefix string) string {
	_, port, err := net.SplitHostPort(*listenAddress)
	if err != nil {
		panic(err)
	}
	hostname, err := getHostname()
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("http://%s:%s%s", hostname, port, pathPrefix)
}

func getHostname() (string, error) {
	if *hostname != "" {
		return *hostname, nil
	}
	return os.Hostname()
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"bytes"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/keys"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util"
)

type metaAction func(*client.Batch, roachpb.Key, *roachpb.RangeDescriptor)

func putMeta(b *client.Batch, key roachpb.Key, desc *roachpb.RangeDescriptor) {
	b.Put(key, desc)
}

func delMeta(b *client.Batch, key roachpb.Key, desc *roachpb.RangeDescriptor) {
	b.Del(key)
}

// splitRangeAddressing creates (or overwrites if necessary) the meta1
// and meta2 range addressing records for the left and right ranges
// caused by a split.
func splitRangeAddressing(b *client.Batch, left, right *roachpb.RangeDescriptor) error {
	if err := rangeAddressing(b, left, putMeta); err != nil {
		return err
	}
	return rangeAddressing(b, right, putMeta)
}

// mergeRangeAddressing removes subsumed meta1 and meta2 range
// addressing records caused by merging and updates the records for
// the new merged range. Left is the range descriptor for the "left"
// range before merging and merged describes the left to right merge.
func mergeRangeAddressing(b *client.Batch, left, merged *roachpb.RangeDescriptor) error {
	if err := rangeAddressing(b, left, delMeta); err != nil {
		return err
	}
	return rangeAddressing(b, merged, putMeta)
}

// updateRangeAddressing overwrites the meta1 and meta2 range addressing
// records for the descriptor.
func updateRangeAddressing(b *client.Batch, desc *roachpb.RangeDescriptor) error {
	return rangeAddressing(b, desc, putMeta)
}

func makeMetaKey(prefix roachpb.Key, key roachpb.RKey) roachpb.Key {
	res := make(roachpb.Key, 0, len(prefix)+len(key))
	res = append(res, prefix...)
	res = append(res, key...)
	return res
}

// rangeAddressing updates or deletes the range addressing metadata
// for the range specified by desc. The action to take is specified by
// the supplied metaAction function.
//
// The rules for meta1 and meta2 records are as follows:
//
//  1. If desc.StartKey or desc.EndKey is meta1:
//     - ERROR
//  2. If desc.EndKey is meta2:
//     - meta1(desc.EndKey)
//  3. If desc.EndKey is normal user key:
//     - meta2(desc.EndKey)
//     3a. If desc.StartKey is KeyMin or meta2:
//         - meta1(KeyMax)
func rangeAddressing(b *client.Batch, desc *roachpb.RangeDescriptor, action metaAction) error {
	// 1. handle illegal case of start or end key being meta1.
	if bytes.HasPrefix(desc.EndKey, keys.Meta1Prefix) ||
		bytes.HasPrefix(desc.StartKey, keys.Meta1Prefix) {
		return util.Errorf("meta1 addressing records cannot be split: %+v", desc)
	}
	// 2. the case of the range ending with a meta2 prefix. This means
	// the range is full of meta2. We must update the relevant meta1
	// entry pointing to the end of this range.
	if bytes.HasPrefix(desc.EndKey, keys.Meta2Prefix) {
		action(b, keys.RangeMetaKey(desc.EndKey), desc)
	} else {
		// 3. the range ends with a normal user key, so we must update the
		// relevant meta2 entry pointing to the end of this range.
		action(b, makeMetaKey(keys.Meta2Prefix, desc.EndKey), desc)
		// 3a. the range starts with KeyMin or a meta2 addressing record,
		// update the meta1 entry for KeyMax.
		if bytes.Equal(desc.StartKey, roachpb.RKeyMin) ||
			bytes.HasPrefix(desc.StartKey, keys.Meta2Prefix) {
			action(b, makeMetaKey(keys.Meta1Prefix, roachpb.RKeyMax), desc)
		}
	}
	return nil
}
<commit_msg>Simplify rangeAddressing.<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"bytes"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/keys"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util"
)

type metaAction func(*client.Batch, roachpb.Key, *roachpb.RangeDescriptor)

func putMeta(b *client.Batch, key roachpb.Key, desc *roachpb.RangeDescriptor) {
	b.Put(key, desc)
}

func delMeta(b *client.Batch, key roachpb.Key, desc *roachpb.RangeDescriptor) {
	b.Del(key)
}

// splitRangeAddressing creates (or overwrites if necessary) the meta1
// and meta2 range addressing records for the left and right ranges
// caused by a split.
func splitRangeAddressing(b *client.Batch, left, right *roachpb.RangeDescriptor) error {
	if err := rangeAddressing(b, left, putMeta); err != nil {
		return err
	}
	return rangeAddressing(b, right, putMeta)
}

// mergeRangeAddressing removes subsumed meta1 and meta2 range
// addressing records caused by merging and updates the records for
// the new merged range. Left is the range descriptor for the "left"
// range before merging and merged describes the left to right merge.
func mergeRangeAddressing(b *client.Batch, left, merged *roachpb.RangeDescriptor) error {
	if err := rangeAddressing(b, left, delMeta); err != nil {
		return err
	}
	return rangeAddressing(b, merged, putMeta)
}

// updateRangeAddressing overwrites the meta1 and meta2 range addressing
// records for the descriptor.
func updateRangeAddressing(b *client.Batch, desc *roachpb.RangeDescriptor) error {
	return rangeAddressing(b, desc, putMeta)
}

// rangeAddressing updates or deletes the range addressing metadata
// for the range specified by desc. The action to take is specified by
// the supplied metaAction function.
//
// The rules for meta1 and meta2 records are as follows:
//
//  1. If desc.StartKey or desc.EndKey is meta1:
//     - ERROR
//  2. If desc.EndKey is meta2:
//     - meta1(desc.EndKey)
//  3. If desc.EndKey is normal user key:
//     - meta2(desc.EndKey)
//     3a. If desc.StartKey is KeyMin or meta2:
//         - meta1(KeyMax)
func rangeAddressing(b *client.Batch, desc *roachpb.RangeDescriptor, action metaAction) error {
	// 1. handle illegal case of start or end key being meta1.
	if bytes.HasPrefix(desc.EndKey, keys.Meta1Prefix) ||
		bytes.HasPrefix(desc.StartKey, keys.Meta1Prefix) {
		return util.Errorf("meta1 addressing records cannot be split: %+v", desc)
	}

	// Note that both cases 2 and 3 are handled by keys.RangeMetaKey.
	//
	// 2. the case of the range ending with a meta2 prefix. This means
	// the range is full of meta2. We must update the relevant meta1
	// entry pointing to the end of this range.
	//
	// 3. the range ends with a normal user key, so we must update the
	// relevant meta2 entry pointing to the end of this range.
	action(b, keys.RangeMetaKey(desc.EndKey), desc)

	if !bytes.HasPrefix(desc.EndKey, keys.Meta2Prefix) {
		// 3a. the range starts with KeyMin or a meta2 addressing record,
		// update the meta1 entry for KeyMax.
		if bytes.Equal(desc.StartKey, roachpb.RKeyMin) ||
			bytes.HasPrefix(desc.StartKey, keys.Meta2Prefix) {
			action(b, keys.Meta1KeyMax, desc)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package picker

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"
)

var device Device
var port Port
var config Env

func Init(ymlFile string) error {
	data, err := ioutil.ReadFile(ymlFile)
	if err != nil {
		return errors.New("Picker: Open file: " + err.Error())
	}
	config = Env{}
	configErr := config.configure([]byte(data))
	if configErr != nil {
		return configErr
	}
	fmt.Printf("%+v\n", config)
	return nil
}

func MakeFirmWare() error {

	cmd := `platformio
../sources/create_env.sh
platformio run --target upload --project-dir .picker --upload-port ` + config.Device.Port

	hFile, err := config.Device.makeHeader()
	if err != nil {
		return err
	}
	ioutil.WriteFile("../sources/arduino/config.h", []byte(hFile), 0666)

	cmds := strings.Split(cmd, "\n")
	for _, line := range cmds {
		fmt.Println(line)
		c := strings.Split(line, " ")
		create := exec.Command(c[0], c[1:]...)
		var out bytes.Buffer
		create.Stdout = &out
		err := create.Run()
		if err != nil {
			return errors.New(err.Error() + ":\n" + out.String())
		}
		fmt.Println(out.String())

	}
	return nil
}

func Create() error {
	port = Port{Name: config.Device.Port, Baud: config.Device.Baud, Timeout: config.Device.TimeOut}
	device = Device{address: config.Device.Address, port: &port, sensors: &sensors{}, dtrReset: config.Device.DTRReset}
	initDeviceError := device.init()
	if initDeviceError != nil {
		return initDeviceError
	}
	pingError := device.ping()
	if pingError != nil {
		return pingError
	}
	return nil
}

func Destroy() error {
	closeDeviceError := device.close()
	return closeDeviceError
}

func ReadSensors() error {
	for _, sensor := range config.Device.Sensors {
		switch sensor.Type {
		case "ds18b20":
			tempErr := device.updateDS1820Sensors()
			if tempErr != nil {
				return tempErr
			}
		case "bmp085":
			pressErr := device.updateBMP085Sensors()
			if pressErr != nil {
				return pressErr
			}
		}
	}
	return nil
}

func GetSensorsRef() *sensors {
	return device.sensors
}
<commit_msg>added creating a project<commit_after>package picker

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"
)

var device Device
var port Port
var config Env

func Init(ymlFile string) error {
	data, err := ioutil.ReadFile(ymlFile)
	if err != nil {
		return errors.New("Picker: Open file: " + err.Error())
	}
	config = Env{}
	configErr := config.configure([]byte(data))
	if configErr != nil {
		return configErr
	}
	fmt.Printf("%+v\n", config)
	return nil
}

func MakeFirmWare() error {

	cmd := "platformio" + "\n"
	cmd += "../sources/create_env.sh" + "\n"
	cmd += "platformio run --target upload --project-dir .picker --upload-port " + config.Device.Port

	hFile, err := config.Device.makeHeader()
	if err != nil {
		return err
	}
	ioutil.WriteFile("../sources/arduino/config.h", []byte(hFile), 0666)

	cmds := strings.Split(cmd, "\n")
	for _, line := range cmds {
		fmt.Println(line)
		c := strings.Split(line, " ")
		create := exec.Command(c[0], c[1:]...)
		var out bytes.Buffer
		create.Stdout = &out
		err := create.Run()
		if err != nil {
			return errors.New(err.Error() + ":\n" + out.String())
		}
		fmt.Println(out.String())

	}
	return nil
}

func Create() error {
	port = Port{Name: config.Device.Port, Baud: config.Device.Baud, Timeout: config.Device.TimeOut}
	device = Device{address: config.Device.Address, port: &port, sensors: &sensors{}, dtrReset: config.Device.DTRReset}
	initDeviceError := device.init()
	if initDeviceError != nil {
		return initDeviceError
	}
	pingError := device.ping()
	if pingError != nil {
		return pingError
	}
	return nil
}

func Destroy() error {
	closeDeviceError := device.close()
	return closeDeviceError
}

func ReadSensors() error {
	for _, sensor := range config.Device.Sensors {
		switch sensor.Type {
		case "ds18b20":
			tempErr := device.updateDS1820Sensors()
			if tempErr != nil {
				return tempErr
			}
		case "bmp085":
			pressErr := device.updateBMP085Sensors()
			if pressErr != nil {
				return pressErr
			}
		}
	}
	return nil
}

func GetSensorsRef() *sensors {
	return device.sensors
}
<|endoftext|>


<commit_before>package proc

import (
	"fmt"
	"testing"
)

func TestGetStatusByPID(t *testing.T) {
	rs, err := GetStatusByPID(1)
	if err != nil {
		t.Error(err)
	}
	fmt.Println(rs.VmRSS)
	fmt.Println(rs.VmRSSBytesN)
	fmt.Println(rs.VmRSSParsedBytes)
	fmt.Printf("GetStatusByPID: %+v\n", rs)
}

func TestGetProgram(t *testing.T) {
	fds, err := ListFds()
	if err != nil {
		t.Fatal(err)
	}

	fd := fds[len(fds)/2]
	fmt.Println("Chosen FD:", fd)

	pid, err := pidFromFd(fd)
	if err != nil {
		t.Fatal(err)
	}
	fmt.Println("Chosen PID:", pid)

	nm, err := GetProgram(pid)
	if err != nil {
		t.Fatal(err)
	}
	fmt.Println("GetProgram:", nm)
}
<commit_msg>proc: skip on 'GetProgram' test error<commit_after>package proc

import (
	"fmt"
	"testing"
)

func TestGetStatusByPID(t *testing.T) {
	rs, err := GetStatusByPID(1)
	if err != nil {
		t.Error(err)
	}
	fmt.Println(rs.VmRSS)
	fmt.Println(rs.VmRSSBytesN)
	fmt.Println(rs.VmRSSParsedBytes)
	fmt.Printf("GetStatusByPID: %+v\n", rs)
}

func TestGetProgram(t *testing.T) {
	fds, err := ListFds()
	if err != nil {
		t.Skip(err)
	}

	fd := fds[len(fds)/2]
	fmt.Println("Chosen FD:", fd)

	pid, err := pidFromFd(fd)
	if err != nil {
		t.Fatal(err)
	}
	fmt.Println("Chosen PID:", pid)

	nm, err := GetProgram(pid)
	if err != nil {
		t.Fatal(err)
	}
	fmt.Println("GetProgram:", nm)
}
<|endoftext|>


<commit_before>package autonat

import (
	"context"
	"errors"
	"math/rand"
	"sync"
	"time"

	host "github.com/libp2p/go-libp2p-host"
	peer "github.com/libp2p/go-libp2p-peer"
	ma "github.com/multiformats/go-multiaddr"
)

// attach to any host and auto-discovery autonat servers
// periodically query them to deal with changing NAT situation

type NATStatus int

const (
	NATStatusUnknown = iota
	NATStatusPublic
	NATStatusPrivate
)

type AutoNAT interface {
	Status() NATStatus
	PublicAddr() (ma.Multiaddr, error)
}

type AutoNATState struct {
	ctx    context.Context
	host   host.Host
	peers  map[peer.ID]bool
	status NATStatus
	addr   ma.Multiaddr
	mx     sync.Mutex
}

func NewAutoNAT(ctx context.Context, h host.Host) AutoNAT {
	as := &AutoNATState{
		ctx:    ctx,
		host:   h,
		peers:  make(map[peer.ID]bool),
		status: NATStatusUnknown,
	}

	go as.background()
	return as
}

func (as *AutoNATState) Status() NATStatus {
	return as.status
}

func (as *AutoNATState) PublicAddr() (ma.Multiaddr, error) {
	as.mx.Lock()
	defer as.mx.Unlock()

	if as.status != NATStatusPublic {
		return nil, errors.New("NAT Status is not public")
	}

	return as.addr, nil
}

func (as *AutoNATState) background() {
	// wait a bit for the node to come online and establish some connections
	// before starting autodetection
	time.Sleep(10 * time.Second)
	for {
		as.autodetect()
		select {
		case <-time.After(15 * time.Minute):
		case <-as.ctx.Done():
			return
		}
	}
}

func (as *AutoNATState) autodetect() {
	if len(as.peers) == 0 {
		log.Debugf("skipping NAT auto detection; no autonat peers")
		return
	}

	as.mx.Lock()
	peers := make([]peer.ID, 0, len(as.peers))
	for p, c := range as.peers {
		if c {
			peers = append(peers, p)
		}
	}

	if len(peers) == 0 {
		// we don't have any open connections, try any autonat peer that we know about
		for p := range as.peers {
			peers = append(peers, p)
		}
	}

	as.mx.Unlock()

	shufflePeers(peers)

	for _, p := range peers {
		cli := NewAutoNATClient(as.host, p)
		ctx, cancel := context.WithTimeout(as.ctx, 60*time.Second)
		a, err := cli.Dial(ctx)
		cancel()

		switch {
		case err == nil:
			log.Debugf("NAT status is public; address through %s: %s", p.Pretty(), a.String())
			as.mx.Lock()
			as.addr = a
			as.status = NATStatusPublic
			as.mx.Unlock()
			return

		case IsDialError(err):
			log.Debugf("NAT status is private; dial error through %s: %s", p.Pretty(), err.Error())
			as.mx.Lock()
			as.status = NATStatusPrivate
			as.mx.Unlock()
			return

		default:
			log.Debugf("Error dialing through %s: %s", p.Pretty(), err.Error())
		}
	}

	as.mx.Lock()
	as.status = NATStatusUnknown
	as.mx.Unlock()
}

func shufflePeers(peers []peer.ID) {
	for i := range peers {
		j := rand.Intn(i + 1)
		peers[i], peers[j] = peers[j], peers[i]
	}
}
<commit_msg>remove left-over design notes<commit_after>package autonat

import (
	"context"
	"errors"
	"math/rand"
	"sync"
	"time"

	host "github.com/libp2p/go-libp2p-host"
	peer "github.com/libp2p/go-libp2p-peer"
	ma "github.com/multiformats/go-multiaddr"
)

type NATStatus int

const (
	NATStatusUnknown = iota
	NATStatusPublic
	NATStatusPrivate
)

type AutoNAT interface {
	Status() NATStatus
	PublicAddr() (ma.Multiaddr, error)
}

type AutoNATState struct {
	ctx    context.Context
	host   host.Host
	peers  map[peer.ID]bool
	status NATStatus
	addr   ma.Multiaddr
	mx     sync.Mutex
}

func NewAutoNAT(ctx context.Context, h host.Host) AutoNAT {
	as := &AutoNATState{
		ctx:    ctx,
		host:   h,
		peers:  make(map[peer.ID]bool),
		status: NATStatusUnknown,
	}

	go as.background()
	return as
}

func (as *AutoNATState) Status() NATStatus {
	return as.status
}

func (as *AutoNATState) PublicAddr() (ma.Multiaddr, error) {
	as.mx.Lock()
	defer as.mx.Unlock()

	if as.status != NATStatusPublic {
		return nil, errors.New("NAT Status is not public")
	}

	return as.addr, nil
}

func (as *AutoNATState) background() {
	// wait a bit for the node to come online and establish some connections
	// before starting autodetection
	time.Sleep(10 * time.Second)
	for {
		as.autodetect()
		select {
		case <-time.After(15 * time.Minute):
		case <-as.ctx.Done():
			return
		}
	}
}

func (as *AutoNATState) autodetect() {
	if len(as.peers) == 0 {
		log.Debugf("skipping NAT auto detection; no autonat peers")
		return
	}

	as.mx.Lock()
	peers := make([]peer.ID, 0, len(as.peers))
	for p, c := range as.peers {
		if c {
			peers = append(peers, p)
		}
	}

	if len(peers) == 0 {
		// we don't have any open connections, try any autonat peer that we know about
		for p := range as.peers {
			peers = append(peers, p)
		}
	}

	as.mx.Unlock()

	shufflePeers(peers)

	for _, p := range peers {
		cli := NewAutoNATClient(as.host, p)
		ctx, cancel := context.WithTimeout(as.ctx, 60*time.Second)
		a, err := cli.Dial(ctx)
		cancel()

		switch {
		case err == nil:
			log.Debugf("NAT status is public; address through %s: %s", p.Pretty(), a.String())
			as.mx.Lock()
			as.addr = a
			as.status = NATStatusPublic
			as.mx.Unlock()
			return

		case IsDialError(err):
			log.Debugf("NAT status is private; dial error through %s: %s", p.Pretty(), err.Error())
			as.mx.Lock()
			as.status = NATStatusPrivate
			as.mx.Unlock()
			return

		default:
			log.Debugf("Error dialing through %s: %s", p.Pretty(), err.Error())
		}
	}

	as.mx.Lock()
	as.status = NATStatusUnknown
	as.mx.Unlock()
}

func shufflePeers(peers []peer.ID) {
	for i := range peers {
		j := rand.Intn(i + 1)
		peers[i], peers[j] = peers[j], peers[i]
	}
}
<|endoftext|>