<commit_before>package dendrite

import (
	"bytes"
	"fmt"
	//"log"
	"sort"
	"time"
)

type MsgType byte
type ChordMsg struct {
	Type             MsgType
	Data             []byte
	TransportMsg     interface{} // unmarshalled data, depending on transport
	TransportHandler func(*ChordMsg, chan *ChordMsg)
}

type Transport interface {
	// Gets a list of the vnodes on the box
	ListVnodes(string) ([]*Vnode, error)

	// Ping a Vnode, check for liveness
	Ping(*Vnode) (bool, error)

	// Request a vnode's predecessor
	GetPredecessor(*Vnode) (*Vnode, error)

	// Notify our successor of ourselves
	Notify(dest, self *Vnode) ([]*Vnode, error)

	// Find a successors for vnode key
	FindSuccessors(*Vnode, int, []byte) ([]*Vnode, error)

	// Clears a predecessor if it matches a given vnode. Used to leave.
	//ClearPredecessor(target, self *Vnode) error

	// Instructs a node to skip a given successor. Used to leave.
	//SkipSuccessor(target, self *Vnode) error

	// Register vnode handlers
	Register(*Vnode, VnodeHandler)

	// encode encodes dendrite msg into two frame byte stream
	// first byte is message type, and the rest is protobuf data
	Encode(MsgType, []byte) []byte
	// decode reverses the above process
	Decode([]byte) (*ChordMsg, error)
}

type Config struct {
	Hostname      string
	NumVnodes     int // num of vnodes to create
	StabilizeMin  time.Duration
	StabilizeMax  time.Duration
	NumSuccessors int // number of successor to keep in self log
}

func DefaultConfig(hostname string) *Config {
	return &Config{
		Hostname: hostname,
		// NumVnodes should be set around logN
		// N is approximate number of real nodes in cluster
		// this way we get O(logN) lookup speed
		NumVnodes:     3,
		StabilizeMin:  7 * time.Second,
		StabilizeMax:  10 * time.Second,
		NumSuccessors: 8, // number of known successors to keep track with
	}
}

type Ring struct {
	config         *Config
	transport      Transport
	vnodes         []*localVnode
	shutdown       chan bool
	Stabilizations int
}

// implement sort.Interface (Len(), Less() and Swap())
func (r *Ring) Less(i, j int) bool {
	return bytes.Compare(r.vnodes[i].Id, r.vnodes[j].Id) == -1
}

func (r *Ring) Swap(i, j int) {
	r.vnodes[i], r.vnodes[j] = r.vnodes[j], r.vnodes[i]
}

func (r *Ring) Len() int {
	return len(r.vnodes)
}

// Initializes the vnodes with their local successors
// Vnodes need to be sorted before this method is called
func (r *Ring) setLocalSuccessors() {
	numV := len(r.vnodes)
	if numV == 1 {
		for _, vnode := range r.vnodes {
			vnode.successors[0] = &vnode.Vnode
		}
		return
	}
	// we use numV-1 in order to avoid setting ourselves as last successor
	numSuc := min(r.config.NumSuccessors, numV-1)
	for idx, vnode := range r.vnodes {
		for i := 0; i < numSuc; i++ {
			vnode.successors[i] = &r.vnodes[(idx+i+1)%numV].Vnode
		}
	}

}

func (r *Ring) init(config *Config, transport Transport) {
	r.config = config
	r.transport = InitLocalTransport(transport)
	r.vnodes = make([]*localVnode, config.NumVnodes)
	r.shutdown = make(chan bool)
	// initialize vnodes
	for i := 0; i < config.NumVnodes; i++ {
		vn := &localVnode{}
		r.vnodes[i] = vn
		vn.ring = r
		vn.init(i)
	}
	sort.Sort(r)
}

func (r *Ring) schedule() {
	for i := 0; i < len(r.vnodes); i++ {
		r.vnodes[i].schedule()
	}
}
func CreateRing(config *Config, transport Transport) (*Ring, error) {
	// initialize the ring and sort vnodes
	r := &Ring{}
	r.init(config, transport)

	// for each vnode, setup local successors
	r.setLocalSuccessors()

	// schedule vnode stabilizers
	r.schedule()

	return r, nil
}

func JoinRing(config *Config, transport Transport, existing string) (*Ring, error) {
	hosts, err := transport.ListVnodes(existing)
	if err != nil {
		return nil, err
	}
	if hosts == nil || len(hosts) == 0 {
		return nil, fmt.Errorf("Remote host has no vnodes registered yet")
	}

	// initialize the ring and sort vnodes
	r := &Ring{}
	r.init(config, transport)

	// for each vnode, get the new list of live successors from remote
	for _, vn := range r.vnodes {
		resolved := false
		var last_error error
		// go through each host until we get successor list from one of them
		for _, remote_host := range hosts {
			succs, err := transport.FindSuccessors(remote_host, config.NumSuccessors, vn.Id)
			if err != nil {
				last_error = err
				continue
			}
			if succs == nil || len(succs) == 0 {
				return nil, fmt.Errorf("Failed to find successors for vnode, got empty list")
			}
			for idx, s := range succs {
				vn.successors[idx] = s
			}
			resolved = true
		}
		if !resolved {
			return nil, fmt.Errorf("Exhausted all remote vnodes while trying to get the list of successors. Last error: %s", last_error.Error())
		}

	}
	r.transport.Ping(&Vnode{Host: existing})

	// We can now initiate stabilization protocol
	for _, vn := range r.vnodes {
		vn.stabilize()
	}
	return r, nil
}
<commit_msg>lowers stablization wait time<commit_after>package dendrite

import (
	"bytes"
	"fmt"
	//"log"
	"sort"
	"time"
)

type MsgType byte
type ChordMsg struct {
	Type             MsgType
	Data             []byte
	TransportMsg     interface{} // unmarshalled data, depending on transport
	TransportHandler func(*ChordMsg, chan *ChordMsg)
}

type Transport interface {
	// Gets a list of the vnodes on the box
	ListVnodes(string) ([]*Vnode, error)

	// Ping a Vnode, check for liveness
	Ping(*Vnode) (bool, error)

	// Request a vnode's predecessor
	GetPredecessor(*Vnode) (*Vnode, error)

	// Notify our successor of ourselves
	Notify(dest, self *Vnode) ([]*Vnode, error)

	// Find a successors for vnode key
	FindSuccessors(*Vnode, int, []byte) ([]*Vnode, error)

	// Clears a predecessor if it matches a given vnode. Used to leave.
	//ClearPredecessor(target, self *Vnode) error

	// Instructs a node to skip a given successor. Used to leave.
	//SkipSuccessor(target, self *Vnode) error

	// Register vnode handlers
	Register(*Vnode, VnodeHandler)

	// encode encodes dendrite msg into two frame byte stream
	// first byte is message type, and the rest is protobuf data
	Encode(MsgType, []byte) []byte
	// decode reverses the above process
	Decode([]byte) (*ChordMsg, error)
}

type Config struct {
	Hostname      string
	NumVnodes     int // num of vnodes to create
	StabilizeMin  time.Duration
	StabilizeMax  time.Duration
	NumSuccessors int // number of successor to keep in self log
}

func DefaultConfig(hostname string) *Config {
	return &Config{
		Hostname: hostname,
		// NumVnodes should be set around logN
		// N is approximate number of real nodes in cluster
		// this way we get O(logN) lookup speed
		NumVnodes:     3,
		StabilizeMin:  1 * time.Second,
		StabilizeMax:  3 * time.Second,
		NumSuccessors: 8, // number of known successors to keep track with
	}
}

type Ring struct {
	config         *Config
	transport      Transport
	vnodes         []*localVnode
	shutdown       chan bool
	Stabilizations int
}

// implement sort.Interface (Len(), Less() and Swap())
func (r *Ring) Less(i, j int) bool {
	return bytes.Compare(r.vnodes[i].Id, r.vnodes[j].Id) == -1
}

func (r *Ring) Swap(i, j int) {
	r.vnodes[i], r.vnodes[j] = r.vnodes[j], r.vnodes[i]
}

func (r *Ring) Len() int {
	return len(r.vnodes)
}

// Initializes the vnodes with their local successors
// Vnodes need to be sorted before this method is called
func (r *Ring) setLocalSuccessors() {
	numV := len(r.vnodes)
	if numV == 1 {
		for _, vnode := range r.vnodes {
			vnode.successors[0] = &vnode.Vnode
		}
		return
	}
	// we use numV-1 in order to avoid setting ourselves as last successor
	numSuc := min(r.config.NumSuccessors, numV-1)
	for idx, vnode := range r.vnodes {
		for i := 0; i < numSuc; i++ {
			vnode.successors[i] = &r.vnodes[(idx+i+1)%numV].Vnode
		}
	}

}

func (r *Ring) init(config *Config, transport Transport) {
	r.config = config
	r.transport = InitLocalTransport(transport)
	r.vnodes = make([]*localVnode, config.NumVnodes)
	r.shutdown = make(chan bool)
	// initialize vnodes
	for i := 0; i < config.NumVnodes; i++ {
		vn := &localVnode{}
		r.vnodes[i] = vn
		vn.ring = r
		vn.init(i)
	}
	sort.Sort(r)
}

func (r *Ring) schedule() {
	for i := 0; i < len(r.vnodes); i++ {
		r.vnodes[i].schedule()
	}
}
func CreateRing(config *Config, transport Transport) (*Ring, error) {
	// initialize the ring and sort vnodes
	r := &Ring{}
	r.init(config, transport)

	// for each vnode, setup local successors
	r.setLocalSuccessors()

	// schedule vnode stabilizers
	r.schedule()

	return r, nil
}

func JoinRing(config *Config, transport Transport, existing string) (*Ring, error) {
	hosts, err := transport.ListVnodes(existing)
	if err != nil {
		return nil, err
	}
	if hosts == nil || len(hosts) == 0 {
		return nil, fmt.Errorf("Remote host has no vnodes registered yet")
	}

	// initialize the ring and sort vnodes
	r := &Ring{}
	r.init(config, transport)

	// for each vnode, get the new list of live successors from remote
	for _, vn := range r.vnodes {
		resolved := false
		var last_error error
		// go through each host until we get successor list from one of them
		for _, remote_host := range hosts {
			succs, err := transport.FindSuccessors(remote_host, config.NumSuccessors, vn.Id)
			if err != nil {
				last_error = err
				continue
			}
			if succs == nil || len(succs) == 0 {
				return nil, fmt.Errorf("Failed to find successors for vnode, got empty list")
			}
			for idx, s := range succs {
				vn.successors[idx] = s
			}
			resolved = true
		}
		if !resolved {
			return nil, fmt.Errorf("Exhausted all remote vnodes while trying to get the list of successors. Last error: %s", last_error.Error())
		}

	}
	r.transport.Ping(&Vnode{Host: existing})

	// We can now initiate stabilization protocol
	for _, vn := range r.vnodes {
		vn.stabilize()
	}
	return r, nil
}
<|endoftext|>


<commit_before>package ntp

import (
	"testing"
)

const (
	host  = "0.pool.ntp.org"
	delta = 1.0
)

func TestVersionSelection(t *testing.T) {
	timeV4, err := Time(host)
	if err != nil {
		t.Errorf("NTP V4 request failed: %s", err)
	}
	t.Logf("Got current time from %s %s for NTP version %d", host, timeV4, 4)

	timeV3, err := TimeV(host, 3)
	if err != nil {
		t.Errorf("NTP V3 request failed: %s", err)
	}
	t.Logf("Got current time from %s %s for NTP version %d", host, timeV3, 3)

	timeV2, err := TimeV(host, 2)
	if err != nil {
		t.Errorf("NTP V3 request failed: %s", err)
	}
	t.Logf("Got current time from %s %s for NTP version %d", host, timeV2, 2)

	if timeV2.Sub(timeV3).Seconds() > delta {
		t.Errorf("Difference between NTP version %d and %d time values greaten than %f seconds",
			2, 3, delta)
	}

	if timeV3.Sub(timeV4).Seconds() > delta {
		t.Errorf("Difference between NTP version %d and %d time values greaten than %f seconds",
			3, 4, delta)
	}

	if timeV2.Sub(timeV4).Seconds() > delta {
		t.Errorf("Difference between NTP version %d and %d time values greaten than %f seconds",
			2, 4, delta)
	}
}
<commit_msg>Fix test message typo<commit_after>package ntp

import (
	"testing"
)

const (
	host  = "0.pool.ntp.org"
	delta = 1.0
)

func TestVersionSelection(t *testing.T) {
	timeV4, err := Time(host)
	if err != nil {
		t.Errorf("NTP V4 request failed: %s", err)
	}
	t.Logf("Got current time from %s %s for NTP version %d", host, timeV4, 4)

	timeV3, err := TimeV(host, 3)
	if err != nil {
		t.Errorf("NTP V3 request failed: %s", err)
	}
	t.Logf("Got current time from %s %s for NTP version %d", host, timeV3, 3)

	timeV2, err := TimeV(host, 2)
	if err != nil {
		t.Errorf("NTP V2 request failed: %s", err)
	}
	t.Logf("Got current time from %s %s for NTP version %d", host, timeV2, 2)

	if timeV2.Sub(timeV3).Seconds() > delta {
		t.Errorf("Difference between NTP version %d and %d time values greaten than %f seconds",
			2, 3, delta)
	}

	if timeV3.Sub(timeV4).Seconds() > delta {
		t.Errorf("Difference between NTP version %d and %d time values greaten than %f seconds",
			3, 4, delta)
	}

	if timeV2.Sub(timeV4).Seconds() > delta {
		t.Errorf("Difference between NTP version %d and %d time values greaten than %f seconds",
			2, 4, delta)
	}
}
<|endoftext|>


<commit_before>package chain

import (
	"bytes"
	"container/list"
	"errors"
	"fmt"
	"math/big"
	"os"
	"sync"
	"time"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
	"github.com/ethereum/go-ethereum/wire"
)

var statelogger = logger.NewLogger("BLOCK")

type Peer interface {
	Inbound() bool
	LastSend() time.Time
	LastPong() int64
	Host() []byte
	Port() uint16
	Version() string
	PingTime() string
	Connected() *int32
	Caps() *ethutil.Value
}

type EthManager interface {
	BlockManager() *BlockManager
	ChainManager() *ChainManager
	TxPool() *TxPool
	Broadcast(msgType wire.MsgType, data []interface{})
	PeerCount() int
	IsMining() bool
	IsListening() bool
	Peers() *list.List
	KeyManager() *crypto.KeyManager
	ClientIdentity() wire.ClientIdentity
	Db() ethutil.Database
	EventMux() *event.TypeMux
}

type BlockManager struct {
	// Mutex for locking the block processor. Blocks can only be handled one at a time
	mutex sync.Mutex
	// Canonical block chain
	bc *ChainManager
	// non-persistent key/value memory storage
	mem map[string]*big.Int
	// Proof of work used for validating
	Pow PoW
	// The ethereum manager interface
	eth EthManager
	// The managed states
	// Transiently state. The trans state isn't ever saved, validated and
	// it could be used for setting account nonces without effecting
	// the main states.
	transState *state.State
	// Mining state. The mining state is used purely and solely by the mining
	// operation.
	miningState *state.State

	// The last attempted block is mainly used for debugging purposes
	// This does not have to be a valid block and will be set during
	// 'Process' & canonical validation.
	lastAttemptedBlock *Block

	events event.Subscription
}

func NewBlockManager(ethereum EthManager) *BlockManager {
	sm := &BlockManager{
		mem: make(map[string]*big.Int),
		Pow: &EasyPow{},
		eth: ethereum,
		bc:  ethereum.ChainManager(),
	}
	sm.transState = ethereum.ChainManager().CurrentBlock.State().Copy()
	sm.miningState = ethereum.ChainManager().CurrentBlock.State().Copy()

	return sm
}

func (self *BlockManager) Start() {
	statelogger.Debugln("Starting state manager")
	self.events = self.eth.EventMux().Subscribe(Blocks(nil))
	go self.updateThread()
}

func (self *BlockManager) Stop() {
	statelogger.Debugln("Stopping state manager")
	self.events.Unsubscribe()
}

func (self *BlockManager) updateThread() {
	for ev := range self.events.Chan() {
		for _, block := range ev.(Blocks) {
			_, err := self.Process(block)
			if err != nil {
				statelogger.Infoln(err)
				statelogger.Debugf("Block #%v failed (%x...)\n", block.Number, block.Hash()[0:4])
				statelogger.Debugln(block)
				break
			}
		}
	}
}

func (sm *BlockManager) CurrentState() *state.State {
	return sm.eth.ChainManager().CurrentBlock.State()
}

func (sm *BlockManager) TransState() *state.State {
	return sm.transState
}

func (sm *BlockManager) MiningState() *state.State {
	return sm.miningState
}

func (sm *BlockManager) NewMiningState() *state.State {
	sm.miningState = sm.eth.ChainManager().CurrentBlock.State().Copy()

	return sm.miningState
}

func (sm *BlockManager) ChainManager() *ChainManager {
	return sm.bc
}

func (self *BlockManager) ProcessTransactions(coinbase *state.StateObject, state *state.State, block, parent *Block, txs Transactions) (Receipts, Transactions, Transactions, Transactions, error) {
	var (
		receipts           Receipts
		handled, unhandled Transactions
		erroneous          Transactions
		totalUsedGas       = big.NewInt(0)
		err                error
	)

done:
	for i, tx := range txs {
		// If we are mining this block and validating we want to set the logs back to 0
		state.EmptyLogs()

		txGas := new(big.Int).Set(tx.Gas)

		cb := state.GetStateObject(coinbase.Address())
		st := NewStateTransition(cb, tx, state, block)
		err = st.TransitionState()
		if err != nil {
			statelogger.Infoln(err)
			switch {
			case IsNonceErr(err):
				err = nil // ignore error
				continue
			case IsGasLimitErr(err):
				unhandled = txs[i:]

				break done
			default:
				statelogger.Infoln(err)
				erroneous = append(erroneous, tx)
				err = nil
				continue
			}
		}

		// Update the state with pending changes
		state.Update()

		txGas.Sub(txGas, st.gas)
		cumulative := new(big.Int).Set(totalUsedGas.Add(totalUsedGas, txGas))
		receipt := &Receipt{ethutil.CopyBytes(state.Root().([]byte)), cumulative, LogsBloom(state.Logs()).Bytes(), state.Logs()}

		if i < len(block.Receipts()) {
			original := block.Receipts()[i]
			if !original.Cmp(receipt) {
				if ethutil.Config.Diff {
					os.Exit(1)
				}

				err := fmt.Errorf("#%d receipt failed (r) %v ~ %x  <=>  (c) %v ~ %x (%x...)", i+1, original.CumulativeGasUsed, original.PostState[0:4], receipt.CumulativeGasUsed, receipt.PostState[0:4], tx.Hash()[0:4])

				return nil, nil, nil, nil, err
			}
		}

		// Notify all subscribers
		go self.eth.EventMux().Post(TxPostEvent{tx})

		receipts = append(receipts, receipt)
		handled = append(handled, tx)

		if ethutil.Config.Diff && ethutil.Config.DiffType == "all" {
			state.CreateOutputForDiff()
		}
	}

	parent.GasUsed = totalUsedGas

	return receipts, handled, unhandled, erroneous, err
}

func (sm *BlockManager) Process(block *Block) (td *big.Int, err error) {
	// Processing a blocks may never happen simultaneously
	sm.mutex.Lock()
	defer sm.mutex.Unlock()

	if sm.bc.HasBlock(block.Hash()) {
		return nil, nil
	}

	if !sm.bc.HasBlock(block.PrevHash) {
		return nil, ParentError(block.PrevHash)
	}
	parent := sm.bc.GetBlock(block.PrevHash)

	return sm.ProcessWithParent(block, parent)
}

func (sm *BlockManager) ProcessWithParent(block, parent *Block) (td *big.Int, err error) {
	sm.lastAttemptedBlock = block

	state := parent.State().Copy()

	// Defer the Undo on the Trie. If the block processing happened
	// we don't want to undo but since undo only happens on dirty
	// nodes this won't happen because Commit would have been called
	// before that.
	defer state.Reset()

	if ethutil.Config.Diff && ethutil.Config.DiffType == "all" {
		fmt.Printf("## %x %x ##\n", block.Hash(), block.Number)
	}

	_, err = sm.ApplyDiff(state, parent, block)
	if err != nil {
		return nil, err
	}

	/* Go and C++ don't have consensus here. FIXME
	txSha := DeriveSha(block.transactions)
	if bytes.Compare(txSha, block.TxSha) != 0 {
		return nil, fmt.Errorf("Error validating transaction sha. Received %x, got %x", block.TxSha, txSha)
	}

	receiptSha := DeriveSha(receipts)
	if bytes.Compare(receiptSha, block.ReceiptSha) != 0 {
		return nil, fmt.Errorf("Error validating receipt sha. Received %x, got %x", block.ReceiptSha, receiptSha)
	}
	*/

	// Block validation
	if err = sm.ValidateBlock(block, parent); err != nil {
		statelogger.Errorln("Error validating block:", err)
		return nil, err
	}

	if err = sm.AccumelateRewards(state, block, parent); err != nil {
		statelogger.Errorln("Error accumulating reward", err)
		return nil, err
	}

	if bytes.Compare(CreateBloom(block), block.LogsBloom) != 0 {
		return nil, errors.New("Unable to replicate block's bloom")
	}

	state.Update()

	if !block.State().Cmp(state) {
		err = fmt.Errorf("Invalid merkle root.\nrec: %x\nis:  %x", block.State().Trie.Root, state.Trie.Root)
		return
	}

	// Calculate the new total difficulty and sync back to the db
	if td, ok := sm.CalculateTD(block); ok {
		// Sync the current block's state to the database and cancelling out the deferred Undo
		state.Sync()

		// TODO at this point we should also insert LOGS in to a database

		sm.transState = state.Copy()

		state.Manifest().Reset()

		sm.eth.TxPool().RemoveSet(block.Transactions())

		return td, nil
	} else {
		return nil, errors.New("total diff failed")
	}
}

func (sm *BlockManager) ApplyDiff(state *state.State, parent, block *Block) (receipts Receipts, err error) {
	coinbase := state.GetOrNewStateObject(block.Coinbase)
	coinbase.SetGasPool(block.CalcGasLimit(parent))

	// Process the transactions on to current block
	receipts, _, _, _, err = sm.ProcessTransactions(coinbase, state, block, parent, block.Transactions())
	if err != nil {
		return nil, err
	}

	return receipts, nil
}

func (sm *BlockManager) CalculateTD(block *Block) (*big.Int, bool) {
	uncleDiff := new(big.Int)
	for _, uncle := range block.Uncles {
		uncleDiff = uncleDiff.Add(uncleDiff, uncle.Difficulty)
	}

	// TD(genesis_block) = 0 and TD(B) = TD(B.parent) + sum(u.difficulty for u in B.uncles) + B.difficulty
	td := new(big.Int)
	td = td.Add(sm.bc.TD, uncleDiff)
	td = td.Add(td, block.Difficulty)

	// The new TD will only be accepted if the new difficulty is
	// is greater than the previous.
	if td.Cmp(sm.bc.TD) > 0 {
		return td, true

		// Set the new total difficulty back to the block chain
		//sm.bc.SetTotalDifficulty(td)
	}

	return nil, false
}

// Validates the current block. Returns an error if the block was invalid,
// an uncle or anything that isn't on the current block chain.
// Validation validates easy over difficult (dagger takes longer time = difficult)
func (sm *BlockManager) ValidateBlock(block, parent *Block) error {
	expd := CalcDifficulty(block, parent)
	if expd.Cmp(block.Difficulty) < 0 {
		return fmt.Errorf("Difficulty check failed for block %v, %v", block.Difficulty, expd)
	}

	diff := block.Time - parent.Time
	if diff < 0 {
		return ValidationError("Block timestamp less then prev block %v (%v - %v)", diff, block.Time, sm.bc.CurrentBlock.Time)
	}

	/* XXX
	// New blocks must be within the 15 minute range of the last block.
	if diff > int64(15*time.Minute) {
		return ValidationError("Block is too far in the future of last block (> 15 minutes)")
	}
	*/

	// Verify the nonce of the block. Return an error if it's not valid
	if !sm.Pow.Verify(block.HashNoNonce(), block.Difficulty, block.Nonce) {
		return ValidationError("Block's nonce is invalid (= %v)", ethutil.Bytes2Hex(block.Nonce))
	}

	return nil
}

func (sm *BlockManager) AccumelateRewards(state *state.State, block, parent *Block) error {
	reward := new(big.Int).Set(BlockReward)

	knownUncles := ethutil.Set(parent.Uncles)
	nonces := ethutil.NewSet(block.Nonce)
	for _, uncle := range block.Uncles {
		if nonces.Include(uncle.Nonce) {
			// Error not unique
			return UncleError("Uncle not unique")
		}

		uncleParent := sm.bc.GetBlock(uncle.PrevHash)
		if uncleParent == nil {
			return UncleError(fmt.Sprintf("Uncle's parent unknown (%x)", uncle.PrevHash[0:4]))
		}

		if uncleParent.Number.Cmp(new(big.Int).Sub(parent.Number, big.NewInt(6))) < 0 {
			return UncleError("Uncle too old")
		}

		if knownUncles.Include(uncle.Hash()) {
			return UncleError("Uncle in chain")
		}

		nonces.Insert(uncle.Nonce)

		r := new(big.Int)
		r.Mul(BlockReward, big.NewInt(15)).Div(r, big.NewInt(16))

		uncleAccount := state.GetAccount(uncle.Coinbase)
		uncleAccount.AddAmount(r)

		reward.Add(reward, new(big.Int).Div(BlockReward, big.NewInt(32)))
	}

	// Get the account associated with the coinbase
	account := state.GetAccount(block.Coinbase)
	// Reward amount of ether to the coinbase address
	account.AddAmount(reward)

	return nil
}

func (sm *BlockManager) GetMessages(block *Block) (messages []*state.Message, err error) {
	if !sm.bc.HasBlock(block.PrevHash) {
		return nil, ParentError(block.PrevHash)
	}

	sm.lastAttemptedBlock = block

	var (
		parent = sm.bc.GetBlock(block.PrevHash)
		state  = parent.State().Copy()
	)

	defer state.Reset()

	sm.ApplyDiff(state, parent, block)

	sm.AccumelateRewards(state, block, parent)

	return state.Manifest().Messages, nil
}
<commit_msg>Fixed bug on gas and setting of receipts and roots<commit_after>package chain

import (
	"bytes"
	"container/list"
	"errors"
	"fmt"
	"math/big"
	"sync"
	"time"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
	"github.com/ethereum/go-ethereum/wire"
)

var statelogger = logger.NewLogger("BLOCK")

type Peer interface {
	Inbound() bool
	LastSend() time.Time
	LastPong() int64
	Host() []byte
	Port() uint16
	Version() string
	PingTime() string
	Connected() *int32
	Caps() *ethutil.Value
}

type EthManager interface {
	BlockManager() *BlockManager
	ChainManager() *ChainManager
	TxPool() *TxPool
	Broadcast(msgType wire.MsgType, data []interface{})
	PeerCount() int
	IsMining() bool
	IsListening() bool
	Peers() *list.List
	KeyManager() *crypto.KeyManager
	ClientIdentity() wire.ClientIdentity
	Db() ethutil.Database
	EventMux() *event.TypeMux
}

type BlockManager struct {
	// Mutex for locking the block processor. Blocks can only be handled one at a time
	mutex sync.Mutex
	// Canonical block chain
	bc *ChainManager
	// non-persistent key/value memory storage
	mem map[string]*big.Int
	// Proof of work used for validating
	Pow PoW
	// The ethereum manager interface
	eth EthManager
	// The managed states
	// Transiently state. The trans state isn't ever saved, validated and
	// it could be used for setting account nonces without effecting
	// the main states.
	transState *state.State
	// Mining state. The mining state is used purely and solely by the mining
	// operation.
	miningState *state.State

	// The last attempted block is mainly used for debugging purposes
	// This does not have to be a valid block and will be set during
	// 'Process' & canonical validation.
	lastAttemptedBlock *Block

	events event.Subscription
}

func NewBlockManager(ethereum EthManager) *BlockManager {
	sm := &BlockManager{
		mem: make(map[string]*big.Int),
		Pow: &EasyPow{},
		eth: ethereum,
		bc:  ethereum.ChainManager(),
	}
	sm.transState = ethereum.ChainManager().CurrentBlock.State().Copy()
	sm.miningState = ethereum.ChainManager().CurrentBlock.State().Copy()

	return sm
}

func (self *BlockManager) Start() {
	statelogger.Debugln("Starting block manager")
}

func (self *BlockManager) Stop() {
	statelogger.Debugln("Stopping state manager")
}

func (sm *BlockManager) CurrentState() *state.State {
	return sm.eth.ChainManager().CurrentBlock.State()
}

func (sm *BlockManager) TransState() *state.State {
	return sm.transState
}

func (sm *BlockManager) MiningState() *state.State {
	return sm.miningState
}

func (sm *BlockManager) NewMiningState() *state.State {
	sm.miningState = sm.eth.ChainManager().CurrentBlock.State().Copy()

	return sm.miningState
}

func (sm *BlockManager) ChainManager() *ChainManager {
	return sm.bc
}

func (self *BlockManager) ProcessTransactions(coinbase *state.StateObject, state *state.State, block, parent *Block, txs Transactions) (Receipts, Transactions, Transactions, Transactions, error) {
	var (
		receipts           Receipts
		handled, unhandled Transactions
		erroneous          Transactions
		totalUsedGas       = big.NewInt(0)
		err                error
	)

done:
	for i, tx := range txs {
		// If we are mining this block and validating we want to set the logs back to 0
		state.EmptyLogs()

		txGas := new(big.Int).Set(tx.Gas)

		cb := state.GetStateObject(coinbase.Address())
		st := NewStateTransition(cb, tx, state, block)
		err = st.TransitionState()
		if err != nil {
			statelogger.Infoln(err)
			switch {
			case IsNonceErr(err):
				err = nil // ignore error
				continue
			case IsGasLimitErr(err):
				unhandled = txs[i:]

				break done
			default:
				statelogger.Infoln(err)
				erroneous = append(erroneous, tx)
				err = nil
				continue
			}
		}

		// Update the state with pending changes
		state.Update()

		txGas.Sub(txGas, st.gas)
		cumulative := new(big.Int).Set(totalUsedGas.Add(totalUsedGas, txGas))
		receipt := &Receipt{ethutil.CopyBytes(state.Root().([]byte)), cumulative, LogsBloom(state.Logs()).Bytes(), state.Logs()}

		// Notify all subscribers
		go self.eth.EventMux().Post(TxPostEvent{tx})

		receipts = append(receipts, receipt)
		handled = append(handled, tx)

		if ethutil.Config.Diff && ethutil.Config.DiffType == "all" {
			state.CreateOutputForDiff()
		}
	}

	block.GasUsed = totalUsedGas

	return receipts, handled, unhandled, erroneous, err
}

func (sm *BlockManager) Process(block *Block) (td *big.Int, err error) {
	// Processing a blocks may never happen simultaneously
	sm.mutex.Lock()
	defer sm.mutex.Unlock()

	if sm.bc.HasBlock(block.Hash()) {
		return nil, nil
	}

	if !sm.bc.HasBlock(block.PrevHash) {
		return nil, ParentError(block.PrevHash)
	}
	parent := sm.bc.GetBlock(block.PrevHash)

	return sm.ProcessWithParent(block, parent)
}

func (sm *BlockManager) ProcessWithParent(block, parent *Block) (td *big.Int, err error) {
	sm.lastAttemptedBlock = block

	state := parent.State().Copy()

	// Defer the Undo on the Trie. If the block processing happened
	// we don't want to undo but since undo only happens on dirty
	// nodes this won't happen because Commit would have been called
	// before that.
	defer state.Reset()

	if ethutil.Config.Diff && ethutil.Config.DiffType == "all" {
		fmt.Printf("## %x %x ##\n", block.Hash(), block.Number)
	}

	receipts, err := sm.ApplyDiff(state, parent, block)
	if err != nil {
		return nil, err
	}
	block.SetReceipts(receipts)

	txSha := DeriveSha(block.transactions)
	if bytes.Compare(txSha, block.TxSha) != 0 {
		return nil, fmt.Errorf("Error validating transaction sha. Received %x, got %x", block.TxSha, txSha)
	}

	receiptSha := DeriveSha(receipts)
	if bytes.Compare(receiptSha, block.ReceiptSha) != 0 {
		return nil, fmt.Errorf("Error validating receipt sha. Received %x, got %x", block.ReceiptSha, receiptSha)
	}

	// Block validation
	if err = sm.ValidateBlock(block, parent); err != nil {
		statelogger.Errorln("Error validating block:", err)
		return nil, err
	}

	if err = sm.AccumelateRewards(state, block, parent); err != nil {
		statelogger.Errorln("Error accumulating reward", err)
		return nil, err
	}

	if bytes.Compare(CreateBloom(block), block.LogsBloom) != 0 {
		return nil, errors.New("Unable to replicate block's bloom")
	}

	state.Update()

	if !block.State().Cmp(state) {
		err = fmt.Errorf("Invalid merkle root.\nrec: %x\nis:  %x", block.State().Trie.Root, state.Trie.Root)
		return
	}

	// Calculate the new total difficulty and sync back to the db
	if td, ok := sm.CalculateTD(block); ok {
		// Sync the current block's state to the database and cancelling out the deferred Undo
		state.Sync()

		// TODO at this point we should also insert LOGS in to a database

		sm.transState = state.Copy()

		state.Manifest().Reset()

		sm.eth.TxPool().RemoveSet(block.Transactions())

		return td, nil
	} else {
		return nil, errors.New("total diff failed")
	}
}

func (sm *BlockManager) ApplyDiff(state *state.State, parent, block *Block) (receipts Receipts, err error) {
	coinbase := state.GetOrNewStateObject(block.Coinbase)
	coinbase.SetGasPool(block.CalcGasLimit(parent))

	// Process the transactions on to current block
	receipts, _, _, _, err = sm.ProcessTransactions(coinbase, state, block, parent, block.Transactions())
	if err != nil {
		return nil, err
	}

	return receipts, nil
}

func (sm *BlockManager) CalculateTD(block *Block) (*big.Int, bool) {
	uncleDiff := new(big.Int)
	for _, uncle := range block.Uncles {
		uncleDiff = uncleDiff.Add(uncleDiff, uncle.Difficulty)
	}

	// TD(genesis_block) = 0 and TD(B) = TD(B.parent) + sum(u.difficulty for u in B.uncles) + B.difficulty
	td := new(big.Int)
	td = td.Add(sm.bc.TD, uncleDiff)
	td = td.Add(td, block.Difficulty)

	// The new TD will only be accepted if the new difficulty is
	// is greater than the previous.
	if td.Cmp(sm.bc.TD) > 0 {
		return td, true

		// Set the new total difficulty back to the block chain
		//sm.bc.SetTotalDifficulty(td)
	}

	return nil, false
}

// Validates the current block. Returns an error if the block was invalid,
// an uncle or anything that isn't on the current block chain.
// Validation validates easy over difficult (dagger takes longer time = difficult)
func (sm *BlockManager) ValidateBlock(block, parent *Block) error {
	expd := CalcDifficulty(block, parent)
	if expd.Cmp(block.Difficulty) < 0 {
		return fmt.Errorf("Difficulty check failed for block %v, %v", block.Difficulty, expd)
	}

	diff := block.Time - parent.Time
	if diff < 0 {
		return ValidationError("Block timestamp less then prev block %v (%v - %v)", diff, block.Time, sm.bc.CurrentBlock.Time)
	}

	/* XXX
	// New blocks must be within the 15 minute range of the last block.
	if diff > int64(15*time.Minute) {
		return ValidationError("Block is too far in the future of last block (> 15 minutes)")
	}
	*/

	// Verify the nonce of the block. Return an error if it's not valid
	if !sm.Pow.Verify(block.HashNoNonce(), block.Difficulty, block.Nonce) {
		return ValidationError("Block's nonce is invalid (= %v)", ethutil.Bytes2Hex(block.Nonce))
	}

	return nil
}

func (sm *BlockManager) AccumelateRewards(state *state.State, block, parent *Block) error {
	reward := new(big.Int).Set(BlockReward)

	knownUncles := ethutil.Set(parent.Uncles)
	nonces := ethutil.NewSet(block.Nonce)
	for _, uncle := range block.Uncles {
		if nonces.Include(uncle.Nonce) {
			// Error not unique
			return UncleError("Uncle not unique")
		}

		uncleParent := sm.bc.GetBlock(uncle.PrevHash)
		if uncleParent == nil {
			return UncleError(fmt.Sprintf("Uncle's parent unknown (%x)", uncle.PrevHash[0:4]))
		}

		if uncleParent.Number.Cmp(new(big.Int).Sub(parent.Number, big.NewInt(6))) < 0 {
			return UncleError("Uncle too old")
		}

		if knownUncles.Include(uncle.Hash()) {
			return UncleError("Uncle in chain")
		}

		nonces.Insert(uncle.Nonce)

		r := new(big.Int)
		r.Mul(BlockReward, big.NewInt(15)).Div(r, big.NewInt(16))

		uncleAccount := state.GetAccount(uncle.Coinbase)
		uncleAccount.AddAmount(r)

		reward.Add(reward, new(big.Int).Div(BlockReward, big.NewInt(32)))
	}

	// Get the account associated with the coinbase
	account := state.GetAccount(block.Coinbase)
	// Reward amount of ether to the coinbase address
	account.AddAmount(reward)

	return nil
}

func (sm *BlockManager) GetMessages(block *Block) (messages []*state.Message, err error) {
	if !sm.bc.HasBlock(block.PrevHash) {
		return nil, ParentError(block.PrevHash)
	}

	sm.lastAttemptedBlock = block

	var (
		parent = sm.bc.GetBlock(block.PrevHash)
		state  = parent.State().Copy()
	)

	defer state.Reset()

	sm.ApplyDiff(state, parent, block)

	sm.AccumelateRewards(state, block, parent)

	return state.Manifest().Messages, nil
}
<|endoftext|>


<commit_before>package dht

import (
	"encoding/hex"
	"math/big"
	"math/rand"
	"net"
	"testing"

	"github.com/anacrolix/missinggo"
)

func TestSetNilBigInt(t *testing.T) {
	i := new(big.Int)
	i.SetBytes(make([]byte, 2))
}

func TestMarshalCompactNodeInfo(t *testing.T) {
	cni := NodeInfo{
		ID: [20]byte{'a', 'b', 'c'},
	}
	addr, err := net.ResolveUDPAddr("udp4", "1.2.3.4:5")
	if err != nil {
		t.Fatal(err)
	}
	cni.Addr = newDHTAddr(addr)
	var b [CompactNodeInfoLen]byte
	cni.PutCompact(b[:])
	if err != nil {
		t.Fatal(err)
	}
	var bb [26]byte
	copy(bb[:], []byte("abc"))
	copy(bb[20:], []byte("\x01\x02\x03\x04\x00\x05"))
	if b != bb {
		t.FailNow()
	}
}

func recoverPanicOrDie(t *testing.T, f func()) {
	defer func() {
		r := recover()
		if r == nil {
			t.Fatal("expected panic")
		}
	}()
	f()
}

const zeroID = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"

var testIDs []nodeID

func init() {
	for _, s := range []string{
		zeroID,
		"\x03" + zeroID[1:],
		"\x03" + zeroID[1:18] + "\x55\xf0",
		"\x55" + zeroID[1:17] + "\xff\x55\x0f",
		"\x54" + zeroID[1:18] + "\x50\x0f",
		"",
	} {
		testIDs = append(testIDs, nodeIDFromString(s))
	}
}

func TestDistances(t *testing.T) {
	expectBitcount := func(i big.Int, count int) {
		if bitCount(i) != count {
			t.Fatalf("expected bitcount of %d: got %d", count, bitCount(i))
		}
	}
	expectBitcount(testIDs[3].Distance(&testIDs[0]), 4+8+4+4)
	expectBitcount(testIDs[3].Distance(&testIDs[1]), 4+8+4+4)
	expectBitcount(testIDs[3].Distance(&testIDs[2]), 4+8+8)
	for i := 0; i < 5; i++ {
		dist := testIDs[i].Distance(&testIDs[5])
		if dist.Cmp(&maxDistance) != 0 {
			t.Fatal("expected max distance for comparison with unset node id")
		}
	}
}

func TestMaxDistanceString(t *testing.T) {
	if string(maxDistance.Bytes()) != "\x01"+zeroID {
		t.FailNow()
	}
}

func TestClosestNodes(t *testing.T) {
	cn := newKClosestNodesSelector(2, testIDs[3])
	for _, i := range rand.Perm(len(testIDs)) {
		cn.Push(testIDs[i])
	}
	if len(cn.IDs()) != 2 {
		t.FailNow()
	}
	m := map[string]bool{}
	for _, id := range cn.IDs() {
		m[id.ByteString()] = true
	}
	if !m[testIDs[3].ByteString()] || !m[testIDs[4].ByteString()] {
		t.FailNow()
	}
}

// func TestUnmarshalGetPeersResponse(t *testing.T) {
// 	var gpr getPeersResponse
// 	if err != nil {
// 		t.Fatal(err)
// 	}
// 	err = dec.Decode(map[string]interface{}{
// 		"values": []string{"\x01\x02\x03\x04\x05\x06", "\x07\x08\x09\x0a\x0b\x0c"},
// 		"nodes":  "\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d",
// 	})
// 	if err != nil {
// 		t.Fatal(err)
// 	}
// 	if len(gpr.Values) != 2 {
// 		t.FailNow()
// 	}
// 	if len(gpr.Nodes) != 2 {
// 		t.FailNow()
// 	}
// }

func TestDHTDefaultConfig(t *testing.T) {
	s, err := NewServer(nil)
	if err != nil {
		t.Fatal(err)
	}
	s.Close()
}

func TestPing(t *testing.T) {
	srv, err := NewServer(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer srv.Close()
	srv0, err := NewServer(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer srv0.Close()
	tn, err := srv.Ping(&net.UDPAddr{
		IP:   []byte{127, 0, 0, 1},
		Port: srv0.Addr().(*net.UDPAddr).Port,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer tn.Close()
	ok := make(chan bool)
	tn.SetResponseHandler(func(msg Msg) {
		ok <- msg.ID() == srv0.ID()
	})
	if !<-ok {
		t.FailNow()
	}
}

func TestDHTSec(t *testing.T) {
	for _, case_ := range []struct {
		ipStr     string
		nodeIDHex string
		valid     bool
	}{
		// These 5 are from the spec example. They are all valid.
		{"124.31.75.21", "5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee401", true},
		{"21.75.31.124", "5a3ce9c14e7a08645677bbd1cfe7d8f956d53256", true},
		{"65.23.51.170", "a5d43220bc8f112a3d426c84764f8c2a1150e616", true},
		{"84.124.73.14", "1b0321dd1bb1fe518101ceef99462b947a01ff41", true},
		{"43.213.53.83", "e56f6cbf5b7c4be0237986d5243b87aa6d51305a", true},
		// spec[0] with one of the rand() bytes changed. Valid.
		{"124.31.75.21", "5fbfbff10c5d7a4ec8a88e4c6ab4c28b95eee401", true},
		// spec[1] with the 21st leading bit changed. Not Valid.
		{"21.75.31.124", "5a3ce1c14e7a08645677bbd1cfe7d8f956d53256", false},
		// spec[2] with the 22nd leading bit changed. Valid.
		{"65.23.51.170", "a5d43620bc8f112a3d426c84764f8c2a1150e616", true},
		// spec[3] with the 4th last bit changed. Valid.
		{"84.124.73.14", "1b0321dd1bb1fe518101ceef99462b947a01fe01", true},
		// spec[4] with the 3rd last bit changed. Not valid.
		{"43.213.53.83", "e56f6cbf5b7c4be0237986d5243b87aa6d51303e", false},
	} {
		ip := net.ParseIP(case_.ipStr)
		id, err := hex.DecodeString(case_.nodeIDHex)
		if err != nil {
			t.Fatal(err)
		}
		secure := NodeIdSecure(string(id), ip)
		if secure != case_.valid {
			t.Fatalf("case failed: %v", case_)
		}
		if !secure {
			SecureNodeId(id, ip)
			if !NodeIdSecure(string(id), ip) {
				t.Fatal("failed to secure node id")
			}
		}
	}
}

func TestServerDefaultNodeIdSecure(t *testing.T) {
	s, err := NewServer(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer s.Close()
	if !NodeIdSecure(s.ID(), missinggo.AddrIP(s.Addr())) {
		t.Fatal("not secure")
	}
}
<commit_msg>dht: Revive the UnmarshalGetPeersResponse test<commit_after>package dht

import (
	"encoding/hex"
	"math/big"
	"math/rand"
	"net"
	"testing"

	"github.com/anacrolix/missinggo"
	"github.com/stretchr/testify/assert"
)

func TestSetNilBigInt(t *testing.T) {
	i := new(big.Int)
	i.SetBytes(make([]byte, 2))
}

func TestMarshalCompactNodeInfo(t *testing.T) {
	cni := NodeInfo{
		ID: [20]byte{'a', 'b', 'c'},
	}
	addr, err := net.ResolveUDPAddr("udp4", "1.2.3.4:5")
	if err != nil {
		t.Fatal(err)
	}
	cni.Addr = newDHTAddr(addr)
	var b [CompactNodeInfoLen]byte
	cni.PutCompact(b[:])
	if err != nil {
		t.Fatal(err)
	}
	var bb [26]byte
	copy(bb[:], []byte("abc"))
	copy(bb[20:], []byte("\x01\x02\x03\x04\x00\x05"))
	if b != bb {
		t.FailNow()
	}
}

func recoverPanicOrDie(t *testing.T, f func()) {
	defer func() {
		r := recover()
		if r == nil {
			t.Fatal("expected panic")
		}
	}()
	f()
}

const zeroID = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"

var testIDs []nodeID

func init() {
	for _, s := range []string{
		zeroID,
		"\x03" + zeroID[1:],
		"\x03" + zeroID[1:18] + "\x55\xf0",
		"\x55" + zeroID[1:17] + "\xff\x55\x0f",
		"\x54" + zeroID[1:18] + "\x50\x0f",
		"",
	} {
		testIDs = append(testIDs, nodeIDFromString(s))
	}
}

func TestDistances(t *testing.T) {
	expectBitcount := func(i big.Int, count int) {
		if bitCount(i) != count {
			t.Fatalf("expected bitcount of %d: got %d", count, bitCount(i))
		}
	}
	expectBitcount(testIDs[3].Distance(&testIDs[0]), 4+8+4+4)
	expectBitcount(testIDs[3].Distance(&testIDs[1]), 4+8+4+4)
	expectBitcount(testIDs[3].Distance(&testIDs[2]), 4+8+8)
	for i := 0; i < 5; i++ {
		dist := testIDs[i].Distance(&testIDs[5])
		if dist.Cmp(&maxDistance) != 0 {
			t.Fatal("expected max distance for comparison with unset node id")
		}
	}
}

func TestMaxDistanceString(t *testing.T) {
	if string(maxDistance.Bytes()) != "\x01"+zeroID {
		t.FailNow()
	}
}

func TestClosestNodes(t *testing.T) {
	cn := newKClosestNodesSelector(2, testIDs[3])
	for _, i := range rand.Perm(len(testIDs)) {
		cn.Push(testIDs[i])
	}
	if len(cn.IDs()) != 2 {
		t.FailNow()
	}
	m := map[string]bool{}
	for _, id := range cn.IDs() {
		m[id.ByteString()] = true
	}
	if !m[testIDs[3].ByteString()] || !m[testIDs[4].ByteString()] {
		t.FailNow()
	}
}

func TestUnmarshalGetPeersResponse(t *testing.T) {
	gpr := Msg{
		"r": map[string]interface{}{
			"values": []interface{}{"\x01\x02\x03\x04\x05\x06", "\x07\x08\x09\x0a\x0b\x0c"},
			"nodes":  "\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x02\x03\x04\x05\x06\x07\x08\x09\x02\x03\x04\x05\x06\x07" + "\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x02\x03\x04\x05\x06\x07\x08\x09\x02\x03\x04\x05\x06\x07",
		},
	}
	assert.EqualValues(t, 2, len(gpr.Values()))
	assert.EqualValues(t, 2, len(gpr.Nodes()))
}

func TestDHTDefaultConfig(t *testing.T) {
	s, err := NewServer(nil)
	if err != nil {
		t.Fatal(err)
	}
	s.Close()
}

func TestPing(t *testing.T) {
	srv, err := NewServer(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer srv.Close()
	srv0, err := NewServer(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer srv0.Close()
	tn, err := srv.Ping(&net.UDPAddr{
		IP:   []byte{127, 0, 0, 1},
		Port: srv0.Addr().(*net.UDPAddr).Port,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer tn.Close()
	ok := make(chan bool)
	tn.SetResponseHandler(func(msg Msg) {
		ok <- msg.ID() == srv0.ID()
	})
	if !<-ok {
		t.FailNow()
	}
}

func TestDHTSec(t *testing.T) {
	for _, case_ := range []struct {
		ipStr     string
		nodeIDHex string
		valid     bool
	}{
		// These 5 are from the spec example. They are all valid.
		{"124.31.75.21", "5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee401", true},
		{"21.75.31.124", "5a3ce9c14e7a08645677bbd1cfe7d8f956d53256", true},
		{"65.23.51.170", "a5d43220bc8f112a3d426c84764f8c2a1150e616", true},
		{"84.124.73.14", "1b0321dd1bb1fe518101ceef99462b947a01ff41", true},
		{"43.213.53.83", "e56f6cbf5b7c4be0237986d5243b87aa6d51305a", true},
		// spec[0] with one of the rand() bytes changed. Valid.
		{"124.31.75.21", "5fbfbff10c5d7a4ec8a88e4c6ab4c28b95eee401", true},
		// spec[1] with the 21st leading bit changed. Not Valid.
		{"21.75.31.124", "5a3ce1c14e7a08645677bbd1cfe7d8f956d53256", false},
		// spec[2] with the 22nd leading bit changed. Valid.
		{"65.23.51.170", "a5d43620bc8f112a3d426c84764f8c2a1150e616", true},
		// spec[3] with the 4th last bit changed. Valid.
		{"84.124.73.14", "1b0321dd1bb1fe518101ceef99462b947a01fe01", true},
		// spec[4] with the 3rd last bit changed. Not valid.
		{"43.213.53.83", "e56f6cbf5b7c4be0237986d5243b87aa6d51303e", false},
	} {
		ip := net.ParseIP(case_.ipStr)
		id, err := hex.DecodeString(case_.nodeIDHex)
		if err != nil {
			t.Fatal(err)
		}
		secure := NodeIdSecure(string(id), ip)
		if secure != case_.valid {
			t.Fatalf("case failed: %v", case_)
		}
		if !secure {
			SecureNodeId(id, ip)
			if !NodeIdSecure(string(id), ip) {
				t.Fatal("failed to secure node id")
			}
		}
	}
}

func TestServerDefaultNodeIdSecure(t *testing.T) {
	s, err := NewServer(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer s.Close()
	if !NodeIdSecure(s.ID(), missinggo.AddrIP(s.Addr())) {
		t.Fatal("not secure")
	}
}
<|endoftext|>


<commit_before>package main

import (
    "log"
    "jvmgo/rtda"
    "jvmgo/instructions"
)

// todo
func loop(thread *rtda.Thread) {
    bcr := instructions.NewBytecodeReader()
    for {
        frame := thread.CurrentFrame()

        // decode
        bcr.SetPC(thread.PC())
        bcr.SetCode(frame.Method().Code())
        _, inst := instructions.Decode(bcr)
        frame.SetNextPC(bcr.PC())

        // execute
        //pc := thread.PC()
        //logInstruction(frame, pc, opcode, inst)
        inst.Execute(frame)
        if !thread.IsStackEmpty() {
            topFrame := thread.TopFrame()
            thread.SetPC(topFrame.NextPC())
        } else {
            break;
        }
    }
}

func logInstruction(frame *rtda.Frame, pc int, opcode uint8, inst instructions.Instruction) {
    className := frame.Method().Class().Name()
    methodName := frame.Method().Name()
    log.Printf("exec: %v.%v #%v 0x%x %v", className, methodName, pc, opcode, inst)
}
<commit_msg>code refactor<commit_after>package main

import (
    "log"
    "jvmgo/rtda"
    "jvmgo/instructions"
)

// todo
func loop(thread *rtda.Thread) {
    bcr := instructions.NewBytecodeReader()
    for {
        frame := thread.CurrentFrame()

        // decode
        bcr.SetPC(thread.PC())
        bcr.SetCode(frame.Method().Code())
        _, inst := instructions.Decode(bcr)
        frame.SetNextPC(bcr.PC())

        // execute
        //logInstruction(frame, thread.PC(), opcode, inst)
        inst.Execute(frame)
        if !thread.IsStackEmpty() {
            topFrame := thread.TopFrame()
            thread.SetPC(topFrame.NextPC())
        } else {
            break;
        }
    }
}

func logInstruction(frame *rtda.Frame, pc int, opcode uint8, inst instructions.Instruction) {
    className := frame.Method().Class().Name()
    methodName := frame.Method().Name()
    log.Printf("exec: %v.%v #%v 0x%x %v", className, methodName, pc, opcode, inst)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Matthew Lord (mattalord@gmail.com)

WARNING: This is experimental and for demonstration purposes only!

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package main

import (
  "os"
  "log"
  "github.com/mattlord/myarbitratord/group_replication/instances"
  "time"
  "flag"
  "sort"
  "encoding/json"
  "io/ioutil"
)

type MembersByOnlineNodes []instances.Instance
var debug = false

var InfoLog = log.New(os.Stderr,
              "INFO: ",
              log.Ldate|log.Ltime|log.Lshortfile)

var DebugLog = log.New(os.Stderr,
               "DEBUG: ",
               log.Ldate|log.Ltime|log.Lshortfile)

func main(){
  var seed_host string
  var seed_port string
  var mysql_user string
  var mysql_pass string
  var mysql_auth_file string
  type json_mysql_auth struct {
    User string
    Password string
  }

  flag.StringVar( &seed_host, "seed_host", "", "IP/Hostname of the seed node used to start monitoring the Group Replication cluster" )
  flag.StringVar( &seed_port, "seed_port", "3306", "Port of the seed node used to start monitoring the Group Replication cluster" )
  flag.BoolVar( &debug, "debug", false, "Execute in debug mode with all debug logging enabled" )
  flag.StringVar( &mysql_user, "mysql_user", "root", "The mysql user account to be used when connecting to any node in the cluster" )
  flag.StringVar( &mysql_pass, "mysql_password", "", "The mysql user account password to be used when connecting to any node in the cluster" )
  flag.StringVar( &mysql_auth_file, "mysql_auth_file", "", "The JSON encoded file containining user and password entities for the mysql account to be used when connecting to any node in the cluster" )

  flag.Parse()

  // ToDo: I need to handle the password on the command-line more securely
  //       I need to do some data masking for the processlist

  // A host is required, the default port of 3306 will then be attempted
  if( seed_host == "" ){
    log.Fatal( "myarbitratord usage: myarbitratord -seed_host=<seed_host> [-seed_port=<seed_port>] [-mysql_user=<mysql_user>] [-mysql_password=<mysql_pass>] [-mysql_auth_file=<path to json file>] [-debug=true]" )
  }

  if( debug ){
    instances.Debug = true
  }

  if( mysql_auth_file != "" && mysql_pass == "" ){
    if( debug ){
      DebugLog.Printf( "Reading MySQL credentials from file: %s\n", mysql_auth_file )
    }

    jsonfile, err := ioutil.ReadFile( mysql_auth_file )

    if( err != nil ){
      log.Fatal( "Could not read mysql credentials from specified file: " + mysql_auth_file )
    }

    var jsonauth json_mysql_auth
    json.Unmarshal( jsonfile, &jsonauth )

    if( debug ){
      DebugLog.Printf( "Unmarshaled mysql auth file contents: %+v\n", jsonauth )
    }

    mysql_user = jsonauth.User
    mysql_pass = jsonauth.Password

    if( mysql_user == "" || mysql_pass == "" ){
      errstr := "Failed to read user and password from " + mysql_auth_file + ". Ensure that the file contents are in the required format: \n{\n  \"user\": \"myser\",\n  \"password\": \"mypass\"\n}"
      log.Fatal( errstr )
    }

    if( debug ){
      DebugLog.Printf( "Read mysql auth info from file. user: %s, password: %s\n", mysql_user, mysql_pass )
    }
  }

  InfoLog.Println( "Welcome to the MySQL Group Replication Arbitrator!" )

  InfoLog.Printf( "Starting operations from seed node: '%s:%s'\n", seed_host, seed_port )

  seed_node := instances.New( seed_host, seed_port, mysql_user, mysql_pass )
  err := MonitorCluster( seed_node )

  if( err != nil ){
    log.Fatal( err )
    os.Exit( 100 )
  } else {
    os.Exit( 0 )
  }
}

func MonitorCluster( seed_node *instances.Instance ) error {
  loop := true
  var err error
  last_view := []instances.Instance{}

  for( loop == true ){
    // let's check the status of the current seed node
    // if the seed node
    err = seed_node.Connect()

    if( err != nil || seed_node.Member_state != "ONLINE" ){
      // if we couldn't connect to the current seed node or it's no longer part of the group
      // let's try and get a new seed node from the last known membership view
      InfoLog.Println( "Attempting to get a new seed node..." )

      for i := 0; i < len(last_view); i++ {
        err = last_view[i].Connect()
        if( err == nil && last_view[i].Member_state == "ONLINE" ){
          seed_node = &last_view[i]
          InfoLog.Printf( "Updated seed node! New seed node is: '%s:%s'\n", seed_node.Mysql_host, seed_node.Mysql_port )
          break
        }
      }
    }

    members, err := seed_node.GetMembers()

    if( err != nil ){
      // something is up with our current seed node, let's loop again
      continue
    }

    // save this view in case the seed node is no longer valid next time
    //last_view = copy( last_view, members[:] )
    last_view = *members

    quorum, err := seed_node.HasQuorum()

    if( err != nil ){
      // something is up with our current seed node, let's loop again
      continue
    }

    if( debug ){
      DebugLog.Printf( "Seed node details: %+v", seed_node )
    }

    if( quorum ){
      // Let's try and shutdown the nodes NOT in the primary partition if we can reach them from the arbitrator

      for _, member := range *members {
        if( member.Member_state == "ERROR" || member.Member_state == "UNREACHABLE" ){
          InfoLog.Printf( "Shutting down non-healthy node: '%s:%s'\n", member.Mysql_host, member.Mysql_port )

          err = member.Connect()

          if( err != nil ){
            InfoLog.Printf( "Could not connect to '%s:%s' in order to shut it down\n", member.Mysql_host, member.Mysql_port )
          } else {
            err = member.Shutdown()
          }

          if( err != nil ){
            InfoLog.Printf( "Could not shutdown instance: '%s:%s'\n", member.Mysql_host, member.Mysql_port )
          }
        }
      }
    } else {
      // handling other network partitions and split brain scenarios will be much trickier... I'll need to try and
      // contact each member in the last seen view and try to determine which partition should become the
      // primary one. We'll then need to contact 1 node in the new primary partition and explicitly set the new
      // membership with 'set global group_replication_force_members="<node_list>"'. Finally we'll need to try
      // and connect to the nodes on the losing side(s) of the partition and attempt to shutdown the mysqlds

      InfoLog.Println( "Network partition detected! Attempting to handle... " )

      // does anyone have a quorum? Let's double check before forcing the membership
      primary_partition := false

      for i := 0; i < len(last_view); i++ {
        var err error

        err = last_view[i].Connect()

        if( err == nil ){
          quorum, err = last_view[i].HasQuorum()
          // let's make sure that the Online_participants is up to date
          _, err = last_view[i].GetMembers()
        }

        if( err == nil && quorum ){
          seed_node = &last_view[i]
          primary_partition = true
          break
        }
      }

      // If no one in fact has a quorum, then let's see which partition has the most
      // online/participating/communicating members. The participants in that partition
      // will then be the ones that we use to force the new membership and unlock the cluster

      if( primary_partition == false ){
        InfoLog.Println( "No primary partition found! Attempting to choose and force a new one ... " )

        sort.Sort( MembersByOnlineNodes(last_view) )

        // now the last element in the array is the one to use as it's coordinating with the most nodes
        view_len := len(last_view)-1
        seed_node = &last_view[view_len]

        // *BUT*, if there's no clear winner based on sub-partition size, then we should pick the sub-partition (which
        // can be 1 node) that has executed the most GTIDs
        if( last_view[view_len].Online_participants == last_view[view_len-1].Online_participants ){
          bestmemberpos := view_len
          bestmembertrxcnt := 0
          curtrxcnt := 0
          bestmembertrxcnt, err = last_view[view_len].TransactionCount()

          // let's loop backwards through the array as it's sorted by online participants / partition size now
          // skipping the last one as we already have the info for it
          for i := view_len-1; i >= 0; i-- {
            if( last_view[i].Online_participants == last_view[bestmemberpos].Online_participants ){
              curtrxcnt, err = last_view[i].TransactionCount()

              if( curtrxcnt > bestmembertrxcnt ){
                bestmembertrxcnt = curtrxcnt
                bestmemberpos = i
              }
            } else {
              // otherwise we've gone backwards far enough and we have the best option
              break
            }
          }

          seed_node = &last_view[bestmemberpos]
        }

        err = seed_node.Connect()

        if( err != nil ){
          // let's just loop again
          continue
        }

        if( debug ){
          DebugLog.Printf( "Member view sorted by number of online nodes: %+v\n", last_view )
        }

        // let's build a string of '<host>:<port>' combinations that we want to use for the new membership view
        members, _ := seed_node.GetMembers()

        force_member_string := ""

        for i, member := range *members {
          err = member.Connect()

          if( err == nil && member.Member_state == "ONLINE" ){
            if( i != 0 ){
              force_member_string = force_member_string + ","
            }

            force_member_string = force_member_string + member.Mysql_host + ":" + member.Mysql_port
          } else {
            member.Member_state = "SHOOT_ME"
          }
        }

        if( force_member_string != "" ){
          InfoLog.Printf( "Forcing group membership to form new primary partition! Using: '%s'\n", force_member_string )

          err := seed_node.ForceMembers( force_member_string )

          if( err != nil ){
            InfoLog.Printf( "Error forcing group membership: %v\n", err )
          } else {
            // We successfully unblocked the group, now let's try and politely STONITH the nodes in the losing partition
            for _, member := range *members {
              if( member.Member_state == "SHOOT_ME" ){
                member.Shutdown()
              }

              if( err != nil ){
                InfoLog.Printf( "Could not shutdown instance: '%s:%s'\n", member.Mysql_host, member.Mysql_port )
              }
            }
          }
        } else {
          InfoLog.Println( "No valid group membership to force!" )
        }
      }
    }

    if( err != nil ){
      loop = false
    } else {
      time.Sleep( time.Millisecond * 2000 )
    }
  }

  return err
}

// The remaining functions are used to sort our membership slice
func (a MembersByOnlineNodes) Len() int {
  return len(a)
}

func (a MembersByOnlineNodes) Swap( i, j int ) {
  a[i], a[j] = a[j], a[i]
}

func (a MembersByOnlineNodes) Less( i, j int ) bool {
  return a[i].Online_participants < a[j].Online_participants
}
<commit_msg>Improved handling of Usage output<commit_after>/*
Copyright 2017 Matthew Lord (mattalord@gmail.com)

WARNING: This is experimental and for demonstration purposes only!

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package main

import (
  "os"
  "log"
  "github.com/mattlord/myarbitratord/group_replication/instances"
  "time"
  "flag"
  "sort"
  "fmt"
  "encoding/json"
  "io/ioutil"
)

type MembersByOnlineNodes []instances.Instance
var debug = false

var InfoLog = log.New(os.Stderr,
              "INFO: ",
              log.Ldate|log.Ltime|log.Lshortfile)

var DebugLog = log.New(os.Stderr,
               "DEBUG: ",
               log.Ldate|log.Ltime|log.Lshortfile)

func main(){
  var seed_host string
  var seed_port string
  var mysql_user string
  var mysql_pass string
  var mysql_auth_file string
  type json_mysql_auth struct {
    User string
    Password string
  }

  flag.StringVar( &seed_host, "seed_host", "", "IP/Hostname of the seed node used to start monitoring the Group Replication cluster (Required Parameter!)" )
  flag.StringVar( &seed_port, "seed_port", "3306", "Port of the seed node used to start monitoring the Group Replication cluster" )
  flag.BoolVar( &debug, "debug", false, "Execute in debug mode with all debug logging enabled" )
  flag.StringVar( &mysql_user, "mysql_user", "root", "The mysql user account to be used when connecting to any node in the cluster" )
  flag.StringVar( &mysql_pass, "mysql_password", "", "The mysql user account password to be used when connecting to any node in the cluster" )
  flag.StringVar( &mysql_auth_file, "mysql_auth_file", "", "The JSON encoded file containining user and password entities for the mysql account to be used when connecting to any node in the cluster" )

  flag.Parse()

  // ToDo: I need to handle the password on the command-line more securely
  //       I need to do some data masking for the processlist

  // A host is required, the default port of 3306 will then be attempted
  if( seed_host == "" ){
    fmt.Fprintf( os.Stderr, "No value specified for required flag: -seed_host\n" )
    fmt.Fprintf( os.Stderr, "Usage of %s:\n", os.Args[0] )
    flag.PrintDefaults()
    os.Exit( 1 )
  }

  if( debug ){
    instances.Debug = true
  }

  if( mysql_auth_file != "" && mysql_pass == "" ){
    if( debug ){
      DebugLog.Printf( "Reading MySQL credentials from file: %s\n", mysql_auth_file )
    }

    jsonfile, err := ioutil.ReadFile( mysql_auth_file )

    if( err != nil ){
      log.Fatal( "Could not read mysql credentials from specified file: " + mysql_auth_file )
    }

    var jsonauth json_mysql_auth
    json.Unmarshal( jsonfile, &jsonauth )

    if( debug ){
      DebugLog.Printf( "Unmarshaled mysql auth file contents: %+v\n", jsonauth )
    }

    mysql_user = jsonauth.User
    mysql_pass = jsonauth.Password

    if( mysql_user == "" || mysql_pass == "" ){
      errstr := "Failed to read user and password from " + mysql_auth_file + ". Ensure that the file contents are in the required format: \n{\n  \"user\": \"myser\",\n  \"password\": \"mypass\"\n}"
      log.Fatal( errstr )
    }

    if( debug ){
      DebugLog.Printf( "Read mysql auth info from file. user: %s, password: %s\n", mysql_user, mysql_pass )
    }
  }

  InfoLog.Println( "Welcome to the MySQL Group Replication Arbitrator!" )

  InfoLog.Printf( "Starting operations from seed node: '%s:%s'\n", seed_host, seed_port )

  seed_node := instances.New( seed_host, seed_port, mysql_user, mysql_pass )
  err := MonitorCluster( seed_node )

  if( err != nil ){
    log.Fatal( err )
    os.Exit( 100 )
  } else {
    os.Exit( 0 )
  }
}

func MonitorCluster( seed_node *instances.Instance ) error {
  loop := true
  var err error
  last_view := []instances.Instance{}

  for( loop == true ){
    // let's check the status of the current seed node
    // if the seed node
    err = seed_node.Connect()

    if( err != nil || seed_node.Member_state != "ONLINE" ){
      // if we couldn't connect to the current seed node or it's no longer part of the group
      // let's try and get a new seed node from the last known membership view
      InfoLog.Println( "Attempting to get a new seed node..." )

      for i := 0; i < len(last_view); i++ {
        err = last_view[i].Connect()
        if( err == nil && last_view[i].Member_state == "ONLINE" ){
          seed_node = &last_view[i]
          InfoLog.Printf( "Updated seed node! New seed node is: '%s:%s'\n", seed_node.Mysql_host, seed_node.Mysql_port )
          break
        }
      }
    }

    members, err := seed_node.GetMembers()

    if( err != nil ){
      // something is up with our current seed node, let's loop again
      continue
    }

    // save this view in case the seed node is no longer valid next time
    //last_view = copy( last_view, members[:] )
    last_view = *members

    quorum, err := seed_node.HasQuorum()

    if( err != nil ){
      // something is up with our current seed node, let's loop again
      continue
    }

    if( debug ){
      DebugLog.Printf( "Seed node details: %+v", seed_node )
    }

    if( quorum ){
      // Let's try and shutdown the nodes NOT in the primary partition if we can reach them from the arbitrator

      for _, member := range *members {
        if( member.Member_state == "ERROR" || member.Member_state == "UNREACHABLE" ){
          InfoLog.Printf( "Shutting down non-healthy node: '%s:%s'\n", member.Mysql_host, member.Mysql_port )

          err = member.Connect()

          if( err != nil ){
            InfoLog.Printf( "Could not connect to '%s:%s' in order to shut it down\n", member.Mysql_host, member.Mysql_port )
          } else {
            err = member.Shutdown()
          }

          if( err != nil ){
            InfoLog.Printf( "Could not shutdown instance: '%s:%s'\n", member.Mysql_host, member.Mysql_port )
          }
        }
      }
    } else {
      // handling other network partitions and split brain scenarios will be much trickier... I'll need to try and
      // contact each member in the last seen view and try to determine which partition should become the
      // primary one. We'll then need to contact 1 node in the new primary partition and explicitly set the new
      // membership with 'set global group_replication_force_members="<node_list>"'. Finally we'll need to try
      // and connect to the nodes on the losing side(s) of the partition and attempt to shutdown the mysqlds

      InfoLog.Println( "Network partition detected! Attempting to handle... " )

      // does anyone have a quorum? Let's double check before forcing the membership
      primary_partition := false

      for i := 0; i < len(last_view); i++ {
        var err error

        err = last_view[i].Connect()

        if( err == nil ){
          quorum, err = last_view[i].HasQuorum()
          // let's make sure that the Online_participants is up to date
          _, err = last_view[i].GetMembers()
        }

        if( err == nil && quorum ){
          seed_node = &last_view[i]
          primary_partition = true
          break
        }
      }

      // If no one in fact has a quorum, then let's see which partition has the most
      // online/participating/communicating members. The participants in that partition
      // will then be the ones that we use to force the new membership and unlock the cluster

      if( primary_partition == false ){
        InfoLog.Println( "No primary partition found! Attempting to choose and force a new one ... " )

        sort.Sort( MembersByOnlineNodes(last_view) )

        // now the last element in the array is the one to use as it's coordinating with the most nodes
        view_len := len(last_view)-1
        seed_node = &last_view[view_len]

        // *BUT*, if there's no clear winner based on sub-partition size, then we should pick the sub-partition (which
        // can be 1 node) that has executed the most GTIDs
        if( last_view[view_len].Online_participants == last_view[view_len-1].Online_participants ){
          bestmemberpos := view_len
          bestmembertrxcnt := 0
          curtrxcnt := 0
          bestmembertrxcnt, err = last_view[view_len].TransactionCount()

          // let's loop backwards through the array as it's sorted by online participants / partition size now
          // skipping the last one as we already have the info for it
          for i := view_len-1; i >= 0; i-- {
            if( last_view[i].Online_participants == last_view[bestmemberpos].Online_participants ){
              curtrxcnt, err = last_view[i].TransactionCount()

              if( curtrxcnt > bestmembertrxcnt ){
                bestmembertrxcnt = curtrxcnt
                bestmemberpos = i
              }
            } else {
              // otherwise we've gone backwards far enough and we have the best option
              break
            }
          }

          seed_node = &last_view[bestmemberpos]
        }

        err = seed_node.Connect()

        if( err != nil ){
          // let's just loop again
          continue
        }

        if( debug ){
          DebugLog.Printf( "Member view sorted by number of online nodes: %+v\n", last_view )
        }

        // let's build a string of '<host>:<port>' combinations that we want to use for the new membership view
        members, _ := seed_node.GetMembers()

        force_member_string := ""

        for i, member := range *members {
          err = member.Connect()

          if( err == nil && member.Member_state == "ONLINE" ){
            if( i != 0 ){
              force_member_string = force_member_string + ","
            }

            force_member_string = force_member_string + member.Mysql_host + ":" + member.Mysql_port
          } else {
            member.Member_state = "SHOOT_ME"
          }
        }

        if( force_member_string != "" ){
          InfoLog.Printf( "Forcing group membership to form new primary partition! Using: '%s'\n", force_member_string )

          err := seed_node.ForceMembers( force_member_string )

          if( err != nil ){
            InfoLog.Printf( "Error forcing group membership: %v\n", err )
          } else {
            // We successfully unblocked the group, now let's try and politely STONITH the nodes in the losing partition
            for _, member := range *members {
              if( member.Member_state == "SHOOT_ME" ){
                member.Shutdown()
              }

              if( err != nil ){
                InfoLog.Printf( "Could not shutdown instance: '%s:%s'\n", member.Mysql_host, member.Mysql_port )
              }
            }
          }
        } else {
          InfoLog.Println( "No valid group membership to force!" )
        }
      }
    }

    if( err != nil ){
      loop = false
    } else {
      time.Sleep( time.Millisecond * 2000 )
    }
  }

  return err
}

// The remaining functions are used to sort our membership slice
func (a MembersByOnlineNodes) Len() int {
  return len(a)
}

func (a MembersByOnlineNodes) Swap( i, j int ) {
  a[i], a[j] = a[j], a[i]
}

func (a MembersByOnlineNodes) Less( i, j int ) bool {
  return a[i].Online_participants < a[j].Online_participants
}
<|endoftext|>


<commit_before>/*
 * mysql-sniffer.go
 *
 * A straightforward program for sniffing MySQL query streams and providing
 * diagnostic information on the realtime queries your database is handling.
 *
 * FIXME: this assumes IPv4.
 * FIXME: tokenizer doesn't handle negative numbers or floating points.
 * FIXME: canonicalizer should collapse "IN (?,?,?,?)" and "VALUES (?,?,?,?)"
 * FIXME: tokenizer breaks on '"' or similarly embedded quotes
 * FIXME: canonicalizer doesn't strip newlines or collapse space
 * FIXME: tokenizer parses numbers in words wrong, i.e. s2compiled -> s?compiled
 *
 * written by Mark Smith <mark@qq.is>
 *
 * requires the gopcap library to be installed from:
 *   https://github.com/xb95/gopcap
 *
 */

package main

import (
	"flag"
	"fmt"
	"log"
	"pcap"
	"sort"
	"strings"
	"time"
)

var start int64 = time.Seconds()
var qbuf map[string]int = make(map[string]int)
var querycount int

const TOKEN_DEFAULT = 0
const TOKEN_QUOTE = 1
const TOKEN_NUMBER = 2

func main() {
	var port *int = flag.Int("P", 3306, "MySQL port to use")
	var eth *string = flag.String("i", "eth0", "Interface to sniff")
	var snaplen *int = flag.Int("s", 1024, "Bytes of each packet to sniff")
	var dirty *bool = flag.Bool("u", false, "Unsanitized -- do not canonicalize queries")
	var period *int = flag.Int("t", 10, "Seconds between outputting status")
	var displaycount *int = flag.Int("d", 25, "Display this many queries in status updates")
	var verbose *bool = flag.Bool("v", false, "Print every query received (spammy)")
	flag.Parse()

	log.SetPrefix("")
	log.SetFlags(0)

	log.Printf("Initializing MySQL sniffing on %s:%d...", *eth, *port)
	iface, err := pcap.Openlive(*eth, int32(*snaplen), false, 0)
	if iface == nil || err != "" {
		if err == "" {
			err = "unknown error"
		}
		log.Fatalf("Failed to open device: %s", err)
	}

	err = iface.Setfilter(fmt.Sprintf("tcp dst port %d", *port))
	if err != "" {
		log.Fatalf("Failed to set port filter: %s", err)
	}

	last := time.Seconds()
	for pkt := iface.Next(); pkt != nil; pkt = iface.Next() {
		handlePacket(pkt, *dirty, *verbose)

		// simple output printer... this should be super fast since we expect that a
		// system like this will have relatively few unique queries once they're
		// canonicalized.
		if !*verbose && querycount%100 == 0 && last < time.Seconds()-int64(*period) {
			last = time.Seconds()
			handleStatusUpdate(*displaycount)
		}
	}
}

func handleStatusUpdate(displaycount int) {
	elapsed := float64(time.Seconds() - start)

	// print status bar
	log.Printf("\n")
	log.SetFlags(log.Ldate | log.Ltime)
	log.Printf("%d total queries, %0.2f per second", querycount, float64(querycount)/elapsed)
	log.SetFlags(0)

	// we cheat so badly here...
	var tmp sort.StringArray = make([]string, 0, len(qbuf))
	for q, c := range qbuf {
		tmp = append(tmp, fmt.Sprintf("%6d  %0.2f/s  %s", c, float64(c)/elapsed, q))
	}
	sort.Sort(tmp)

	// now print top to bottom, since our sorted list is sorted backwards
	// from what we want
	if len(tmp) < displaycount {
		displaycount = len(tmp)
	}
	for i := 1; i <= displaycount; i++ {
		log.Printf(tmp[len(tmp)-i])
	}
}

// extract the data... we have to figure out where it is, which means extracting data
// from the various headers until we get the location we want.  this is crude, but
// functional and it should be fast.
func handlePacket(pkt *pcap.Packet, dirty, verbose bool) {
	// Ethernet frame has 14 bytes of stuff to ignore, so we start our root position here
	var pos uint8 = 14

	// The IP frame has the header length in bits 4-7 of byte 0 (relative)
	pos += pkt.Data[pos] & 0x0F * 4

	// The TCP frame has the data offset in bits 4-7 of byte 12 (relative)
	pos += uint8(pkt.Data[pos+12]) >> 4 * 4

	// must be at least 6 bytes of payload
	if len(pkt.Data)-int(pos) <= 6 {
		return
	}

	// MySQL packet type is the fifth byte... COM_QUERY is 0x03, if it's not that
	// then we don't care, move on
	if pkt.Data[pos+4] != 0x03 {
		return
	}

	// the query is now in query ... easier to deal with than always offsetting
	handleQuery(pkt.Data[pos+5:], dirty, verbose)
}

// scans forward in the query given the current type and returns when we encounter
// a new type and need to stop scanning.  returns the size of the last token and
// the type of it.
func scanToken(query []byte) (length int, thistype int) {
	if len(query) < 1 {
		log.Fatalf("scanToken called with empty query")
	}

	// peek at the first byte, then loop
	switch {
	case query[0] == 39 || query[0] == 34: // '"
		escaped := false
		for i := 1; i < len(query); i++ {
			switch query[i] {
			case 39, 34:
				if escaped {
					escaped = false
					continue
				}
				return i, TOKEN_QUOTE
			case 92:
				escaped = true
			default:
				escaped = false
			}
		}
		return len(query), TOKEN_QUOTE

	case query[0] >= 48 && query[0] <= 57: // 0-9
		for i := 1; i < len(query); i++ {
			switch {
			case query[i] >= 48 && query[i] <= 57: // 0-9
			default:
				return i, TOKEN_NUMBER
			}
		}
		return len(query), TOKEN_NUMBER

	default:
		for i := 1; i < len(query); i++ {
			switch {
			case query[i] == 39 || query[i] == 34 || (query[i] >= 48 && query[i] <= 57):
				return i, TOKEN_DEFAULT
			default:
			}
		}
		return len(query), TOKEN_DEFAULT
	}

	// shouldn't get here
	log.Fatalf("scanToken failure: [%s]", query)
	return
}

func cleanupQuery(query []byte) string {
	// iterate until we hit the end of the query...
	var qspace []string
	for i := 0; i < len(query); {
		length, toktype := scanToken(query[i:])

		switch toktype {
		case TOKEN_DEFAULT:
			qspace = append(qspace, string(query[i:i+length]))

		case TOKEN_NUMBER, TOKEN_QUOTE:
			qspace = append(qspace, "?")

		default:
			log.Fatalf("scanToken returned invalid token type %d", toktype)
		}

		i += length
	}

	// store in our global structure
	return strings.Join(qspace, "")
}

func handleQuery(query []byte, dirty bool, verbose bool) {
	var qstr string
	if dirty {
		qstr = string(query)
	} else {
		qstr = cleanupQuery(query)
	}

	// if verbose, just print it
	if verbose {
		log.Print(qstr)
		return
	}

	// map for later...
	querycount++
	qbuf[qstr]++
}
<commit_msg>Escape binary in unsanitized queries. Add option to leave them as is.<commit_after>/*
 * mysql-sniffer.go
 *
 * A straightforward program for sniffing MySQL query streams and providing
 * diagnostic information on the realtime queries your database is handling.
 *
 * FIXME: this assumes IPv4.
 * FIXME: tokenizer doesn't handle negative numbers or floating points.
 * FIXME: canonicalizer should collapse "IN (?,?,?,?)" and "VALUES (?,?,?,?)"
 * FIXME: tokenizer breaks on '"' or similarly embedded quotes
 * FIXME: canonicalizer doesn't strip newlines or collapse space
 * FIXME: tokenizer parses numbers in words wrong, i.e. s2compiled -> s?compiled
 *
 * written by Mark Smith <mark@qq.is>
 *
 * requires the gopcap library to be installed from:
 *   https://github.com/xb95/gopcap
 *
 */

package main

import (
	"flag"
	"fmt"
	"log"
	"pcap"
	"sort"
	"strings"
	"time"
)

var start int64 = time.Seconds()
var qbuf map[string]int = make(map[string]int)
var querycount int

const TOKEN_DEFAULT = 0
const TOKEN_QUOTE = 1
const TOKEN_NUMBER = 2

func main() {
	var port *int = flag.Int("P", 3306, "MySQL port to use")
	var eth *string = flag.String("i", "eth0", "Interface to sniff")
	var snaplen *int = flag.Int("s", 1024, "Bytes of each packet to sniff")
	var dirty *bool = flag.Bool("u", false, "Unsanitized -- do not canonicalize queries")
	var binary *bool = flag.Bool("b", false, "Output binary -- do not escape binary in queries")
	var period *int = flag.Int("t", 10, "Seconds between outputting status")
	var displaycount *int = flag.Int("d", 25, "Display this many queries in status updates")
	var verbose *bool = flag.Bool("v", false, "Print every query received (spammy)")
	flag.Parse()

	log.SetPrefix("")
	log.SetFlags(0)

	log.Printf("Initializing MySQL sniffing on %s:%d...", *eth, *port)
	iface, err := pcap.Openlive(*eth, int32(*snaplen), false, 0)
	if iface == nil || err != "" {
		if err == "" {
			err = "unknown error"
		}
		log.Fatalf("Failed to open device: %s", err)
	}

	err = iface.Setfilter(fmt.Sprintf("tcp dst port %d", *port))
	if err != "" {
		log.Fatalf("Failed to set port filter: %s", err)
	}

	last := time.Seconds()
	for pkt := iface.Next(); pkt != nil; pkt = iface.Next() {
		handlePacket(pkt, *dirty, *binary, *verbose)

		// simple output printer... this should be super fast since we expect that a
		// system like this will have relatively few unique queries once they're
		// canonicalized.
		if !*verbose && querycount%100 == 0 && last < time.Seconds()-int64(*period) {
			last = time.Seconds()
			handleStatusUpdate(*displaycount)
		}
	}
}

func handleStatusUpdate(displaycount int) {
	elapsed := float64(time.Seconds() - start)

	// print status bar
	log.Printf("\n")
	log.SetFlags(log.Ldate | log.Ltime)
	log.Printf("%d total queries, %0.2f per second", querycount, float64(querycount)/elapsed)
	log.SetFlags(0)

	// we cheat so badly here...
	var tmp sort.StringArray = make([]string, 0, len(qbuf))
	for q, c := range qbuf {
		tmp = append(tmp, fmt.Sprintf("%6d  %0.2f/s  %s", c, float64(c)/elapsed, q))
	}
	sort.Sort(tmp)

	// now print top to bottom, since our sorted list is sorted backwards
	// from what we want
	if len(tmp) < displaycount {
		displaycount = len(tmp)
	}
	for i := 1; i <= displaycount; i++ {
		log.Printf(tmp[len(tmp)-i])
	}
}

// extract the data... we have to figure out where it is, which means extracting data
// from the various headers until we get the location we want.  this is crude, but
// functional and it should be fast.
func handlePacket(pkt *pcap.Packet, dirty, binary, verbose bool) {
	// Ethernet frame has 14 bytes of stuff to ignore, so we start our root position here
	var pos uint8 = 14

	// The IP frame has the header length in bits 4-7 of byte 0 (relative)
	pos += pkt.Data[pos] & 0x0F * 4

	// The TCP frame has the data offset in bits 4-7 of byte 12 (relative)
	pos += uint8(pkt.Data[pos+12]) >> 4 * 4

	// must be at least 6 bytes of payload
	if len(pkt.Data)-int(pos) <= 6 {
		return
	}

	// MySQL packet type is the fifth byte... COM_QUERY is 0x03, if it's not that
	// then we don't care, move on
	if pkt.Data[pos+4] != 0x03 {
		return
	}

	// the query is now in query ... easier to deal with than always offsetting
	handleQuery(pkt.Data[pos+5:], dirty, binary, verbose)
}

// scans forward in the query given the current type and returns when we encounter
// a new type and need to stop scanning.  returns the size of the last token and
// the type of it.
func scanToken(query []byte) (length int, thistype int) {
	if len(query) < 1 {
		log.Fatalf("scanToken called with empty query")
	}

	// peek at the first byte, then loop
	switch {
	case query[0] == 39 || query[0] == 34: // '"
		escaped := false
		for i := 1; i < len(query); i++ {
			switch query[i] {
			case 39, 34:
				if escaped {
					escaped = false
					continue
				}
				return i, TOKEN_QUOTE
			case 92:
				escaped = true
			default:
				escaped = false
			}
		}
		return len(query), TOKEN_QUOTE

	case query[0] >= 48 && query[0] <= 57: // 0-9
		for i := 1; i < len(query); i++ {
			switch {
			case query[i] >= 48 && query[i] <= 57: // 0-9
			default:
				return i, TOKEN_NUMBER
			}
		}
		return len(query), TOKEN_NUMBER

	default:
		for i := 1; i < len(query); i++ {
			switch {
			case query[i] == 39 || query[i] == 34 || (query[i] >= 48 && query[i] <= 57):
				return i, TOKEN_DEFAULT
			default:
			}
		}
		return len(query), TOKEN_DEFAULT
	}

	// shouldn't get here
	log.Fatalf("scanToken failure: [%s]", query)
	return
}

func cleanupRawQuery(query []byte) string {
	var qspace []string
	var theByte byte
	for i := 0; i < len(query); i++ {
		theByte = query[i]
		if (theByte >= 0x20 && theByte <= 0x7E) || theByte == 0x0A || theByte == 0x0D {
			qspace = append(qspace, string(theByte))
		} else {
			qspace = append(qspace, fmt.Sprintf("\\x%02x", theByte))
		}
	}
	return strings.Join(qspace, "")
}

func cleanupQuery(query []byte) string {
	// iterate until we hit the end of the query...
	var qspace []string
	for i := 0; i < len(query); {
		length, toktype := scanToken(query[i:])

		switch toktype {
		case TOKEN_DEFAULT:
			qspace = append(qspace, string(query[i:i+length]))

		case TOKEN_NUMBER, TOKEN_QUOTE:
			qspace = append(qspace, "?")

		default:
			log.Fatalf("scanToken returned invalid token type %d", toktype)
		}

		i += length
	}

	// store in our global structure
	return strings.Join(qspace, "")
}

func handleQuery(query []byte, dirty bool, binary bool, verbose bool) {
	var qstr string
	if dirty && binary {
		qstr = string(query)
	} else if dirty {
		qstr = cleanupRawQuery(query)
	} else {
		qstr = cleanupQuery(query)
	}

	// if verbose, just print it
	if verbose {
		log.Print(qstr)
		return
	}

	// map for later...
	querycount++
	qbuf[qstr]++
}
<|endoftext|>


<commit_before>package dbus

import (
	"github.com/guelfey/go.dbus"
)

func (c *Conn) initJobs() {
	c.jobListener.jobs = make(map[dbus.ObjectPath]chan string)
}

func (c *Conn) jobComplete(signal *dbus.Signal) {
	var id uint32
	var job dbus.ObjectPath
	var unit string
	var result string
	dbus.Store(signal.Body, &id, &job, &unit, &result)
	c.jobListener.Lock()
	out, ok := c.jobListener.jobs[job]
	if ok {
		out <- result
	}
	c.jobListener.Unlock()
}

func (c *Conn) startJob(job string, args ...interface{}) (<-chan string, error) {
	c.jobListener.Lock()
	defer c.jobListener.Unlock()

	ch := make(chan string, 1)
	var path dbus.ObjectPath
	err := c.sysobj.Call(job, 0, args...).Store(&path)
	if err != nil {
		return nil, err
	}
	c.jobListener.jobs[path] = ch
	return ch, nil
}

func (c *Conn) runJob(job string, args ...interface{}) (string, error) {
	respCh, err := c.startJob(job, args...)
	if err != nil {
		return "", err
	}
	return <-respCh, nil
}

// StartUnit enqeues a start job and depending jobs, if any (unless otherwise
// specified by the mode string).
//
// Takes the unit to activate, plus a mode string. The mode needs to be one of
// replace, fail, isolate, ignore-dependencies, ignore-requirements. If
// "replace" the call will start the unit and its dependencies, possibly
// replacing already queued jobs that conflict with this. If "fail" the call
// will start the unit and its dependencies, but will fail if this would change
// an already queued job. If "isolate" the call will start the unit in question
// and terminate all units that aren't dependencies of it. If
// "ignore-dependencies" it will start a unit but ignore all its dependencies.
// If "ignore-requirements" it will start a unit but only ignore the
// requirement dependencies. It is not recommended to make use of the latter
// two options.
//
// Result string: one of done, canceled, timeout, failed, dependency, skipped.
// done indicates successful execution of a job. canceled indicates that a job
// has been canceled  before it finished execution. timeout indicates that the
// job timeout was reached. failed indicates that the job failed. dependency
// indicates that a job this job has been depending on failed and the job hence
// has been removed too. skipped indicates that a job was skipped because it
// didn't apply to the units current state.
func (c *Conn) StartUnit(name string, mode string) (string, error) {
	return c.runJob("StartUnit", name, mode)
}

// StopUnit is similar to StartUnit but stops the specified unit rather
// than starting it.
func (c *Conn) StopUnit(name string, mode string) (string, error) {
	return c.runJob("StopUnit", name, mode)
}

// ReloadUnit reloads a unit.  Reloading is done only if the unit is already running and fails otherwise.
func (c *Conn) ReloadUnit(name string, mode string) (string, error) {
	return c.runJob("ReloadUnit", name, mode)
}

// RestartUnit restarts a service.  If a service is restarted that isn't
// running it will be started.
func (c *Conn) RestartUnit(name string, mode string) (string, error) {
	return c.runJob("RestartUnit", name, mode)
}

// TryRestartUnit is like RestartUnit, except that a service that isn't running
// is not affected by the restart.
func (c *Conn) TryRestartUnit(name string, mode string) (string, error) {
	return c.runJob("TryRestartUnit", name, mode)
}

// ReloadOrRestart attempts a reload if the unit supports it and use a restart
// otherwise.
func (c *Conn) ReloadOrRestartUnit(name string, mode string) (string, error) {
	return c.runJob("ReloadOrRestartUnit", name, mode)
}

// ReloadOrTryRestart attempts a reload if the unit supports it and use a "Try"
// flavored restart otherwise.
func (c *Conn) ReloadOrTryRestartUnit(name string, mode string) (string, error) {
	return c.runJob("ReloadOrTryRestartUnit", name, mode)
}

// StartTransientUnit() may be used to create and start a transient unit, which
// will be released as soon as it is not running or referenced anymore or the
// system is rebooted. name is the unit name including suffix, and must be
// unique. mode is the same as in StartUnit(), properties contains properties
// of the unit.
func (c *Conn) StartTransientUnit(name string, mode string, properties ...Property) (string, error) {
	// the dbus interface for this method does not use the last argument and
	// should simply be given an empty list.  We use a concrete type here
	// (instead of the more appropriate interface{}) to satisfy the dbus library.
	return c.runJob("StartTransientUnit", name, mode, properties, make([]string, 0))
}

// KillUnit takes the unit name and a UNIX signal number to send.  All of the unit's
// processes are killed.
func (c *Conn) KillUnit(name string, signal int32) {
	c.sysobj.Call("KillUnit", 0, name, "all", signal).Store()
}

// ListUnits returns an array with all currently loaded units. Note that
// units may be known by multiple names at the same time, and hence there might
// be more unit names loaded than actual units behind them.
func (c *Conn) ListUnits() ([]UnitStatus, error) {
	result := make([][]interface{}, 0)
	err := c.sysobj.Call("ListUnits", 0).Store(&result)
	if err != nil {
		return nil, err
	}

	resultInterface := make([]interface{}, len(result))
	for i := range result {
		resultInterface[i] = result[i]
	}

	status := make([]UnitStatus, len(result))
	statusInterface := make([]interface{}, len(status))
	for i := range status {
		statusInterface[i] = &status[i]
	}

	err = dbus.Store(resultInterface, statusInterface...)
	if err != nil {
		return nil, err
	}

	return status, nil
}

type UnitStatus struct {
	Name        string          // The primary unit name as string
	Description string          // The human readable description string
	LoadState   string          // The load state (i.e. whether the unit file has been loaded successfully)
	ActiveState string          // The active state (i.e. whether the unit is currently started or not)
	SubState    string          // The sub state (a more fine-grained version of the active state that is specific to the unit type, which the active state is not)
	Followed    string          // A unit that is being followed in its state by this unit, if there is any, otherwise the empty string.
	Path        dbus.ObjectPath // The unit object path
	JobId       uint32          // If there is a job queued for the job unit the numeric job id, 0 otherwise
	JobType     string          // The job type as string
	JobPath     dbus.ObjectPath // The job object path
}
<commit_msg>feat(EnableUnitFiles): Add EnableUnitFiles method<commit_after>package dbus

import (
	"github.com/guelfey/go.dbus"
)

func (c *Conn) initJobs() {
	c.jobListener.jobs = make(map[dbus.ObjectPath]chan string)
}

func (c *Conn) jobComplete(signal *dbus.Signal) {
	var id uint32
	var job dbus.ObjectPath
	var unit string
	var result string
	dbus.Store(signal.Body, &id, &job, &unit, &result)
	c.jobListener.Lock()
	out, ok := c.jobListener.jobs[job]
	if ok {
		out <- result
	}
	c.jobListener.Unlock()
}

func (c *Conn) startJob(job string, args ...interface{}) (<-chan string, error) {
	c.jobListener.Lock()
	defer c.jobListener.Unlock()

	ch := make(chan string, 1)
	var path dbus.ObjectPath
	err := c.sysobj.Call(job, 0, args...).Store(&path)
	if err != nil {
		return nil, err
	}
	c.jobListener.jobs[path] = ch
	return ch, nil
}

func (c *Conn) runJob(job string, args ...interface{}) (string, error) {
	respCh, err := c.startJob(job, args...)
	if err != nil {
		return "", err
	}
	return <-respCh, nil
}

// StartUnit enqeues a start job and depending jobs, if any (unless otherwise
// specified by the mode string).
//
// Takes the unit to activate, plus a mode string. The mode needs to be one of
// replace, fail, isolate, ignore-dependencies, ignore-requirements. If
// "replace" the call will start the unit and its dependencies, possibly
// replacing already queued jobs that conflict with this. If "fail" the call
// will start the unit and its dependencies, but will fail if this would change
// an already queued job. If "isolate" the call will start the unit in question
// and terminate all units that aren't dependencies of it. If
// "ignore-dependencies" it will start a unit but ignore all its dependencies.
// If "ignore-requirements" it will start a unit but only ignore the
// requirement dependencies. It is not recommended to make use of the latter
// two options.
//
// Result string: one of done, canceled, timeout, failed, dependency, skipped.
// done indicates successful execution of a job. canceled indicates that a job
// has been canceled  before it finished execution. timeout indicates that the
// job timeout was reached. failed indicates that the job failed. dependency
// indicates that a job this job has been depending on failed and the job hence
// has been removed too. skipped indicates that a job was skipped because it
// didn't apply to the units current state.
func (c *Conn) StartUnit(name string, mode string) (string, error) {
	return c.runJob("StartUnit", name, mode)
}

// StopUnit is similar to StartUnit but stops the specified unit rather
// than starting it.
func (c *Conn) StopUnit(name string, mode string) (string, error) {
	return c.runJob("StopUnit", name, mode)
}

// ReloadUnit reloads a unit.  Reloading is done only if the unit is already running and fails otherwise.
func (c *Conn) ReloadUnit(name string, mode string) (string, error) {
	return c.runJob("ReloadUnit", name, mode)
}

// RestartUnit restarts a service.  If a service is restarted that isn't
// running it will be started.
func (c *Conn) RestartUnit(name string, mode string) (string, error) {
	return c.runJob("RestartUnit", name, mode)
}

// TryRestartUnit is like RestartUnit, except that a service that isn't running
// is not affected by the restart.
func (c *Conn) TryRestartUnit(name string, mode string) (string, error) {
	return c.runJob("TryRestartUnit", name, mode)
}

// ReloadOrRestart attempts a reload if the unit supports it and use a restart
// otherwise.
func (c *Conn) ReloadOrRestartUnit(name string, mode string) (string, error) {
	return c.runJob("ReloadOrRestartUnit", name, mode)
}

// ReloadOrTryRestart attempts a reload if the unit supports it and use a "Try"
// flavored restart otherwise.
func (c *Conn) ReloadOrTryRestartUnit(name string, mode string) (string, error) {
	return c.runJob("ReloadOrTryRestartUnit", name, mode)
}

// StartTransientUnit() may be used to create and start a transient unit, which
// will be released as soon as it is not running or referenced anymore or the
// system is rebooted. name is the unit name including suffix, and must be
// unique. mode is the same as in StartUnit(), properties contains properties
// of the unit.
func (c *Conn) StartTransientUnit(name string, mode string, properties ...Property) (string, error) {
	// the dbus interface for this method does not use the last argument and
	// should simply be given an empty list.  We use a concrete type here
	// (instead of the more appropriate interface{}) to satisfy the dbus library.
	return c.runJob("StartTransientUnit", name, mode, properties, make([]string, 0))
}

// KillUnit takes the unit name and a UNIX signal number to send.  All of the unit's
// processes are killed.
func (c *Conn) KillUnit(name string, signal int32) {
	c.sysobj.Call("KillUnit", 0, name, "all", signal).Store()
}

// ListUnits returns an array with all currently loaded units. Note that
// units may be known by multiple names at the same time, and hence there might
// be more unit names loaded than actual units behind them.
func (c *Conn) ListUnits() ([]UnitStatus, error) {
	result := make([][]interface{}, 0)
	err := c.sysobj.Call("ListUnits", 0).Store(&result)
	if err != nil {
		return nil, err
	}

	resultInterface := make([]interface{}, len(result))
	for i := range result {
		resultInterface[i] = result[i]
	}

	status := make([]UnitStatus, len(result))
	statusInterface := make([]interface{}, len(status))
	for i := range status {
		statusInterface[i] = &status[i]
	}

	err = dbus.Store(resultInterface, statusInterface...)
	if err != nil {
		return nil, err
	}

	return status, nil
}

type UnitStatus struct {
	Name        string          // The primary unit name as string
	Description string          // The human readable description string
	LoadState   string          // The load state (i.e. whether the unit file has been loaded successfully)
	ActiveState string          // The active state (i.e. whether the unit is currently started or not)
	SubState    string          // The sub state (a more fine-grained version of the active state that is specific to the unit type, which the active state is not)
	Followed    string          // A unit that is being followed in its state by this unit, if there is any, otherwise the empty string.
	Path        dbus.ObjectPath // The unit object path
	JobId       uint32          // If there is a job queued for the job unit the numeric job id, 0 otherwise
	JobType     string          // The job type as string
	JobPath     dbus.ObjectPath // The job object path
}

// EnableUnitFiles() may be used to enable one or more units in the system (by
// creating symlinks to them in /etc or /run).
//
// It takes a list of unit files to enable (either just file names or full
// absolute paths if the unit files are residing outside the usual unit
// search paths), and two booleans: the first controls whether the unit shall
// be enabled for runtime only (true, /run), or persistently (false, /etc).
// The second one controls whether symlinks pointing to other units shall
// be replaced if necessary.
//
// This call returns one boolean and an array with the changes made. The
// boolean signals whether the unit files contained any enablement
// information (i.e. an [Install]) section. The changes list consists of
// structures with three strings: the type of the change (one of symlink
// or unlink), the file name of the symlink and the destination of the
// symlink.
func (c *Conn) EnableUnitFiles(files []string, runtime bool, force bool) (bool, []EnableUnitFileChange, error) {
	var carries_install_info bool

	result := make([][]interface{}, 0)
	err := c.sysobj.Call("EnableUnitFiles", 0, files, runtime, force).Store(&carries_install_info, &result)
	if err != nil {
		return false, nil, err
	}

	resultInterface := make([]interface{}, len(result))
	for i := range result {
		resultInterface[i] = result[i]
	}

	changes := make([]EnableUnitFileChange, len(result))
	changesInterface := make([]interface{}, len(changes))
	for i := range changes {
		changesInterface[i] = &changes[i]
	}

	err = dbus.Store(resultInterface, changesInterface...)
	if err != nil {
		return false, nil, err
	}

	return carries_install_info, changes, nil
}

type EnableUnitFileChange struct {
	Type        string // Type of the change (one of symlink or unlink)
	Filename    string // File name of the symlink
	Destination string // Destination of the symlink
}
<|endoftext|>