<commit_before>package state

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"sync"
	"time"

	abci "github.com/tendermint/abci/types"

	cmn "github.com/tendermint/tmlibs/common"
	dbm "github.com/tendermint/tmlibs/db"
	"github.com/tendermint/tmlibs/log"

	wire "github.com/tendermint/go-wire"

	"github.com/tendermint/tendermint/types"
)

var (
	stateKey         = []byte("stateKey")
	abciResponsesKey = []byte("abciResponsesKey")
)

func calcValidatorsKey(height int64) []byte {
	return []byte(cmn.Fmt("validatorsKey:%v", height))
}

//-----------------------------------------------------------------------------

// State represents the latest committed state of the Tendermint consensus,
// including the last committed block and validator set.
// Newly committed blocks are validated and executed against the State.
// NOTE: not goroutine-safe.
type State struct {
	// mtx for writing to db
	mtx sync.Mutex
	db  dbm.DB

	ChainID string
	// Consensus parameters used for validating blocks
	Params types.ConsensusParams

	// These fields are updated by SetBlockAndValidators.
	// LastBlockHeight=0 at genesis (ie. block(H=0) does not exist)
	// LastValidators is used to validate block.LastCommit.
	LastBlockHeight  int64
	LastBlockTotalTx int64
	LastBlockID      types.BlockID
	LastBlockTime    time.Time
	Validators       *types.ValidatorSet
	LastValidators   *types.ValidatorSet
	// When a block returns a validator set change via EndBlock,
	// the change only applies to the next block.
	// So, if s.LastBlockHeight causes a valset change,
	// we set s.LastHeightValidatorsChanged = s.LastBlockHeight + 1
	LastHeightValidatorsChanged int64

	// AppHash is updated after Commit
	AppHash []byte

	logger log.Logger
}

// GetState loads the most recent state from the database,
// or creates a new one from the given genesisFile and persists the result
// to the database.
func GetState(stateDB dbm.DB, genesisFile string) (*State, error) {
	state := LoadState(stateDB)
	if state == nil {
		var err error
		state, err = MakeGenesisStateFromFile(stateDB, genesisFile)
		if err != nil {
			return nil, err
		}
		state.Save()
	}

	return state, nil
}

// LoadState loads the State from the database.
func LoadState(db dbm.DB) *State {
	return loadState(db, stateKey)
}

func loadState(db dbm.DB, key []byte) *State {
	buf := db.Get(key)
	if len(buf) == 0 {
		return nil
	}

	s := &State{db: db}
	r, n, err := bytes.NewReader(buf), new(int), new(error)
	wire.ReadBinaryPtr(&s, r, 0, n, err)
	if *err != nil {
		// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
		cmn.Exit(cmn.Fmt(`LoadState: Data has been corrupted or its spec has changed:
                %v\n`, *err))
	}
	// TODO: ensure that buf is completely read.

	return s
}

// SetLogger sets the logger on the State.
func (s *State) SetLogger(l log.Logger) {
	s.logger = l
}

// Copy makes a copy of the State for mutating.
func (s *State) Copy() *State {
	return &State{
		db:                          s.db,
		LastBlockHeight:             s.LastBlockHeight,
		LastBlockTotalTx:            s.LastBlockTotalTx,
		LastBlockID:                 s.LastBlockID,
		LastBlockTime:               s.LastBlockTime,
		Validators:                  s.Validators.Copy(),
		LastValidators:              s.LastValidators.Copy(),
		AppHash:                     s.AppHash,
		LastHeightValidatorsChanged: s.LastHeightValidatorsChanged,
		logger:  s.logger,
		ChainID: s.ChainID,
		Params:  s.Params,
	}
}

// Save persists the State to the database.
func (s *State) Save() {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	s.saveValidatorsInfo()
	s.db.SetSync(stateKey, s.Bytes())
}

// SaveABCIResponses persists the ABCIResponses to the database.
// This is useful in case we crash after app.Commit and before s.Save().
func (s *State) SaveABCIResponses(abciResponses *ABCIResponses) {
	s.db.SetSync(abciResponsesKey, abciResponses.Bytes())
}

// LoadABCIResponses loads the ABCIResponses from the database.
// This is useful for recovering from crashes where we called app.Commit and before we called
// s.Save()
func (s *State) LoadABCIResponses() *ABCIResponses {
	buf := s.db.Get(abciResponsesKey)
	if len(buf) == 0 {
		return nil
	}

	abciResponses := new(ABCIResponses)
	r, n, err := bytes.NewReader(buf), new(int), new(error)
	wire.ReadBinaryPtr(abciResponses, r, 0, n, err)
	if *err != nil {
		// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
		cmn.Exit(cmn.Fmt(`LoadABCIResponses: Data has been corrupted or its spec has
                changed: %v\n`, *err))
	}
	// TODO: ensure that buf is completely read.

	return abciResponses
}

// LoadValidators loads the ValidatorSet for a given height.
func (s *State) LoadValidators(height int64) (*types.ValidatorSet, error) {
	valInfo := s.loadValidators(height)
	if valInfo == nil {
		return nil, ErrNoValSetForHeight{height}
	}

	if valInfo.ValidatorSet == nil {
		valInfo = s.loadValidators(valInfo.LastHeightChanged)
		if valInfo == nil {
			cmn.PanicSanity(fmt.Sprintf(`Couldn't find validators at height %d as
                        last changed from height %d`, valInfo.LastHeightChanged, height))
		}
	}

	return valInfo.ValidatorSet, nil
}

func (s *State) loadValidators(height int64) *ValidatorsInfo {
	buf := s.db.Get(calcValidatorsKey(height))
	if len(buf) == 0 {
		return nil
	}

	v := new(ValidatorsInfo)
	r, n, err := bytes.NewReader(buf), new(int), new(error)
	wire.ReadBinaryPtr(v, r, 0, n, err)
	if *err != nil {
		// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
		cmn.Exit(cmn.Fmt(`LoadValidators: Data has been corrupted or its spec has changed:
                %v\n`, *err))
	}
	// TODO: ensure that buf is completely read.

	return v
}

// saveValidatorsInfo persists the validator set for the next block to disk.
// It should be called from s.Save(), right before the state itself is persisted.
// If the validator set did not change after processing the latest block,
// only the last height for which the validators changed is persisted.
func (s *State) saveValidatorsInfo() {
	changeHeight := s.LastHeightValidatorsChanged
	nextHeight := s.LastBlockHeight + 1
	valInfo := &ValidatorsInfo{
		LastHeightChanged: changeHeight,
	}
	if changeHeight == nextHeight {
		valInfo.ValidatorSet = s.Validators
	}
	s.db.SetSync(calcValidatorsKey(nextHeight), valInfo.Bytes())
}

// Equals returns true if the States are identical.
func (s *State) Equals(s2 *State) bool {
	return bytes.Equal(s.Bytes(), s2.Bytes())
}

// Bytes serializes the State using go-wire.
func (s *State) Bytes() []byte {
	return wire.BinaryBytes(s)
}

// SetBlockAndValidators mutates State variables
// to update block and validators after running EndBlock.
func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader,
	abciResponses *ABCIResponses) {

	// copy the valset so we can apply changes from EndBlock
	// and update s.LastValidators and s.Validators
	prevValSet := s.Validators.Copy()
	nextValSet := prevValSet.Copy()

	// update the validator set with the latest abciResponses
	if len(abciResponses.EndBlock.Changes) > 0 {
		err := updateValidators(nextValSet, abciResponses.EndBlock.Changes)
		if err != nil {
			s.logger.Error("Error changing validator set", "err", err)
			// TODO: err or carry on?
		}
		// change results from this height but only applies to the next height
		s.LastHeightValidatorsChanged = header.Height + 1
	}

	// Update validator accums and set state variables
	nextValSet.IncrementAccum(1)

	nextParams := applyChanges(s.Params,
		abciResponses.EndBlock.ConsensusParamChanges)

	s.setBlockAndValidators(header.Height,
		header.NumTxs,
		types.BlockID{header.Hash(), blockPartsHeader},
		header.Time,
		prevValSet, nextValSet,
		nextParams)

}

// applyChanges returns a new param set, overriding any
// parameter that is non-zero in argument
func applyChanges(p types.ConsensusParams,
	c *abci.ConsensusParams) types.ConsensusParams {

	if c == nil {
		return p
	}
	res := p
	// we must defensively consider any structs may be nil
	if c.BlockSizeParams != nil {

		if c.BlockSizeParams.MaxBytes != 0 {
			res.BlockSizeParams.MaxBytes = int(c.BlockSizeParams.MaxBytes)
		}
		if c.BlockSizeParams.MaxTxs != 0 {
			res.BlockSizeParams.MaxTxs = int(c.BlockSizeParams.MaxTxs)
		}
		if c.BlockSizeParams.MaxGas != 0 {
			res.BlockSizeParams.MaxGas = int(c.BlockSizeParams.MaxGas)
		}
	}
	if c.TxSizeParams != nil {
		if c.TxSizeParams.MaxBytes != 0 {
			res.TxSizeParams.MaxBytes = int(c.TxSizeParams.MaxBytes)
		}
		if c.TxSizeParams.MaxGas != 0 {
			res.TxSizeParams.MaxGas = int(c.TxSizeParams.MaxGas)
		}
	}
	if c.BlockGossipParams != nil {
		if c.BlockGossipParams.BlockPartSizeBytes != 0 {
			res.BlockGossipParams.BlockPartSizeBytes = int(c.BlockGossipParams.BlockPartSizeBytes)
		}
	}
	return res
}

func (s *State) setBlockAndValidators(height int64,
	newTxs int64, blockID types.BlockID, blockTime time.Time,
	prevValSet, nextValSet *types.ValidatorSet,
	nextParams types.ConsensusParams) {

	s.LastBlockHeight = height
	s.LastBlockTotalTx += newTxs
	s.LastBlockID = blockID
	s.LastBlockTime = blockTime
	s.Validators = nextValSet
	s.LastValidators = prevValSet
	s.Params = nextParams
}

// GetValidators returns the last and current validator sets.
func (s *State) GetValidators() (last *types.ValidatorSet, current *types.ValidatorSet) {
	return s.LastValidators, s.Validators
}

//------------------------------------------------------------------------

// ABCIResponses retains the responses of the various ABCI calls during block processing.
// It is persisted to disk before calling Commit.
type ABCIResponses struct {
	Height int64

	DeliverTx []*abci.ResponseDeliverTx
	EndBlock  *abci.ResponseEndBlock

	txs types.Txs // reference for indexing results by hash
}

// NewABCIResponses returns a new ABCIResponses
func NewABCIResponses(block *types.Block) *ABCIResponses {
	return &ABCIResponses{
		Height:    block.Height,
		DeliverTx: make([]*abci.ResponseDeliverTx, block.NumTxs),
		txs:       block.Data.Txs,
	}
}

// Bytes serializes the ABCIResponse using go-wire
func (a *ABCIResponses) Bytes() []byte {
	return wire.BinaryBytes(*a)
}

//-----------------------------------------------------------------------------

// ValidatorsInfo represents the latest validator set, or the last height it changed
type ValidatorsInfo struct {
	ValidatorSet      *types.ValidatorSet
	LastHeightChanged int64
}

// Bytes serializes the ValidatorsInfo using go-wire
func (valInfo *ValidatorsInfo) Bytes() []byte {
	return wire.BinaryBytes(*valInfo)
}

//------------------------------------------------------------------------
// Genesis

// MakeGenesisStateFromFile reads and unmarshals state from the given
// file.
//
// Used during replay and in tests.
func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) (*State, error) {
	genDoc, err := MakeGenesisDocFromFile(genDocFile)
	if err != nil {
		return nil, err
	}
	return MakeGenesisState(db, genDoc)
}

// MakeGenesisDocFromFile reads and unmarshals genesis doc from the given file.
func MakeGenesisDocFromFile(genDocFile string) (*types.GenesisDoc, error) {
	genDocJSON, err := ioutil.ReadFile(genDocFile)
	if err != nil {
		return nil, fmt.Errorf("Couldn't read GenesisDoc file: %v", err)
	}
	genDoc, err := types.GenesisDocFromJSON(genDocJSON)
	if err != nil {
		return nil, fmt.Errorf("Error reading GenesisDoc: %v", err)
	}
	return genDoc, nil
}

// MakeGenesisState creates state from types.GenesisDoc.
func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) (*State, error) {
	err := genDoc.ValidateAndComplete()
	if err != nil {
		return nil, fmt.Errorf("Error in genesis file: %v", err)
	}

	// Make validators slice
	validators := make([]*types.Validator, len(genDoc.Validators))
	for i, val := range genDoc.Validators {
		pubKey := val.PubKey
		address := pubKey.Address()

		// Make validator
		validators[i] = &types.Validator{
			Address:     address,
			PubKey:      pubKey,
			VotingPower: val.Power,
		}
	}

	return &State{
		db: db,

		ChainID: genDoc.ChainID,
		Params:  *genDoc.ConsensusParams,

		LastBlockHeight:             0,
		LastBlockID:                 types.BlockID{},
		LastBlockTime:               genDoc.GenesisTime,
		Validators:                  types.NewValidatorSet(validators),
		LastValidators:              types.NewValidatorSet(nil),
		AppHash:                     genDoc.AppHash,
		LastHeightValidatorsChanged: 1,
	}, nil
}
<commit_msg>Validate ConsensusParams returned from abci app<commit_after>package state

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"sync"
	"time"

	abci "github.com/tendermint/abci/types"

	cmn "github.com/tendermint/tmlibs/common"
	dbm "github.com/tendermint/tmlibs/db"
	"github.com/tendermint/tmlibs/log"

	wire "github.com/tendermint/go-wire"

	"github.com/tendermint/tendermint/types"
)

var (
	stateKey         = []byte("stateKey")
	abciResponsesKey = []byte("abciResponsesKey")
)

func calcValidatorsKey(height int64) []byte {
	return []byte(cmn.Fmt("validatorsKey:%v", height))
}

//-----------------------------------------------------------------------------

// State represents the latest committed state of the Tendermint consensus,
// including the last committed block and validator set.
// Newly committed blocks are validated and executed against the State.
// NOTE: not goroutine-safe.
type State struct {
	// mtx for writing to db
	mtx sync.Mutex
	db  dbm.DB

	ChainID string
	// Consensus parameters used for validating blocks
	Params types.ConsensusParams

	// These fields are updated by SetBlockAndValidators.
	// LastBlockHeight=0 at genesis (ie. block(H=0) does not exist)
	// LastValidators is used to validate block.LastCommit.
	LastBlockHeight  int64
	LastBlockTotalTx int64
	LastBlockID      types.BlockID
	LastBlockTime    time.Time
	Validators       *types.ValidatorSet
	LastValidators   *types.ValidatorSet
	// When a block returns a validator set change via EndBlock,
	// the change only applies to the next block.
	// So, if s.LastBlockHeight causes a valset change,
	// we set s.LastHeightValidatorsChanged = s.LastBlockHeight + 1
	LastHeightValidatorsChanged int64

	// AppHash is updated after Commit
	AppHash []byte

	logger log.Logger
}

// GetState loads the most recent state from the database,
// or creates a new one from the given genesisFile and persists the result
// to the database.
func GetState(stateDB dbm.DB, genesisFile string) (*State, error) {
	state := LoadState(stateDB)
	if state == nil {
		var err error
		state, err = MakeGenesisStateFromFile(stateDB, genesisFile)
		if err != nil {
			return nil, err
		}
		state.Save()
	}

	return state, nil
}

// LoadState loads the State from the database.
func LoadState(db dbm.DB) *State {
	return loadState(db, stateKey)
}

func loadState(db dbm.DB, key []byte) *State {
	buf := db.Get(key)
	if len(buf) == 0 {
		return nil
	}

	s := &State{db: db}
	r, n, err := bytes.NewReader(buf), new(int), new(error)
	wire.ReadBinaryPtr(&s, r, 0, n, err)
	if *err != nil {
		// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
		cmn.Exit(cmn.Fmt(`LoadState: Data has been corrupted or its spec has changed:
                %v\n`, *err))
	}
	// TODO: ensure that buf is completely read.

	return s
}

// SetLogger sets the logger on the State.
func (s *State) SetLogger(l log.Logger) {
	s.logger = l
}

// Copy makes a copy of the State for mutating.
func (s *State) Copy() *State {
	return &State{
		db:                          s.db,
		LastBlockHeight:             s.LastBlockHeight,
		LastBlockTotalTx:            s.LastBlockTotalTx,
		LastBlockID:                 s.LastBlockID,
		LastBlockTime:               s.LastBlockTime,
		Validators:                  s.Validators.Copy(),
		LastValidators:              s.LastValidators.Copy(),
		AppHash:                     s.AppHash,
		LastHeightValidatorsChanged: s.LastHeightValidatorsChanged,
		logger:  s.logger,
		ChainID: s.ChainID,
		Params:  s.Params,
	}
}

// Save persists the State to the database.
func (s *State) Save() {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	s.saveValidatorsInfo()
	s.db.SetSync(stateKey, s.Bytes())
}

// SaveABCIResponses persists the ABCIResponses to the database.
// This is useful in case we crash after app.Commit and before s.Save().
func (s *State) SaveABCIResponses(abciResponses *ABCIResponses) {
	s.db.SetSync(abciResponsesKey, abciResponses.Bytes())
}

// LoadABCIResponses loads the ABCIResponses from the database.
// This is useful for recovering from crashes where we called app.Commit and before we called
// s.Save()
func (s *State) LoadABCIResponses() *ABCIResponses {
	buf := s.db.Get(abciResponsesKey)
	if len(buf) == 0 {
		return nil
	}

	abciResponses := new(ABCIResponses)
	r, n, err := bytes.NewReader(buf), new(int), new(error)
	wire.ReadBinaryPtr(abciResponses, r, 0, n, err)
	if *err != nil {
		// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
		cmn.Exit(cmn.Fmt(`LoadABCIResponses: Data has been corrupted or its spec has
                changed: %v\n`, *err))
	}
	// TODO: ensure that buf is completely read.

	return abciResponses
}

// LoadValidators loads the ValidatorSet for a given height.
func (s *State) LoadValidators(height int64) (*types.ValidatorSet, error) {
	valInfo := s.loadValidators(height)
	if valInfo == nil {
		return nil, ErrNoValSetForHeight{height}
	}

	if valInfo.ValidatorSet == nil {
		valInfo = s.loadValidators(valInfo.LastHeightChanged)
		if valInfo == nil {
			cmn.PanicSanity(fmt.Sprintf(`Couldn't find validators at height %d as
                        last changed from height %d`, valInfo.LastHeightChanged, height))
		}
	}

	return valInfo.ValidatorSet, nil
}

func (s *State) loadValidators(height int64) *ValidatorsInfo {
	buf := s.db.Get(calcValidatorsKey(height))
	if len(buf) == 0 {
		return nil
	}

	v := new(ValidatorsInfo)
	r, n, err := bytes.NewReader(buf), new(int), new(error)
	wire.ReadBinaryPtr(v, r, 0, n, err)
	if *err != nil {
		// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
		cmn.Exit(cmn.Fmt(`LoadValidators: Data has been corrupted or its spec has changed:
                %v\n`, *err))
	}
	// TODO: ensure that buf is completely read.

	return v
}

// saveValidatorsInfo persists the validator set for the next block to disk.
// It should be called from s.Save(), right before the state itself is persisted.
// If the validator set did not change after processing the latest block,
// only the last height for which the validators changed is persisted.
func (s *State) saveValidatorsInfo() {
	changeHeight := s.LastHeightValidatorsChanged
	nextHeight := s.LastBlockHeight + 1
	valInfo := &ValidatorsInfo{
		LastHeightChanged: changeHeight,
	}
	if changeHeight == nextHeight {
		valInfo.ValidatorSet = s.Validators
	}
	s.db.SetSync(calcValidatorsKey(nextHeight), valInfo.Bytes())
}

// Equals returns true if the States are identical.
func (s *State) Equals(s2 *State) bool {
	return bytes.Equal(s.Bytes(), s2.Bytes())
}

// Bytes serializes the State using go-wire.
func (s *State) Bytes() []byte {
	return wire.BinaryBytes(s)
}

// SetBlockAndValidators mutates State variables
// to update block and validators after running EndBlock.
func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader,
	abciResponses *ABCIResponses) {

	// copy the valset so we can apply changes from EndBlock
	// and update s.LastValidators and s.Validators
	prevValSet := s.Validators.Copy()
	nextValSet := prevValSet.Copy()

	// update the validator set with the latest abciResponses
	if len(abciResponses.EndBlock.Changes) > 0 {
		err := updateValidators(nextValSet, abciResponses.EndBlock.Changes)
		if err != nil {
			s.logger.Error("Error changing validator set", "err", err)
			// TODO: err or carry on?
		}
		// change results from this height but only applies to the next height
		s.LastHeightValidatorsChanged = header.Height + 1
	}

	// Update validator accums and set state variables
	nextValSet.IncrementAccum(1)

	nextParams := applyChanges(s.Params,
		abciResponses.EndBlock.ConsensusParamChanges)
	err := nextParams.Validate()
	if err != nil {
		s.logger.Error("Error updating consensus params", "err", err)
		// TODO: err or carry on?
		nextParams = s.Params
	}

	s.setBlockAndValidators(header.Height,
		header.NumTxs,
		types.BlockID{header.Hash(), blockPartsHeader},
		header.Time,
		prevValSet, nextValSet,
		nextParams)

}

// applyChanges returns a new param set, overriding any
// parameter that is non-zero in argument
func applyChanges(p types.ConsensusParams,
	c *abci.ConsensusParams) types.ConsensusParams {

	if c == nil {
		return p
	}
	res := p
	// we must defensively consider any structs may be nil
	if c.BlockSizeParams != nil {

		if c.BlockSizeParams.MaxBytes != 0 {
			res.BlockSizeParams.MaxBytes = int(c.BlockSizeParams.MaxBytes)
		}
		if c.BlockSizeParams.MaxTxs != 0 {
			res.BlockSizeParams.MaxTxs = int(c.BlockSizeParams.MaxTxs)
		}
		if c.BlockSizeParams.MaxGas != 0 {
			res.BlockSizeParams.MaxGas = int(c.BlockSizeParams.MaxGas)
		}
	}
	if c.TxSizeParams != nil {
		if c.TxSizeParams.MaxBytes != 0 {
			res.TxSizeParams.MaxBytes = int(c.TxSizeParams.MaxBytes)
		}
		if c.TxSizeParams.MaxGas != 0 {
			res.TxSizeParams.MaxGas = int(c.TxSizeParams.MaxGas)
		}
	}
	if c.BlockGossipParams != nil {
		if c.BlockGossipParams.BlockPartSizeBytes != 0 {
			res.BlockGossipParams.BlockPartSizeBytes = int(c.BlockGossipParams.BlockPartSizeBytes)
		}
	}
	return res
}

func (s *State) setBlockAndValidators(height int64,
	newTxs int64, blockID types.BlockID, blockTime time.Time,
	prevValSet, nextValSet *types.ValidatorSet,
	nextParams types.ConsensusParams) {

	s.LastBlockHeight = height
	s.LastBlockTotalTx += newTxs
	s.LastBlockID = blockID
	s.LastBlockTime = blockTime
	s.Validators = nextValSet
	s.LastValidators = prevValSet
	s.Params = nextParams
}

// GetValidators returns the last and current validator sets.
func (s *State) GetValidators() (last *types.ValidatorSet, current *types.ValidatorSet) {
	return s.LastValidators, s.Validators
}

//------------------------------------------------------------------------

// ABCIResponses retains the responses of the various ABCI calls during block processing.
// It is persisted to disk before calling Commit.
type ABCIResponses struct {
	Height int64

	DeliverTx []*abci.ResponseDeliverTx
	EndBlock  *abci.ResponseEndBlock

	txs types.Txs // reference for indexing results by hash
}

// NewABCIResponses returns a new ABCIResponses
func NewABCIResponses(block *types.Block) *ABCIResponses {
	return &ABCIResponses{
		Height:    block.Height,
		DeliverTx: make([]*abci.ResponseDeliverTx, block.NumTxs),
		txs:       block.Data.Txs,
	}
}

// Bytes serializes the ABCIResponse using go-wire
func (a *ABCIResponses) Bytes() []byte {
	return wire.BinaryBytes(*a)
}

//-----------------------------------------------------------------------------

// ValidatorsInfo represents the latest validator set, or the last height it changed
type ValidatorsInfo struct {
	ValidatorSet      *types.ValidatorSet
	LastHeightChanged int64
}

// Bytes serializes the ValidatorsInfo using go-wire
func (valInfo *ValidatorsInfo) Bytes() []byte {
	return wire.BinaryBytes(*valInfo)
}

//------------------------------------------------------------------------
// Genesis

// MakeGenesisStateFromFile reads and unmarshals state from the given
// file.
//
// Used during replay and in tests.
func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) (*State, error) {
	genDoc, err := MakeGenesisDocFromFile(genDocFile)
	if err != nil {
		return nil, err
	}
	return MakeGenesisState(db, genDoc)
}

// MakeGenesisDocFromFile reads and unmarshals genesis doc from the given file.
func MakeGenesisDocFromFile(genDocFile string) (*types.GenesisDoc, error) {
	genDocJSON, err := ioutil.ReadFile(genDocFile)
	if err != nil {
		return nil, fmt.Errorf("Couldn't read GenesisDoc file: %v", err)
	}
	genDoc, err := types.GenesisDocFromJSON(genDocJSON)
	if err != nil {
		return nil, fmt.Errorf("Error reading GenesisDoc: %v", err)
	}
	return genDoc, nil
}

// MakeGenesisState creates state from types.GenesisDoc.
func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) (*State, error) {
	err := genDoc.ValidateAndComplete()
	if err != nil {
		return nil, fmt.Errorf("Error in genesis file: %v", err)
	}

	// Make validators slice
	validators := make([]*types.Validator, len(genDoc.Validators))
	for i, val := range genDoc.Validators {
		pubKey := val.PubKey
		address := pubKey.Address()

		// Make validator
		validators[i] = &types.Validator{
			Address:     address,
			PubKey:      pubKey,
			VotingPower: val.Power,
		}
	}

	return &State{
		db: db,

		ChainID: genDoc.ChainID,
		Params:  *genDoc.ConsensusParams,

		LastBlockHeight:             0,
		LastBlockID:                 types.BlockID{},
		LastBlockTime:               genDoc.GenesisTime,
		Validators:                  types.NewValidatorSet(validators),
		LastValidators:              types.NewValidatorSet(nil),
		AppHash:                     genDoc.AppHash,
		LastHeightValidatorsChanged: 1,
	}, nil
}
<|endoftext|>


<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/util"
	apt_models "github.com/APTrust/exchange/models"
	"github.com/nsqio/go-nsq"
//	"os"
	"os/exec"
//	"path/filepath"
	"strconv"
//	"time"
)

// dpn_copier copies tarred bags from other nodes via rsync.
// This is used when replicating content from other nodes.
// For putting together DPN bags from APTrust files, see fetcher.go.

type Copier struct {
	CopyChannel         chan *CopyManifest
	ChecksumChannel     chan *CopyManifest
	Context             *context.Context
	LocalClient         *network.DPNRestClient
	RemoteClients       map[string]*network.DPNRestClient
}

type CopyManifest struct {
	NsqMessage          *nsq.Message `json:"-"`
	DPNWorkItem         *apt_models.DPNWorkItem
	ReplicationTransfer *models.ReplicationTransfer
	DPNBag              *models.DPNBag
	WorkSummary         *apt_models.WorkSummary
	LocalPath           string
	RsyncStdout         string
	RsyncStderr         string
}

func NewCopyManifest() (*CopyManifest) {
	return &CopyManifest{
		WorkSummary: apt_models.NewWorkSummary(),
	}
}

func NewCopier(_context *context.Context) (*Copier, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, err
	}
	copier := &Copier {
		Context: _context,
		LocalClient: localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := _context.Config.DPN.DPNCopyWorker.Workers * 4
	copier.CopyChannel = make(chan *CopyManifest, workerBufferSize)
	copier.ChecksumChannel = make(chan *CopyManifest, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNCopyWorker.Workers; i++ {
		go copier.doCopy()
		go copier.verifyChecksum()
	}
	return copier, nil
}

func (copier *Copier) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	// Get the DPNWorkItem, the ReplicationTransfer, and the DPNBag
	copyManifest := copier.buildCopyManifest(message)

	// Start processing.
	copier.CopyChannel <- copyManifest
	copier.Context.MessageLog.Info("Put xfer request %s (bag %s) from %s " +
		" into the copy channel", copyManifest.ReplicationTransfer.ReplicationId,
		copyManifest.ReplicationTransfer, copyManifest.ReplicationTransfer.FromNode)
	return nil
}

// Copy the file from the remote node to our local staging area.
func (copier *Copier) doCopy() {
	for copyManifest := range copier.CopyChannel {
		localPath := "?"
		rsyncCommand := GetRsyncCommand(copyManifest.ReplicationTransfer.Link,
			localPath, copier.Context.Config.DPN.UseSSHWithRsync)

		// Touch message on both sides of rsync, so NSQ doesn't time out.
		if copyManifest.NsqMessage != nil {
			copyManifest.NsqMessage.Touch()
		}
		output, err := rsyncCommand.CombinedOutput()
		copier.Context.MessageLog.Info("Rsync Output: %s", output)
		if copyManifest.NsqMessage != nil {
			copyManifest.NsqMessage.Touch()
		}
		if err != nil {
			// Something went wrong
		} else {
			// OK
		}
	}
}

// Run a checksum on the tag manifest and send that back to the
// FromNode. If the checksum is good, the FromNode will set
// the ReplicationTransfer's StoreRequested attribute to true,
// and we should store the bag. If the checksum is bad, the remote
// node will set StoreRequested to false, and we should delete
// the tar file.
func (copier *Copier) verifyChecksum() {
	//for copyManifest := range copier.ChecksumChannel {
		// 1. Calculate the sha256 digest of the tag manifest.
		// 2. Send the result the ReplicationTransfer.FromNode.
		// 3. If the updated ReplicationTransfer.StoreRequested is true,
		//    push this item into the validation queue. Otherwise,
		//    delete the bag from the local staging area.
	//}
}

// buildCopyManifest creates a CopyManifest for this job.
func (copier *Copier) buildCopyManifest(message *nsq.Message) (*CopyManifest) {
	// 1. Get the DPNWorkItem from Pharos.
	//    Stop if it's marked complete.
	// 2. Get the ReplicationTransfer from the remote node.
	//    Stop if it's completed or cancelled.
	// 3. Get the DPNBag record from the remote node.
	//    We need to know its size.
	// 4. Build and return the CopyManifest.
	copyManifest := NewCopyManifest()
	copyManifest.NsqMessage = message
	copier.getDPNWorkItem(copyManifest)
	if copyManifest.WorkSummary.HasErrors() {
		return copyManifest
	}
	copier.getXferRequest(copyManifest)
	if copyManifest.WorkSummary.HasErrors() {
		return copyManifest
	}
	copier.getDPNBag(copyManifest)
	return copyManifest
}

// getDPNWorkItem returns the DPNWorkItem associated with this message,
// and a boolean indicating whether or not processing should continue.
func (copier *Copier) getDPNWorkItem(copyManifest *CopyManifest) {
	workItemId, err := strconv.Atoi(string(copyManifest.NsqMessage.Body))
	if err != nil {
		msg := fmt.Sprintf("Could not get DPNWorkItemId from" +
			"NSQ message body '%s': %v", copyManifest.NsqMessage.Body, err)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	resp := copier.Context.PharosClient.DPNWorkItemGet(workItemId)
	if resp.Error != nil {
		msg := fmt.Sprintf("Could not get DPNWorkItem (id %d) " +
			"from Pharos: %v", workItemId, resp.Error)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	dpnWorkItem := resp.DPNWorkItem()
	copyManifest.DPNWorkItem = dpnWorkItem
	if dpnWorkItem == nil {
		msg := fmt.Sprintf("Pharos returned nil for DPNWorkItem %d",
			workItemId)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	if dpnWorkItem.Task != constants.DPNTaskReplication {
		msg := fmt.Sprintf("DPNWorkItem %d has task type %s, " +
			"and does not belong in this queue!", workItemId, dpnWorkItem.Task)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
	}
	if !util.LooksLikeUUID(dpnWorkItem.Identifier) {
		msg := fmt.Sprintf("DPNWorkItem %d has identifier '%s', " +
			"which does not look like a UUID", workItemId, dpnWorkItem.Identifier)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
	}
}

func (copier *Copier) getXferRequest(copyManifest *CopyManifest) {
	if copyManifest == nil || copyManifest.DPNWorkItem == nil {
		msg := fmt.Sprintf("getXferRequest: CopyManifest.DPNWorkItem cannot be nil.")
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	resp := copier.LocalClient.ReplicationTransferGet(copyManifest.DPNWorkItem.Identifier)
	if resp.Error != nil {
		msg := fmt.Sprintf("Could not get ReplicationTransfer %s " +
			"from DPN server: %v", copyManifest.DPNWorkItem.Identifier, resp.Error)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	xfer := resp.ReplicationTransfer()
	copyManifest.ReplicationTransfer = xfer
	if xfer == nil {
		msg := fmt.Sprintf("DPN server returned nil for ReplicationId %s",
			copyManifest.DPNWorkItem.Identifier)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	if xfer.Stored {
		msg := fmt.Sprintf("ReplicationId %s is already marked as Stored. Nothing left to do.",
			copyManifest.DPNWorkItem.Identifier)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	if xfer.Cancelled {
		msg := fmt.Sprintf("ReplicationId %s was cancelled. Nothing left to do.",
			copyManifest.DPNWorkItem.Identifier)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
	}
}

func (copier *Copier) getDPNBag(copyManifest *CopyManifest) {
	if copyManifest == nil || copyManifest.ReplicationTransfer == nil {
		msg := fmt.Sprintf("getDPNBag: CopyManifest.ReplicationTransfer cannot be nil.")
		copyManifest.WorkSummary.ErrorIsFatal = true
		copyManifest.WorkSummary.AddError(msg)
		return
	}
	resp := copier.LocalClient.DPNBagGet(copyManifest.ReplicationTransfer.Bag)
	if resp.Error != nil {
		msg := fmt.Sprintf("Could not get ReplicationTransfer %s " +
			"from DPN server: %v", copyManifest.DPNWorkItem.Identifier, resp.Error)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	dpnBag := resp.Bag()
	copyManifest.DPNBag = dpnBag
	if dpnBag == nil {
		msg := fmt.Sprintf("DPN server returned nil for Bag %s",
			copyManifest.ReplicationTransfer.Bag)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
}

// Make sure we have space to copy this item from the remote node.
// We will be validating this bag in a later step without untarring it,
// so we just have to reserve enough room for the tar file.
func (copier *Copier) reserveSpaceOnVolume(copyManifest *CopyManifest) (bool) {
	okToCopy := false
	err := copier.Context.VolumeClient.Ping(500)
	if err == nil {
		path := copyManifest.LocalPath
		ok, err := copier.Context.VolumeClient.Reserve(path, uint64(copyManifest.DPNBag.Size))
		if err != nil {
			copier.Context.MessageLog.Warning("Volume service returned an error. " +
				"Will requeue ReplicationTransfer %s bag (%s) because we may not " +
				"have enough space to copy %d bytes from %s.",
				copyManifest.ReplicationTransfer.ReplicationId,
				copyManifest.ReplicationTransfer.Bag,
				copyManifest.DPNBag.Size,
				copyManifest.ReplicationTransfer.FromNode)
		} else if ok {
			// VolumeService says we have enough space for this.
			okToCopy = ok
		}
	} else {
		copier.Context.MessageLog.Warning("Volume service is not running or returned an error. " +
			"Continuing as if we have enough space to download %d bytes.",
			copyManifest.DPNBag.Size,)
		okToCopy = true
	}
	return okToCopy
}

// GetRsyncCommand returns a command object for copying from the remote
// location to the local filesystem. The copy is done via rsync over ssh,
// and the command will capture stdout and stderr. The copyFrom param
// should be a valid scp target in this format:
//
// remoteuser@remotehost:/remote/dir/bag.tar
//
// The copyTo param should be an absolute path on a locally-accessible
// file system, such as:
//
// /mnt/dpn/data/bag.tar
//
// Using this assumes a few things:
//
// 1. You have rsync installed.
// 2. You have an ssh client installed.
// 3. You have an entry in your ~/.ssh/config file specifying
//    connection and key information for the remote host.
//
// Usage:
//
// command := GetRsyncCommand("aptrust@tdr:bag.tar", "/mnt/dpn/bag.tar")
// err := command.Run()
// if err != nil {
//    ... do something ...
// }
//
// -- OR --
//
// output, err := command.CombinedOutput()
// if err != nil {
//    fmt.Println(err.Error())
//    fmt.Println(string(output))
// }
func GetRsyncCommand(copyFrom, copyTo string, useSSH bool) (*exec.Cmd) {
	//rsync -avz -e ssh remoteuser@remotehost:/remote/dir /this/dir/
	if useSSH {
		return exec.Command("rsync", "-avzW", "-e",  "ssh", copyFrom, copyTo, "--inplace")
	}
	return exec.Command("rsync", "-avzW", "--inplace", copyFrom, copyTo)
}
<commit_msg>Working on DPN copier<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/util"
	apt_models "github.com/APTrust/exchange/models"
	"github.com/nsqio/go-nsq"
//	"os"
	"os/exec"
//	"path/filepath"
	"strconv"
	"time"
)

// dpn_copier copies tarred bags from other nodes via rsync.
// This is used when replicating content from other nodes.
// For putting together DPN bags from APTrust files, see fetcher.go.

type Copier struct {
	CopyChannel         chan *CopyManifest
	ChecksumChannel     chan *CopyManifest
	Context             *context.Context
	LocalClient         *network.DPNRestClient
	RemoteClients       map[string]*network.DPNRestClient
}

type CopyManifest struct {
	NsqMessage          *nsq.Message `json:"-"`
	DPNWorkItem         *apt_models.DPNWorkItem
	ReplicationTransfer *models.ReplicationTransfer
	DPNBag              *models.DPNBag
	WorkSummary         *apt_models.WorkSummary
	LocalPath           string
	RsyncStdout         string
	RsyncStderr         string
}

func NewCopyManifest() (*CopyManifest) {
	return &CopyManifest{
		WorkSummary: apt_models.NewWorkSummary(),
	}
}

func NewCopier(_context *context.Context) (*Copier, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, err
	}
	copier := &Copier {
		Context: _context,
		LocalClient: localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := _context.Config.DPN.DPNCopyWorker.Workers * 4
	copier.CopyChannel = make(chan *CopyManifest, workerBufferSize)
	copier.ChecksumChannel = make(chan *CopyManifest, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNCopyWorker.Workers; i++ {
		go copier.doCopy()
		go copier.verifyChecksum()
	}
	return copier, nil
}

func (copier *Copier) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	// Get the DPNWorkItem, the ReplicationTransfer, and the DPNBag
	copyManifest := copier.buildCopyManifest(message)
	if copyManifest.WorkSummary.HasErrors() {
		copier.finishWithError(copyManifest)
		return nil
	}

	if !copier.reserveSpaceOnVolume(copyManifest) {
		copyManifest.WorkSummary.AddError("Cannot reserve disk space to process this bag.")
		copyManifest.WorkSummary.Finish()
		message.Requeue(10 * time.Minute)
	}

	// Start processing.
	copier.CopyChannel <- copyManifest
	copier.Context.MessageLog.Info("Put xfer request %s (bag %s) from %s " +
		" into the copy channel", copyManifest.ReplicationTransfer.ReplicationId,
		copyManifest.ReplicationTransfer, copyManifest.ReplicationTransfer.FromNode)
	return nil
}

// Copy the file from the remote node to our local staging area.
func (copier *Copier) doCopy() {
	for copyManifest := range copier.CopyChannel {
		localPath := "?"
		rsyncCommand := GetRsyncCommand(copyManifest.ReplicationTransfer.Link,
			localPath, copier.Context.Config.DPN.UseSSHWithRsync)

		// Touch message on both sides of rsync, so NSQ doesn't time out.
		if copyManifest.NsqMessage != nil {
			copyManifest.NsqMessage.Touch()
		}
		output, err := rsyncCommand.CombinedOutput()
		copier.Context.MessageLog.Info("Rsync Output: %s", output)
		if copyManifest.NsqMessage != nil {
			copyManifest.NsqMessage.Touch()
		}
		if err != nil {
			// Something went wrong
		} else {
			// OK
		}
	}
}

// Run a checksum on the tag manifest and send that back to the
// FromNode. If the checksum is good, the FromNode will set
// the ReplicationTransfer's StoreRequested attribute to true,
// and we should store the bag. If the checksum is bad, the remote
// node will set StoreRequested to false, and we should delete
// the tar file.
func (copier *Copier) verifyChecksum() {
	//for copyManifest := range copier.ChecksumChannel {
		// 1. Calculate the sha256 digest of the tag manifest.
		// 2. Send the result the ReplicationTransfer.FromNode.
		// 3. If the updated ReplicationTransfer.StoreRequested is true,
		//    push this item into the validation queue. Otherwise,
		//    delete the bag from the local staging area.
	//}
}

// buildCopyManifest creates a CopyManifest for this job.
func (copier *Copier) buildCopyManifest(message *nsq.Message) (*CopyManifest) {
	copyManifest := NewCopyManifest()
	copyManifest.NsqMessage = message
	copyManifest.WorkSummary.Attempted = true
	copyManifest.WorkSummary.AttemptNumber = 1
	copyManifest.WorkSummary.Start()
	copier.getDPNWorkItem(copyManifest)
	if copyManifest.WorkSummary.HasErrors() {
		return copyManifest
	}
	copier.getXferRequest(copyManifest)
	if copyManifest.WorkSummary.HasErrors() {
		return copyManifest
	}
	copier.getDPNBag(copyManifest)
	return copyManifest
}

// getDPNWorkItem returns the DPNWorkItem associated with this message,
// and a boolean indicating whether or not processing should continue.
func (copier *Copier) getDPNWorkItem(copyManifest *CopyManifest) {
	workItemId, err := strconv.Atoi(string(copyManifest.NsqMessage.Body))
	if err != nil {
		msg := fmt.Sprintf("Could not get DPNWorkItemId from" +
			"NSQ message body '%s': %v", copyManifest.NsqMessage.Body, err)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	resp := copier.Context.PharosClient.DPNWorkItemGet(workItemId)
	if resp.Error != nil {
		msg := fmt.Sprintf("Could not get DPNWorkItem (id %d) " +
			"from Pharos: %v", workItemId, resp.Error)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	dpnWorkItem := resp.DPNWorkItem()
	copyManifest.DPNWorkItem = dpnWorkItem
	if dpnWorkItem == nil {
		msg := fmt.Sprintf("Pharos returned nil for DPNWorkItem %d",
			workItemId)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	if dpnWorkItem.Task != constants.DPNTaskReplication {
		msg := fmt.Sprintf("DPNWorkItem %d has task type %s, " +
			"and does not belong in this queue!", workItemId, dpnWorkItem.Task)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
	}
	if !util.LooksLikeUUID(dpnWorkItem.Identifier) {
		msg := fmt.Sprintf("DPNWorkItem %d has identifier '%s', " +
			"which does not look like a UUID", workItemId, dpnWorkItem.Identifier)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
	}
}

func (copier *Copier) getXferRequest(copyManifest *CopyManifest) {
	if copyManifest == nil || copyManifest.DPNWorkItem == nil {
		msg := fmt.Sprintf("getXferRequest: CopyManifest.DPNWorkItem cannot be nil.")
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	resp := copier.LocalClient.ReplicationTransferGet(copyManifest.DPNWorkItem.Identifier)
	if resp.Error != nil {
		msg := fmt.Sprintf("Could not get ReplicationTransfer %s " +
			"from DPN server: %v", copyManifest.DPNWorkItem.Identifier, resp.Error)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	xfer := resp.ReplicationTransfer()
	copyManifest.ReplicationTransfer = xfer
	if xfer == nil {
		msg := fmt.Sprintf("DPN server returned nil for ReplicationId %s",
			copyManifest.DPNWorkItem.Identifier)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	if xfer.Stored {
		msg := fmt.Sprintf("ReplicationId %s is already marked as Stored. Nothing left to do.",
			copyManifest.DPNWorkItem.Identifier)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	if xfer.Cancelled {
		msg := fmt.Sprintf("ReplicationId %s was cancelled. Nothing left to do.",
			copyManifest.DPNWorkItem.Identifier)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
	}
}

func (copier *Copier) getDPNBag(copyManifest *CopyManifest) {
	if copyManifest == nil || copyManifest.ReplicationTransfer == nil {
		msg := fmt.Sprintf("getDPNBag: CopyManifest.ReplicationTransfer cannot be nil.")
		copyManifest.WorkSummary.ErrorIsFatal = true
		copyManifest.WorkSummary.AddError(msg)
		return
	}
	resp := copier.LocalClient.DPNBagGet(copyManifest.ReplicationTransfer.Bag)
	if resp.Error != nil {
		msg := fmt.Sprintf("Could not get ReplicationTransfer %s " +
			"from DPN server: %v", copyManifest.DPNWorkItem.Identifier, resp.Error)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
	dpnBag := resp.Bag()
	copyManifest.DPNBag = dpnBag
	if dpnBag == nil {
		msg := fmt.Sprintf("DPN server returned nil for Bag %s",
			copyManifest.ReplicationTransfer.Bag)
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.WorkSummary.ErrorIsFatal = true
		return
	}
}

// Make sure we have space to copy this item from the remote node.
// We will be validating this bag in a later step without untarring it,
// so we just have to reserve enough room for the tar file.
func (copier *Copier) reserveSpaceOnVolume(copyManifest *CopyManifest) (bool) {
	okToCopy := false
	err := copier.Context.VolumeClient.Ping(500)
	if err == nil {
		path := copyManifest.LocalPath
		ok, err := copier.Context.VolumeClient.Reserve(path, uint64(copyManifest.DPNBag.Size))
		if err != nil {
			copier.Context.MessageLog.Warning("Volume service returned an error. " +
				"Will requeue ReplicationTransfer %s bag (%s) because we may not " +
				"have enough space to copy %d bytes from %s.",
				copyManifest.ReplicationTransfer.ReplicationId,
				copyManifest.ReplicationTransfer.Bag,
				copyManifest.DPNBag.Size,
				copyManifest.ReplicationTransfer.FromNode)
		} else if ok {
			// VolumeService says we have enough space for this.
			okToCopy = ok
		}
	} else {
		copier.Context.MessageLog.Warning("Volume service is not running or returned an error. " +
			"Continuing as if we have enough space to download %d bytes.",
			copyManifest.DPNBag.Size,)
		okToCopy = true
	}
	return okToCopy
}

func (copier *Copier) finishWithError(copyManifest *CopyManifest) {
	xferId := "[unknown]"
	if copyManifest.ReplicationTransfer != nil {
		xferId = copyManifest.ReplicationTransfer.ReplicationId
	} else if copyManifest.DPNWorkItem != nil {
		xferId = copyManifest.DPNWorkItem.Identifier
	}
	if copyManifest.WorkSummary.ErrorIsFatal {
		msg := fmt.Sprintf("Xfer %s has fatal error: %s",
			xferId, copyManifest.WorkSummary.Errors[0])
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.NsqMessage.Finish()
	} else {
		msg := fmt.Sprintf("Xfer %s has non-fatal error: %s",
			xferId, copyManifest.WorkSummary.Errors[0])
		copyManifest.WorkSummary.AddError(msg)
		copyManifest.NsqMessage.Requeue(1 * time.Minute)
	}
	copyManifest.WorkSummary.Finish()
}

// GetRsyncCommand returns a command object for copying from the remote
// location to the local filesystem. The copy is done via rsync over ssh,
// and the command will capture stdout and stderr. The copyFrom param
// should be a valid scp target in this format:
//
// remoteuser@remotehost:/remote/dir/bag.tar
//
// The copyTo param should be an absolute path on a locally-accessible
// file system, such as:
//
// /mnt/dpn/data/bag.tar
//
// Using this assumes a few things:
//
// 1. You have rsync installed.
// 2. You have an ssh client installed.
// 3. You have an entry in your ~/.ssh/config file specifying
//    connection and key information for the remote host.
//
// Usage:
//
// command := GetRsyncCommand("aptrust@tdr:bag.tar", "/mnt/dpn/bag.tar")
// err := command.Run()
// if err != nil {
//    ... do something ...
// }
//
// -- OR --
//
// output, err := command.CombinedOutput()
// if err != nil {
//    fmt.Println(err.Error())
//    fmt.Println(string(output))
// }
func GetRsyncCommand(copyFrom, copyTo string, useSSH bool) (*exec.Cmd) {
	//rsync -avz -e ssh remoteuser@remotehost:/remote/dir /this/dir/
	if useSSH {
		return exec.Command("rsync", "-avzW", "-e",  "ssh", copyFrom, copyTo, "--inplace")
	}
	return exec.Command("rsync", "-avzW", "--inplace", copyFrom, copyTo)
}
<|endoftext|>


<commit_before>package gautomator

import (
	"encoding/json"
	"fmt"
	"log"
	"io/ioutil"
)

func ParseNode(filename *string) TaskDefinition {
	taskDefJson, err := ioutil.ReadFile(*filename)

	if err != nil {
		fmt.Println("Err is ", err)
	}

	var taskDef taskDefs
	taskDefinition := make(map[string]TaskInstance, 0)

	err = json.Unmarshal(taskDefJson, &taskDef)
	if err != nil {
		log.Panic(err)
	}
	for _, task := range taskDef {
		taskDefinition[task.Taskname] = task
	}
	return taskDefinition
}

func (this *TaskGraphStructure) InstanciateTaskStructure(taskInstances TaskDefinition) {
		allSubTasks := make(map[int]*TaskGraphStructure, 0)
		index := 0
		for _, taskInstance := range taskInstances {
			for _, node := range taskInstance.Hosts {
			    doNotDuplicate := false
			    for _, task := range this.Tasks {
				    if task.Origin == taskInstance.Taskname && task.Node == "null" {
					// Setting the node to node
					task.Node = node
					doNotDuplicate = true
				    }
			    }

			    if doNotDuplicate != true {
				subTasks := this.GetSubstructure(taskInstance.Taskname)
				// If there is subtask
				if subTasks != nil {
					// TODO,  if a subtask exists with dummy, set it the hostname and no not add it
					for i, _ := range subTasks.Tasks {
						subTasks.Tasks[i].Node = node
					}
					allSubTasks[index] = subTasks
					index += 1
				} else {
					for _, task := range this.Tasks {
					    if task.Name == taskInstance.Taskname {
						if task.Node == "null" || task.Node == node {
							task.Node = node
						} else {
						    newIds := this.DuplicateTask(taskInstance.Taskname)
						    for _, newId := range newIds {
							if newId != -1 {
							    this.Tasks[newId].Node = node
							}
						    }
						}
					    }
					}
				}
			    }
			}
		}

		for _, subTask := range allSubTasks {
			//subTask.PrintAdjacencyMatrix()
			this = this.AugmentTaskStructure(subTask)
		}
		this.Relink()
		// Now, for each task, assign module, hosts and co...
		for _, task := range this.Tasks {
			if _, ok := taskInstances[task.Name]; ok {
				if taskInstances[task.Name].Module != "" {
					task.Module = taskInstances[task.Name].Module
				}
				if taskInstances[task.Name].Args != nil {
					task.Args = taskInstances[task.Name].Args
				}
			}
		}

}
<commit_msg>Formating<commit_after>package gautomator

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
)

func ParseNode(filename *string) TaskDefinition {
	taskDefJson, err := ioutil.ReadFile(*filename)

	if err != nil {
		fmt.Println("Err is ", err)
	}

	var taskDef taskDefs
	taskDefinition := make(map[string]TaskInstance, 0)

	err = json.Unmarshal(taskDefJson, &taskDef)
	if err != nil {
		log.Panic(err)
	}
	for _, task := range taskDef {
		taskDefinition[task.Taskname] = task
	}
	return taskDefinition
}

func (this *TaskGraphStructure) InstanciateTaskStructure(taskInstances TaskDefinition) {
	allSubTasks := make(map[int]*TaskGraphStructure, 0)
	index := 0
	for _, taskInstance := range taskInstances {
		for _, node := range taskInstance.Hosts {
			doNotDuplicate := false
			for _, task := range this.Tasks {
				if task.Origin == taskInstance.Taskname && task.Node == "null" {
					// Setting the node to node
					task.Node = node
					doNotDuplicate = true
				}
			}

			if doNotDuplicate != true {
				subTasks := this.GetSubstructure(taskInstance.Taskname)
				// If there is subtask
				if subTasks != nil {
					// TODO,  if a subtask exists with dummy, set it the hostname and no not add it
					for i, _ := range subTasks.Tasks {
						subTasks.Tasks[i].Node = node
					}
					allSubTasks[index] = subTasks
					index += 1
				} else {
					for _, task := range this.Tasks {
						if task.Name == taskInstance.Taskname {
							if task.Node == "null" || task.Node == node {
								task.Node = node
							} else {
								newIds := this.DuplicateTask(taskInstance.Taskname)
								for _, newId := range newIds {
									if newId != -1 {
										this.Tasks[newId].Node = node
									}
								}
							}
						}
					}
				}
			}
		}
	}

	for _, subTask := range allSubTasks {
		//subTask.PrintAdjacencyMatrix()
		this = this.AugmentTaskStructure(subTask)
	}
	this.Relink()
	// Now, for each task, assign module, hosts and co...
	for _, task := range this.Tasks {
		if _, ok := taskInstances[task.Name]; ok {
			if taskInstances[task.Name].Module != "" {
				task.Module = taskInstances[task.Name].Module
			}
			if taskInstances[task.Name].Args != nil {
				task.Args = taskInstances[task.Name].Args
			}
		}
	}

}
<|endoftext|>


<commit_before>package store

import (
	"github.com/boltdb/bolt"
	"github.com/pkg/errors"
)

type Version string

const (
	VerNone       = Version("")
	VerAlpha001_2 = Version("0.0.1-alpha-2")

	VerCurrent = VerAlpha001_2
)

var (
	VersionBucket = []byte("version")

	migrations = map[Version]map[Version]func(*bolt.Tx) error{
		VerNone: {VerAlpha001_2: PutV(VerCurrent)},
	}
)

// Bucket is an identifier for a package constant to define the BoltDB
// bucket where a resource is stored.
//
// TODO: nested Buckets?
type Bucket []byte

func Prep(buckets ...Bucket) func(*bolt.Tx) error {
	return Wrap(
		Migrate(VerCurrent),
		SetupBuckets(buckets...),
	)
}

func SetupBuckets(buckets ...Bucket) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		for _, bucket := range buckets {
			_, err := tx.CreateBucketIfNotExists(bucket)
			if err != nil {
				return err
			}
		}

		return nil
	}
}

func Migrate(v Version) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		b := tx.Bucket(VersionBucket)
		if b != nil {
			oldVer := Version(b.Get([]byte("version")))
			if v != oldVer {
				return MigrateFrom(tx, oldVer, v)
			}
			return nil
		}
		return MigrateFrom(tx, VerNone, v)
	}
}

func MigrateFrom(tx *bolt.Tx, from, to Version) error {
	msFrom, ok := migrations[from]
	if !ok {
		return errors.Errorf("no migration defined from version %#q", from)
	}
	if from == to {
		return nil
	}
	mTo, ok := msFrom[to]
	if !ok {
		return errors.Errorf("no migration defined from version %#q to %#q", from, to)
	}
	return mTo(tx)
}

func PutV(v Version) func(*bolt.Tx) error {
	return Wrap(
		func(tx *bolt.Tx) error {
			_, err := tx.CreateBucketIfNotExists(VersionBucket)
			return err
		},
		Put(VersionBucket, []byte("version"), []byte(v)),
	)
}

func Wrap(apps ...func(*bolt.Tx) error) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		for _, app := range apps {
			if err := app(tx); err != nil {
				return err
			}
		}
		return nil
	}
}
<commit_msg>Update store version and add migration for v0.0.1<commit_after>package store

import (
	"log"

	"github.com/boltdb/bolt"
	"github.com/pkg/errors"
)

type Version string

const (
	Ver001        = Version("0.0.1")
	VerAlpha001_2 = Version("0.0.1-alpha-2")
	VerNone       = Version("")

	VerCurrent = Ver001
)

var (
	VersionBucket = []byte("version")

	migrations = map[Version]map[Version]func(*bolt.Tx) error{
		VerNone: {
			VerAlpha001_2: PutV(VerAlpha001_2),
			Ver001:        PutV(Ver001),
		},
		VerAlpha001_2: {Ver001: PutV(Ver001)},
	}
)

// Bucket is an identifier for a package constant to define the BoltDB
// bucket where a resource is stored.
//
// TODO: nested Buckets?
type Bucket []byte

func Prep(buckets ...Bucket) func(*bolt.Tx) error {
	return Wrap(
		Migrate(VerCurrent),
		SetupBuckets(buckets...),
	)
}

func SetupBuckets(buckets ...Bucket) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		for _, bucket := range buckets {
			_, err := tx.CreateBucketIfNotExists(bucket)
			if err != nil {
				return err
			}
		}

		return nil
	}
}

func Migrate(v Version) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		b := tx.Bucket(VersionBucket)
		if b != nil {
			oldVer := Version(b.Get([]byte("version")))
			if v != oldVer {
				log.Printf("migrating from %q to %q",
					oldVer, v)
				return MigrateFrom(tx, oldVer, v)
			}
			return nil
		}
		return MigrateFrom(tx, VerNone, v)
	}
}

func MigrateFrom(tx *bolt.Tx, from, to Version) error {
	msFrom, ok := migrations[from]
	if !ok {
		return errors.Errorf("no migration defined from version %#q", from)
	}
	if from == to {
		return nil
	}
	mTo, ok := msFrom[to]
	if !ok {
		return errors.Errorf("no migration defined from version %#q to %#q", from, to)
	}
	return mTo(tx)
}

func PutV(v Version) func(*bolt.Tx) error {
	return Wrap(
		func(tx *bolt.Tx) error {
			_, err := tx.CreateBucketIfNotExists(VersionBucket)
			return err
		},
		Put(VersionBucket, []byte("version"), []byte(v)),
	)
}

func Wrap(apps ...func(*bolt.Tx) error) func(*bolt.Tx) error {
	return func(tx *bolt.Tx) error {
		for _, app := range apps {
			if err := app(tx); err != nil {
				return err
			}
		}
		return nil
	}
}
<|endoftext|>


<commit_before>// 22 august 2012
package main

import (
	"fmt"
	"encoding/xml"
	"net/url"
	"strings"
)

type Scan struct {
	Console		string
	Region		string
	Front		string
	Back			string
	Spine		string
	SpineMissing	bool
	SpineCard		string
	Cart			string
	Disc			string
	Manual		string
}

type gamepage struct {
	Source		string	`xml:"query>pages>page>revisions>rev"`
}

func urlForGame(game string) string {
//	return "/api.php?format=xml&action=query&titles=" + url.QueryEscape(game) + "&prop=revisions&rvparse&rvgeneratexml&rvprop=content"
	return "/api.php?action=query&prop=revisions&rvprop=content&format=xml&titles=" + url.QueryEscape(game)
}

func GetScans(game string) ([]Scan, error) {
	var scans []Scan
	var gp gamepage

	r, err := getWikiAPIData(urlForGame(game))
	if err != nil {
		return nil, fmt.Errorf("error retrieving game %s: %v", game, err)
	}
	err = xml.Unmarshal(r, &gp)
	if err != nil {
		return nil, fmt.Errorf("error processing games: %v\ndata: %s", err, r)
	}
	scanboxes := GetScanboxes(gp.Source)
	for _, v := range scanboxes {
		var s Scan

		for _, p := range v {
			pname := strings.ToLower(strings.TrimSpace(p.Name))
			pvalue := strings.TrimSpace(p.Value)
			switch pname {
			case "console":
				s.Console = pvalue
			case "region":
				s.Region = pvalue
			case "front":
				s.Front = pvalue
			case "back":
				s.Back = pvalue
			case "spine":
				s.Spine = pvalue
			case "spinemissing":
				s.SpineMissing = (pvalue == "yes")
			case "spinecard":
				s.SpineCard = pvalue
			case "cart":
				s.Cart = pvalue
			case "disc", "disk":
				s.Disc = pvalue
			case "manual":
				s.Manual = pvalue
			case "square", "spine2":
				// ignore
				// TODO what to do about spine2?
			default:	// ignore item* and jewelcase*
				if !strings.HasPrefix(pname, "item") &&
					!strings.HasPrefix(pname, "jewelcase") {
					return nil, fmt.Errorf("unknown parameter %s=%s", pname, pvalue)
				}
			}
		}
		scans = append(scans, s)
	}
	return scans, err
}

/*
// test
func main() {
//	scans, err := GetScans("Thunder Force IV")
//	scans, err := GetScans("Light Crusader")
//	scans, err := GetScans("Crusader of Centy")
	scans, err := GetScans("The Lucky Dime Caper Starring Donald Duck")
	if err != nil {
		fmt.Printf("error: %v\n", err)
		return
	}
	for _, v := range scans {
		fmt.Printf("%#v\n", v)
		fmt.Printf("box scan state: %v\n", v.BoxScanState())
		fmt.Printf("cart scan state: %v\n", v.CartScanState())
	}
}
*/
<commit_msg>More parameter name exceptions for Ecco Jr. on Pico.<commit_after>// 22 august 2012
package main

import (
	"fmt"
	"encoding/xml"
	"net/url"
	"strings"
)

type Scan struct {
	Console		string
	Region		string
	Front		string
	Back			string
	Spine		string
	SpineMissing	bool
	SpineCard		string
	Cart			string
	Disc			string
	Manual		string
}

type gamepage struct {
	Source		string	`xml:"query>pages>page>revisions>rev"`
}

func urlForGame(game string) string {
//	return "/api.php?format=xml&action=query&titles=" + url.QueryEscape(game) + "&prop=revisions&rvparse&rvgeneratexml&rvprop=content"
	return "/api.php?action=query&prop=revisions&rvprop=content&format=xml&titles=" + url.QueryEscape(game)
}

func GetScans(game string) ([]Scan, error) {
	var scans []Scan
	var gp gamepage

	r, err := getWikiAPIData(urlForGame(game))
	if err != nil {
		return nil, fmt.Errorf("error retrieving game %s: %v", game, err)
	}
	err = xml.Unmarshal(r, &gp)
	if err != nil {
		return nil, fmt.Errorf("error processing games: %v\ndata: %s", err, r)
	}
	scanboxes := GetScanboxes(gp.Source)
	for _, v := range scanboxes {
		var s Scan

		for _, p := range v {
			pname := strings.ToLower(strings.TrimSpace(p.Name))
			pvalue := strings.TrimSpace(p.Value)
			switch pname {
			case "console":
				s.Console = pvalue
			case "region":
				s.Region = pvalue
			case "front":
				s.Front = pvalue
			case "back":
				s.Back = pvalue
			case "spine":
				s.Spine = pvalue
			case "spinemissing":
				s.SpineMissing = (pvalue == "yes")
			case "spinecard":
				s.SpineCard = pvalue
			case "cart":
				s.Cart = pvalue
			case "disc", "disk":
				s.Disc = pvalue
			case "manual":
				s.Manual = pvalue
			case "square", "spine2":
				// ignore
				// TODO what to do about spine2?
			default:	// ignore item* and jewelcase*... top* and bottom* too?
				if !strings.HasPrefix(pname, "item") &&
					!strings.HasPrefix(pname, "jewelcase") &&
					!strings.HasPrefix(pname, "top") &&
					!strings.HasPrefix(pname, "bottom") {
					return nil, fmt.Errorf("unknown parameter %s=%s", pname, pvalue)
				}
			}
		}
		scans = append(scans, s)
	}
	return scans, err
}

/*
// test
func main() {
//	scans, err := GetScans("Thunder Force IV")
//	scans, err := GetScans("Light Crusader")
//	scans, err := GetScans("Crusader of Centy")
	scans, err := GetScans("The Lucky Dime Caper Starring Donald Duck")
	if err != nil {
		fmt.Printf("error: %v\n", err)
		return
	}
	for _, v := range scans {
		fmt.Printf("%#v\n", v)
		fmt.Printf("box scan state: %v\n", v.BoxScanState())
		fmt.Printf("cart scan state: %v\n", v.CartScanState())
	}
}
*/
<|endoftext|>


<commit_before>package store

import (
	"crypto/tls"
	"errors"
	"time"
)

// Backend represents a KV Store Backend
type Backend string

const (
	// CONSUL backend
	CONSUL Backend = "consul"
	// ETCD backend
	ETCD Backend = "etcd"
	// ZK backend
	ZK Backend = "zk"
)

var (
	// ErrNotSupported is thrown when the backend k/v store is not supported by libkv
	ErrNotSupported = errors.New("Backend storage not supported yet, please choose another one")
	// ErrNotImplemented is thrown when a method is not implemented by the current backend
	ErrNotImplemented = errors.New("Call not implemented in current backend")
	// ErrNotReachable is thrown when the API cannot be reached for issuing common store operations
	ErrNotReachable = errors.New("Api not reachable")
	// ErrCannotLock is thrown when there is an error acquiring a lock on a key
	ErrCannotLock = errors.New("Error acquiring the lock")
	// ErrKeyModified is thrown during an atomic operation if the index does not match the one in the store
	ErrKeyModified = errors.New("Unable to complete atomic operation, key modified")
	// ErrKeyNotFound is thrown when the key is not found in the store during a Get operation
	ErrKeyNotFound = errors.New("Key not found in store")
	// ErrPreviousNotSpecified is thrown when the previous value is not specified for an atomic operation
	ErrPreviousNotSpecified = errors.New("Previous K/V pair should be provided for the Atomic operation")
)

// Config contains the options for a storage client
type Config struct {
	TLS               *tls.Config
	ConnectionTimeout time.Duration
	EphemeralTTL      time.Duration
}

// Store represents the backend K/V storage
// Each store should support every call listed
// here. Or it couldn't be implemented as a K/V
// backend for libkv
type Store interface {
	// Put a value at the specified key
	Put(key string, value []byte, options *WriteOptions) error

	// Get a value given its key
	Get(key string) (*KVPair, error)

	// Delete the value at the specified key
	Delete(key string) error

	// Verify if a Key exists in the store
	Exists(key string) (bool, error)

	// Watch for changes on a key
	Watch(key string, stopCh <-chan struct{}) (<-chan *KVPair, error)

	// WatchTree watches for changes on child nodes under
	// a given a directory
	WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*KVPair, error)

	// CreateLock for a given key.
	// The returned Locker is not held and must be acquired
	// with `.Lock`. The Value is optional.
	NewLock(key string, options *LockOptions) (Locker, error)

	// List the content of a given prefix
	List(directory string) ([]*KVPair, error)

	// DeleteTree deletes a range of keys under a given directory
	DeleteTree(directory string) error

	// Atomic CAS operation on a single value.
	// Pass previous = nil to create a new key.
	AtomicPut(key string, value []byte, previous *KVPair, options *WriteOptions) (bool, *KVPair, error)

	// Atomic delete of a single value
	AtomicDelete(key string, previous *KVPair) (bool, error)

	// Close the store connection
	Close()
}

// KVPair represents {Key, Value, Lastindex} tuple
type KVPair struct {
	Key       string
	Value     []byte
	LastIndex uint64
}

// WriteOptions contains optional request parameters
type WriteOptions struct {
	Heartbeat time.Duration
	Ephemeral bool
}

// LockOptions contains optional request parameters
type LockOptions struct {
	Value []byte        // Optional, value to associate with the lock
	TTL   time.Duration // Optional, expiration ttl associated with the lock
}

// WatchCallback is used for watch methods on keys
// and is triggered on key change
type WatchCallback func(entries ...*KVPair)

// Locker provides locking mechanism on top of the store.
// Similar to `sync.Lock` except it may return errors.
type Locker interface {
	Lock() (<-chan struct{}, error)
	Unlock() error
}
<commit_msg>Remove unused type WatchCallback<commit_after>package store

import (
	"crypto/tls"
	"errors"
	"time"
)

// Backend represents a KV Store Backend
type Backend string

const (
	// CONSUL backend
	CONSUL Backend = "consul"
	// ETCD backend
	ETCD Backend = "etcd"
	// ZK backend
	ZK Backend = "zk"
)

var (
	// ErrNotSupported is thrown when the backend k/v store is not supported by libkv
	ErrNotSupported = errors.New("Backend storage not supported yet, please choose another one")
	// ErrNotImplemented is thrown when a method is not implemented by the current backend
	ErrNotImplemented = errors.New("Call not implemented in current backend")
	// ErrNotReachable is thrown when the API cannot be reached for issuing common store operations
	ErrNotReachable = errors.New("Api not reachable")
	// ErrCannotLock is thrown when there is an error acquiring a lock on a key
	ErrCannotLock = errors.New("Error acquiring the lock")
	// ErrKeyModified is thrown during an atomic operation if the index does not match the one in the store
	ErrKeyModified = errors.New("Unable to complete atomic operation, key modified")
	// ErrKeyNotFound is thrown when the key is not found in the store during a Get operation
	ErrKeyNotFound = errors.New("Key not found in store")
	// ErrPreviousNotSpecified is thrown when the previous value is not specified for an atomic operation
	ErrPreviousNotSpecified = errors.New("Previous K/V pair should be provided for the Atomic operation")
)

// Config contains the options for a storage client
type Config struct {
	TLS               *tls.Config
	ConnectionTimeout time.Duration
	EphemeralTTL      time.Duration
}

// Store represents the backend K/V storage
// Each store should support every call listed
// here. Or it couldn't be implemented as a K/V
// backend for libkv
type Store interface {
	// Put a value at the specified key
	Put(key string, value []byte, options *WriteOptions) error

	// Get a value given its key
	Get(key string) (*KVPair, error)

	// Delete the value at the specified key
	Delete(key string) error

	// Verify if a Key exists in the store
	Exists(key string) (bool, error)

	// Watch for changes on a key
	Watch(key string, stopCh <-chan struct{}) (<-chan *KVPair, error)

	// WatchTree watches for changes on child nodes under
	// a given a directory
	WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*KVPair, error)

	// CreateLock for a given key.
	// The returned Locker is not held and must be acquired
	// with `.Lock`. The Value is optional.
	NewLock(key string, options *LockOptions) (Locker, error)

	// List the content of a given prefix
	List(directory string) ([]*KVPair, error)

	// DeleteTree deletes a range of keys under a given directory
	DeleteTree(directory string) error

	// Atomic CAS operation on a single value.
	// Pass previous = nil to create a new key.
	AtomicPut(key string, value []byte, previous *KVPair, options *WriteOptions) (bool, *KVPair, error)

	// Atomic delete of a single value
	AtomicDelete(key string, previous *KVPair) (bool, error)

	// Close the store connection
	Close()
}

// KVPair represents {Key, Value, Lastindex} tuple
type KVPair struct {
	Key       string
	Value     []byte
	LastIndex uint64
}

// WriteOptions contains optional request parameters
type WriteOptions struct {
	Heartbeat time.Duration
	Ephemeral bool
}

// LockOptions contains optional request parameters
type LockOptions struct {
	Value []byte        // Optional, value to associate with the lock
	TTL   time.Duration // Optional, expiration ttl associated with the lock
}

// Locker provides locking mechanism on top of the store.
// Similar to `sync.Lock` except it may return errors.
type Locker interface {
	Lock() (<-chan struct{}, error)
	Unlock() error
}
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"context"
	"fmt"
	"path/filepath"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var (
	starWarsDemoLinkRoot = "https://raw.githubusercontent.com/cilium/star-wars-demo/v1.0"
)

func getStarWarsResourceLink(file string) string {
	// Cannot use filepath.Join because it removes one of the '/' from
	// https:// and results in a malformed URL.
	return fmt.Sprintf("%s/%s", starWarsDemoLinkRoot, file)
}

var _ = Describe("K8sDemosTest", func() {

	var (
		kubectl          *helpers.Kubectl
		microscopeErr    error
		microscopeCancel = func() error { return nil }

		backgroundCancel context.CancelFunc = func() { return }
		backgroundError  error

		deathStarYAMLLink = getStarWarsResourceLink("01-deathstar.yaml")
		xwingYAMLLink     = getStarWarsResourceLink("02-xwing.yaml")
		l7PolicyYAMLLink  = getStarWarsResourceLink("policy/l7_policy.yaml")
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		err := kubectl.CiliumInstall(helpers.CiliumDefaultDSPatch, helpers.CiliumConfigMapPatch)
		Expect(err).To(BeNil(), "Cilium cannot be installed")

		ExpectCiliumReady(kubectl)
		ExpectKubeDNSReady(kubectl)
	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace,
			"cilium endpoint list",
			"cilium service list")
	})

	JustBeforeEach(func() {
		microscopeErr, microscopeCancel = kubectl.MicroscopeStart()
		Expect(microscopeErr).To(BeNil(), "Microscope cannot be started")

		backgroundCancel, backgroundError = kubectl.BackgroundReport("uptime")
		Expect(backgroundError).To(BeNil(), "Cannot start background report process")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
		Expect(microscopeCancel()).To(BeNil(), "cannot stop microscope")
		backgroundCancel()
	})

	AfterEach(func() {
		By("Deleting all resources created during test")
		kubectl.Delete(l7PolicyYAMLLink)
		kubectl.Delete(deathStarYAMLLink)
		kubectl.Delete(xwingYAMLLink)

		By("Waiting for all pods to finish terminating")
		ExpectAllPodsTerminated(kubectl)
	})

	It("Tests Star Wars Demo", func() {

		allianceLabel := "org=alliance"
		deathstarServiceName := "deathstar"
		deathstarFQDN := fmt.Sprintf("%s.%s.svc.cluster.local", deathstarServiceName, helpers.DefaultNamespace)

		exhaustPortPath := filepath.Join(deathstarFQDN, "/v1/exhaust-port")

		By("Applying deployments")

		res := kubectl.Apply(deathStarYAMLLink)
		res.ExpectSuccess("unable to apply %s: %s", deathStarYAMLLink, res.CombineOutput())

		res = kubectl.Apply(xwingYAMLLink)
		res.ExpectSuccess("unable to apply %s: %s", xwingYAMLLink, res.CombineOutput())

		By("Waiting for pods to be ready")
		err := kubectl.WaitforPods(helpers.DefaultNamespace, "", 300)
		Expect(err).Should(BeNil(), "Pods are not ready after timeout")

		By("Getting xwing pod names")
		xwingPods, err := kubectl.GetPodNames(helpers.DefaultNamespace, allianceLabel)
		Expect(err).Should(BeNil())
		Expect(xwingPods[0]).ShouldNot(Equal(""), "unable to get xwing pod names")

		// Test only needs to access one of the pods.
		xwingPod := xwingPods[0]

		By("Making sure all endpoints are in ready state")
		err = kubectl.CiliumEndpointWaitReady()
		Expect(err).To(BeNil(), "Endpoints are not ready after timeout")

		By("Showing how alliance can execute REST API call to main API endpoint")

		err = kubectl.WaitForKubeDNSEntry(deathstarServiceName, helpers.DefaultNamespace)
		Expect(err).To(BeNil(), "DNS entry is not ready after timeout")

		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, xwingPod,
			helpers.CurlFail("http://%s/v1", deathstarFQDN))
		res.ExpectSuccess("unable to curl %s/v1: %s", deathstarFQDN, res.Output())

		By("Importing L7 Policy which restricts access to %q", exhaustPortPath)
		_, err = kubectl.CiliumPolicyAction(
			helpers.KubeSystemNamespace, l7PolicyYAMLLink, helpers.KubectlApply, 300)
		Expect(err).Should(BeNil(), "Unable to apply %s", l7PolicyYAMLLink)

		By("Waiting for endpoints to be ready after importing policy")
		err = kubectl.CiliumEndpointWaitReady()
		Expect(err).To(BeNil(), "Endpoints are not ready after timeout")

		By("Showing how alliance cannot access %q without force header in API request after importing L7 Policy", exhaustPortPath)
		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, xwingPod,
			helpers.CurlWithHTTPCode("-X PUT http://%s", exhaustPortPath))
		res.ExpectContains("403", "able to access %s when policy disallows it; %s", exhaustPortPath, res.Output())

		By("Showing how alliance can access %q with force header in API request to attack the deathstar", exhaustPortPath)
		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, xwingPod,
			helpers.CurlWithHTTPCode("-X PUT -H 'X-Has-Force: True' http://%s", exhaustPortPath))
		By("Expecting 503 to be returned when using force header to attack the deathstar")
		res.ExpectContains("503", "unable to access %s when policy allows it; %s", exhaustPortPath, res.Output())
	})
})
<commit_msg>Test/Demos: Make assert more robust.<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"context"
	"fmt"
	"path/filepath"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var (
	starWarsDemoLinkRoot = "https://raw.githubusercontent.com/cilium/star-wars-demo/v1.0"
)

func getStarWarsResourceLink(file string) string {
	// Cannot use filepath.Join because it removes one of the '/' from
	// https:// and results in a malformed URL.
	return fmt.Sprintf("%s/%s", starWarsDemoLinkRoot, file)
}

var _ = Describe("K8sDemosTest", func() {

	var (
		kubectl          *helpers.Kubectl
		microscopeErr    error
		microscopeCancel = func() error { return nil }

		backgroundCancel context.CancelFunc = func() { return }
		backgroundError  error

		deathStarYAMLLink = getStarWarsResourceLink("01-deathstar.yaml")
		xwingYAMLLink     = getStarWarsResourceLink("02-xwing.yaml")
		l7PolicyYAMLLink  = getStarWarsResourceLink("policy/l7_policy.yaml")
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		err := kubectl.CiliumInstall(helpers.CiliumDefaultDSPatch, helpers.CiliumConfigMapPatch)
		Expect(err).To(BeNil(), "Cilium cannot be installed")

		ExpectCiliumReady(kubectl)
		ExpectKubeDNSReady(kubectl)
	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace,
			"cilium endpoint list",
			"cilium service list")
	})

	JustBeforeEach(func() {
		microscopeErr, microscopeCancel = kubectl.MicroscopeStart()
		Expect(microscopeErr).To(BeNil(), "Microscope cannot be started")

		backgroundCancel, backgroundError = kubectl.BackgroundReport("uptime")
		Expect(backgroundError).To(BeNil(), "Cannot start background report process")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
		Expect(microscopeCancel()).To(BeNil(), "cannot stop microscope")
		backgroundCancel()
	})

	AfterEach(func() {
		By("Deleting all resources created during test")
		kubectl.Delete(l7PolicyYAMLLink)
		kubectl.Delete(deathStarYAMLLink)
		kubectl.Delete(xwingYAMLLink)

		By("Waiting for all pods to finish terminating")
		ExpectAllPodsTerminated(kubectl)
	})

	It("Tests Star Wars Demo", func() {

		allianceLabel := "org=alliance"
		deathstarServiceName := "deathstar"
		deathstarFQDN := fmt.Sprintf("%s.%s.svc.cluster.local", deathstarServiceName, helpers.DefaultNamespace)

		exhaustPortPath := filepath.Join(deathstarFQDN, "/v1/exhaust-port")

		By("Applying deployments")

		res := kubectl.Apply(deathStarYAMLLink)
		res.ExpectSuccess("unable to apply %s: %s", deathStarYAMLLink, res.CombineOutput())

		res = kubectl.Apply(xwingYAMLLink)
		res.ExpectSuccess("unable to apply %s: %s", xwingYAMLLink, res.CombineOutput())

		By("Waiting for pods to be ready")
		err := kubectl.WaitforPods(helpers.DefaultNamespace, "", 300)
		Expect(err).Should(BeNil(), "Pods are not ready after timeout")

		By("Getting xwing pod names")
		xwingPods, err := kubectl.GetPodNames(helpers.DefaultNamespace, allianceLabel)
		Expect(err).Should(BeNil())
		Expect(xwingPods).ShouldNot(BeEmpty(), "Unable to get xwing pod names")

		// Test only needs to access one of the pods.
		xwingPod := xwingPods[0]

		By("Making sure all endpoints are in ready state")
		err = kubectl.CiliumEndpointWaitReady()
		Expect(err).To(BeNil(), "Endpoints are not ready after timeout")

		By("Showing how alliance can execute REST API call to main API endpoint")

		err = kubectl.WaitForKubeDNSEntry(deathstarServiceName, helpers.DefaultNamespace)
		Expect(err).To(BeNil(), "DNS entry is not ready after timeout")

		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, xwingPod,
			helpers.CurlFail("http://%s/v1", deathstarFQDN))
		res.ExpectSuccess("unable to curl %s/v1: %s", deathstarFQDN, res.Output())

		By("Importing L7 Policy which restricts access to %q", exhaustPortPath)
		_, err = kubectl.CiliumPolicyAction(
			helpers.KubeSystemNamespace, l7PolicyYAMLLink, helpers.KubectlApply, 300)
		Expect(err).Should(BeNil(), "Unable to apply %s", l7PolicyYAMLLink)

		By("Waiting for endpoints to be ready after importing policy")
		err = kubectl.CiliumEndpointWaitReady()
		Expect(err).To(BeNil(), "Endpoints are not ready after timeout")

		By("Showing how alliance cannot access %q without force header in API request after importing L7 Policy", exhaustPortPath)
		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, xwingPod,
			helpers.CurlWithHTTPCode("-X PUT http://%s", exhaustPortPath))
		res.ExpectContains("403", "able to access %s when policy disallows it; %s", exhaustPortPath, res.Output())

		By("Showing how alliance can access %q with force header in API request to attack the deathstar", exhaustPortPath)
		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, xwingPod,
			helpers.CurlWithHTTPCode("-X PUT -H 'X-Has-Force: True' http://%s", exhaustPortPath))
		By("Expecting 503 to be returned when using force header to attack the deathstar")
		res.ExpectContains("503", "unable to access %s when policy allows it; %s", exhaustPortPath, res.Output())
	})
})
<|endoftext|>


<commit_before>package strict

import (
	"net/http"
	"net/http/httptest"
	"testing"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func TestAccepts(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Martini Strict tests")
}

var _ = Describe("Negotiator", func() {
	var n Negotiator
	var r *http.Request
	BeforeEach(func() {
		var err error
		r, err = http.NewRequest("POST", "http://example.com/", nil)
		Expect(err).
			NotTo(HaveOccured())
		n = &negotiator{r}
	})
	It("should parse the Accept header correctly", func() {
		r.Header.Set("Accept", "application/json,text/xml;q=0.8")
		Expect(n.Accepts("application/json")).
			To(Equal(1.0))
		Expect(n.Accepts("text/xml")).
			To(Equal(0.8))
		Expect(n.Accepts("text/csv")).
			To(Equal(0.0))
	})
	It("should parse the Content-Type header correctly", func() {
		r.Header.Set("Content-Type", "application/json; charset=UTF-8")
		Expect(n.ContentType("application/json")).
			To(BeTrue())
		Expect(n.ContentType("text/plain")).
			To(BeFalse())
	})
})

var _ = Describe("ContentType", func() {
	var w *httptest.ResponseRecorder
	var r *http.Request
	BeforeEach(func() {
		var err error
		r, err = http.NewRequest("POST", "http://example.com/", nil)
		Expect(err).
			NotTo(HaveOccured())
		w = httptest.NewRecorder()
	})
	It("should accept requests with a matching content type", func() {
		r.Header.Set("Content-Type", "application/json")
		ContentType("application/json")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with a matching content type with extra values", func() {
		r.Header.Set("Content-Type", "application/json; charset=UTF-8")
		ContentType("application/json")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with a matching content type when multiple content types are supported", func() {
		r.Header.Set("Content-Type", "text/xml; charset=UTF-8")
		ContentType("application/json", "text/xml")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with no content type if empty content type headers are allowed", func() {
		ContentType("application/json", "text/xml", "")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with no content type if empty content type headers are not allowed", func() {
		ContentType("application/json", "text/xml")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with a mismatching content type", func() {
		r.Header.Set("Content-Type", "text/plain")
		ContentType("application/json", "text/xml")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with a mismatching content type even if empty content types are allowed", func() {
		r.Header.Set("Content-Type", "text/plain")
		ContentType("application/json", "text/xml", "")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
})

var _ = Describe("ContentCharset", func() {
	var w *httptest.ResponseRecorder
	var r *http.Request
	BeforeEach(func() {
		var err error
		r, err = http.NewRequest("POST", "http://example.com/", nil)
		Expect(err).
			NotTo(HaveOccured())
		w = httptest.NewRecorder()
	})
	It("should accept requests with a matching charset", func() {
		r.Header.Set("Content-Type", "application/json; charset=UTF-8")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should be case-insensitive", func() {
		r.Header.Set("Content-Type", "application/json; charset=utf-8")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with a matching charset with extra values", func() {
		r.Header.Set("Content-Type", "application/json; foo=bar; charset=UTF-8; spam=eggs")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with a matching charset when multiple charsets are supported", func() {
		r.Header.Set("Content-Type", "text/xml; charset=UTF-8")
		ContentCharset("UTF-8", "Latin-1")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with no charset if empty charset headers are allowed", func() {
		r.Header.Set("Content-Type", "text/xml")
		ContentCharset("UTF-8", "")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with no charset if empty charset headers are not allowed", func() {
		r.Header.Set("Content-Type", "text/xml")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with a mismatching charset", func() {
		r.Header.Set("Content-Type", "text/plain; charset=Latin-1")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with a mismatching charset even if empty charsets are allowed", func() {
		r.Header.Set("Content-Type", "text/plain; charset=Latin-1")
		ContentCharset("UTF-8", "")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
})

var _ = Describe("Accept", func() {
	var w *httptest.ResponseRecorder
	var r *http.Request
	BeforeEach(func() {
		var err error
		r, err = http.NewRequest("POST", "http://example.com/", nil)
		Expect(err).
			NotTo(HaveOccured())
		w = httptest.NewRecorder()
	})
	It("should accept requests with a matching content type", func() {
		r.Header.Set("Accept", "application/json")
		Accept("application/json")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusNotAcceptable))
	})
	It("should accept requests with a matching content type when multiple content types are supported", func() {
		r.Header.Set("Accept", "text/xml")
		Accept("application/json", "text/xml")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusNotAcceptable))
	})
	It("should accept requests with a matching content type when multiple content types are acceptable", func() {
		r.Header.Set("Accept", "text/xml,application/json")
		Accept("application/json")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusNotAcceptable))
	})
	It("should not accept requests when no matching pairs are found", func() {
		r.Header.Set("Accept", "image/webp,image/png")
		Accept("application/json", "text/xml")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusNotAcceptable))
	})
})

var _ = Describe("accepts", func() {
	It("should return the correct q value", func() {
		a := "text/html,application/xhtml+xml;q=0.9,image/webp,image/*;q=0.8;,*/*;q=0.6"
		Expect(accepts(a, "text/html")).
			To(Equal(1.0))
		Expect(accepts(a, "image/webp")).
			To(Equal(1.0))
		Expect(accepts(a, "application/xhtml+xml")).
			To(Equal(0.9))
		Expect(accepts(a, "image/png")).
			To(Equal(0.8))
		Expect(accepts(a, "text/csv")).
			To(Equal(0.6))
	})
	It("should return the correct q value even if not acceptable", func() {
		a := "text/html,application/json;level=2;q=0.2"
		Expect(accepts(a, "text/html")).
			To(Equal(1.0))
		Expect(accepts(a, "application/json")).
			To(Equal(0.2))
		Expect(accepts(a, "image/png")).
			To(Equal(0.0))
	})
	It("should return the correct q value when everything is acceptable", func() {
		a := ""
		Expect(accepts(a, "text/html")).
			To(Equal(1.0))
		Expect(accepts(a, "application/json")).
			To(Equal(1.0))
		Expect(accepts(a, "image/png")).
			To(Equal(1.0))
	})
})
<commit_msg>add more tests<commit_after>package strict

import (
	"net/http"
	"net/http/httptest"
	"testing"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func TestAccepts(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Martini Strict tests")
}

var _ = Describe("Negotiator", func() {
	var n Negotiator
	var r *http.Request
	BeforeEach(func() {
		var err error
		r, err = http.NewRequest("POST", "http://example.com/", nil)
		Expect(err).
			NotTo(HaveOccured())
		n = &negotiator{r}
	})
	It("should parse the Accept header correctly", func() {
		r.Header.Set("Accept", "application/json,text/xml;q=0.8")
		Expect(n.Accepts("application/json")).
			To(Equal(1.0))
		Expect(n.Accepts("text/xml")).
			To(Equal(0.8))
		Expect(n.Accepts("text/csv")).
			To(Equal(0.0))
	})
	It("should parse the Content-Type header correctly", func() {
		r.Header.Set("Content-Type", "application/json; charset=UTF-8")
		Expect(n.ContentType("application/json")).
			To(BeTrue())
		Expect(n.ContentType("text/plain")).
			To(BeFalse())
	})
})

var _ = Describe("ContentType", func() {
	var w *httptest.ResponseRecorder
	var r *http.Request
	BeforeEach(func() {
		var err error
		r, err = http.NewRequest("POST", "http://example.com/", nil)
		Expect(err).
			NotTo(HaveOccured())
		w = httptest.NewRecorder()
	})
	It("should accept requests with a matching content type", func() {
		r.Header.Set("Content-Type", "application/json")
		ContentType("application/json")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with a matching content type with extra values", func() {
		r.Header.Set("Content-Type", "application/json; charset=UTF-8")
		ContentType("application/json")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with a matching content type when multiple content types are supported", func() {
		r.Header.Set("Content-Type", "text/xml; charset=UTF-8")
		ContentType("application/json", "text/xml")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with no content type if empty content type headers are allowed", func() {
		ContentType("application/json", "text/xml", "")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with no content type if empty content type headers are not allowed", func() {
		ContentType("application/json", "text/xml")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with a mismatching content type", func() {
		r.Header.Set("Content-Type", "text/plain")
		ContentType("application/json", "text/xml")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with a mismatching content type even if empty content types are allowed", func() {
		r.Header.Set("Content-Type", "text/plain")
		ContentType("application/json", "text/xml", "")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should act on block POST, PATCH and PUT requests", func() {
		var err error
		for _, m := range []string{"POST", "PATCH", "PUT"} {
			r, err = http.NewRequest(m, "http://example.com/", nil)
			Expect(err).
				NotTo(HaveOccured())
			r.Header.Set("Content-Type", "text/plain")
			ContentType("application/json", "text/xml", "")(w, r)
			Expect(w.Code).
				To(Equal(http.StatusUnsupportedMediaType))
		}
	})
	It("should not block GET, HEAD, OPTIONS and DELETE requests", func() {
		var err error
		for _, m := range []string{"GET", "HEAD", "OPTIONS", "DELETE"} {
			r, err = http.NewRequest(m, "http://example.com/", nil)
			Expect(err).
				NotTo(HaveOccured())
			r.Header.Set("Content-Type", "text/plain")
			ContentType("application/json", "text/xml", "")(w, r)
			Expect(w.Code).
				NotTo(Equal(http.StatusUnsupportedMediaType))
		}
	})
})

var _ = Describe("ContentCharset", func() {
	var w *httptest.ResponseRecorder
	var r *http.Request
	BeforeEach(func() {
		var err error
		r, err = http.NewRequest("POST", "http://example.com/", nil)
		Expect(err).
			NotTo(HaveOccured())
		w = httptest.NewRecorder()
	})
	It("should accept requests with a matching charset", func() {
		r.Header.Set("Content-Type", "application/json; charset=UTF-8")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should be case-insensitive", func() {
		r.Header.Set("Content-Type", "application/json; charset=utf-8")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with a matching charset with extra values", func() {
		r.Header.Set("Content-Type", "application/json; foo=bar; charset=UTF-8; spam=eggs")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with a matching charset when multiple charsets are supported", func() {
		r.Header.Set("Content-Type", "text/xml; charset=UTF-8")
		ContentCharset("UTF-8", "Latin-1")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should accept requests with no charset if empty charset headers are allowed", func() {
		r.Header.Set("Content-Type", "text/xml")
		ContentCharset("UTF-8", "")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with no charset if empty charset headers are not allowed", func() {
		r.Header.Set("Content-Type", "text/xml")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with a mismatching charset", func() {
		r.Header.Set("Content-Type", "text/plain; charset=Latin-1")
		ContentCharset("UTF-8")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should not accept requests with a mismatching charset even if empty charsets are allowed", func() {
		r.Header.Set("Content-Type", "text/plain; charset=Latin-1")
		ContentCharset("UTF-8", "")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusUnsupportedMediaType))
	})
	It("should act on block POST, PATCH and PUT requests", func() {
		var err error
		for _, m := range []string{"POST", "PATCH", "PUT"} {
			r, err = http.NewRequest(m, "http://example.com/", nil)
			Expect(err).
				NotTo(HaveOccured())
			r.Header.Set("Content-Type", "text/plain")
			ContentType("application/json", "text/xml", "")(w, r)
			Expect(w.Code).
				To(Equal(http.StatusUnsupportedMediaType))
		}
	})
	It("should not block GET, HEAD, OPTIONS and DELETE requests", func() {
		var err error
		for _, m := range []string{"GET", "HEAD", "OPTIONS", "DELETE"} {
			r, err = http.NewRequest(m, "http://example.com/", nil)
			Expect(err).
				NotTo(HaveOccured())
			r.Header.Set("Content-Type", "text/plain")
			ContentType("application/json", "text/xml", "")(w, r)
			Expect(w.Code).
				NotTo(Equal(http.StatusUnsupportedMediaType))
		}
	})
})

var _ = Describe("Accept", func() {
	var w *httptest.ResponseRecorder
	var r *http.Request
	BeforeEach(func() {
		var err error
		r, err = http.NewRequest("POST", "http://example.com/", nil)
		Expect(err).
			NotTo(HaveOccured())
		w = httptest.NewRecorder()
	})
	It("should accept requests with a matching content type", func() {
		r.Header.Set("Accept", "application/json")
		Accept("application/json")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusNotAcceptable))
	})
	It("should accept requests with a matching content type when multiple content types are supported", func() {
		r.Header.Set("Accept", "text/xml")
		Accept("application/json", "text/xml")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusNotAcceptable))
	})
	It("should accept requests with a matching content type when multiple content types are acceptable", func() {
		r.Header.Set("Accept", "text/xml,application/json")
		Accept("application/json")(w, r)
		Expect(w.Code).
			ToNot(Equal(http.StatusNotAcceptable))
	})
	It("should not accept requests when no matching pairs are found", func() {
		r.Header.Set("Accept", "image/webp,image/png")
		Accept("application/json", "text/xml")(w, r)
		Expect(w.Code).
			To(Equal(http.StatusNotAcceptable))
	})
})

var _ = Describe("accepts", func() {
	It("should return the correct q value", func() {
		a := "text/html,application/xhtml+xml;q=0.9,image/webp,image/*;q=0.8;,*/*;q=0.6"
		Expect(accepts(a, "text/html")).
			To(Equal(1.0))
		Expect(accepts(a, "image/webp")).
			To(Equal(1.0))
		Expect(accepts(a, "application/xhtml+xml")).
			To(Equal(0.9))
		Expect(accepts(a, "image/png")).
			To(Equal(0.8))
		Expect(accepts(a, "text/csv")).
			To(Equal(0.6))
	})
	It("should return the correct q value even if not acceptable", func() {
		a := "text/html,application/json;level=2;q=0.2"
		Expect(accepts(a, "text/html")).
			To(Equal(1.0))
		Expect(accepts(a, "application/json")).
			To(Equal(0.2))
		Expect(accepts(a, "image/png")).
			To(Equal(0.0))
	})
	It("should return the correct q value when everything is acceptable", func() {
		a := ""
		Expect(accepts(a, "text/html")).
			To(Equal(1.0))
		Expect(accepts(a, "application/json")).
			To(Equal(1.0))
		Expect(accepts(a, "image/png")).
			To(Equal(1.0))
	})
})
<|endoftext|>