<commit_before>package app

import (
	"bytes"
	"fmt"
	"path"

	abci "github.com/tendermint/abci/types"
	"github.com/tendermint/go-crypto"
	"github.com/tendermint/go-wire"
	"github.com/tendermint/merkleeyes/iavl"
	cmn "github.com/tendermint/tmlibs/common"
	dbm "github.com/tendermint/tmlibs/db"
	"github.com/tendermint/tmlibs/merkle"
)

// MerkleEyesApp is a Merkle KV-store served as an ABCI app
type MerkleEyesApp struct {
	abci.BaseApplication

	state  State
	db     dbm.DB
	height uint64

	changes []*abci.Validator
}

// just make sure we really are an application, if the interface
// ever changes in the future
func (app *MerkleEyesApp) assertApplication() abci.Application {
	return app
}

var eyesStateKey = []byte("merkleeyes:state") // Database key for merkle tree save value db values

// MerkleEyesState contains the latest Merkle root hash and the number of times `Commit` has been called
type MerkleEyesState struct {
	Hash   []byte
	Height uint64
}

// Transaction type bytes
const (
	TxTypeSet           byte = 0x01
	TxTypeRm            byte = 0x02
	TxTypeGet           byte = 0x03
	TxTypeCompareAndSet byte = 0x04
	TxTypeValSetChange  byte = 0x05

	NonceLength = 12
)

// NewMerkleEyesApp initializes the database, loads any existing state, and returns a new MerkleEyesApp
func NewMerkleEyesApp(dbName string, cacheSize int) *MerkleEyesApp {
	// start at 1 so the height returned by query is for the
	// next block, ie. the one that includes the AppHash for our current state
	initialHeight := uint64(1)

	// Non-persistent case
	if dbName == "" {
		tree := iavl.NewIAVLTree(
			0,
			nil,
		)
		return &MerkleEyesApp{
			state:  NewState(tree, false),
			db:     nil,
			height: initialHeight,
		}
	}

	// Setup the persistent merkle tree
	empty, _ := cmn.IsDirEmpty(path.Join(dbName, dbName+".db"))

	// Open the db, if the db doesn't exist it will be created
	db := dbm.NewDB(dbName, dbm.LevelDBBackendStr, dbName)

	// Load Tree
	tree := iavl.NewIAVLTree(cacheSize, db)

	if empty {
		fmt.Println("no existing db, creating new db")
		db.Set(eyesStateKey, wire.BinaryBytes(MerkleEyesState{
			Hash:   tree.Save(),
			Height: initialHeight,
		}))
	} else {
		fmt.Println("loading existing db")
	}

	// Load merkle state
	eyesStateBytes := db.Get(eyesStateKey)
	var eyesState MerkleEyesState
	err := wire.ReadBinaryBytes(eyesStateBytes, &eyesState)
	if err != nil {
		fmt.Println("error reading MerkleEyesState")
		panic(err.Error())
	}
	tree.Load(eyesState.Hash)

	return &MerkleEyesApp{
		state:  NewState(tree, true),
		db:     db,
		height: eyesState.Height,
	}
}

// CloseDB closes the database
func (app *MerkleEyesApp) CloseDB() {
	if app.db != nil {
		app.db.Close()
	}
}

// Info implements abci.Application
func (app *MerkleEyesApp) Info() abci.ResponseInfo {
	return abci.ResponseInfo{Data: "merkleeyes"}
}

// SetOption implements abci.Application
func (app *MerkleEyesApp) SetOption(key string, value string) (log string) {
	return "No options are supported yet"
}

// DeliverTx implements abci.Application
func (app *MerkleEyesApp) DeliverTx(tx []byte) abci.Result {
	tree := app.state.Deliver()
	r := app.doTx(tree, tx)
	if r.IsErr() {
		fmt.Println("DeliverTx Err", r)
	}
	return r
}

// CheckTx implements abci.Application
func (app *MerkleEyesApp) CheckTx(tx []byte) abci.Result {
	return abci.OK
	// tree := app.state.Check()
	// return app.doTx(tree, tx)
}

func nonceKey(nonce []byte) []byte {
	return append([]byte("/nonce/"), nonce...)
}

func storeKey(key []byte) []byte {
	return append([]byte("/key/"), key...)
}

func (app *MerkleEyesApp) doTx(tree merkle.Tree, tx []byte) abci.Result {
	// minimum length is 12 (nonce) + 1 (type byte) + 1 (len of len) + 1 (len) + 1 (arg) = 16
	minTxLen := NonceLength + 4
	if len(tx) < minTxLen {
		return abci.ErrEncodingError.SetLog(fmt.Sprintf("Tx length must be at least %d", minTxLen))
	}

	nonce := tx[:12]
	tx = tx[12:]

	// check nonce
	_, _, exists := tree.Get(nonceKey(nonce))
	if exists {
		return abci.ErrBadNonce.AppendLog(fmt.Sprintf("Nonce %X already exists", nonce))
	}

	// set nonce
	tree.Set(nonceKey(nonce), []byte("found"))

	typeByte := tx[0]
	tx = tx[1:]
	switch typeByte {
	case TxTypeSet: // Set
		key, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error()))
		}
		tx = tx[n:]
		value, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading value: %v", err.Error()))
		}
		tx = tx[n:]
		if len(tx) != 0 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over"))
		}

		tree.Set(storeKey(key), value)

		fmt.Println("SET", cmn.Fmt("%X", key), cmn.Fmt("%X", value))
	case TxTypeRm: // Remove
		key, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error()))
		}
		tx = tx[n:]
		if len(tx) != 0 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over"))
		}
		tree.Remove(storeKey(key))
		fmt.Println("RM", cmn.Fmt("%X", key))
	case TxTypeGet: // Get
		key, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error()))
		}
		tx = tx[n:]
		if len(tx) != 0 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over"))
		}

		_, value, exists := tree.Get(storeKey(key))
		if exists {
			fmt.Println("GET", cmn.Fmt("%X", key), cmn.Fmt("%X", value))
			return abci.OK.SetData(value)
		} else {
			return abci.ErrBaseUnknownAddress.AppendLog(fmt.Sprintf("Cannot find key: %X", key))
		}
	case TxTypeCompareAndSet: // Compare and Set
		key, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error()))
		}
		tx = tx[n:]

		compareValue, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading compare value: %v", err.Error()))
		}
		tx = tx[n:]

		setValue, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading set value: %v", err.Error()))
		}
		tx = tx[n:]

		if len(tx) != 0 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over"))
		}

		_, value, exists := tree.Get(storeKey(key))
		if !exists {
			return abci.ErrBaseUnknownAddress.AppendLog(fmt.Sprintf("Cannot find key: %X", key))
		}
		if !bytes.Equal(value, compareValue) {
			return abci.ErrUnauthorized.AppendLog(fmt.Sprintf("Value was %X, not %X", value, compareValue))
		}
		tree.Set(storeKey(key), setValue)

		fmt.Println("CAS-SET", cmn.Fmt("%X", key), cmn.Fmt("%X", compareValue), cmn.Fmt("%X", setValue))
	case TxTypeValSetChange:
		pubKey, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading pubkey: %v", err.Error()))
		}
		if len(pubKey) != 32 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Pubkey must be 32 bytes: %X is %d bytes", pubKey, len(pubKey)))
		}
		tx = tx[n:]

		power := wire.GetInt64(tx)

		// copy to PubKeyEd25519 so we can go-wire encode properly
		var pubKeyEd crypto.PubKeyEd25519
		copy(pubKeyEd[:], pubKey)
		return app.updateValidator(&abci.Validator{pubKeyEd.Bytes(), uint64(power)})

	default:
		return abci.ErrUnknownRequest.SetLog(cmn.Fmt("Unexpected Tx type byte %X", typeByte))
	}
	return abci.OK
}

func (app *MerkleEyesApp) updateValidator(v *abci.Validator) abci.Result {
	key := []byte("val:" + string(v.PubKey))
	if v.Power == 0 {
		// remove validator
		if !app.state.deliverTx.Has(key) {
			return abci.ErrUnauthorized.SetLog(cmn.Fmt("Cannot remove non-existent validator %X", key))
		}
		app.state.deliverTx.Remove(key)
	} else {
		// add or update validator
		value := bytes.NewBuffer(make([]byte, 0))
		if err := abci.WriteMessage(v, value); err != nil {
			return abci.ErrInternalError.SetLog(cmn.Fmt("Error encoding validator: %v", err))
		}
		app.state.deliverTx.Set(key, value.Bytes())
	}

	// we only update the changes array if we succesfully updated the tree
	app.changes = append(app.changes, v)

	return abci.OK
}

func (app *MerkleEyesApp) InitChain(validators []*abci.Validator) {
	for _, v := range validators {
		r := app.updateValidator(v)
		if r.IsErr() {
			fmt.Println("Error updating validators", r)
		}
	}
}

func (app *MerkleEyesApp) BeginBlock(hash []byte, header *abci.Header) {
	// reset valset changes
	app.changes = make([]*abci.Validator, 0)
}

func (app *MerkleEyesApp) EndBlock(height uint64) (resEndBlock abci.ResponseEndBlock) {
	return abci.ResponseEndBlock{Diffs: app.changes}
}

// Commit implements abci.Application
func (app *MerkleEyesApp) Commit() abci.Result {

	hash := app.state.Commit()

	app.height++
	if app.db != nil {
		app.db.Set(eyesStateKey, wire.BinaryBytes(MerkleEyesState{
			Hash:   hash,
			Height: app.height,
		}))
	}

	if app.state.Committed().Size() == 0 {
		return abci.NewResultOK(nil, "Empty hash for empty tree")
	}
	return abci.NewResultOK(hash, "")
}

// Query implements abci.Application
func (app *MerkleEyesApp) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) {
	if len(reqQuery.Data) == 0 {
		return
	}
	tree := app.state.Committed()

	if reqQuery.Height != 0 {
		// TODO: support older commits
		resQuery.Code = abci.CodeType_InternalError
		resQuery.Log = "merkleeyes only supports queries on latest commit"
		return
	}

	// set the query response height
	resQuery.Height = app.height

	switch reqQuery.Path {
	case "/store", "/key": // Get by key
		key := reqQuery.Data // Data holds the key bytes
		resQuery.Key = key
		if reqQuery.Prove {
			value, proof, exists := tree.Proof(storeKey(key))
			if !exists {
				resQuery.Log = "Key not found"
			}
			resQuery.Value = value
			resQuery.Proof = proof
			// TODO: return index too?
		} else {
			index, value, _ := tree.Get(storeKey(key))
			resQuery.Value = value
			resQuery.Index = int64(index)
		}

	case "/index": // Get by Index
		index := wire.GetInt64(reqQuery.Data)
		key, value := tree.GetByIndex(int(index))
		resQuery.Key = key
		resQuery.Index = int64(index)
		resQuery.Value = value

	case "/size": // Get size
		size := tree.Size()
		sizeBytes := wire.BinaryBytes(size)
		resQuery.Value = sizeBytes

	default:
		resQuery.Code = abci.CodeType_UnknownRequest
		resQuery.Log = cmn.Fmt("Unexpected Query path: %v", reqQuery.Path)
	}
	return
}
<commit_msg>ensure power is right size<commit_after>package app

import (
	"bytes"
	"fmt"
	"path"

	abci "github.com/tendermint/abci/types"
	"github.com/tendermint/go-crypto"
	"github.com/tendermint/go-wire"
	"github.com/tendermint/merkleeyes/iavl"
	cmn "github.com/tendermint/tmlibs/common"
	dbm "github.com/tendermint/tmlibs/db"
	"github.com/tendermint/tmlibs/merkle"
)

// MerkleEyesApp is a Merkle KV-store served as an ABCI app
type MerkleEyesApp struct {
	abci.BaseApplication

	state  State
	db     dbm.DB
	height uint64

	changes []*abci.Validator
}

// just make sure we really are an application, if the interface
// ever changes in the future
func (app *MerkleEyesApp) assertApplication() abci.Application {
	return app
}

var eyesStateKey = []byte("merkleeyes:state") // Database key for merkle tree save value db values

// MerkleEyesState contains the latest Merkle root hash and the number of times `Commit` has been called
type MerkleEyesState struct {
	Hash   []byte
	Height uint64
}

// Transaction type bytes
const (
	TxTypeSet           byte = 0x01
	TxTypeRm            byte = 0x02
	TxTypeGet           byte = 0x03
	TxTypeCompareAndSet byte = 0x04
	TxTypeValSetChange  byte = 0x05

	NonceLength = 12
)

// NewMerkleEyesApp initializes the database, loads any existing state, and returns a new MerkleEyesApp
func NewMerkleEyesApp(dbName string, cacheSize int) *MerkleEyesApp {
	// start at 1 so the height returned by query is for the
	// next block, ie. the one that includes the AppHash for our current state
	initialHeight := uint64(1)

	// Non-persistent case
	if dbName == "" {
		tree := iavl.NewIAVLTree(
			0,
			nil,
		)
		return &MerkleEyesApp{
			state:  NewState(tree, false),
			db:     nil,
			height: initialHeight,
		}
	}

	// Setup the persistent merkle tree
	empty, _ := cmn.IsDirEmpty(path.Join(dbName, dbName+".db"))

	// Open the db, if the db doesn't exist it will be created
	db := dbm.NewDB(dbName, dbm.LevelDBBackendStr, dbName)

	// Load Tree
	tree := iavl.NewIAVLTree(cacheSize, db)

	if empty {
		fmt.Println("no existing db, creating new db")
		db.Set(eyesStateKey, wire.BinaryBytes(MerkleEyesState{
			Hash:   tree.Save(),
			Height: initialHeight,
		}))
	} else {
		fmt.Println("loading existing db")
	}

	// Load merkle state
	eyesStateBytes := db.Get(eyesStateKey)
	var eyesState MerkleEyesState
	err := wire.ReadBinaryBytes(eyesStateBytes, &eyesState)
	if err != nil {
		fmt.Println("error reading MerkleEyesState")
		panic(err.Error())
	}
	tree.Load(eyesState.Hash)

	return &MerkleEyesApp{
		state:  NewState(tree, true),
		db:     db,
		height: eyesState.Height,
	}
}

// CloseDB closes the database
func (app *MerkleEyesApp) CloseDB() {
	if app.db != nil {
		app.db.Close()
	}
}

// Info implements abci.Application
func (app *MerkleEyesApp) Info() abci.ResponseInfo {
	return abci.ResponseInfo{Data: "merkleeyes"}
}

// SetOption implements abci.Application
func (app *MerkleEyesApp) SetOption(key string, value string) (log string) {
	return "No options are supported yet"
}

// DeliverTx implements abci.Application
func (app *MerkleEyesApp) DeliverTx(tx []byte) abci.Result {
	tree := app.state.Deliver()
	r := app.doTx(tree, tx)
	if r.IsErr() {
		fmt.Println("DeliverTx Err", r)
	}
	return r
}

// CheckTx implements abci.Application
func (app *MerkleEyesApp) CheckTx(tx []byte) abci.Result {
	return abci.OK
	// tree := app.state.Check()
	// return app.doTx(tree, tx)
}

func nonceKey(nonce []byte) []byte {
	return append([]byte("/nonce/"), nonce...)
}

func storeKey(key []byte) []byte {
	return append([]byte("/key/"), key...)
}

func (app *MerkleEyesApp) doTx(tree merkle.Tree, tx []byte) abci.Result {
	// minimum length is 12 (nonce) + 1 (type byte) + 1 (len of len) + 1 (len) + 1 (arg) = 16
	minTxLen := NonceLength + 4
	if len(tx) < minTxLen {
		return abci.ErrEncodingError.SetLog(fmt.Sprintf("Tx length must be at least %d", minTxLen))
	}

	nonce := tx[:12]
	tx = tx[12:]

	// check nonce
	_, _, exists := tree.Get(nonceKey(nonce))
	if exists {
		return abci.ErrBadNonce.AppendLog(fmt.Sprintf("Nonce %X already exists", nonce))
	}

	// set nonce
	tree.Set(nonceKey(nonce), []byte("found"))

	typeByte := tx[0]
	tx = tx[1:]
	switch typeByte {
	case TxTypeSet: // Set
		key, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error()))
		}
		tx = tx[n:]
		value, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading value: %v", err.Error()))
		}
		tx = tx[n:]
		if len(tx) != 0 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over"))
		}

		tree.Set(storeKey(key), value)

		fmt.Println("SET", cmn.Fmt("%X", key), cmn.Fmt("%X", value))
	case TxTypeRm: // Remove
		key, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error()))
		}
		tx = tx[n:]
		if len(tx) != 0 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over"))
		}
		tree.Remove(storeKey(key))
		fmt.Println("RM", cmn.Fmt("%X", key))
	case TxTypeGet: // Get
		key, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error()))
		}
		tx = tx[n:]
		if len(tx) != 0 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over"))
		}

		_, value, exists := tree.Get(storeKey(key))
		if exists {
			fmt.Println("GET", cmn.Fmt("%X", key), cmn.Fmt("%X", value))
			return abci.OK.SetData(value)
		} else {
			return abci.ErrBaseUnknownAddress.AppendLog(fmt.Sprintf("Cannot find key: %X", key))
		}
	case TxTypeCompareAndSet: // Compare and Set
		key, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error()))
		}
		tx = tx[n:]

		compareValue, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading compare value: %v", err.Error()))
		}
		tx = tx[n:]

		setValue, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading set value: %v", err.Error()))
		}
		tx = tx[n:]

		if len(tx) != 0 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over"))
		}

		_, value, exists := tree.Get(storeKey(key))
		if !exists {
			return abci.ErrBaseUnknownAddress.AppendLog(fmt.Sprintf("Cannot find key: %X", key))
		}
		if !bytes.Equal(value, compareValue) {
			return abci.ErrUnauthorized.AppendLog(fmt.Sprintf("Value was %X, not %X", value, compareValue))
		}
		tree.Set(storeKey(key), setValue)

		fmt.Println("CAS-SET", cmn.Fmt("%X", key), cmn.Fmt("%X", compareValue), cmn.Fmt("%X", setValue))
	case TxTypeValSetChange:
		pubKey, n, err := wire.GetByteSlice(tx)
		if err != nil {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading pubkey: %v", err.Error()))
		}
		if len(pubKey) != 32 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Pubkey must be 32 bytes: %X is %d bytes", pubKey, len(pubKey)))
		}
		tx = tx[n:]
		if len(tx) != 8 {
			return abci.ErrEncodingError.SetLog(cmn.Fmt("Power must be 8 bytes: %X is %d bytes", tx, len(tx)))
		}
		power := wire.GetUint64(tx)

		// copy to PubKeyEd25519 so we can go-wire encode properly
		var pubKeyEd crypto.PubKeyEd25519
		copy(pubKeyEd[:], pubKey)
		return app.updateValidator(&abci.Validator{pubKeyEd.Bytes(), uint64(power)})

	default:
		return abci.ErrUnknownRequest.SetLog(cmn.Fmt("Unexpected Tx type byte %X", typeByte))
	}
	return abci.OK
}

func (app *MerkleEyesApp) updateValidator(v *abci.Validator) abci.Result {
	key := []byte("val:" + string(v.PubKey))
	if v.Power == 0 {
		// remove validator
		if !app.state.deliverTx.Has(key) {
			return abci.ErrUnauthorized.SetLog(cmn.Fmt("Cannot remove non-existent validator %X", key))
		}
		app.state.deliverTx.Remove(key)
	} else {
		// add or update validator
		value := bytes.NewBuffer(make([]byte, 0))
		if err := abci.WriteMessage(v, value); err != nil {
			return abci.ErrInternalError.SetLog(cmn.Fmt("Error encoding validator: %v", err))
		}
		app.state.deliverTx.Set(key, value.Bytes())
	}

	// we only update the changes array if we succesfully updated the tree
	app.changes = append(app.changes, v)

	return abci.OK
}

func (app *MerkleEyesApp) InitChain(validators []*abci.Validator) {
	for _, v := range validators {
		r := app.updateValidator(v)
		if r.IsErr() {
			fmt.Println("Error updating validators", r)
		}
	}
}

func (app *MerkleEyesApp) BeginBlock(hash []byte, header *abci.Header) {
	// reset valset changes
	app.changes = make([]*abci.Validator, 0)
}

func (app *MerkleEyesApp) EndBlock(height uint64) (resEndBlock abci.ResponseEndBlock) {
	return abci.ResponseEndBlock{Diffs: app.changes}
}

// Commit implements abci.Application
func (app *MerkleEyesApp) Commit() abci.Result {

	hash := app.state.Commit()

	app.height++
	if app.db != nil {
		app.db.Set(eyesStateKey, wire.BinaryBytes(MerkleEyesState{
			Hash:   hash,
			Height: app.height,
		}))
	}

	if app.state.Committed().Size() == 0 {
		return abci.NewResultOK(nil, "Empty hash for empty tree")
	}
	return abci.NewResultOK(hash, "")
}

// Query implements abci.Application
func (app *MerkleEyesApp) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) {
	if len(reqQuery.Data) == 0 {
		return
	}
	tree := app.state.Committed()

	if reqQuery.Height != 0 {
		// TODO: support older commits
		resQuery.Code = abci.CodeType_InternalError
		resQuery.Log = "merkleeyes only supports queries on latest commit"
		return
	}

	// set the query response height
	resQuery.Height = app.height

	switch reqQuery.Path {
	case "/store", "/key": // Get by key
		key := reqQuery.Data // Data holds the key bytes
		resQuery.Key = key
		if reqQuery.Prove {
			value, proof, exists := tree.Proof(storeKey(key))
			if !exists {
				resQuery.Log = "Key not found"
			}
			resQuery.Value = value
			resQuery.Proof = proof
			// TODO: return index too?
		} else {
			index, value, _ := tree.Get(storeKey(key))
			resQuery.Value = value
			resQuery.Index = int64(index)
		}

	case "/index": // Get by Index
		index := wire.GetInt64(reqQuery.Data)
		key, value := tree.GetByIndex(int(index))
		resQuery.Key = key
		resQuery.Index = int64(index)
		resQuery.Value = value

	case "/size": // Get size
		size := tree.Size()
		sizeBytes := wire.BinaryBytes(size)
		resQuery.Value = sizeBytes

	default:
		resQuery.Code = abci.CodeType_UnknownRequest
		resQuery.Log = cmn.Fmt("Unexpected Query path: %v", reqQuery.Path)
	}
	return
}
<|endoftext|>


<commit_before>// Package app is the "runtime" for the ps-top / ps-stats application packages
//
// This file contains the library routines related to running the app.
package app

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/sjmudd/anonymiser"
	"github.com/sjmudd/ps-top/connector"
	"github.com/sjmudd/ps-top/context"
	"github.com/sjmudd/ps-top/display"
	"github.com/sjmudd/ps-top/event"
	fsbi "github.com/sjmudd/ps-top/file_io_latency"
	"github.com/sjmudd/ps-top/global"
	"github.com/sjmudd/ps-top/lib"
	"github.com/sjmudd/ps-top/logger"
	"github.com/sjmudd/ps-top/memory_usage"
	ewsgben "github.com/sjmudd/ps-top/mutex_latency"
	"github.com/sjmudd/ps-top/p_s/ps_table"
	"github.com/sjmudd/ps-top/setup_instruments"
	essgben "github.com/sjmudd/ps-top/stages_latency"
	tiwsbt "github.com/sjmudd/ps-top/table_io_latency"
	tlwsbt "github.com/sjmudd/ps-top/table_lock_latency"
	"github.com/sjmudd/ps-top/user_latency"
	"github.com/sjmudd/ps-top/view"
	"github.com/sjmudd/ps-top/wait_info"
)

// Flags for initialising the app
type Settings struct {
	Anonymise bool
	Conn      *connector.Connector
	Interval  int
	Count     int
	Stdout    bool
	View      string
	Disp      display.Display
}

// App holds the data needed by an application
type App struct {
	ctx                *context.Context
	count              int
	display            display.Display
	done               chan struct{}
	sigChan            chan os.Signal
	wi                 wait_info.WaitInfo
	finished           bool
	stdout             bool
	dbh                *sql.DB
	help               bool
	fsbi               ps_table.Tabler // ufsbi.File_summary_by_instance
	tiwsbt             tiwsbt.Object
	tlwsbt             ps_table.Tabler // tlwsbt.Table_lock_waits_summary_by_table
	ewsgben            ps_table.Tabler // ewsgben.Events_waits_summary_global_by_event_name
	essgben            ps_table.Tabler // essgben.Events_stages_summary_global_by_event_name
	memory             memory_usage.Object
	users              user_latency.Object
	view               view.View
	wait_info.WaitInfo // embedded
	setupInstruments   setup_instruments.SetupInstruments
	wantRelativeStats  bool
}

// ensure performance_schema is enabled
// - if not will not return and will exit
func ensurePerformanceSchemaEnabled(variables *global.Variables) {
	if variables == nil {
		log.Fatal("ensurePerformanceSchemaEnabled() variables is nil")
	}

	// check that performance_schema = ON
	if value := variables.Get("performance_schema"); value != "ON" {
		log.Fatal(fmt.Sprintf("ensurePerformanceSchemaEnabled(): performance_schema = '%s'. Please configure performance_schema = 1 in /etc/my.cnf (or equivalent) and restart mysqld to use %s.",
			value, lib.MyName()))
	} else {
		logger.Println("performance_schema = ON check succeeds")
	}
}

// NewApp sets up the application given various parameters.
func NewApp(settings Settings) *App {
	logger.Println("app.NewApp()")
	app := new(App)

	anonymiser.Enable(settings.Anonymise) // not dynamic at the moment
	app.dbh = settings.Conn.Handle()

	status := global.NewStatus(app.dbh)
	variables := global.NewVariables(app.dbh)
	// Prior to setting up screen check that performance_schema is enabled.
	// On MariaDB this is not the default setting so it will confuse people.
	ensurePerformanceSchemaEnabled(variables)

	app.ctx = context.NewContext(status, variables)
	app.count = settings.Count
	app.finished = false

	app.stdout = settings.Stdout
	app.display = settings.Disp
	app.display.SetContext(app.ctx)
	app.SetHelp(false)

	if err := view.ValidateViews(app.dbh); err != nil {
		log.Fatal(err)
	}

	logger.Println("app.Setup() Setting the default view to:", settings.View)
	app.view.SetByName(settings.View) // if empty will use the default

	app.setupInstruments = setup_instruments.NewSetupInstruments(app.dbh)
	app.setupInstruments.EnableMonitoring()

	app.wi.SetWaitInterval(time.Second * time.Duration(settings.Interval))

	// setup to their initial types/values
	app.fsbi = fsbi.NewFileSummaryByInstance(variables)
	app.tlwsbt = new(tlwsbt.Object)
	app.ewsgben = new(ewsgben.Object)
	app.essgben = new(essgben.Object)

	app.SetWantRelativeStats(true)
	app.fixLatencySetting() // adjust to see ops/latency

	app.resetDBStatistics()

	return app
}

// Finished tells us if we have finished
func (app App) Finished() bool {
	return app.finished
}

// CollectAll collects all the stats together in one go
func (app *App) collectAll() {
	app.fsbi.Collect(app.dbh)
	app.tlwsbt.Collect(app.dbh)
	app.tiwsbt.Collect(app.dbh)
	app.users.Collect(app.dbh)
	app.essgben.Collect(app.dbh)
	app.ewsgben.Collect(app.dbh)
	app.memory.Collect(app.dbh)
}

func (app *App) WantRelativeStats() bool {
	return app.wantRelativeStats
}

// do a fresh collection of data and then update the initial values based on that.
func (app *App) resetDBStatistics() {
	app.collectAll()
	app.setInitialFromCurrent()
}

func (app *App) setInitialFromCurrent() {
	start := time.Now()
	app.fsbi.SetInitialFromCurrent()
	app.tlwsbt.SetInitialFromCurrent()
	app.tiwsbt.SetInitialFromCurrent()
	app.essgben.SetInitialFromCurrent()
	app.ewsgben.SetInitialFromCurrent()
	app.memory.SetInitialFromCurrent()
	logger.Println("app.setInitialFromCurrent() took", time.Duration(time.Since(start)).String())
}

// Collect the data we are looking at.
func (app *App) Collect() {
	start := time.Now()

	switch app.view.Get() {
	case view.ViewLatency, view.ViewOps:
		app.tiwsbt.Collect(app.dbh)
	case view.ViewIO:
		app.fsbi.Collect(app.dbh)
	case view.ViewLocks:
		app.tlwsbt.Collect(app.dbh)
	case view.ViewUsers:
		app.users.Collect(app.dbh)
	case view.ViewMutex:
		app.ewsgben.Collect(app.dbh)
	case view.ViewStages:
		app.essgben.Collect(app.dbh)
	case view.ViewMemory:
		app.memory.Collect(app.dbh)
	}
	app.wi.CollectedNow()
	logger.Println("app.Collect() took", time.Duration(time.Since(start)).String())
}

// SetHelp determines if we need to display help
func (app *App) SetHelp(newHelp bool) {
	app.help = newHelp

	app.display.ClearScreen()
}

// Help returns the internal help variable
func (app App) Help() bool {
	return app.help
}

// Display shows the output appropriate to the corresponding view and device
func (app *App) Display() {
	if app.help {
		app.display.DisplayHelp() // shouldn't get here if in --stdout mode
	} else {
		switch app.view.Get() {
		case view.ViewLatency, view.ViewOps:
			app.display.Display(app.tiwsbt)
		case view.ViewIO:
			app.display.Display(app.fsbi)
		case view.ViewLocks:
			app.display.Display(app.tlwsbt)
		case view.ViewUsers:
			app.display.Display(app.users)
		case view.ViewMutex:
			app.display.Display(app.ewsgben)
		case view.ViewStages:
			app.display.Display(app.essgben)
		case view.ViewMemory:
			app.display.Display(app.memory)
		}
	}
}

// fixLatencySetting() ensures the SetWantsLatency() value is
// correct. This needs to be done more cleanly.
func (app *App) fixLatencySetting() {
	if app.view.Get() == view.ViewLatency {
		app.tiwsbt.SetWantsLatency(true)
	}
	if app.view.Get() == view.ViewOps {
		app.tiwsbt.SetWantsLatency(false)
	}
}

// change to the previous display mode
func (app *App) displayPrevious() {
	app.view.SetPrev()
	app.fixLatencySetting()
	app.display.ClearScreen()
	app.Display()
}

// change to the next display mode
func (app *App) displayNext() {
	app.view.SetNext()
	app.fixLatencySetting()
	app.display.ClearScreen()
	app.Display()
}

// SetWantRelativeStats sets whether we want to see data that's relative or absolute
func (app *App) SetWantRelativeStats(want bool) {
	app.wantRelativeStats = want

	app.fsbi.SetWantRelativeStats(want)
	app.tlwsbt.SetWantRelativeStats(want)
	app.tiwsbt.SetWantRelativeStats(want)
	app.users.SetWantRelativeStats(want) // ignored
	app.essgben.SetWantRelativeStats(want)
	app.ewsgben.SetWantRelativeStats(want) // ignored
	app.memory.SetWantRelativeStats(want)
}

// Cleanup prepares  the application prior to shutting down
func (app *App) Cleanup() {
	app.display.Close()
	if app.dbh != nil {
		app.setupInstruments.RestoreConfiguration()
		_ = app.dbh.Close()
	}
	logger.Println("App.Cleanup completed")
}

// Run runs the application in a loop until we're ready to finish
func (app *App) Run() {
	logger.Println("app.Run()")

	app.sigChan = make(chan os.Signal, 10) // 10 entries
	signal.Notify(app.sigChan, syscall.SIGINT, syscall.SIGTERM)

	eventChan := app.display.EventChan()

	for !app.Finished() {
		select {
		case sig := <-app.sigChan:
			fmt.Println("Caught signal: ", sig)
			app.finished = true
		case <-app.wi.WaitNextPeriod():
			app.Collect()
			app.Display()
			if app.stdout {
				app.setInitialFromCurrent()
			}
		case inputEvent := <-eventChan:
			switch inputEvent.Type {
			case event.EventAnonymise:
				anonymiser.Enable(!anonymiser.Enabled()) // toggle current behaviour
			case event.EventFinished:
				app.finished = true
			case event.EventViewNext:
				app.displayNext()
			case event.EventViewPrev:
				app.displayPrevious()
			case event.EventDecreasePollTime:
				if app.wi.WaitInterval() > time.Second {
					app.wi.SetWaitInterval(app.wi.WaitInterval() - time.Second)
				}
			case event.EventIncreasePollTime:
				app.wi.SetWaitInterval(app.wi.WaitInterval() + time.Second)
			case event.EventHelp:
				app.SetHelp(!app.Help())
			case event.EventToggleWantRelative:
				app.SetWantRelativeStats(!app.WantRelativeStats())
				app.Display()
			case event.EventResetStatistics:
				app.resetDBStatistics()
				app.Display()
			case event.EventResizeScreen:
				width, height := inputEvent.Width, inputEvent.Height
				app.display.Resize(width, height)
				app.Display()
			case event.EventError:
				log.Fatalf("Quitting because of EventError error")
			}
		}
		// provide a hook to stop the application if the counter goes down to zero
		if app.stdout && app.count > 0 {
			app.count--
			if app.count == 0 {
				app.finished = true
			}
		}
	}
}
<commit_msg>Rename App.view to App.currentView for clarity<commit_after>// Package app is the "runtime" for the ps-top / ps-stats application packages
//
// This file contains the library routines related to running the app.
package app

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/sjmudd/anonymiser"
	"github.com/sjmudd/ps-top/connector"
	"github.com/sjmudd/ps-top/context"
	"github.com/sjmudd/ps-top/display"
	"github.com/sjmudd/ps-top/event"
	fsbi "github.com/sjmudd/ps-top/file_io_latency"
	"github.com/sjmudd/ps-top/global"
	"github.com/sjmudd/ps-top/lib"
	"github.com/sjmudd/ps-top/logger"
	"github.com/sjmudd/ps-top/memory_usage"
	ewsgben "github.com/sjmudd/ps-top/mutex_latency"
	"github.com/sjmudd/ps-top/p_s/ps_table"
	"github.com/sjmudd/ps-top/setup_instruments"
	essgben "github.com/sjmudd/ps-top/stages_latency"
	tiwsbt "github.com/sjmudd/ps-top/table_io_latency"
	tlwsbt "github.com/sjmudd/ps-top/table_lock_latency"
	"github.com/sjmudd/ps-top/user_latency"
	"github.com/sjmudd/ps-top/view"
	"github.com/sjmudd/ps-top/wait_info"
)

// Flags for initialising the app
type Settings struct {
	Anonymise bool
	Conn      *connector.Connector
	Interval  int
	Count     int
	Stdout    bool
	View      string
	Disp      display.Display
}

// App holds the data needed by an application
type App struct {
	ctx                *context.Context
	count              int
	display            display.Display
	done               chan struct{}
	sigChan            chan os.Signal
	wi                 wait_info.WaitInfo
	finished           bool
	stdout             bool
	dbh                *sql.DB
	help               bool
	fsbi               ps_table.Tabler // ufsbi.File_summary_by_instance
	tiwsbt             tiwsbt.Object
	tlwsbt             ps_table.Tabler // tlwsbt.Table_lock_waits_summary_by_table
	ewsgben            ps_table.Tabler // ewsgben.Events_waits_summary_global_by_event_name
	essgben            ps_table.Tabler // essgben.Events_stages_summary_global_by_event_name
	memory             memory_usage.Object
	users              user_latency.Object
	currentView        view.View
	wait_info.WaitInfo // embedded
	setupInstruments   setup_instruments.SetupInstruments
	wantRelativeStats  bool
}

// ensure performance_schema is enabled
// - if not will not return and will exit
func ensurePerformanceSchemaEnabled(variables *global.Variables) {
	if variables == nil {
		log.Fatal("ensurePerformanceSchemaEnabled() variables is nil")
	}

	// check that performance_schema = ON
	if value := variables.Get("performance_schema"); value != "ON" {
		log.Fatal(fmt.Sprintf("ensurePerformanceSchemaEnabled(): performance_schema = '%s'. Please configure performance_schema = 1 in /etc/my.cnf (or equivalent) and restart mysqld to use %s.",
			value, lib.MyName()))
	} else {
		logger.Println("performance_schema = ON check succeeds")
	}
}

// NewApp sets up the application given various parameters.
func NewApp(settings Settings) *App {
	logger.Println("app.NewApp()")
	app := new(App)

	anonymiser.Enable(settings.Anonymise) // not dynamic at the moment
	app.dbh = settings.Conn.Handle()

	status := global.NewStatus(app.dbh)
	variables := global.NewVariables(app.dbh)
	// Prior to setting up screen check that performance_schema is enabled.
	// On MariaDB this is not the default setting so it will confuse people.
	ensurePerformanceSchemaEnabled(variables)

	app.ctx = context.NewContext(status, variables)
	app.count = settings.Count
	app.finished = false

	app.stdout = settings.Stdout
	app.display = settings.Disp
	app.display.SetContext(app.ctx)
	app.SetHelp(false)

	if err := view.ValidateViews(app.dbh); err != nil {
		log.Fatal(err)
	}

	logger.Println("app.Setup() Setting the default view to:", settings.View)
	app.currentView.SetByName(settings.View) // if empty will use the default

	app.setupInstruments = setup_instruments.NewSetupInstruments(app.dbh)
	app.setupInstruments.EnableMonitoring()

	app.wi.SetWaitInterval(time.Second * time.Duration(settings.Interval))

	// setup to their initial types/values
	app.fsbi = fsbi.NewFileSummaryByInstance(variables)
	app.tlwsbt = new(tlwsbt.Object)
	app.ewsgben = new(ewsgben.Object)
	app.essgben = new(essgben.Object)

	app.SetWantRelativeStats(true)
	app.fixLatencySetting() // adjust to see ops/latency

	app.resetDBStatistics()

	return app
}

// Finished tells us if we have finished
func (app App) Finished() bool {
	return app.finished
}

// CollectAll collects all the stats together in one go
func (app *App) collectAll() {
	app.fsbi.Collect(app.dbh)
	app.tlwsbt.Collect(app.dbh)
	app.tiwsbt.Collect(app.dbh)
	app.users.Collect(app.dbh)
	app.essgben.Collect(app.dbh)
	app.ewsgben.Collect(app.dbh)
	app.memory.Collect(app.dbh)
}

func (app *App) WantRelativeStats() bool {
	return app.wantRelativeStats
}

// do a fresh collection of data and then update the initial values based on that.
func (app *App) resetDBStatistics() {
	app.collectAll()
	app.setInitialFromCurrent()
}

func (app *App) setInitialFromCurrent() {
	start := time.Now()
	app.fsbi.SetInitialFromCurrent()
	app.tlwsbt.SetInitialFromCurrent()
	app.tiwsbt.SetInitialFromCurrent()
	app.essgben.SetInitialFromCurrent()
	app.ewsgben.SetInitialFromCurrent()
	app.memory.SetInitialFromCurrent()
	logger.Println("app.setInitialFromCurrent() took", time.Duration(time.Since(start)).String())
}

// Collect the data we are looking at.
func (app *App) Collect() {
	start := time.Now()

	switch app.currentView.Get() {
	case view.ViewLatency, view.ViewOps:
		app.tiwsbt.Collect(app.dbh)
	case view.ViewIO:
		app.fsbi.Collect(app.dbh)
	case view.ViewLocks:
		app.tlwsbt.Collect(app.dbh)
	case view.ViewUsers:
		app.users.Collect(app.dbh)
	case view.ViewMutex:
		app.ewsgben.Collect(app.dbh)
	case view.ViewStages:
		app.essgben.Collect(app.dbh)
	case view.ViewMemory:
		app.memory.Collect(app.dbh)
	}
	app.wi.CollectedNow()
	logger.Println("app.Collect() took", time.Duration(time.Since(start)).String())
}

// SetHelp determines if we need to display help
func (app *App) SetHelp(newHelp bool) {
	app.help = newHelp

	app.display.ClearScreen()
}

// Help returns the internal help variable
func (app App) Help() bool {
	return app.help
}

// Display shows the output appropriate to the corresponding view and device
func (app *App) Display() {
	if app.help {
		app.display.DisplayHelp() // shouldn't get here if in --stdout mode
	} else {
		switch app.currentView.Get() {
		case view.ViewLatency, view.ViewOps:
			app.display.Display(app.tiwsbt)
		case view.ViewIO:
			app.display.Display(app.fsbi)
		case view.ViewLocks:
			app.display.Display(app.tlwsbt)
		case view.ViewUsers:
			app.display.Display(app.users)
		case view.ViewMutex:
			app.display.Display(app.ewsgben)
		case view.ViewStages:
			app.display.Display(app.essgben)
		case view.ViewMemory:
			app.display.Display(app.memory)
		}
	}
}

// fixLatencySetting() ensures the SetWantsLatency() value is
// correct. This needs to be done more cleanly.
func (app *App) fixLatencySetting() {
	if app.currentView.Get() == view.ViewLatency {
		app.tiwsbt.SetWantsLatency(true)
	}
	if app.currentView.Get() == view.ViewOps {
		app.tiwsbt.SetWantsLatency(false)
	}
}

// change to the previous display mode
func (app *App) displayPrevious() {
	app.currentView.SetPrev()
	app.fixLatencySetting()
	app.display.ClearScreen()
	app.Display()
}

// change to the next display mode
func (app *App) displayNext() {
	app.currentView.SetNext()
	app.fixLatencySetting()
	app.display.ClearScreen()
	app.Display()
}

// SetWantRelativeStats sets whether we want to see data that's relative or absolute
func (app *App) SetWantRelativeStats(want bool) {
	app.wantRelativeStats = want

	app.fsbi.SetWantRelativeStats(want)
	app.tlwsbt.SetWantRelativeStats(want)
	app.tiwsbt.SetWantRelativeStats(want)
	app.users.SetWantRelativeStats(want) // ignored
	app.essgben.SetWantRelativeStats(want)
	app.ewsgben.SetWantRelativeStats(want) // ignored
	app.memory.SetWantRelativeStats(want)
}

// Cleanup prepares  the application prior to shutting down
func (app *App) Cleanup() {
	app.display.Close()
	if app.dbh != nil {
		app.setupInstruments.RestoreConfiguration()
		_ = app.dbh.Close()
	}
	logger.Println("App.Cleanup completed")
}

// Run runs the application in a loop until we're ready to finish
func (app *App) Run() {
	logger.Println("app.Run()")

	app.sigChan = make(chan os.Signal, 10) // 10 entries
	signal.Notify(app.sigChan, syscall.SIGINT, syscall.SIGTERM)

	eventChan := app.display.EventChan()

	for !app.Finished() {
		select {
		case sig := <-app.sigChan:
			fmt.Println("Caught signal: ", sig)
			app.finished = true
		case <-app.wi.WaitNextPeriod():
			app.Collect()
			app.Display()
			if app.stdout {
				app.setInitialFromCurrent()
			}
		case inputEvent := <-eventChan:
			switch inputEvent.Type {
			case event.EventAnonymise:
				anonymiser.Enable(!anonymiser.Enabled()) // toggle current behaviour
			case event.EventFinished:
				app.finished = true
			case event.EventViewNext:
				app.displayNext()
			case event.EventViewPrev:
				app.displayPrevious()
			case event.EventDecreasePollTime:
				if app.wi.WaitInterval() > time.Second {
					app.wi.SetWaitInterval(app.wi.WaitInterval() - time.Second)
				}
			case event.EventIncreasePollTime:
				app.wi.SetWaitInterval(app.wi.WaitInterval() + time.Second)
			case event.EventHelp:
				app.SetHelp(!app.Help())
			case event.EventToggleWantRelative:
				app.SetWantRelativeStats(!app.WantRelativeStats())
				app.Display()
			case event.EventResetStatistics:
				app.resetDBStatistics()
				app.Display()
			case event.EventResizeScreen:
				width, height := inputEvent.Width, inputEvent.Height
				app.display.Resize(width, height)
				app.Display()
			case event.EventError:
				log.Fatalf("Quitting because of EventError error")
			}
		}
		// provide a hook to stop the application if the counter goes down to zero
		if app.stdout && app.count > 0 {
			app.count--
			if app.count == 0 {
				app.finished = true
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"strings"
	"testing"

	"knative.dev/networking/test"
)

var stableTests = map[string]func(t *testing.T){
	"basics":                       TestBasics,
	"basics/http2":                 TestBasicsHTTP2,
	"grpc":                         TestGRPC,
	"grpc/split":                   TestGRPCSplit,
	"headers/pre-split":            TestPreSplitSetHeaders,
	"headers/post-split":           TestPostSplitSetHeaders,
	"headers/probe":                TestProbeHeaders,
	"hosts/multiple":               TestMultipleHosts,
	"dispatch/path":                TestPath,
	"dispatch/percentage":          TestPercentage,
	"dispatch/path_and_percentage": TestPathAndPercentageSplit,
	"dispatch/rule":                TestRule,
	"retry":                        TestRetry,
	"timeout":                      TestTimeout,
	"tls":                          TestIngressTLS,
	"update":                       TestUpdate,
	"visibility":                   TestVisibility,
	"visibility/split":             TestVisibilitySplit,
	"visibility/path":              TestVisibilityPath,
	"ingressclass":                 TestIngressClass,
	"websocket":                    TestWebsocket,
	"websocket/split":              TestWebsocketSplit,
}

var betaTests = map[string]func(t *testing.T){
	// Add your conformance test for beta features
	"host-rewrite": TestRewriteHost,
	"headers/tags": TestTagHeaders,
}

var alphaTests = map[string]func(t *testing.T){
	// Add your conformance test for alpha features
	"httpoption": TestHTTPOption,
}

// RunConformance will run ingress conformance tests
//
// Depending on the options it may test alpha and beta features
func RunConformance(t *testing.T) {

	for name, test := range stableTests {
		t.Run(name, test)
	}

	skipTests := skipTests()

	// TODO(dprotaso) we'll need something more robust
	// in the long term that lets downstream
	// implementations to better select which tests
	// should be run -  selection across various
	// dimensions
	// ie. state - alpha, beta, ga
	// ie. requirement - must, should, may
	if test.NetworkingFlags.EnableBetaFeatures {
		for name, test := range betaTests {
			if _, ok := skipTests[name]; ok {
				t.Run(name, skipFunc)
				continue
			}
			t.Run(name, test)
		}
	}

	if test.NetworkingFlags.EnableAlphaFeatures {
		for name, test := range alphaTests {
			if _, ok := skipTests[name]; ok {
				t.Run(name, skipFunc)
				continue
			}
			t.Run(name, test)
		}
	}
}

var skipFunc = func(t *testing.T) {
	t.Skip("Skipping the test in skip-test flag")
}

func skipTests() map[string]struct{} {
	skipArray := strings.Split(test.NetworkingFlags.SkipTests, ",")
	skipMap := make(map[string]struct{}, len(skipArray))
	for _, name := range skipArray {
		skipMap[name] = struct{}{}
	}
	return skipMap
}
<commit_msg>Allow to skip non-alpha/beta test (#527)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"strings"
	"testing"

	"knative.dev/networking/test"
)

var stableTests = map[string]func(t *testing.T){
	"basics":                       TestBasics,
	"basics/http2":                 TestBasicsHTTP2,
	"grpc":                         TestGRPC,
	"grpc/split":                   TestGRPCSplit,
	"headers/pre-split":            TestPreSplitSetHeaders,
	"headers/post-split":           TestPostSplitSetHeaders,
	"headers/probe":                TestProbeHeaders,
	"hosts/multiple":               TestMultipleHosts,
	"dispatch/path":                TestPath,
	"dispatch/percentage":          TestPercentage,
	"dispatch/path_and_percentage": TestPathAndPercentageSplit,
	"dispatch/rule":                TestRule,
	"retry":                        TestRetry,
	"timeout":                      TestTimeout,
	"tls":                          TestIngressTLS,
	"update":                       TestUpdate,
	"visibility":                   TestVisibility,
	"visibility/split":             TestVisibilitySplit,
	"visibility/path":              TestVisibilityPath,
	"ingressclass":                 TestIngressClass,
	"websocket":                    TestWebsocket,
	"websocket/split":              TestWebsocketSplit,
}

var betaTests = map[string]func(t *testing.T){
	// Add your conformance test for beta features
	"host-rewrite": TestRewriteHost,
	"headers/tags": TestTagHeaders,
}

var alphaTests = map[string]func(t *testing.T){
	// Add your conformance test for alpha features
	"httpoption": TestHTTPOption,
}

// RunConformance will run ingress conformance tests
//
// Depending on the options it may test alpha and beta features
func RunConformance(t *testing.T) {
	skipTests := skipTests()

	for name, test := range stableTests {
		if _, ok := skipTests[name]; ok {
			t.Run(name, skipFunc)
			continue
		}
		t.Run(name, test)
	}

	// TODO(dprotaso) we'll need something more robust
	// in the long term that lets downstream
	// implementations to better select which tests
	// should be run -  selection across various
	// dimensions
	// ie. state - alpha, beta, ga
	// ie. requirement - must, should, may
	if test.NetworkingFlags.EnableBetaFeatures {
		for name, test := range betaTests {
			if _, ok := skipTests[name]; ok {
				t.Run(name, skipFunc)
				continue
			}
			t.Run(name, test)
		}
	}

	if test.NetworkingFlags.EnableAlphaFeatures {
		for name, test := range alphaTests {
			if _, ok := skipTests[name]; ok {
				t.Run(name, skipFunc)
				continue
			}
			t.Run(name, test)
		}
	}
}

var skipFunc = func(t *testing.T) {
	t.Skip("Skipping the test in skip-test flag")
}

func skipTests() map[string]struct{} {
	skipArray := strings.Split(test.NetworkingFlags.SkipTests, ",")
	skipMap := make(map[string]struct{}, len(skipArray))
	for _, name := range skipArray {
		skipMap[name] = struct{}{}
	}
	return skipMap
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"context"
	"encoding/json"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"

	"github.com/onsi/ginkgo"
)

var _ = ginkgo.Describe("[sig-architecture] PodTemplates", func() {
	f := framework.NewDefaultFramework("podtemplate")
	/*
	   Release : v1.18
	   Testname: ConfigMap lifecycle
	   Description: Attempt to create a ConfigMap. Patch the created ConfigMap. Fetching the ConfigMap MUST reflect changes.
	          By fetching all the ConfigMaps via a Label selector it MUST find the ConfigMap by it's static label and updated value. The ConfigMap must be deleted by Collection.
	*/
	framework.ConformanceIt("should run the lifecycle of PodTemplates", func() {
		testNamespaceName := f.Namespace.Name
		podTemplateName := "nginx-pod-template-" + string(uuid.NewUUID())

		// get a list of PodTemplates (in all namespaces to hit endpoint)
		podTemplateList, err := f.ClientSet.CoreV1().PodTemplates("").List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-static=true",
		})
		framework.ExpectNoError(err, "failed to list all PodTemplates")
		framework.ExpectEqual(len(podTemplateList.Items), 0, "unable to find templates")

		// create a PodTemplate
		_, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Create(context.TODO(), &v1.PodTemplate{
			ObjectMeta: metav1.ObjectMeta{
				Name: podTemplateName,
				Labels: map[string]string{
					"podtemplate-static": "true",
				},
			},
			Template: v1.PodTemplateSpec{
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{Name: "nginx", Image: "nginx"},
					},
				},
			},
		}, metav1.CreateOptions{})
		framework.ExpectNoError(err, "failed to create PodTemplate")

		// get template
		podTemplateRead, err := f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Get(context.TODO(), podTemplateName, metav1.GetOptions{})
		framework.ExpectNoError(err, "failed to get created PodTemplate")
		framework.ExpectEqual(podTemplateRead.ObjectMeta.Name, podTemplateName)

		// patch template
		podTemplatePatch, err := json.Marshal(map[string]interface{}{
			"metadata": map[string]interface{}{
				"labels": map[string]string{
					"podtemplate": "patched",
				},
			},
		})
		framework.ExpectNoError(err, "failed to marshal patch data")
		_, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Patch(context.TODO(), podTemplateName, types.StrategicMergePatchType, []byte(podTemplatePatch), metav1.PatchOptions{})
		framework.ExpectNoError(err, "failed to patch PodTemplate")

		// get template (ensure label is there)
		podTemplateRead, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Get(context.TODO(), podTemplateName, metav1.GetOptions{})
		framework.ExpectNoError(err, "failed to get PodTemplate")
		framework.ExpectEqual(podTemplateRead.ObjectMeta.Labels["podtemplate"], "patched", "failed to patch template, new label not found")

		// delete the PodTemplate
		err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Delete(context.TODO(), podTemplateName, &metav1.DeleteOptions{})
		framework.ExpectNoError(err, "failed to delete PodTemplate")

		// list the PodTemplates
		podTemplateList, err = f.ClientSet.CoreV1().PodTemplates("").List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-static=true",
		})
		framework.ExpectNoError(err, "failed to list PodTemplate")
		framework.ExpectEqual(len(podTemplateList.Items), 0, "PodTemplate list returned items, failed to delete PodTemplate")
	})
})
<commit_msg>Fix: metadata<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"context"
	"encoding/json"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"

	"github.com/onsi/ginkgo"
)

var _ = ginkgo.Describe("[sig-architecture] PodTemplates", func() {
	f := framework.NewDefaultFramework("podtemplate")
	/*
	   Release : v1.18
	   Testname: PodTemplate lifecycle
	   Description: Attempt to create a PodTemplate. Patch the created PodTemplate. Fetching the PodTemplate MUST reflect changes.
	          By fetching all the PodTemplates via a Label selector it MUST find the PodTemplate by it's static label and updated value. The PodTemplate must be deleted by Collection.
	*/
	framework.ConformanceIt("should run the lifecycle of PodTemplates", func() {
		testNamespaceName := f.Namespace.Name
		podTemplateName := "nginx-pod-template-" + string(uuid.NewUUID())

		// get a list of PodTemplates (in all namespaces to hit endpoint)
		podTemplateList, err := f.ClientSet.CoreV1().PodTemplates("").List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-static=true",
		})
		framework.ExpectNoError(err, "failed to list all PodTemplates")
		framework.ExpectEqual(len(podTemplateList.Items), 0, "unable to find templates")

		// create a PodTemplate
		_, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Create(context.TODO(), &v1.PodTemplate{
			ObjectMeta: metav1.ObjectMeta{
				Name: podTemplateName,
				Labels: map[string]string{
					"podtemplate-static": "true",
				},
			},
			Template: v1.PodTemplateSpec{
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{Name: "nginx", Image: "nginx"},
					},
				},
			},
		}, metav1.CreateOptions{})
		framework.ExpectNoError(err, "failed to create PodTemplate")

		// get template
		podTemplateRead, err := f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Get(context.TODO(), podTemplateName, metav1.GetOptions{})
		framework.ExpectNoError(err, "failed to get created PodTemplate")
		framework.ExpectEqual(podTemplateRead.ObjectMeta.Name, podTemplateName)

		// patch template
		podTemplatePatch, err := json.Marshal(map[string]interface{}{
			"metadata": map[string]interface{}{
				"labels": map[string]string{
					"podtemplate": "patched",
				},
			},
		})
		framework.ExpectNoError(err, "failed to marshal patch data")
		_, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Patch(context.TODO(), podTemplateName, types.StrategicMergePatchType, []byte(podTemplatePatch), metav1.PatchOptions{})
		framework.ExpectNoError(err, "failed to patch PodTemplate")

		// get template (ensure label is there)
		podTemplateRead, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Get(context.TODO(), podTemplateName, metav1.GetOptions{})
		framework.ExpectNoError(err, "failed to get PodTemplate")
		framework.ExpectEqual(podTemplateRead.ObjectMeta.Labels["podtemplate"], "patched", "failed to patch template, new label not found")

		// delete the PodTemplate
		err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Delete(context.TODO(), podTemplateName, &metav1.DeleteOptions{})
		framework.ExpectNoError(err, "failed to delete PodTemplate")

		// list the PodTemplates
		podTemplateList, err = f.ClientSet.CoreV1().PodTemplates("").List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-static=true",
		})
		framework.ExpectNoError(err, "failed to list PodTemplate")
		framework.ExpectEqual(len(podTemplateList.Items), 0, "PodTemplate list returned items, failed to delete PodTemplate")
	})
})
<|endoftext|>


<commit_before>package alert

import (
	"context"
	"fmt"
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"github.com/prometheus/common/model"

	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/disruption"
	helper "github.com/openshift/origin/test/extended/util/prometheus"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/upgrades"
)

// UpgradeTest runs verifies invariants regarding what alerts are allowed to fire
// during the upgrade process.
type UpgradeTest struct {
	url         string
	bearerToken string
	oc          *exutil.CLI
}

func (UpgradeTest) Name() string { return "check-for-alerts" }
func (UpgradeTest) DisplayName() string {
	return "[sig-arch] Check if alerts are firing during or after upgrade success"
}

// Setup creates parameters to query Prometheus
func (t *UpgradeTest) Setup(f *framework.Framework) {
	g.By("Setting up upgrade alert test")

	oc := exutil.NewCLIWithFramework(f)
	url, _, bearerToken, ok := helper.LocatePrometheus(oc)
	if !ok {
		framework.Failf("Prometheus could not be located on this cluster, failing test %s", t.Name())
	}
	t.url = url
	t.bearerToken = bearerToken
	t.oc = oc
	framework.Logf("Post-upgrade alert test setup complete")
}

// Test checks if alerts are firing at various points during upgrade.
// An alert firing during an upgrade is a high severity bug - it either points to a real issue in
// a dependency, or a failure of the component, and therefore must be fixed.
func (t *UpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	tolerateDuringSkew := exutil.TolerateVersionSkewInTests()
	firingAlertsWithBugs := helper.MetricConditions{
		{
			Selector: map[string]string{"alertname": "KubePodNotReady", "namespace": "openshift-kube-apiserver-operator"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "KubePodNotReady", "namespace": "openshift-kube-apiserver-operator"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "ClusterOperatorDegraded", "name": "openshift-apiserver"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "ClusterOperatorDown", "name": "authentication"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "ClusterOperatorDown", "name": "machine-config"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1955300",
		},
		{
			Selector: map[string]string{"alertname": "ClusterOperatorDegraded", "name": "authentication"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "KubeDaemonSetRolloutStuck"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1943667",
		},
		{
			Selector: map[string]string{"alertname": "KubeAPIErrorBudgetBurn"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1953798",
			Matches: func(_ *model.Sample) bool {
				return framework.ProviderIs("gce")
			},
		},
		{
			Selector: map[string]string{"alertname": "AggregatedAPIDown", "namespace": "default", "name": "v1beta1.metrics.k8s.io"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1970624",
			Matches: func(_ *model.Sample) bool {
				return framework.ProviderIs("gce")
			},
		},
		{
			// Should be removed one release after the attached bugzilla is fixed.
			Selector: map[string]string{"alertname": "HighlyAvailableWorkloadIncorrectlySpread", "namespace": "openshift-monitoring", "workload": "prometheus-k8s"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1949262",
		},
		{
			// Should be removed one release after the attached bugzilla is fixed.
			Selector: map[string]string{"alertname": "HighlyAvailableWorkloadIncorrectlySpread", "namespace": "openshift-monitoring", "workload": "alertmanager-main"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1955489",
		},
		{
			// Should be removed one release after the attached bugzilla is fixed, or after that bug is fixed in a backport to the previous minor.
			Selector: map[string]string{"alertname": "ExtremelyHighIndividualControlPlaneCPU"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1985073",
			Matches: func(_ *model.Sample) bool {
				return framework.ProviderIs("gce")
			},
		},
	}

	pendingAlertsWithBugs := helper.MetricConditions{
		{
			Selector: map[string]string{"alertname": "ClusterMonitoringOperatorReconciliationErrors"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1932624",
		},
		{
			Selector: map[string]string{"alertname": "KubeClientErrors"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1925698",
		},
	}
	allowedPendingAlerts := helper.MetricConditions{
		{
			Selector: map[string]string{"alertname": "etcdMemberCommunicationSlow"},
			Text:     "Excluded because it triggers during upgrade (detects ~5m of high latency immediately preceeding the end of the test), and we don't want to change the alert because it is correct",
		},
	}

	knownViolations := sets.NewString()
	unexpectedViolations := sets.NewString()
	unexpectedViolationsAsFlakes := sets.NewString()
	debug := sets.NewString()

	g.By("Checking for alerts")

	start := time.Now()

	// Block until upgrade is done
	g.By("Waiting for upgrade to finish before checking for alerts")
	<-done

	// Additonal delay after upgrade completion to allow pending alerts to settle
	g.By("Waiting before checking for alerts")
	time.Sleep(1 * time.Minute)

	ns := t.oc.SetupNamespace()
	execPod := exutil.CreateExecPodOrFail(t.oc.AdminKubeClient(), ns, "execpod")
	defer func() {
		t.oc.AdminKubeClient().CoreV1().Pods(ns).Delete(context.Background(), execPod.Name, *metav1.NewDeleteOptions(1))
	}()

	testDuration := time.Now().Sub(start).Truncate(time.Second)

	// Invariant: The watchdog alert should be firing continuously during the whole upgrade via the thanos
	// querier (which should have no gaps when it queries the individual stores). Allow zero or one changes
	// to the presence of this series (zero if data is preserved over upgrade, one if data is lost on upgrade).
	// This would not catch the alert stopping firing, but we catch that in other places and tests.
	watchdogQuery := fmt.Sprintf(`changes((max((ALERTS{alertstate="firing",alertname="Watchdog",severity="none"}) or (absent(ALERTS{alertstate="firing",alertname="Watchdog",severity="none"})*0)))[%s:1s]) > 1`, testDuration)
	result, err := helper.RunQuery(watchdogQuery, ns, execPod.Name, t.url, t.bearerToken)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to check watchdog alert over upgrade window")
	if len(result.Data.Result) > 0 {
		if result.Data.Result[0].Value <= 8 {
			unexpectedViolations.Insert(fmt.Sprintf("Watchdog alert had %s changes during the run, which may be a sign of a Prometheus outage in violation of the prometheus query SLO of 100%% uptime during upgrade", result.Data.Result[0].Value))
		} else {
			knownViolations.Insert(fmt.Sprintf("Watchdog alert had %s changes during the run, which may be a sign of a Prometheus outage in violation of the prometheus query SLO of 100%% uptime during upgrade, but is being tracked in https://bugzilla.redhat.com/show_bug.cgi?id=1949262 and is acceptable while that is open", result.Data.Result[0].Value))
		}
	}

	// Invariant: No non-info level alerts should have fired during the upgrade
	firingAlertQuery := fmt.Sprintf(`
sort_desc(
count_over_time(ALERTS{alertstate="firing",severity!="info",alertname!~"Watchdog|AlertmanagerReceiversNotConfigured"}[%[1]s:1s])
) > 0
`, testDuration)
	result, err = helper.RunQuery(firingAlertQuery, ns, execPod.Name, t.url, t.bearerToken)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to check firing alerts during upgrade")
	for _, series := range result.Data.Result {
		labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
		violation := fmt.Sprintf("alert %s fired for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
		if cause := firingAlertsWithBugs.Matches(series); cause != nil {
			knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
		} else {
			unexpectedViolations.Insert(violation)
		}
	}

	// Invariant: There should be no pending alerts 1m after the upgrade completes
	pendingAlertQuery := fmt.Sprintf(`
sort_desc(
  time() * ALERTS + 1
  -
  last_over_time((
    time() * ALERTS{alertname!~"Watchdog|AlertmanagerReceiversNotConfigured",alertstate="pending",severity!="info"}
    unless
    ALERTS offset 1s
  )[%[1]s:1s])
)
`, testDuration)
	result, err = helper.RunQuery(pendingAlertQuery, ns, execPod.Name, t.url, t.bearerToken)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to retrieve pending alerts after upgrade")
	for _, series := range result.Data.Result {
		labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
		violation := fmt.Sprintf("alert %s pending for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
		if cause := allowedPendingAlerts.Matches(series); cause != nil {
			debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text))
			continue
		}
		if cause := pendingAlertsWithBugs.Matches(series); cause != nil {
			knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
		} else {
			// treat pending errors as a flake right now because we are still trying to determine the scope
			// TODO: move this to unexpectedViolations later
			unexpectedViolationsAsFlakes.Insert(violation)
		}
	}

	if len(debug) > 0 {
		framework.Logf("Alerts were detected during upgrade which are allowed:\n\n%s", strings.Join(debug.List(), "\n"))
	}
	if len(unexpectedViolations) > 0 {
		if !tolerateDuringSkew {
			framework.Failf("Unexpected alerts fired or pending during the upgrade:\n\n%s", strings.Join(unexpectedViolations.List(), "\n"))
		}
	}
	if flakes := sets.NewString().Union(knownViolations).Union(unexpectedViolations).Union(unexpectedViolationsAsFlakes); len(flakes) > 0 {
		disruption.FrameworkFlakef(f, "Unexpected alert behavior during upgrade:\n\n%s", strings.Join(flakes.List(), "\n"))
	}
	framework.Logf("No alerts fired during upgrade")
}

// Teardown cleans up any remaining resources.
func (t *UpgradeTest) Teardown(f *framework.Framework) {
	// rely on the namespace deletion to clean up everything
}
<commit_msg>suppress NetworkPodsCrashLooping while we work out whether its a bug or tolerable<commit_after>package alert

import (
	"context"
	"fmt"
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"github.com/prometheus/common/model"

	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/disruption"
	helper "github.com/openshift/origin/test/extended/util/prometheus"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/upgrades"
)

// UpgradeTest runs verifies invariants regarding what alerts are allowed to fire
// during the upgrade process.
type UpgradeTest struct {
	url         string
	bearerToken string
	oc          *exutil.CLI
}

func (UpgradeTest) Name() string { return "check-for-alerts" }
func (UpgradeTest) DisplayName() string {
	return "[sig-arch] Check if alerts are firing during or after upgrade success"
}

// Setup creates parameters to query Prometheus
func (t *UpgradeTest) Setup(f *framework.Framework) {
	g.By("Setting up upgrade alert test")

	oc := exutil.NewCLIWithFramework(f)
	url, _, bearerToken, ok := helper.LocatePrometheus(oc)
	if !ok {
		framework.Failf("Prometheus could not be located on this cluster, failing test %s", t.Name())
	}
	t.url = url
	t.bearerToken = bearerToken
	t.oc = oc
	framework.Logf("Post-upgrade alert test setup complete")
}

// Test checks if alerts are firing at various points during upgrade.
// An alert firing during an upgrade is a high severity bug - it either points to a real issue in
// a dependency, or a failure of the component, and therefore must be fixed.
func (t *UpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	tolerateDuringSkew := exutil.TolerateVersionSkewInTests()
	firingAlertsWithBugs := helper.MetricConditions{
		{
			Selector: map[string]string{"alertname": "KubePodNotReady", "namespace": "openshift-kube-apiserver-operator"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "KubePodNotReady", "namespace": "openshift-kube-apiserver-operator"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "ClusterOperatorDegraded", "name": "openshift-apiserver"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "ClusterOperatorDown", "name": "authentication"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "ClusterOperatorDown", "name": "machine-config"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1955300",
		},
		{
			Selector: map[string]string{"alertname": "ClusterOperatorDegraded", "name": "authentication"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1939580",
		},
		{
			Selector: map[string]string{"alertname": "KubeDaemonSetRolloutStuck"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1943667",
		},
		{
			Selector: map[string]string{"alertname": "KubeAPIErrorBudgetBurn"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1953798",
			Matches: func(_ *model.Sample) bool {
				return framework.ProviderIs("gce")
			},
		},
		{
			Selector: map[string]string{"alertname": "AggregatedAPIDown", "namespace": "default", "name": "v1beta1.metrics.k8s.io"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1970624",
			Matches: func(_ *model.Sample) bool {
				return framework.ProviderIs("gce")
			},
		},
		{
			// Should be removed one release after the attached bugzilla is fixed.
			Selector: map[string]string{"alertname": "HighlyAvailableWorkloadIncorrectlySpread", "namespace": "openshift-monitoring", "workload": "prometheus-k8s"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1949262",
		},
		{
			// Should be removed one release after the attached bugzilla is fixed.
			Selector: map[string]string{"alertname": "HighlyAvailableWorkloadIncorrectlySpread", "namespace": "openshift-monitoring", "workload": "alertmanager-main"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1955489",
		},
		{
			// Should be removed one release after the attached bugzilla is fixed, or after that bug is fixed in a backport to the previous minor.
			Selector: map[string]string{"alertname": "ExtremelyHighIndividualControlPlaneCPU"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1985073",
			Matches: func(_ *model.Sample) bool {
				return framework.ProviderIs("gce")
			},
		},
	}

	pendingAlertsWithBugs := helper.MetricConditions{
		{
			Selector: map[string]string{"alertname": "ClusterMonitoringOperatorReconciliationErrors"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1932624",
		},
		{
			Selector: map[string]string{"alertname": "KubeClientErrors"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=1925698",
		},
		{
			Selector: map[string]string{"alertname": "NetworkPodsCrashLooping"},
			Text:     "https://bugzilla.redhat.com/show_bug.cgi?id=2009078",
		},
	}
	allowedPendingAlerts := helper.MetricConditions{
		{
			Selector: map[string]string{"alertname": "etcdMemberCommunicationSlow"},
			Text:     "Excluded because it triggers during upgrade (detects ~5m of high latency immediately preceeding the end of the test), and we don't want to change the alert because it is correct",
		},
	}

	knownViolations := sets.NewString()
	unexpectedViolations := sets.NewString()
	unexpectedViolationsAsFlakes := sets.NewString()
	debug := sets.NewString()

	g.By("Checking for alerts")

	start := time.Now()

	// Block until upgrade is done
	g.By("Waiting for upgrade to finish before checking for alerts")
	<-done

	// Additonal delay after upgrade completion to allow pending alerts to settle
	g.By("Waiting before checking for alerts")
	time.Sleep(1 * time.Minute)

	ns := t.oc.SetupNamespace()
	execPod := exutil.CreateExecPodOrFail(t.oc.AdminKubeClient(), ns, "execpod")
	defer func() {
		t.oc.AdminKubeClient().CoreV1().Pods(ns).Delete(context.Background(), execPod.Name, *metav1.NewDeleteOptions(1))
	}()

	testDuration := time.Now().Sub(start).Truncate(time.Second)

	// Invariant: The watchdog alert should be firing continuously during the whole upgrade via the thanos
	// querier (which should have no gaps when it queries the individual stores). Allow zero or one changes
	// to the presence of this series (zero if data is preserved over upgrade, one if data is lost on upgrade).
	// This would not catch the alert stopping firing, but we catch that in other places and tests.
	watchdogQuery := fmt.Sprintf(`changes((max((ALERTS{alertstate="firing",alertname="Watchdog",severity="none"}) or (absent(ALERTS{alertstate="firing",alertname="Watchdog",severity="none"})*0)))[%s:1s]) > 1`, testDuration)
	result, err := helper.RunQuery(watchdogQuery, ns, execPod.Name, t.url, t.bearerToken)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to check watchdog alert over upgrade window")
	if len(result.Data.Result) > 0 {
		if result.Data.Result[0].Value <= 8 {
			unexpectedViolations.Insert(fmt.Sprintf("Watchdog alert had %s changes during the run, which may be a sign of a Prometheus outage in violation of the prometheus query SLO of 100%% uptime during upgrade", result.Data.Result[0].Value))
		} else {
			knownViolations.Insert(fmt.Sprintf("Watchdog alert had %s changes during the run, which may be a sign of a Prometheus outage in violation of the prometheus query SLO of 100%% uptime during upgrade, but is being tracked in https://bugzilla.redhat.com/show_bug.cgi?id=1949262 and is acceptable while that is open", result.Data.Result[0].Value))
		}
	}

	// Invariant: No non-info level alerts should have fired during the upgrade
	firingAlertQuery := fmt.Sprintf(`
sort_desc(
count_over_time(ALERTS{alertstate="firing",severity!="info",alertname!~"Watchdog|AlertmanagerReceiversNotConfigured"}[%[1]s:1s])
) > 0
`, testDuration)
	result, err = helper.RunQuery(firingAlertQuery, ns, execPod.Name, t.url, t.bearerToken)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to check firing alerts during upgrade")
	for _, series := range result.Data.Result {
		labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
		violation := fmt.Sprintf("alert %s fired for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
		if cause := firingAlertsWithBugs.Matches(series); cause != nil {
			knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
		} else {
			unexpectedViolations.Insert(violation)
		}
	}

	// Invariant: There should be no pending alerts 1m after the upgrade completes
	pendingAlertQuery := fmt.Sprintf(`
sort_desc(
  time() * ALERTS + 1
  -
  last_over_time((
    time() * ALERTS{alertname!~"Watchdog|AlertmanagerReceiversNotConfigured",alertstate="pending",severity!="info"}
    unless
    ALERTS offset 1s
  )[%[1]s:1s])
)
`, testDuration)
	result, err = helper.RunQuery(pendingAlertQuery, ns, execPod.Name, t.url, t.bearerToken)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to retrieve pending alerts after upgrade")
	for _, series := range result.Data.Result {
		labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
		violation := fmt.Sprintf("alert %s pending for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
		if cause := allowedPendingAlerts.Matches(series); cause != nil {
			debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text))
			continue
		}
		if cause := pendingAlertsWithBugs.Matches(series); cause != nil {
			knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
		} else {
			// treat pending errors as a flake right now because we are still trying to determine the scope
			// TODO: move this to unexpectedViolations later
			unexpectedViolationsAsFlakes.Insert(violation)
		}
	}

	if len(debug) > 0 {
		framework.Logf("Alerts were detected during upgrade which are allowed:\n\n%s", strings.Join(debug.List(), "\n"))
	}
	if len(unexpectedViolations) > 0 {
		if !tolerateDuringSkew {
			framework.Failf("Unexpected alerts fired or pending during the upgrade:\n\n%s", strings.Join(unexpectedViolations.List(), "\n"))
		}
	}
	if flakes := sets.NewString().Union(knownViolations).Union(unexpectedViolations).Union(unexpectedViolationsAsFlakes); len(flakes) > 0 {
		disruption.FrameworkFlakef(f, "Unexpected alert behavior during upgrade:\n\n%s", strings.Join(flakes.List(), "\n"))
	}
	framework.Logf("No alerts fired during upgrade")
}

// Teardown cleans up any remaining resources.
func (t *UpgradeTest) Teardown(f *framework.Framework) {
	// rely on the namespace deletion to clean up everything
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2enode

import (
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
	"k8s.io/kubernetes/test/e2e/framework"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	"k8s.io/kubernetes/test/e2e_node/perf/workloads"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
)

// makeNodePerfPod returns a pod with the information provided from the workload.
func makeNodePerfPod(w workloads.NodePerfWorkload) *v1.Pod {
	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: fmt.Sprintf("%s-pod", w.Name()),
		},
		Spec: w.PodSpec(),
	}
}

func setKubeletConfig(f *framework.Framework, cfg *kubeletconfig.KubeletConfiguration) {
	if cfg != nil {
		framework.ExpectNoError(setKubeletConfiguration(f, cfg))
	}

	// Wait for the Kubelet to be ready.
	gomega.Eventually(func() bool {
		nodes, err := e2enode.TotalReady(f.ClientSet)
		framework.ExpectNoError(err)
		return nodes == 1
	}, time.Minute, time.Second).Should(gomega.BeTrue())
}

// Serial because the test updates kubelet configuration.
// Slow by design.
var _ = SIGDescribe("Node Performance Testing [Serial] [Slow] [Flaky]", func() {
	f := framework.NewDefaultFramework("node-performance-testing")
	var (
		wl     workloads.NodePerfWorkload
		oldCfg *kubeletconfig.KubeletConfiguration
		newCfg *kubeletconfig.KubeletConfiguration
		pod    *v1.Pod
	)
	ginkgo.JustBeforeEach(func() {
		err := wl.PreTestExec()
		framework.ExpectNoError(err)
		oldCfg, err = getCurrentKubeletConfig()
		framework.ExpectNoError(err)
		newCfg, err = wl.KubeletConfig(oldCfg)
		framework.ExpectNoError(err)
		setKubeletConfig(f, newCfg)
	})

	cleanup := func() {
		gp := int64(0)
		delOpts := metav1.DeleteOptions{
			GracePeriodSeconds: &gp,
		}
		f.PodClient().DeleteSync(pod.Name, delOpts, framework.DefaultPodDeletionTimeout)

		// We are going to give some more time for the CPU manager to do any clean
		// up it needs to do now that the pod has been deleted. Otherwise we may
		// run into a data race condition in which the PostTestExec function
		// deletes the CPU manager's checkpoint file while the CPU manager is still
		// doing work and we end with a new checkpoint file after PosttestExec has
		// finished. This issues would result in the kubelet panicking after we try
		// and set the kubelet config.
		time.Sleep(15 * time.Second)
		ginkgo.By("running the post test exec from the workload")
		err := wl.PostTestExec()
		framework.ExpectNoError(err)
		setKubeletConfig(f, oldCfg)
	}

	runWorkload := func() {
		ginkgo.By("running the workload and waiting for success")
		// Make the pod for the workload.
		pod = makeNodePerfPod(wl)
		// Create the pod.
		pod = f.PodClient().CreateSync(pod)
		// Wait for pod success.
		f.PodClient().WaitForSuccess(pod.Name, wl.Timeout())
		podLogs, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name)
		framework.ExpectNoError(err)
		perf, err := wl.ExtractPerformanceFromLogs(podLogs)
		framework.ExpectNoError(err)
		framework.Logf("Time to complete workload %s: %v", wl.Name(), perf)
	}

	ginkgo.Context("Run node performance testing with pre-defined workloads", func() {
		ginkgo.BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[0]
		})
		ginkgo.It("NAS parallel benchmark (NPB) suite - Integer Sort (IS) workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
	ginkgo.Context("Run node performance testing with pre-defined workloads", func() {
		ginkgo.BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[1]
		})
		ginkgo.It("NAS parallel benchmark (NPB) suite - Embarrassingly Parallel (EP) workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
	ginkgo.Context("Run node performance testing with pre-defined workloads", func() {
		ginkgo.BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[2]
		})
		ginkgo.It("TensorFlow workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
})
<commit_msg>prepare node performance tests to move to separate job<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2enode

import (
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
	"k8s.io/kubernetes/test/e2e/framework"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
	"k8s.io/kubernetes/test/e2e_node/perf/workloads"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
)

// makeNodePerfPod returns a pod with the information provided from the workload.
func makeNodePerfPod(w workloads.NodePerfWorkload) *v1.Pod {
	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: fmt.Sprintf("%s-pod", w.Name()),
		},
		Spec: w.PodSpec(),
	}
}

func setKubeletConfig(f *framework.Framework, cfg *kubeletconfig.KubeletConfiguration) {
	if cfg != nil {
		framework.ExpectNoError(setKubeletConfiguration(f, cfg))
	}

	// Wait for the Kubelet to be ready.
	gomega.Eventually(func() bool {
		nodes, err := e2enode.TotalReady(f.ClientSet)
		framework.ExpectNoError(err)
		return nodes == 1
	}, time.Minute, time.Second).Should(gomega.BeTrue())
}

// Serial because the test updates kubelet configuration.
// Slow by design.
var _ = SIGDescribe("Node Performance Testing [Serial] [Slow]", func() {
	f := framework.NewDefaultFramework("node-performance-testing")
	var (
		wl     workloads.NodePerfWorkload
		oldCfg *kubeletconfig.KubeletConfiguration
		newCfg *kubeletconfig.KubeletConfiguration
		pod    *v1.Pod
	)
	ginkgo.JustBeforeEach(func() {
		err := wl.PreTestExec()
		framework.ExpectNoError(err)
		oldCfg, err = getCurrentKubeletConfig()
		framework.ExpectNoError(err)
		newCfg, err = wl.KubeletConfig(oldCfg)
		framework.ExpectNoError(err)
		setKubeletConfig(f, newCfg)
	})

	cleanup := func() {
		gp := int64(0)
		delOpts := metav1.DeleteOptions{
			GracePeriodSeconds: &gp,
		}
		f.PodClient().DeleteSync(pod.Name, delOpts, framework.DefaultPodDeletionTimeout)

		// We are going to give some more time for the CPU manager to do any clean
		// up it needs to do now that the pod has been deleted. Otherwise we may
		// run into a data race condition in which the PostTestExec function
		// deletes the CPU manager's checkpoint file while the CPU manager is still
		// doing work and we end with a new checkpoint file after PosttestExec has
		// finished. This issues would result in the kubelet panicking after we try
		// and set the kubelet config.
		time.Sleep(15 * time.Second)
		ginkgo.By("running the post test exec from the workload")
		err := wl.PostTestExec()
		framework.ExpectNoError(err)
		setKubeletConfig(f, oldCfg)
	}

	runWorkload := func() {
		ginkgo.By("running the workload and waiting for success")
		// Make the pod for the workload.
		pod = makeNodePerfPod(wl)
		// Create the pod.
		pod = f.PodClient().CreateSync(pod)
		// Wait for pod success.
		f.PodClient().WaitForSuccess(pod.Name, wl.Timeout())
		podLogs, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name)
		framework.ExpectNoError(err)
		perf, err := wl.ExtractPerformanceFromLogs(podLogs)
		framework.ExpectNoError(err)
		framework.Logf("Time to complete workload %s: %v", wl.Name(), perf)
	}

	ginkgo.BeforeEach(func() {
		ginkgo.By("ensure environment has enough CPU + Memory to run")
		minimumRequiredCPU := resource.MustParse("15")
		minimumRequiredMemory := resource.MustParse("48Gi")
		localNodeCap := getLocalNode(f).Status.Allocatable
		cpuCap := localNodeCap[v1.ResourceCPU]
		memCap := localNodeCap[v1.ResourceMemory]
		if cpuCap.Cmp(minimumRequiredCPU) == -1 {
			e2eskipper.Skipf("Skipping Node Performance Tests due to lack of CPU. Required %v is less than capacity %v.", minimumRequiredCPU, cpuCap)
		}
		if memCap.Cmp(minimumRequiredMemory) == -1 {
			e2eskipper.Skipf("Skipping Node Performance Tests due to lack of memory. Required %v is less than capacity %v.", minimumRequiredMemory, memCap)
		}
	})

	ginkgo.Context("Run node performance testing with pre-defined workloads", func() {
		ginkgo.BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[0]
		})
		ginkgo.It("NAS parallel benchmark (NPB) suite - Integer Sort (IS) workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
	ginkgo.Context("Run node performance testing with pre-defined workloads", func() {
		ginkgo.BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[1]
		})
		ginkgo.It("NAS parallel benchmark (NPB) suite - Embarrassingly Parallel (EP) workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
	ginkgo.Context("Run node performance testing with pre-defined workloads", func() {
		ginkgo.BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[2]
		})
		ginkgo.It("TensorFlow workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
})
<|endoftext|>