<commit_before>package golang

import (
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/ellcrys/util"
	"github.com/ncodes/cocoon/core/common"
	"github.com/ncodes/cocoon/core/stubs/golang/config"
	"github.com/ncodes/cocoon/core/stubs/golang/proto"
	"github.com/ncodes/cocoon/core/types"
	"github.com/op/go-logging"
	cmap "github.com/orcaman/concurrent-map"
	"google.golang.org/grpc"
)

var (

	// serverPort to bind to
	serverPort = util.Env("COCOON_CODE_PORT", "8000")

	// stub logger
	log *logging.Logger

	// default running server
	defaultServer *stubServer

	// stop channel to stop the server/cocoon code
	serverDone chan bool

	// The default ledger is the global ledger.
	defaultLedger = GetGlobalLedgerName()

	// txChannels holds the channels to send transaction responses to
	txRespChannels = cmap.New()

	// ErrAlreadyExist represents an error about an already existing resource
	ErrAlreadyExist = fmt.Errorf("already exists")

	// ErrNotConnected represents an error about the cocoon code not
	// having an active connection with the connector.
	ErrNotConnected = fmt.Errorf("not connected to the connector")

	// Flag to help tell whether cocoon code is running
	running = false

	// Number of transactions per block
	txPerBlock = util.Env("TX_PER_BLOCK", "100")

	// Time between block creation (seconds)
	blockCreationInterval = util.Env("BLOCK_CREATION_INT", "5")

	// blockMaker creates a collection of blockchain transactions at interval
	blockMaker *BlockMaker

	// The cocoon code currently running
	ccode CocoonCode
)

// GetLogger returns the stubs logger.
func GetLogger() *logging.Logger {
	return log
}

// SetDebugLevel sets the default logger debug level
func SetDebugLevel(level logging.Level) {
	logging.SetLevel(level, log.Module)
}

func init() {
	defaultServer = new(stubServer)
	config.ConfigureLogger()
	log = logging.MustGetLogger("ccode.stub")
}

// GetGlobalLedgerName returns the name of the global ledger
func GetGlobalLedgerName() string {
	return types.GetGlobalLedgerName()
}

// Run starts the stub server, takes a cocoon code and attempts to initialize it..
func Run(cc CocoonCode) {

	if running {
		log.Info("cocoon code is already running")
		return
	}

	serverDone = make(chan bool, 1)

	lis, err := net.Listen("tcp", fmt.Sprintf(":%s", serverPort))
	if err != nil {
		log.Fatalf("failed to listen on port=%s", serverPort)
	}

	log.Infof("Started stub service at port=%s", serverPort)
	server := grpc.NewServer()
	proto.RegisterStubServer(server, defaultServer)
	go server.Serve(lis)

	intTxPerBlock, _ := strconv.Atoi(txPerBlock)
	intBlkCreationInt, _ := strconv.Atoi(blockCreationInterval)
	blockMaker = NewBlockMaker(intTxPerBlock, time.Duration(intBlkCreationInt)*time.Second)
	go blockMaker.Begin(blockCommitter)

	ccode = cc

	// run Init() after 1 second to give time for connector to connect
	time.AfterFunc(1*time.Second, func() {
		if err = cc.Init(); err != nil {
			log.Errorf("cocoode Init() returned error: %s", err)
			Stop(1)
		} else {
			running = true
		}
	})

	<-serverDone
	log.Info("Cocoon code stopped")
	os.Exit(0)
}

// blockCommit creates a PUT operation which adds one or many
// transactions to the store and blockchain and returns the block if
// if succeed or error if otherwise.
func blockCommitter(entries []*Entry) interface{} {

	txs := make([]*types.Transaction, len(entries))
	for i, e := range entries {
		txs[i] = e.Tx
	}

	ledgerName := entries[0].Tx.Ledger
	txsJSON, _ := util.ToJSON(txs)

	var respCh = make(chan *proto.Tx)

	txID := util.UUID4()
	err := sendTx(&proto.Tx{
		Id:     txID,
		Invoke: true,
		Name:   types.TxPut,
		Params: []string{ledgerName},
		Body:   txsJSON,
	}, respCh)
	if err != nil {
		return fmt.Errorf("failed to put block transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return err
	}

	if resp.Status != 200 {
		return fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var block types.Block
	if err = util.FromJSON(resp.Body, &block); err != nil {
		return fmt.Errorf("failed to unmarshall response data")
	}

	return &block
}

// sendTx sends a transaction to the cocoon code
// and saves the response channel. The response channel will
// be passed a response when it is available in the Transact loop.
func sendTx(tx *proto.Tx, respCh chan *proto.Tx) error {
	txRespChannels.Set(tx.GetId(), respCh)
	if err := defaultServer.stream.Send(tx); err != nil {
		txRespChannels.Remove(tx.GetId())
		log.Errorf("failed to send transaction [%s] to connector. %s", tx.GetId(), err)
		return err
	}
	log.Debugf("Successfully sent transaction [%s] to connector", tx.GetId())
	return nil
}

// Stop stub and cocoon code
func Stop(exitCode int) {
	defaultServer.stream = nil
	serverDone <- true
	log.Info("Cocoon code exiting with exit code %d", exitCode)
	os.Exit(exitCode)
}

// isConnected checks if connection with the connector
// is active.
func isConnected() bool {
	return defaultServer.stream != nil
}

// SetDefaultLedger sets the default ledger
func SetDefaultLedger(name string) error {
	_, err := GetLedger(name)
	if err != nil {
		return err
	}
	defaultLedger = name
	return nil
}

// GetDefaultLedgerName returns the name of the default ledger.
func GetDefaultLedgerName() string {
	return defaultLedger
}

// CreateLedger creates a new ledger by sending an
// invoke transaction (TxCreateLedger) to the connector.
// If chained is set to true, a blockchain is created and subsequent
// PUT operations to the ledger will be included in the types. Otherwise,
// PUT operations will only be incuded in the types.
func CreateLedger(name string, chained, public bool) (*types.Ledger, error) {

	if name == GetGlobalLedgerName() {
		return nil, fmt.Errorf("cannot use the same name as the global ledger")
	} else if !common.IsValidResName(name) {
		return nil, fmt.Errorf("invalid ledger name")
	}

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)

	txID := util.UUID4()
	err := sendTx(&proto.Tx{
		Id:     txID,
		Invoke: true,
		Name:   types.TxCreateLedger,
		Params: []string{name, fmt.Sprintf("%t", chained), fmt.Sprintf("%t", public)},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to create ledger. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}

	if resp.Status != 200 {
		err = fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
		if strings.Contains(err.Error(), "already exists") {
			return nil, ErrAlreadyExist
		}
		return nil, err
	}

	var ledger types.Ledger
	if err = util.FromJSON(resp.Body, &ledger); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &ledger, nil
}

// GetLedger fetches a ledger
func GetLedger(ledgerName string) (*types.Ledger, error) {

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)

	txID := util.UUID4()
	err := sendTx(&proto.Tx{
		Id:     txID,
		Invoke: true,
		Name:   types.TxGetLedger,
		Params: []string{ledgerName},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to get ledger. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var ledger types.Ledger
	if err = util.FromJSON(resp.Body, &ledger); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &ledger, nil
}

// PutIn adds a new transaction to a ledger
func PutIn(ledgerName string, key string, value []byte) (*types.Transaction, error) {

	start := time.Now()

	if !isConnected() {
		return nil, ErrNotConnected
	}

	ledger, err := GetLedger(ledgerName)
	if err != nil {
		return nil, err
	}

	tx := &types.Transaction{
		ID:        util.UUID4(),
		Ledger:    ledger.Name,
		Key:       key,
		Value:     string(value),
		CreatedAt: time.Now().Unix(),
	}
	tx.Hash = tx.MakeHash()

	if ledger.Chained {
		respChan := make(chan interface{})
		blockMaker.Add(&Entry{
			Tx:       tx,
			RespChan: respChan,
		})
		result := <-respChan

		log.Debug("Put(): Time taken: ", time.Since(start))

		switch v := result.(type) {
		case error:
			return nil, v
		case *types.Block:
			tx.Block = v
			return tx, err
		default:
			return nil, fmt.Errorf("unexpected response %s", err)
		}
	}

	txJSON, _ := util.ToJSON([]*types.Transaction{tx})

	var respCh = make(chan *proto.Tx)
	err = sendTx(&proto.Tx{
		Id:     util.UUID4(),
		Invoke: true,
		Name:   types.TxPut,
		Params: []string{ledgerName},
		Body:   txJSON,
	}, respCh)

	if err != nil {
		return nil, fmt.Errorf("failed to put transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	log.Debug("Put(): Time taken: ", time.Since(start))

	return tx, nil
}

// Put adds a new transaction into the default ledger
func Put(key string, value []byte) (*types.Transaction, error) {
	return PutIn(GetDefaultLedgerName(), key, value)
}

// GetFrom returns a transaction by its key and the ledger it belongs to
func GetFrom(ledgerName, key string) (*types.Transaction, error) {

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)
	err := sendTx(&proto.Tx{
		Id:     util.UUID4(),
		Invoke: true,
		Name:   types.TxGet,
		Params: []string{ledgerName, key},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to get transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var tx types.Transaction
	if err = util.FromJSON(resp.Body, &tx); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &tx, nil
}

// Get returns a transaction that belongs to the default legder by its key.
func Get(key string) (*types.Transaction, error) {
	return GetFrom(GetDefaultLedgerName(), key)
}

// GetByIDFrom returns a transaction by its id and the ledger it belongs to
func GetByIDFrom(ledgerName, id string) (*types.Transaction, error) {

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)
	err := sendTx(&proto.Tx{
		Id:     util.UUID4(),
		Invoke: true,
		Name:   types.TxGetByID,
		Params: []string{ledgerName, id},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to get transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var tx types.Transaction
	if err = util.FromJSON(resp.Body, &tx); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &tx, nil
}

// GetByID returns a transaction that belongs to the default legder by its id.
func GetByID(id string) (*types.Transaction, error) {
	return GetByIDFrom(GetDefaultLedgerName(), id)
}

// GetBlockFrom returns a block from a ledger by its block id
func GetBlockFrom(ledgerName, id string) (*types.Block, error) {

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)
	err := sendTx(&proto.Tx{
		Id:     util.UUID4(),
		Invoke: true,
		Name:   types.TxGetBlockByID,
		Params: []string{ledgerName, id},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to get transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var blk types.Block
	if err = util.FromJSON(resp.Body, &blk); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &blk, nil
}

// GetBlock returns a block from the default ledger by its block id
func GetBlock(id string) (*types.Block, error) {
	return GetBlockFrom(GetDefaultLedgerName(), id)
}
<commit_msg>fix ledger name verification<commit_after>package golang

import (
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/ellcrys/util"
	"github.com/ncodes/cocoon/core/common"
	"github.com/ncodes/cocoon/core/stubs/golang/config"
	"github.com/ncodes/cocoon/core/stubs/golang/proto"
	"github.com/ncodes/cocoon/core/types"
	"github.com/op/go-logging"
	cmap "github.com/orcaman/concurrent-map"
	"google.golang.org/grpc"
)

var (

	// serverPort to bind to
	serverPort = util.Env("COCOON_CODE_PORT", "8000")

	// stub logger
	log *logging.Logger

	// default running server
	defaultServer *stubServer

	// stop channel to stop the server/cocoon code
	serverDone chan bool

	// The default ledger is the global ledger.
	defaultLedger = GetGlobalLedgerName()

	// txChannels holds the channels to send transaction responses to
	txRespChannels = cmap.New()

	// ErrAlreadyExist represents an error about an already existing resource
	ErrAlreadyExist = fmt.Errorf("already exists")

	// ErrNotConnected represents an error about the cocoon code not
	// having an active connection with the connector.
	ErrNotConnected = fmt.Errorf("not connected to the connector")

	// Flag to help tell whether cocoon code is running
	running = false

	// Number of transactions per block
	txPerBlock = util.Env("TX_PER_BLOCK", "100")

	// Time between block creation (seconds)
	blockCreationInterval = util.Env("BLOCK_CREATION_INT", "5")

	// blockMaker creates a collection of blockchain transactions at interval
	blockMaker *BlockMaker

	// The cocoon code currently running
	ccode CocoonCode
)

// GetLogger returns the stubs logger.
func GetLogger() *logging.Logger {
	return log
}

// SetDebugLevel sets the default logger debug level
func SetDebugLevel(level logging.Level) {
	logging.SetLevel(level, log.Module)
}

func init() {
	defaultServer = new(stubServer)
	config.ConfigureLogger()
	log = logging.MustGetLogger("ccode.stub")
}

// GetGlobalLedgerName returns the name of the global ledger
func GetGlobalLedgerName() string {
	return types.GetGlobalLedgerName()
}

// Run starts the stub server, takes a cocoon code and attempts to initialize it..
func Run(cc CocoonCode) {

	if running {
		log.Info("cocoon code is already running")
		return
	}

	serverDone = make(chan bool, 1)

	lis, err := net.Listen("tcp", fmt.Sprintf(":%s", serverPort))
	if err != nil {
		log.Fatalf("failed to listen on port=%s", serverPort)
	}

	log.Infof("Started stub service at port=%s", serverPort)
	server := grpc.NewServer()
	proto.RegisterStubServer(server, defaultServer)
	go server.Serve(lis)

	intTxPerBlock, _ := strconv.Atoi(txPerBlock)
	intBlkCreationInt, _ := strconv.Atoi(blockCreationInterval)
	blockMaker = NewBlockMaker(intTxPerBlock, time.Duration(intBlkCreationInt)*time.Second)
	go blockMaker.Begin(blockCommitter)

	ccode = cc

	// run Init() after 1 second to give time for connector to connect
	time.AfterFunc(1*time.Second, func() {
		if err = cc.Init(); err != nil {
			log.Errorf("cocoode Init() returned error: %s", err)
			Stop(1)
		} else {
			running = true
		}
	})

	<-serverDone
	log.Info("Cocoon code stopped")
	os.Exit(0)
}

// blockCommit creates a PUT operation which adds one or many
// transactions to the store and blockchain and returns the block if
// if succeed or error if otherwise.
func blockCommitter(entries []*Entry) interface{} {

	txs := make([]*types.Transaction, len(entries))
	for i, e := range entries {
		txs[i] = e.Tx
	}

	ledgerName := entries[0].Tx.Ledger
	txsJSON, _ := util.ToJSON(txs)

	var respCh = make(chan *proto.Tx)

	txID := util.UUID4()
	err := sendTx(&proto.Tx{
		Id:     txID,
		Invoke: true,
		Name:   types.TxPut,
		Params: []string{ledgerName},
		Body:   txsJSON,
	}, respCh)
	if err != nil {
		return fmt.Errorf("failed to put block transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return err
	}

	if resp.Status != 200 {
		return fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var block types.Block
	if err = util.FromJSON(resp.Body, &block); err != nil {
		return fmt.Errorf("failed to unmarshall response data")
	}

	return &block
}

// sendTx sends a transaction to the cocoon code
// and saves the response channel. The response channel will
// be passed a response when it is available in the Transact loop.
func sendTx(tx *proto.Tx, respCh chan *proto.Tx) error {
	txRespChannels.Set(tx.GetId(), respCh)
	if err := defaultServer.stream.Send(tx); err != nil {
		txRespChannels.Remove(tx.GetId())
		log.Errorf("failed to send transaction [%s] to connector. %s", tx.GetId(), err)
		return err
	}
	log.Debugf("Successfully sent transaction [%s] to connector", tx.GetId())
	return nil
}

// Stop stub and cocoon code
func Stop(exitCode int) {
	defaultServer.stream = nil
	serverDone <- true
	log.Info("Cocoon code exiting with exit code %d", exitCode)
	os.Exit(exitCode)
}

// isConnected checks if connection with the connector
// is active.
func isConnected() bool {
	return defaultServer.stream != nil
}

// SetDefaultLedger sets the default ledger
func SetDefaultLedger(name string) error {
	_, err := GetLedger(name)
	if err != nil {
		return err
	}
	defaultLedger = name
	return nil
}

// GetDefaultLedgerName returns the name of the default ledger.
func GetDefaultLedgerName() string {
	return defaultLedger
}

// CreateLedger creates a new ledger by sending an
// invoke transaction (TxCreateLedger) to the connector.
// If chained is set to true, a blockchain is created and subsequent
// PUT operations to the ledger will be included in the types. Otherwise,
// PUT operations will only be incuded in the types.
func CreateLedger(name string, chained, public bool) (*types.Ledger, error) {

	if util.Sha256(name) == GetGlobalLedgerName() {
		return nil, fmt.Errorf("cannot use a reserved name")
	} else if !common.IsValidResName(name) {
		return nil, fmt.Errorf("invalid ledger name")
	}

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)

	txID := util.UUID4()
	err := sendTx(&proto.Tx{
		Id:     txID,
		Invoke: true,
		Name:   types.TxCreateLedger,
		Params: []string{name, fmt.Sprintf("%t", chained), fmt.Sprintf("%t", public)},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to create ledger. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}

	if resp.Status != 200 {
		err = fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
		if strings.Contains(err.Error(), "already exists") {
			return nil, ErrAlreadyExist
		}
		return nil, err
	}

	var ledger types.Ledger
	if err = util.FromJSON(resp.Body, &ledger); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &ledger, nil
}

// GetLedger fetches a ledger
func GetLedger(ledgerName string) (*types.Ledger, error) {

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)

	txID := util.UUID4()
	err := sendTx(&proto.Tx{
		Id:     txID,
		Invoke: true,
		Name:   types.TxGetLedger,
		Params: []string{ledgerName},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to get ledger. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var ledger types.Ledger
	if err = util.FromJSON(resp.Body, &ledger); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &ledger, nil
}

// PutIn adds a new transaction to a ledger
func PutIn(ledgerName string, key string, value []byte) (*types.Transaction, error) {

	start := time.Now()

	if !isConnected() {
		return nil, ErrNotConnected
	}

	ledger, err := GetLedger(ledgerName)
	if err != nil {
		return nil, err
	}

	tx := &types.Transaction{
		ID:        util.UUID4(),
		Ledger:    ledger.Name,
		Key:       key,
		Value:     string(value),
		CreatedAt: time.Now().Unix(),
	}
	tx.Hash = tx.MakeHash()

	if ledger.Chained {
		respChan := make(chan interface{})
		blockMaker.Add(&Entry{
			Tx:       tx,
			RespChan: respChan,
		})
		result := <-respChan

		log.Debug("Put(): Time taken: ", time.Since(start))

		switch v := result.(type) {
		case error:
			return nil, v
		case *types.Block:
			tx.Block = v
			return tx, err
		default:
			return nil, fmt.Errorf("unexpected response %s", err)
		}
	}

	txJSON, _ := util.ToJSON([]*types.Transaction{tx})

	var respCh = make(chan *proto.Tx)
	err = sendTx(&proto.Tx{
		Id:     util.UUID4(),
		Invoke: true,
		Name:   types.TxPut,
		Params: []string{ledgerName},
		Body:   txJSON,
	}, respCh)

	if err != nil {
		return nil, fmt.Errorf("failed to put transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	log.Debug("Put(): Time taken: ", time.Since(start))

	return tx, nil
}

// Put adds a new transaction into the default ledger
func Put(key string, value []byte) (*types.Transaction, error) {
	return PutIn(GetDefaultLedgerName(), key, value)
}

// GetFrom returns a transaction by its key and the ledger it belongs to
func GetFrom(ledgerName, key string) (*types.Transaction, error) {

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)
	err := sendTx(&proto.Tx{
		Id:     util.UUID4(),
		Invoke: true,
		Name:   types.TxGet,
		Params: []string{ledgerName, key},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to get transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var tx types.Transaction
	if err = util.FromJSON(resp.Body, &tx); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &tx, nil
}

// Get returns a transaction that belongs to the default legder by its key.
func Get(key string) (*types.Transaction, error) {
	return GetFrom(GetDefaultLedgerName(), key)
}

// GetByIDFrom returns a transaction by its id and the ledger it belongs to
func GetByIDFrom(ledgerName, id string) (*types.Transaction, error) {

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)
	err := sendTx(&proto.Tx{
		Id:     util.UUID4(),
		Invoke: true,
		Name:   types.TxGetByID,
		Params: []string{ledgerName, id},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to get transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var tx types.Transaction
	if err = util.FromJSON(resp.Body, &tx); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &tx, nil
}

// GetByID returns a transaction that belongs to the default legder by its id.
func GetByID(id string) (*types.Transaction, error) {
	return GetByIDFrom(GetDefaultLedgerName(), id)
}

// GetBlockFrom returns a block from a ledger by its block id
func GetBlockFrom(ledgerName, id string) (*types.Block, error) {

	if !isConnected() {
		return nil, ErrNotConnected
	}

	var respCh = make(chan *proto.Tx)
	err := sendTx(&proto.Tx{
		Id:     util.UUID4(),
		Invoke: true,
		Name:   types.TxGetBlockByID,
		Params: []string{ledgerName, id},
	}, respCh)
	if err != nil {
		return nil, fmt.Errorf("failed to get transaction. %s", err)
	}

	resp, err := common.AwaitTxChan(respCh)
	if err != nil {
		return nil, err
	}
	if resp.Status != 200 {
		return nil, fmt.Errorf("%s", common.StripRPCErrorPrefix(resp.Body))
	}

	var blk types.Block
	if err = util.FromJSON(resp.Body, &blk); err != nil {
		return nil, fmt.Errorf("failed to unmarshall response data")
	}

	return &blk, nil
}

// GetBlock returns a block from the default ledger by its block id
func GetBlock(id string) (*types.Block, error) {
	return GetBlockFrom(GetDefaultLedgerName(), id)
}
<|endoftext|>


<commit_before>package types

import (
	"bytes"
	"crypto/ecdsa"
	"fmt"
	"math/big"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/crypto/secp256k1"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/rlp"
)

func IsContractAddr(addr []byte) bool {
	return len(addr) == 0
}

type Transaction struct {
	AccountNonce uint64
	Price        *big.Int
	GasLimit     *big.Int
	Recipient    []byte
	Amount       *big.Int
	Payload      []byte
	V            uint64
	R, S         []byte
}

func NewContractCreationTx(Amount, gasAmount, price *big.Int, data []byte) *Transaction {
	return NewTransactionMessage(nil, Amount, gasAmount, price, data)
}

func NewTransactionMessage(to []byte, Amount, gasAmount, price *big.Int, data []byte) *Transaction {
	return &Transaction{Recipient: to, Amount: Amount, Price: price, GasLimit: gasAmount, Payload: data}
}

func NewTransactionFromBytes(data []byte) *Transaction {
	tx := &Transaction{}
	tx.RlpDecode(data)

	return tx
}

func NewTransactionFromAmount(val *ethutil.Value) *Transaction {
	tx := &Transaction{}
	tx.RlpValueDecode(val)

	return tx
}

func (tx *Transaction) Hash() []byte {
	data := []interface{}{tx.AccountNonce, tx.Price, tx.GasLimit, tx.Recipient, tx.Amount, tx.Payload}

	return crypto.Sha3(ethutil.Encode(data))
}

func (self *Transaction) Data() []byte {
	return self.Payload
}

func (self *Transaction) Gas() *big.Int {
	return self.GasLimit
}

func (self *Transaction) GasPrice() *big.Int {
	return self.Price
}

func (self *Transaction) Value() *big.Int {
	return self.Amount
}

func (self *Transaction) Nonce() uint64 {
	return self.AccountNonce
}

func (self *Transaction) SetNonce(AccountNonce uint64) {
	self.AccountNonce = AccountNonce
}

func (self *Transaction) From() []byte {
	return self.sender()
}

func (self *Transaction) To() []byte {
	return self.Recipient
}

func (tx *Transaction) Curve() (v byte, r []byte, s []byte) {
	v = byte(tx.V)
	r = ethutil.LeftPadBytes(tx.R, 32)
	s = ethutil.LeftPadBytes(tx.S, 32)

	return
}

func (tx *Transaction) Signature(key []byte) []byte {
	hash := tx.Hash()

	sig, _ := secp256k1.Sign(hash, key)

	return sig
}

func (tx *Transaction) PublicKey() []byte {
	hash := tx.Hash()

	v, r, s := tx.Curve()

	sig := append(r, s...)
	sig = append(sig, v-27)

	//pubkey := crypto.Ecrecover(append(hash, sig...))
	pubkey, _ := secp256k1.RecoverPubkey(hash, sig)

	return pubkey
}

func (tx *Transaction) sender() []byte {
	pubkey := tx.PublicKey()

	// Validate the returned key.
	// Return nil if public key isn't in full format
	if len(pubkey) == 0 || pubkey[0] != 4 {
		return nil
	}

	return crypto.Sha3(pubkey[1:])[12:]
}

// TODO: deprecate after new accounts & key stores are integrated
func (tx *Transaction) Sign(privk []byte) error {

	sig := tx.Signature(privk)

	tx.R = sig[:32]
	tx.S = sig[32:64]
	tx.V = uint64(sig[64] + 27)

	return nil
}

func (tx *Transaction) SetSignatureValues(sig []byte) error {
	tx.R = sig[:32]
	tx.S = sig[32:64]
	tx.V = uint64(sig[64] + 27)
	return nil
}

func (tx *Transaction) SignECDSA(key *ecdsa.PrivateKey) error {
	return tx.Sign(crypto.FromECDSA(key))
}

func (tx *Transaction) RlpData() interface{} {
	data := []interface{}{tx.AccountNonce, tx.Price, tx.GasLimit, tx.Recipient, tx.Amount, tx.Payload}

	return append(data, tx.V, new(big.Int).SetBytes(tx.R).Bytes(), new(big.Int).SetBytes(tx.S).Bytes())
}

func (tx *Transaction) RlpEncode() []byte {
	return ethutil.Encode(tx)
}

func (tx *Transaction) RlpDecode(data []byte) {
	rlp.Decode(bytes.NewReader(data), tx)
}

func (tx *Transaction) RlpValueDecode(decoder *ethutil.Value) {
	tx.AccountNonce = decoder.Get(0).Uint()
	tx.Price = decoder.Get(1).BigInt()
	tx.GasLimit = decoder.Get(2).BigInt()
	tx.Recipient = decoder.Get(3).Bytes()
	tx.Amount = decoder.Get(4).BigInt()
	tx.Payload = decoder.Get(5).Bytes()
	tx.V = decoder.Get(6).Uint()
	tx.R = decoder.Get(7).Bytes()
	tx.S = decoder.Get(8).Bytes()
}

func (tx *Transaction) String() string {
	return fmt.Sprintf(`
	TX(%x)
	Contract: %v
	From:     %x
	To:       %x
	Nonce:    %v
	GasPrice: %v
	GasLimit  %v
	Value:    %v
	Data:     0x%x
	V:        0x%x
	R:        0x%x
	S:        0x%x
	Hex:      %x
`,
		tx.Hash(),
		len(tx.Recipient) == 0,
		tx.From(),
		tx.To(),
		tx.AccountNonce,
		tx.Price,
		tx.GasLimit,
		tx.Amount,
		tx.Payload,
		tx.V,
		tx.R,
		tx.S,
		ethutil.Encode(tx),
	)
}

// Transaction slice type for basic sorting
type Transactions []*Transaction

func (self Transactions) RlpData() interface{} {
	// Marshal the transactions of this block
	enc := make([]interface{}, len(self))
	for i, tx := range self {
		// Cast it to a string (safe)
		enc[i] = tx.RlpData()
	}

	return enc
}
func (s Transactions) Len() int            { return len(s) }
func (s Transactions) Swap(i, j int)       { s[i], s[j] = s[j], s[i] }
func (s Transactions) GetRlp(i int) []byte { return ethutil.Rlp(s[i]) }

type TxByNonce struct{ Transactions }

func (s TxByNonce) Less(i, j int) bool {
	return s.Transactions[i].AccountNonce < s.Transactions[j].AccountNonce
}
<commit_msg>Changed V to byte. Closes #456<commit_after>package types

import (
	"bytes"
	"crypto/ecdsa"
	"fmt"
	"math/big"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/crypto/secp256k1"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/rlp"
)

func IsContractAddr(addr []byte) bool {
	return len(addr) == 0
}

type Transaction struct {
	AccountNonce uint64
	Price        *big.Int
	GasLimit     *big.Int
	Recipient    []byte
	Amount       *big.Int
	Payload      []byte
	V            byte
	R, S         []byte
}

func NewContractCreationTx(Amount, gasAmount, price *big.Int, data []byte) *Transaction {
	return NewTransactionMessage(nil, Amount, gasAmount, price, data)
}

func NewTransactionMessage(to []byte, Amount, gasAmount, price *big.Int, data []byte) *Transaction {
	return &Transaction{Recipient: to, Amount: Amount, Price: price, GasLimit: gasAmount, Payload: data}
}

func NewTransactionFromBytes(data []byte) *Transaction {
	tx := &Transaction{}
	tx.RlpDecode(data)

	return tx
}

func NewTransactionFromAmount(val *ethutil.Value) *Transaction {
	tx := &Transaction{}
	tx.RlpValueDecode(val)

	return tx
}

func (tx *Transaction) Hash() []byte {
	data := []interface{}{tx.AccountNonce, tx.Price, tx.GasLimit, tx.Recipient, tx.Amount, tx.Payload}

	return crypto.Sha3(ethutil.Encode(data))
}

func (self *Transaction) Data() []byte {
	return self.Payload
}

func (self *Transaction) Gas() *big.Int {
	return self.GasLimit
}

func (self *Transaction) GasPrice() *big.Int {
	return self.Price
}

func (self *Transaction) Value() *big.Int {
	return self.Amount
}

func (self *Transaction) Nonce() uint64 {
	return self.AccountNonce
}

func (self *Transaction) SetNonce(AccountNonce uint64) {
	self.AccountNonce = AccountNonce
}

func (self *Transaction) From() []byte {
	return self.sender()
}

func (self *Transaction) To() []byte {
	return self.Recipient
}

func (tx *Transaction) Curve() (v byte, r []byte, s []byte) {
	v = byte(tx.V)
	r = ethutil.LeftPadBytes(tx.R, 32)
	s = ethutil.LeftPadBytes(tx.S, 32)

	return
}

func (tx *Transaction) Signature(key []byte) []byte {
	hash := tx.Hash()

	sig, _ := secp256k1.Sign(hash, key)

	return sig
}

func (tx *Transaction) PublicKey() []byte {
	hash := tx.Hash()

	v, r, s := tx.Curve()

	sig := append(r, s...)
	sig = append(sig, v-27)

	//pubkey := crypto.Ecrecover(append(hash, sig...))
	pubkey, _ := secp256k1.RecoverPubkey(hash, sig)

	return pubkey
}

func (tx *Transaction) sender() []byte {
	pubkey := tx.PublicKey()

	// Validate the returned key.
	// Return nil if public key isn't in full format
	if len(pubkey) == 0 || pubkey[0] != 4 {
		return nil
	}

	return crypto.Sha3(pubkey[1:])[12:]
}

// TODO: deprecate after new accounts & key stores are integrated
func (tx *Transaction) Sign(privk []byte) error {

	sig := tx.Signature(privk)

	tx.R = sig[:32]
	tx.S = sig[32:64]
	tx.V = sig[64] + 27

	return nil
}

func (tx *Transaction) SetSignatureValues(sig []byte) error {
	tx.R = sig[:32]
	tx.S = sig[32:64]
	tx.V = sig[64] + 27
	return nil
}

func (tx *Transaction) SignECDSA(key *ecdsa.PrivateKey) error {
	return tx.Sign(crypto.FromECDSA(key))
}

func (tx *Transaction) RlpData() interface{} {
	data := []interface{}{tx.AccountNonce, tx.Price, tx.GasLimit, tx.Recipient, tx.Amount, tx.Payload}

	return append(data, tx.V, new(big.Int).SetBytes(tx.R).Bytes(), new(big.Int).SetBytes(tx.S).Bytes())
}

func (tx *Transaction) RlpEncode() []byte {
	return ethutil.Encode(tx)
}

func (tx *Transaction) RlpDecode(data []byte) {
	rlp.Decode(bytes.NewReader(data), tx)
}

func (tx *Transaction) RlpValueDecode(decoder *ethutil.Value) {
	tx.AccountNonce = decoder.Get(0).Uint()
	tx.Price = decoder.Get(1).BigInt()
	tx.GasLimit = decoder.Get(2).BigInt()
	tx.Recipient = decoder.Get(3).Bytes()
	tx.Amount = decoder.Get(4).BigInt()
	tx.Payload = decoder.Get(5).Bytes()
	tx.V = decoder.Get(6).Byte()
	tx.R = decoder.Get(7).Bytes()
	tx.S = decoder.Get(8).Bytes()
}

func (tx *Transaction) String() string {
	return fmt.Sprintf(`
	TX(%x)
	Contract: %v
	From:     %x
	To:       %x
	Nonce:    %v
	GasPrice: %v
	GasLimit  %v
	Value:    %v
	Data:     0x%x
	V:        0x%x
	R:        0x%x
	S:        0x%x
	Hex:      %x
`,
		tx.Hash(),
		len(tx.Recipient) == 0,
		tx.From(),
		tx.To(),
		tx.AccountNonce,
		tx.Price,
		tx.GasLimit,
		tx.Amount,
		tx.Payload,
		tx.V,
		tx.R,
		tx.S,
		ethutil.Encode(tx),
	)
}

// Transaction slice type for basic sorting
type Transactions []*Transaction

func (self Transactions) RlpData() interface{} {
	// Marshal the transactions of this block
	enc := make([]interface{}, len(self))
	for i, tx := range self {
		// Cast it to a string (safe)
		enc[i] = tx.RlpData()
	}

	return enc
}
func (s Transactions) Len() int            { return len(s) }
func (s Transactions) Swap(i, j int)       { s[i], s[j] = s[j], s[i] }
func (s Transactions) GetRlp(i int) []byte { return ethutil.Rlp(s[i]) }

type TxByNonce struct{ Transactions }

func (s TxByNonce) Less(i, j int) bool {
	return s.Transactions[i].AccountNonce < s.Transactions[j].AccountNonce
}
<|endoftext|>


<commit_before>package VK

import (
	"encoding/json"
	"github.com/gorilla/feeds"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type VK struct{}

type VKPhoto struct {
	Album_id   int
	Owner_id   int
	Photo_75   string
	Photo_130  string
	Photo_604  string
	Photo_807  string
	Photo_1280 string
}

type VKAttachment struct {
	Type  string
	Photo VKPhoto
}

type VKItem struct {
	Id           int
	From_id      int
	Owner_id     int
	Date         int
	Post_type    string
	Text         string
	Copy_history []VKItem
	Attachments  []VKAttachment
}

type VKProfile struct {
	Id          int
	First_name  string
	Last_name   string
	Screen_name string
	Photo_200   string
}

type VKGroup struct {
	Id          int
	Name        string
	Screen_name string
	Is_closed   int
	Type        string
	Photo_200   string
}

type VKResponseBody struct {
	Count    int
	Items    []VKItem
	Profiles []VKProfile
	Groups   []VKGroup
}

type VKResponse struct {
	Response VKResponseBody
}

func processAttachments(attachments []VKAttachment) string {
	if len(attachments) > 0 {
		var result, photo string = "", ""
		for _, attachment := range attachments {
			if attachment.Type == "photo" {

				if attachment.Photo.Photo_1280 != "" {
					photo = attachment.Photo.Photo_1280
				} else if attachment.Photo.Photo_807 != "" {
					photo = attachment.Photo.Photo_807
				} else if attachment.Photo.Photo_604 != "" {
					photo = attachment.Photo.Photo_604
				} else if attachment.Photo.Photo_130 != "" {
					photo = attachment.Photo.Photo_130
				} else if attachment.Photo.Photo_75 != "" {
					photo = attachment.Photo.Photo_75
				}
			}
			result += "
"
		}
		return result
	} else {
		return ""
	}
}

type ResolvedScreenName struct {
	Type      string
	Object_id float64
}

type ResolvedScreenNameResponse struct {
	Response ResolvedScreenName
}

func resolveScreenName(screenName string) ResolvedScreenName {
	var requestUrl = "https://api.vk.com/method/utils.resolveScreenName?v=5.12&screen_name=" + screenName
	resp, err := http.Get(requestUrl)

	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		panic(err)
	}

	var encoded ResolvedScreenNameResponse

	err = json.Unmarshal(body, &encoded)

	if err != nil {
		panic(err)
	}

	return encoded.Response
}

type SourceInfo struct {
	Name        string
	Screen_name string
	First_name  string
	Last_name   string
}

type SourceInfoContainer struct {
	Response []SourceInfo
}

func getSourceInfo(feedId string) (string, string) {
	var isGroup bool = strings.Contains(feedId, "-")
	var groupUrl string = "https://api.vk.com/method/groups.getById?group_id="
	var profileUrl string = "https://api.vk.com/method/users.get?user_ids="
	var requestUrl string

	if isGroup {
		feedId = feedId[1:]
		requestUrl = groupUrl + feedId
	} else {
		requestUrl = profileUrl + feedId
	}

	resp, err := http.Get(requestUrl)

	if err != nil {

	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {

	}
	var encoded SourceInfoContainer
	err = json.Unmarshal(body, &encoded)

	if len(encoded.Response[0].Name) > 0 {
		return encoded.Response[0].Name, encoded.Response[0].Screen_name
	} else {
		return encoded.Response[0].First_name + " " + encoded.Response[0].Last_name, feedId
	}
}

func GetPosts(feedId string) (string, error) {
	var requestUrl string = "https://api.vk.com/method/wall.get?v=5.12&extended=1&owner_id=" + feedId

	resp, err := http.Get(requestUrl)

	if err != nil {
		return "", err
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	var encoded VKResponse

	err = json.Unmarshal(body, &encoded)

	if err != nil {
		return "", err
	}

	name, screenName := getSourceInfo(feedId)

	feed := &feeds.Feed{
		Title: name,
		Link:  &feeds.Link{Href: "https://vk.com/" + screenName},
	}

	for _, elem := range encoded.Response.Items {
		var description string = ""
		var screenName, name string = "", ""
		photo := processAttachments(elem.Attachments)
		description += elem.Text + photo

		if len(elem.Copy_history) > 0 {
			description += "
repost
" + elem.Copy_history[0].Text
			name, screenName = getSourceInfo(strconv.Itoa(elem.Copy_history[0].Owner_id))
		}
		feed.Add(&feeds.Item{
			Author:      &feeds.Author{Name: name, Email: "https://vk.com/" + screenName},
			Title:       strings.Split(elem.Text, ".")[0] + "...",
			Link:        &feeds.Link{Href: "http://vk.com/wall" + strconv.Itoa(elem.Owner_id) + "_" + strconv.Itoa(elem.Id)},
			Description: description,
			Created:     time.Unix(int64(elem.Date), int64(0)),
		})
	}

	return feed.ToRss()
}

func GetPostsByUrl(feedUrl string) (string, error) {
	rp := regexp.MustCompile("vk.com/(\\w+)")
	result := rp.FindAllStringSubmatch(feedUrl, -1)
	screenName := resolveScreenName(result[0][1])
	var resolvedFeedId string

	if screenName.Type != "user" {
		resolvedFeedId = "-" + strconv.Itoa(int(screenName.Object_id))
	} else {
		resolvedFeedId = strconv.Itoa(int(screenName.Object_id))
	}

	return GetPosts(resolvedFeedId)
}
<commit_msg>panic off<commit_after>package VK

import (
	"encoding/json"
	"github.com/gorilla/feeds"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type VK struct{}

type VKPhoto struct {
	Album_id   int
	Owner_id   int
	Photo_75   string
	Photo_130  string
	Photo_604  string
	Photo_807  string
	Photo_1280 string
}

type VKAttachment struct {
	Type  string
	Photo VKPhoto
}

type VKItem struct {
	Id           int
	From_id      int
	Owner_id     int
	Date         int
	Post_type    string
	Text         string
	Copy_history []VKItem
	Attachments  []VKAttachment
}

type VKProfile struct {
	Id          int
	First_name  string
	Last_name   string
	Screen_name string
	Photo_200   string
}

type VKGroup struct {
	Id          int
	Name        string
	Screen_name string
	Is_closed   int
	Type        string
	Photo_200   string
}

type VKResponseBody struct {
	Count    int
	Items    []VKItem
	Profiles []VKProfile
	Groups   []VKGroup
}

type VKResponse struct {
	Response VKResponseBody
}

func processAttachments(attachments []VKAttachment) string {
	if len(attachments) > 0 {
		var result, photo string = "", ""
		for _, attachment := range attachments {
			if attachment.Type == "photo" {

				if attachment.Photo.Photo_1280 != "" {
					photo = attachment.Photo.Photo_1280
				} else if attachment.Photo.Photo_807 != "" {
					photo = attachment.Photo.Photo_807
				} else if attachment.Photo.Photo_604 != "" {
					photo = attachment.Photo.Photo_604
				} else if attachment.Photo.Photo_130 != "" {
					photo = attachment.Photo.Photo_130
				} else if attachment.Photo.Photo_75 != "" {
					photo = attachment.Photo.Photo_75
				}
			}
			result += "
"
		}
		return result
	} else {
		return ""
	}
}

type ResolvedScreenName struct {
	Type      string
	Object_id float64
}

type ResolvedScreenNameResponse struct {
	Response ResolvedScreenName
}

func resolveScreenName(screenName string) ResolvedScreenName {
	var requestUrl = "https://api.vk.com/method/utils.resolveScreenName?v=5.12&screen_name=" + screenName
	resp, err := http.Get(requestUrl)

	if err != nil {

	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {

	}

	var encoded ResolvedScreenNameResponse

	err = json.Unmarshal(body, &encoded)

	if err != nil {

	}

	return encoded.Response
}

type SourceInfo struct {
	Name        string
	Screen_name string
	First_name  string
	Last_name   string
}

type SourceInfoContainer struct {
	Response []SourceInfo
}

func getSourceInfo(feedId string) (string, string) {
	var isGroup bool = strings.Contains(feedId, "-")
	var groupUrl string = "https://api.vk.com/method/groups.getById?group_id="
	var profileUrl string = "https://api.vk.com/method/users.get?user_ids="
	var requestUrl string

	if isGroup {
		feedId = feedId[1:]
		requestUrl = groupUrl + feedId
	} else {
		requestUrl = profileUrl + feedId
	}

	resp, err := http.Get(requestUrl)

	if err != nil {

	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {

	}
	var encoded SourceInfoContainer
	err = json.Unmarshal(body, &encoded)

	if len(encoded.Response[0].Name) > 0 {
		return encoded.Response[0].Name, encoded.Response[0].Screen_name
	} else {
		return encoded.Response[0].First_name + " " + encoded.Response[0].Last_name, feedId
	}
}

func GetPosts(feedId string) (string, error) {
	var requestUrl string = "https://api.vk.com/method/wall.get?v=5.12&extended=1&owner_id=" + feedId

	resp, err := http.Get(requestUrl)

	if err != nil {
		return "", err
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	var encoded VKResponse

	err = json.Unmarshal(body, &encoded)

	if err != nil {
		return "", err
	}

	name, screenName := getSourceInfo(feedId)

	feed := &feeds.Feed{
		Title: name,
		Link:  &feeds.Link{Href: "https://vk.com/" + screenName},
	}

	for _, elem := range encoded.Response.Items {
		var description string = ""
		var screenName, name string = "", ""
		photo := processAttachments(elem.Attachments)
		description += elem.Text + photo

		if len(elem.Copy_history) > 0 {
			description += "
repost
" + elem.Copy_history[0].Text
			name, screenName = getSourceInfo(strconv.Itoa(elem.Copy_history[0].Owner_id))
		}
		feed.Add(&feeds.Item{
			Author:      &feeds.Author{Name: name, Email: "https://vk.com/" + screenName},
			Title:       strings.Split(elem.Text, ".")[0] + "...",
			Link:        &feeds.Link{Href: "http://vk.com/wall" + strconv.Itoa(elem.Owner_id) + "_" + strconv.Itoa(elem.Id)},
			Description: description,
			Created:     time.Unix(int64(elem.Date), int64(0)),
		})
	}

	return feed.ToRss()
}

func GetPostsByUrl(feedUrl string) (string, error) {
	rp := regexp.MustCompile("vk.com/(\\w+)")
	result := rp.FindAllStringSubmatch(feedUrl, -1)
	screenName := resolveScreenName(result[0][1])
	var resolvedFeedId string

	if screenName.Type != "user" {
		resolvedFeedId = "-" + strconv.Itoa(int(screenName.Object_id))
	} else {
		resolvedFeedId = strconv.Itoa(int(screenName.Object_id))
	}

	return GetPosts(resolvedFeedId)
}
<|endoftext|>


<commit_before>package services_test

import (
	"context"
	"testing"

	"github.com/emwalker/digraph/cmd/frontend/services"
	"github.com/volatiletech/sqlboiler/queries/qm"
)

func TestUpsertBadLink(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}

	result, err := c.UpsertLink(context.Background(), defaultRepo, "topic name", nil, []string{})
	if err != nil {
		t.Fatal(err)
	}
	defer result.Cleanup()

	if len(result.Alerts) < 1 {
		t.Fatal("Expected one or more alerts")
	}

	if result.Link != nil {
		t.Fatal("A link should not have been created")
	}
}

func TestLinkHasATopic(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	title := "A title"
	result, err := c.UpsertLink(ctx, defaultRepo, "http://some.url.com/", &title, []string{})
	if err != nil {
		t.Fatal(err)
	}
	defer result.Cleanup()

	if !result.LinkCreated {
		t.Fatal("Expected link to be a new one")
	}

	topics, err := result.Link.ParentTopics().All(ctx, c.Exec)
	if err != nil {
		t.Fatal(err)
	}

	if len(topics) < 1 {
		t.Fatal("Expected the link to be added to the root topic")
	}
}

func TestUpsertExistingLinkWithTopic(t *testing.T) {
	// https://github.com/emwalker/digraph/issues/13
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	topicResult, err := c.UpsertTopic(ctx, defaultRepo, "62ce187241e", nil, []string{})
	if err != nil {
		t.Fatalf("There was a problem upserting the topic: %s", err)
	}
	defer topicResult.Cleanup()

	// Initial creation
	title := "A title"
	linkResult, err := c.UpsertLink(ctx, defaultRepo, "http://some.url.com/", &title, []string{topicResult.Topic.ID})
	if err != nil {
		t.Fatal(err)
	}
	defer linkResult.Cleanup()

	if !linkResult.LinkCreated {
		t.Fatal("Expected link to be a new one")
	}

	// A second upsert
	linkResult, err = c.UpsertLink(ctx, defaultRepo, "http://some.url.com/", &title, []string{})
	if err != nil {
		t.Fatal(err)
	}
	defer linkResult.Cleanup()

	topics, err := linkResult.Link.ParentTopics().All(ctx, c.Exec)
	if err != nil {
		t.Fatal(err)
	}

	for _, topic := range topics {
		if topic.Root {
			t.Fatal("The root topic should not be automatically added to a link that already has a topic")
		}
	}
}

func TestUserLinkHistory(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	result, err := c.UpsertTopic(ctx, defaultRepo, "62ce1872411", nil, []string{})
	if err != nil {
		t.Fatalf("There was a problem upserting the topic: %s", err)
	}
	defer result.Cleanup()

	topic := result.Topic

	prevCount, _ := testActor.UserLinks().Count(ctx, testDB)
	var nextCount int64

	// A log is added for an upsert
	title := "A title"
	upsertResult, err := c.UpsertLink(ctx, defaultRepo, "http://frotz.com/", &title, []string{topic.ID})
	if err != nil {
		t.Fatal(err)
	}
	defer upsertResult.Cleanup()

	nextCount, _ = testActor.UserLinks().Count(ctx, testDB)
	if (prevCount + 1) != nextCount {
		t.Fatal("Expected a new user link record to be created for the upsert")
	}

	userLink, err := testActor.UserLinks(qm.OrderBy("created_at desc")).One(ctx, testDB)
	if err != nil {
		t.Fatal(err)
	}

	linkTopicCount, err := userLink.UserLinkTopics().Count(ctx, testDB)
	if err != nil {
		t.Fatal(err)
	}

	if linkTopicCount < 1 {
		t.Fatal("Expected at least one row to be added to user_link_topics")
	}

	// A log is not added for a delete at this time
	deleteResult, err := c.DeleteLink(ctx, defaultRepo, upsertResult.Link)
	if err != nil {
		t.Fatal(err)
	}
	defer deleteResult.Cleanup()
}

func TestUserLinkReviewAdded(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	prevCount, _ := testActor.UserLinkReviews().Count(ctx, testDB)

	title := "A title"
	upsertResult, err := c.UpsertLink(ctx, defaultRepo, "http://frotz.com/", &title, []string{})
	if err != nil {
		t.Fatalf("There was a problem upserting the topic: %s", err)
	}
	defer upsertResult.Cleanup()

	nextCount, _ := testActor.UserLinkReviews().Count(ctx, testDB)

	if prevCount+1 != nextCount {
		t.Fatalf("Expected a user-link-review record to be created")
	}
}

func TestReviewLink(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	title := "A title"
	upsertResult, err := c.UpsertLink(ctx, defaultRepo, "http://frotz.com/", &title, []string{})
	if err != nil {
		t.Fatal(err)
	}
	defer upsertResult.Cleanup()

	link := upsertResult.Link

	reviews, err := testActor.UserLinkReviews(qm.Where("link_id = ?", link.ID)).All(ctx, c.Exec)
	if err != nil {
		t.Fatal(err)
	}

	if len(reviews) != 1 {
		t.Fatal("Expected there to be a single user-link-review")
	}

	review := reviews[0]
	if !review.ReviewedAt.IsZero() {
		t.Fatal("Expected the review to be pending")
	}

	result, err := c.ReviewLink(ctx, link, true)
	if result.Review.ReviewedAt.IsZero() {
		t.Fatal("Expected the review to be pending")
	}
}
<commit_msg>Add a test for the most recent bugfix<commit_after>package services_test

import (
	"context"
	"testing"

	"github.com/emwalker/digraph/cmd/frontend/services"
	"github.com/volatiletech/sqlboiler/queries/qm"
)

func TestUpsertBadLink(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}

	result, err := c.UpsertLink(context.Background(), defaultRepo, "topic name", nil, []string{})
	if err != nil {
		t.Fatal(err)
	}
	defer result.Cleanup()

	if len(result.Alerts) < 1 {
		t.Fatal("Expected one or more alerts")
	}

	if result.Link != nil {
		t.Fatal("A link should not have been created")
	}
}

func TestLinkHasATopic(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	title := "A title"
	result, err := c.UpsertLink(ctx, defaultRepo, "http://some.url.com/", &title, []string{})
	if err != nil {
		t.Fatal(err)
	}
	defer result.Cleanup()

	if !result.LinkCreated {
		t.Fatal("Expected link to be a new one")
	}

	if result.Link.R != nil {
		t.Fatal("There should be no preloads on the link")
	}

	topics, err := result.Link.ParentTopics().All(ctx, c.Exec)
	if err != nil {
		t.Fatal(err)
	}

	if len(topics) < 1 {
		t.Fatal("Expected the link to be added to the root topic")
	}
}

func TestUpsertExistingLinkWithTopic(t *testing.T) {
	// https://github.com/emwalker/digraph/issues/13
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	topicResult, err := c.UpsertTopic(ctx, defaultRepo, "62ce187241e", nil, []string{})
	if err != nil {
		t.Fatalf("There was a problem upserting the topic: %s", err)
	}
	defer topicResult.Cleanup()

	// Initial creation
	title := "A title"
	linkResult, err := c.UpsertLink(ctx, defaultRepo, "http://some.url.com/", &title, []string{topicResult.Topic.ID})
	if err != nil {
		t.Fatal(err)
	}
	defer linkResult.Cleanup()

	if !linkResult.LinkCreated {
		t.Fatal("Expected link to be a new one")
	}

	// A second upsert
	linkResult, err = c.UpsertLink(ctx, defaultRepo, "http://some.url.com/", &title, []string{})
	if err != nil {
		t.Fatal(err)
	}
	defer linkResult.Cleanup()

	topics, err := linkResult.Link.ParentTopics().All(ctx, c.Exec)
	if err != nil {
		t.Fatal(err)
	}

	for _, topic := range topics {
		if topic.Root {
			t.Fatal("The root topic should not be automatically added to a link that already has a topic")
		}
	}
}

func TestUserLinkHistory(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	result, err := c.UpsertTopic(ctx, defaultRepo, "62ce1872411", nil, []string{})
	if err != nil {
		t.Fatalf("There was a problem upserting the topic: %s", err)
	}
	defer result.Cleanup()

	topic := result.Topic

	prevCount, _ := testActor.UserLinks().Count(ctx, testDB)
	var nextCount int64

	// A log is added for an upsert
	title := "A title"
	upsertResult, err := c.UpsertLink(ctx, defaultRepo, "http://frotz.com/", &title, []string{topic.ID})
	if err != nil {
		t.Fatal(err)
	}
	defer upsertResult.Cleanup()

	nextCount, _ = testActor.UserLinks().Count(ctx, testDB)
	if (prevCount + 1) != nextCount {
		t.Fatal("Expected a new user link record to be created for the upsert")
	}

	userLink, err := testActor.UserLinks(qm.OrderBy("created_at desc")).One(ctx, testDB)
	if err != nil {
		t.Fatal(err)
	}

	linkTopicCount, err := userLink.UserLinkTopics().Count(ctx, testDB)
	if err != nil {
		t.Fatal(err)
	}

	if linkTopicCount < 1 {
		t.Fatal("Expected at least one row to be added to user_link_topics")
	}

	// A log is not added for a delete at this time
	deleteResult, err := c.DeleteLink(ctx, defaultRepo, upsertResult.Link)
	if err != nil {
		t.Fatal(err)
	}
	defer deleteResult.Cleanup()
}

func TestUserLinkReviewAdded(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	prevCount, _ := testActor.UserLinkReviews().Count(ctx, testDB)

	title := "A title"
	upsertResult, err := c.UpsertLink(ctx, defaultRepo, "http://frotz.com/", &title, []string{})
	if err != nil {
		t.Fatalf("There was a problem upserting the topic: %s", err)
	}
	defer upsertResult.Cleanup()

	nextCount, _ := testActor.UserLinkReviews().Count(ctx, testDB)

	if prevCount+1 != nextCount {
		t.Fatalf("Expected a user-link-review record to be created")
	}
}

func TestReviewLink(t *testing.T) {
	c := services.Connection{Exec: testDB, Actor: testActor}
	ctx := context.Background()

	title := "A title"
	upsertResult, err := c.UpsertLink(ctx, defaultRepo, "http://frotz.com/", &title, []string{})
	if err != nil {
		t.Fatal(err)
	}
	defer upsertResult.Cleanup()

	link := upsertResult.Link

	reviews, err := testActor.UserLinkReviews(qm.Where("link_id = ?", link.ID)).All(ctx, c.Exec)
	if err != nil {
		t.Fatal(err)
	}

	if len(reviews) != 1 {
		t.Fatal("Expected there to be a single user-link-review")
	}

	review := reviews[0]
	if !review.ReviewedAt.IsZero() {
		t.Fatal("Expected the review to be pending")
	}

	result, err := c.ReviewLink(ctx, link, true)
	if result.Review.ReviewedAt.IsZero() {
		t.Fatal("Expected the review to be pending")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"time"

	"github.com/golang/protobuf/proto"
	tpb "github.com/google/mako/clients/proto/analyzers/threshold_analyzer_go_proto"
	mpb "github.com/google/mako/spec/proto/mako_go_proto"
	vegeta "github.com/tsenart/vegeta/lib"
	"knative.dev/pkg/test/mako"
)

var (
	// This analyzer validates that the p95 latency talking to pods through a Kubernetes
	// Service falls in the +5ms range.  This does not have Knative or Istio components
	// on the dataplane, and so it is intended as a canary to flag environmental
	// problems that might be causing contemporaneous Knative or Istio runs to fall out of SLA.
	Kubernetes95PercentileLatency = &tpb.ThresholdAnalyzerInput{
		Name: proto.String("Kubernetes baseline"),
		Configs: []*tpb.ThresholdConfig{{
			Min: bound(100 * time.Millisecond),
			Max: bound(105 * time.Millisecond),
			DataFilter: &mpb.DataFilter{
				DataType:            mpb.DataFilter_METRIC_AGGREGATE_PERCENTILE.Enum(),
				PercentileMilliRank: proto.Int32(95000),
				ValueKey:            proto.String("kd"),
			},
		}},
		CrossRunConfig: mako.NewCrossRunConfig(10),
	}

	// This analyzer validates that the p95 latency talking to pods through Istio
	// falls in the +8ms range.  This does not actually have Knative components
	// on the dataplane, and so it is intended as a canary to flag environmental
	// problems that might be causing contemporaneous Knative runs to fall out of SLA.
	Istio95PercentileLatency = &tpb.ThresholdAnalyzerInput{
		Name: proto.String("Istio baseline"),
		Configs: []*tpb.ThresholdConfig{{
			Min: bound(100 * time.Millisecond),
			Max: bound(108 * time.Millisecond),
			DataFilter: &mpb.DataFilter{
				DataType:            mpb.DataFilter_METRIC_AGGREGATE_PERCENTILE.Enum(),
				PercentileMilliRank: proto.Int32(95000),
				ValueKey:            proto.String("id"),
			},
		}},
		CrossRunConfig: mako.NewCrossRunConfig(10),
	}

	// This analyzer validates that the p95 latency hitting a Knative Service
	// going through JUST the queue-proxy falls in the +10ms range.
	Queue95PercentileLatency = &tpb.ThresholdAnalyzerInput{
		Name: proto.String("Queue p95 latency"),
		Configs: []*tpb.ThresholdConfig{{
			Min: bound(100 * time.Millisecond),
			Max: bound(110 * time.Millisecond),
			DataFilter: &mpb.DataFilter{
				DataType:            mpb.DataFilter_METRIC_AGGREGATE_PERCENTILE.Enum(),
				PercentileMilliRank: proto.Int32(95000),
				ValueKey:            proto.String("qp"),
			},
		}},
		CrossRunConfig: mako.NewCrossRunConfig(10),
	}

	// This analyzer validates that the p95 latency hitting a Knative Service
	// going through BOTH the activator and queue-proxy falls in the +10ms range.
	Activator95PercentileLatency = &tpb.ThresholdAnalyzerInput{
		Name: proto.String("Activator p95 latency"),
		Configs: []*tpb.ThresholdConfig{{
			Min: bound(100 * time.Millisecond),
			Max: bound(110 * time.Millisecond),
			DataFilter: &mpb.DataFilter{
				DataType:            mpb.DataFilter_METRIC_AGGREGATE_PERCENTILE.Enum(),
				PercentileMilliRank: proto.Int32(95000),
				ValueKey:            proto.String("a"),
			},
		}},
		CrossRunConfig: mako.NewCrossRunConfig(10),
	}

	// Map the above to our benchmark targets.
	targets = map[string]struct {
		target    vegeta.Target
		stat      string
		estat     string
		analyzers []*tpb.ThresholdAnalyzerInput
	}{
		"deployment": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://deployment.default.svc.cluster.local?sleep=100",
			},
			stat:      "kd",
			estat:     "ke",
			analyzers: []*tpb.ThresholdAnalyzerInput{Kubernetes95PercentileLatency},
		},
		"istio": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://istio.default.svc.cluster.local?sleep=100",
			},
			stat:      "id",
			estat:     "ie",
			analyzers: []*tpb.ThresholdAnalyzerInput{Istio95PercentileLatency},
		},
		"queue": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://queue-proxy.default.svc.cluster.local?sleep=100",
			},
			stat:      "qp",
			estat:     "qe",
			analyzers: []*tpb.ThresholdAnalyzerInput{Queue95PercentileLatency},
		},
		"queue-with-cc": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://queue-proxy-with-cc.default.svc.cluster.local?sleep=100",
			},
			stat:  "qc",
			estat: "re",
			// We use the same threshold analyzer, since we want Breaker to exert minimal latency impact.
			analyzers: []*tpb.ThresholdAnalyzerInput{Queue95PercentileLatency},
		},
		"queue-with-cc-10": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://queue-proxy-with-cc-10.default.svc.cluster.local?sleep=100",
			},
			stat:  "qct",
			estat: "ret",
			// TODO(vagababov): determine values here.
			analyzers: []*tpb.ThresholdAnalyzerInput{Queue95PercentileLatency},
		},
		"queue-with-cc-1": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://queue-proxy-with-cc-1.default.svc.cluster.local?sleep=100",
			},
			stat:  "qc1",
			estat: "re1",
			// TODO(vagababov): determine values here.
			analyzers: []*tpb.ThresholdAnalyzerInput{Queue95PercentileLatency},
		},
		"activator": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://activator.default.svc.cluster.local?sleep=100",
			},
			stat:      "a",
			estat:     "ae",
			analyzers: []*tpb.ThresholdAnalyzerInput{Activator95PercentileLatency},
		},
		"activator-with-cc": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://activator-with-cc.default.svc.cluster.local?sleep=100",
			},
			stat:  "ac",
			estat: "be",
			// We use the same threshold analyzer, since we want Throttler/Breaker to exert minimal latency impact.
			analyzers: []*tpb.ThresholdAnalyzerInput{Activator95PercentileLatency},
		},
		"activator-with-cc-10": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://activator-with-cc-10.default.svc.cluster.local?sleep=100",
			},
			stat:  "act",
			estat: "bet",
			// TODO(vagababov): determine values here.
			analyzers: []*tpb.ThresholdAnalyzerInput{Activator95PercentileLatency},
		},
		"activator-with-cc-1": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://activator-with-cc-1.default.svc.cluster.local?sleep=100",
			},
			stat:  "ac1",
			estat: "be1",
			// TODO(vagababov): determine values here.
			analyzers: []*tpb.ThresholdAnalyzerInput{Activator95PercentileLatency},
		},
	}
)

// bound is a helper for making the inline SLOs more readable by expressing
// them as durations.
func bound(d time.Duration) *float64 {
	return proto.Float64(d.Seconds())
}
<commit_msg>Fix analyzer for dataplane-probe benchmark (#5408)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"time"

	"github.com/golang/protobuf/proto"
	tpb "github.com/google/mako/clients/proto/analyzers/threshold_analyzer_go_proto"
	mpb "github.com/google/mako/spec/proto/mako_go_proto"
	vegeta "github.com/tsenart/vegeta/lib"
	"knative.dev/pkg/test/mako"
)

// This function constructs an analyzer that validates the p95 aggregate value of the given metric.
func new95PercentileLatency(name, valueKey string, min, max time.Duration) *tpb.ThresholdAnalyzerInput {
	return &tpb.ThresholdAnalyzerInput{
		Name: proto.String(name),
		Configs: []*tpb.ThresholdConfig{{
			Min: bound(min),
			Max: bound(max),
			DataFilter: &mpb.DataFilter{
				DataType:            mpb.DataFilter_METRIC_AGGREGATE_PERCENTILE.Enum(),
				PercentileMilliRank: proto.Int32(95000),
				ValueKey:            proto.String(valueKey),
			},
		}},
		CrossRunConfig: mako.NewCrossRunConfig(10),
	}
}

// This analyzer validates that the p95 latency talking to pods through a Kubernetes
// Service falls in the +5ms range.  This does not have Knative or Istio components
// on the dataplane, and so it is intended as a canary to flag environmental
// problems that might be causing contemporaneous Knative or Istio runs to fall out of SLA.
func newKubernetes95PercentileLatency(valueKey string) *tpb.ThresholdAnalyzerInput {
	return new95PercentileLatency("Kubernetes baseline", valueKey, 100*time.Millisecond, 105*time.Millisecond)
}

// This analyzer validates that the p95 latency talking to pods through Istio
// falls in the +8ms range.  This does not actually have Knative components
// on the dataplane, and so it is intended as a canary to flag environmental
// problems that might be causing contemporaneous Knative runs to fall out of SLA.
func newIstio95PercentileLatency(valueKey string) *tpb.ThresholdAnalyzerInput {
	return new95PercentileLatency("Istio baseline", valueKey, 100*time.Millisecond, 108*time.Millisecond)
}

// This analyzer validates that the p95 latency hitting a Knative Service
// going through JUST the queue-proxy falls in the +10ms range.
func newQueue95PercentileLatency(valueKey string) *tpb.ThresholdAnalyzerInput {
	return new95PercentileLatency("Queue p95 latency", valueKey, 100*time.Millisecond, 110*time.Millisecond)
}

// This analyzer validates that the p95 latency hitting a Knative Service
// going through BOTH the activator and queue-proxy falls in the +10ms range.
func newActivator95PercentileLatency(valueKey string) *tpb.ThresholdAnalyzerInput {
	return new95PercentileLatency("Activator p95 latency", valueKey, 100*time.Millisecond, 110*time.Millisecond)
}

var (
	// Map the above to our benchmark targets.
	targets = map[string]struct {
		target    vegeta.Target
		stat      string
		estat     string
		analyzers []*tpb.ThresholdAnalyzerInput
	}{
		"deployment": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://deployment.default.svc.cluster.local?sleep=100",
			},
			stat:      "kd",
			estat:     "ke",
			analyzers: []*tpb.ThresholdAnalyzerInput{newKubernetes95PercentileLatency("kd")},
		},
		"istio": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://istio.default.svc.cluster.local?sleep=100",
			},
			stat:      "id",
			estat:     "ie",
			analyzers: []*tpb.ThresholdAnalyzerInput{newIstio95PercentileLatency("id")},
		},
		"queue": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://queue-proxy.default.svc.cluster.local?sleep=100",
			},
			stat:      "qp",
			estat:     "qe",
			analyzers: []*tpb.ThresholdAnalyzerInput{newQueue95PercentileLatency("qp")},
		},
		"queue-with-cc": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://queue-proxy-with-cc.default.svc.cluster.local?sleep=100",
			},
			stat:  "qc",
			estat: "re",
			// We use the same threshold analyzer, since we want Breaker to exert minimal latency impact.
			analyzers: []*tpb.ThresholdAnalyzerInput{newQueue95PercentileLatency("qc")},
		},
		"queue-with-cc-10": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://queue-proxy-with-cc-10.default.svc.cluster.local?sleep=100",
			},
			stat:  "qct",
			estat: "ret",
			// TODO(vagababov): determine values here.
			analyzers: []*tpb.ThresholdAnalyzerInput{},
		},
		"queue-with-cc-1": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://queue-proxy-with-cc-1.default.svc.cluster.local?sleep=100",
			},
			stat:  "qc1",
			estat: "re1",
			// TODO(vagababov): determine values here.
			analyzers: []*tpb.ThresholdAnalyzerInput{},
		},
		"activator": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://activator.default.svc.cluster.local?sleep=100",
			},
			stat:      "a",
			estat:     "ae",
			analyzers: []*tpb.ThresholdAnalyzerInput{newActivator95PercentileLatency("a")},
		},
		"activator-with-cc": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://activator-with-cc.default.svc.cluster.local?sleep=100",
			},
			stat:  "ac",
			estat: "be",
			// We use the same threshold analyzer, since we want Throttler/Breaker to exert minimal latency impact.
			analyzers: []*tpb.ThresholdAnalyzerInput{newActivator95PercentileLatency("ac")},
		},
		"activator-with-cc-10": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://activator-with-cc-10.default.svc.cluster.local?sleep=100",
			},
			stat:  "act",
			estat: "bet",
			// TODO(vagababov): determine values here.
			analyzers: []*tpb.ThresholdAnalyzerInput{},
		},
		"activator-with-cc-1": {
			target: vegeta.Target{
				Method: "GET",
				URL:    "http://activator-with-cc-1.default.svc.cluster.local?sleep=100",
			},
			stat:  "ac1",
			estat: "be1",
			// TODO(vagababov): determine values here.
			analyzers: []*tpb.ThresholdAnalyzerInput{},
		},
	}
)

// bound is a helper for making the inline SLOs more readable by expressing
// them as durations.
func bound(d time.Duration) *float64 {
	return proto.Float64(d.Seconds())
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"net/http"
	"net/http/httputil"
	"net/url"

	"knative.dev/serving/test"
)

const (
	targetHostEnv  = "TARGET_HOST"
	gatewayHostEnv = "GATEWAY_HOST"
)

var (
	httpProxy *httputil.ReverseProxy
)

func handler(w http.ResponseWriter, r *http.Request) {
	log.Print("HTTP proxy received a request.")
	// Reverse proxy does not automatically reset the Host header.
	// We need to manually reset it.
	r.Host = getTargetHostEnv()
	httpProxy.ServeHTTP(w, r)
}

func getTargetHostEnv() string {
	value := os.Getenv(targetHostEnv)
	if value == "" {
		log.Fatalf("No env %v provided.", targetHostEnv)
	}
	return value
}

func initialHTTPProxy(proxyURL string) *httputil.ReverseProxy {
	target, err := url.Parse(proxyURL)
	if err != nil {
		log.Fatalf("Failed to parse url %v", proxyURL)
	}
	proxy := httputil.NewSingleHostReverseProxy(target)
	proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) {
		log.Printf("error reverse proxying request: %v", err)
		http.Error(w, err.Error(), http.StatusBadGateway)
	}
	return proxy
}

func main() {
	flag.Parse()
	log.Print("HTTP Proxy app started.")

	targetHost := getTargetHostEnv()

	// Gateway is an optional value. It is used only when resolvable domain is not set
	// for external access test, as xip.io is flaky.
	// ref: https://github.com/knative/serving/issues/5389
	gateway := os.Getenv(gatewayHostEnv)
	if gateway != "" {
		targetHost = gateway
	}
	targetURL := fmt.Sprintf("http://%s", targetHost)
	log.Print("target is " + targetURL)
	httpProxy = initialHTTPProxy(targetURL)

	test.ListenAndServeGracefully(":8080", handler)
}
<commit_msg>Update httpproxy with port and header, so it can be used in Ingress conformance tests without queue-proxy (#6560)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"net/http"
	"net/http/httputil"
	"net/url"

	"knative.dev/serving/pkg/network"
	"knative.dev/serving/test"
)

const (
	targetHostEnv  = "TARGET_HOST"
	gatewayHostEnv = "GATEWAY_HOST"
	portEnv        = "PORT" // Allow port to be customized / randomly assigned by tests

	defaultPort = "8080"
)

var (
	httpProxy *httputil.ReverseProxy
)

func handler(w http.ResponseWriter, r *http.Request) {
	log.Print("HTTP proxy received a request.")
	// Reverse proxy does not automatically reset the Host header.
	// We need to manually reset it.
	r.Host = getTargetHostEnv()
	httpProxy.ServeHTTP(w, r)
}

func getPort() string {
	value := os.Getenv(portEnv)
	if value == "" {
		return defaultPort
	}
	return value
}

func getTargetHostEnv() string {
	value := os.Getenv(targetHostEnv)
	if value == "" {
		log.Fatalf("No env %v provided.", targetHostEnv)
	}
	return value
}

func initialHTTPProxy(proxyURL string) *httputil.ReverseProxy {
	target, err := url.Parse(proxyURL)
	if err != nil {
		log.Fatalf("Failed to parse url %v", proxyURL)
	}
	proxy := httputil.NewSingleHostReverseProxy(target)
	proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) {
		log.Printf("error reverse proxying request: %v", err)
		http.Error(w, err.Error(), http.StatusBadGateway)
	}
	return proxy
}

func main() {
	flag.Parse()
	log.Print("HTTP Proxy app started.")

	targetHost := getTargetHostEnv()
	port := getPort()

	// Gateway is an optional value. It is used only when resolvable domain is not set
	// for external access test, as xip.io is flaky.
	// ref: https://github.com/knative/serving/issues/5389
	gateway := os.Getenv(gatewayHostEnv)
	if gateway != "" {
		targetHost = gateway
	}
	targetURL := fmt.Sprintf("http://%s", targetHost)
	log.Print("target is " + targetURL)
	httpProxy = initialHTTPProxy(targetURL)

	address := fmt.Sprintf(":%s", port)
	log.Printf("Listening on address: %s", address)
	// Handle forwarding requests which uses "K-Network-Hash" header.
	probeHandler := network.NewProbeHandler(http.HandlerFunc(handler)).ServeHTTP
	test.ListenAndServeGracefully(address, probeHandler)
}
<|endoftext|>


<commit_before><commit_msg>Lots of changes to install.go - almost a rewrite<commit_after><|endoftext|>


<commit_before>package intfns

import (
	"encoding/xml"
	"fmt"

	"github.com/ChrisTrenkamp/goxpath/goxpath/xpfn"
	"github.com/ChrisTrenkamp/goxpath/tree"
	"github.com/ChrisTrenkamp/goxpath/tree/literals/numlit"
	"github.com/ChrisTrenkamp/goxpath/tree/literals/strlit"
)

func last(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	return []tree.Res{numlit.NumLit(c.Size)}, nil
}

func position(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	return []tree.Res{numlit.NumLit(c.Pos)}, nil
}

func count(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	arg := args[0]

	if len(arg) == 0 {
		return nil, nil
	}

	if _, ok := arg[0].(tree.Node); !ok {
		return nil, fmt.Errorf("Argument is not a node-set")
	}

	ret := 0

	for i := range arg {
		countArg(arg[i], &ret)
	}

	return []tree.Res{numlit.NumLit(ret)}, nil
}

func countArg(r tree.Res, c *int) {
	switch t := r.(type) {
	case tree.Elem:
		for _, i := range t.GetChildren() {
			countArg(i, c)
		}
		(*c)++
	case tree.Node:
		(*c)++
	}
}

func localName(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	var arg []tree.Res

	if len(args) == 0 {
		arg = c.Filter
	} else {
		arg = args[0]
	}

	if len(arg) == 0 {
		return []tree.Res{strlit.StrLit("")}, nil
	}

	node, ok := arg[0].(tree.Node)

	if !ok {
		return nil, fmt.Errorf("Argument is not a node")
	}

	tok := node.GetToken()

	switch t := tok.(type) {
	case xml.StartElement:
		if t.Name.Local == "" {
			return nil, fmt.Errorf("Cannot run local-name on root node.")
		}
		ret := []tree.Res{strlit.StrLit(t.Name.Local)}
		return ret, nil
	case xml.Attr:
		ret := []tree.Res{strlit.StrLit(t.Name.Local)}
		return ret, nil
	case xml.ProcInst:
		ret := []tree.Res{strlit.StrLit(t.Target)}
		return ret, nil
	}

	return []tree.Res{strlit.StrLit("")}, nil
}

func namespaceURI(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	var arg []tree.Res

	if len(args) == 0 {
		arg = c.Filter
	} else {
		arg = args[0]
	}

	if len(arg) == 0 {
		return []tree.Res{strlit.StrLit("")}, nil
	}

	node, ok := arg[0].(tree.Node)

	if !ok {
		return nil, fmt.Errorf("Argument is not a node")
	}

	tok := node.GetToken()

	switch t := tok.(type) {
	case xml.StartElement:
		if t.Name.Local == "" {
			return nil, fmt.Errorf("Cannot run namespace-uri on root node.")
		}
		ret := []tree.Res{strlit.StrLit(t.Name.Space)}
		return ret, nil
	case xml.Attr:
		ret := []tree.Res{strlit.StrLit(t.Name.Space)}
		return ret, nil
	}

	return []tree.Res{strlit.StrLit("")}, nil
}

func name(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	var arg []tree.Res

	if len(args) == 0 {
		arg = c.Filter
	} else {
		arg = args[0]
	}

	if len(arg) == 0 {
		return []tree.Res{strlit.StrLit("")}, nil
	}

	node, ok := arg[0].(tree.Node)

	if !ok {
		return nil, fmt.Errorf("Argument is not a node")
	}

	tok := node.GetToken()

	switch t := tok.(type) {
	case xml.StartElement:
		if t.Name.Local == "" {
			return nil, fmt.Errorf("Cannot run name on root node.")
		}
		space := ""
		if t.Name.Space != "" {
			space = fmt.Sprintf("{%s}", t.Name.Space)
		}
		res := fmt.Sprintf("%s%s", space, t.Name.Local)
		ret := []tree.Res{strlit.StrLit(res)}
		return ret, nil
	case xml.Attr:
		space := ""
		if t.Name.Space != "" {
			space = fmt.Sprintf("{%s}", t.Name.Space)
		}
		res := fmt.Sprintf("%s%s", space, t.Name.Local)
		ret := []tree.Res{strlit.StrLit(res)}
		return ret, nil
	case xml.ProcInst:
		res := fmt.Sprintf("%s", t.Target)
		ret := []tree.Res{strlit.StrLit(res)}
		return ret, nil
	}

	return []tree.Res{strlit.StrLit("")}, nil
}
<commit_msg>Fixing the count() function.<commit_after>package intfns

import (
	"encoding/xml"
	"fmt"

	"github.com/ChrisTrenkamp/goxpath/goxpath/xpfn"
	"github.com/ChrisTrenkamp/goxpath/tree"
	"github.com/ChrisTrenkamp/goxpath/tree/literals/numlit"
	"github.com/ChrisTrenkamp/goxpath/tree/literals/strlit"
)

func last(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	return []tree.Res{numlit.NumLit(c.Size)}, nil
}

func position(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	return []tree.Res{numlit.NumLit(c.Pos)}, nil
}

func count(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	arg := args[0]

	if len(arg) == 0 {
		return []tree.Res{numlit.NumLit(0)}, nil
	}

	if _, ok := arg[0].(tree.Node); !ok {
		return nil, fmt.Errorf("Argument is not a node-set")
	}

	ret := 0

	for i := range arg {
		countArg(arg[i], &ret)
	}

	return []tree.Res{numlit.NumLit(ret)}, nil
}

func countArg(r tree.Res, c *int) {
	switch t := r.(type) {
	case tree.Elem:
		for _, i := range t.GetChildren() {
			countArg(i, c)
		}
		(*c)++
	case tree.Node:
		(*c)++
	}
}

func localName(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	var arg []tree.Res

	if len(args) == 0 {
		arg = c.Filter
	} else {
		arg = args[0]
	}

	if len(arg) == 0 {
		return []tree.Res{strlit.StrLit("")}, nil
	}

	node, ok := arg[0].(tree.Node)

	if !ok {
		return nil, fmt.Errorf("Argument is not a node")
	}

	tok := node.GetToken()

	switch t := tok.(type) {
	case xml.StartElement:
		if t.Name.Local == "" {
			return nil, fmt.Errorf("Cannot run local-name on root node.")
		}
		ret := []tree.Res{strlit.StrLit(t.Name.Local)}
		return ret, nil
	case xml.Attr:
		ret := []tree.Res{strlit.StrLit(t.Name.Local)}
		return ret, nil
	case xml.ProcInst:
		ret := []tree.Res{strlit.StrLit(t.Target)}
		return ret, nil
	}

	return []tree.Res{strlit.StrLit("")}, nil
}

func namespaceURI(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	var arg []tree.Res

	if len(args) == 0 {
		arg = c.Filter
	} else {
		arg = args[0]
	}

	if len(arg) == 0 {
		return []tree.Res{strlit.StrLit("")}, nil
	}

	node, ok := arg[0].(tree.Node)

	if !ok {
		return nil, fmt.Errorf("Argument is not a node")
	}

	tok := node.GetToken()

	switch t := tok.(type) {
	case xml.StartElement:
		if t.Name.Local == "" {
			return nil, fmt.Errorf("Cannot run namespace-uri on root node.")
		}
		ret := []tree.Res{strlit.StrLit(t.Name.Space)}
		return ret, nil
	case xml.Attr:
		ret := []tree.Res{strlit.StrLit(t.Name.Space)}
		return ret, nil
	}

	return []tree.Res{strlit.StrLit("")}, nil
}

func name(c xpfn.Ctx, args ...[]tree.Res) ([]tree.Res, error) {
	var arg []tree.Res

	if len(args) == 0 {
		arg = c.Filter
	} else {
		arg = args[0]
	}

	if len(arg) == 0 {
		return []tree.Res{strlit.StrLit("")}, nil
	}

	node, ok := arg[0].(tree.Node)

	if !ok {
		return nil, fmt.Errorf("Argument is not a node")
	}

	tok := node.GetToken()

	switch t := tok.(type) {
	case xml.StartElement:
		if t.Name.Local == "" {
			return nil, fmt.Errorf("Cannot run name on root node.")
		}
		space := ""
		if t.Name.Space != "" {
			space = fmt.Sprintf("{%s}", t.Name.Space)
		}
		res := fmt.Sprintf("%s%s", space, t.Name.Local)
		ret := []tree.Res{strlit.StrLit(res)}
		return ret, nil
	case xml.Attr:
		space := ""
		if t.Name.Space != "" {
			space = fmt.Sprintf("{%s}", t.Name.Space)
		}
		res := fmt.Sprintf("%s%s", space, t.Name.Local)
		ret := []tree.Res{strlit.StrLit(res)}
		return ret, nil
	case xml.ProcInst:
		res := fmt.Sprintf("%s", t.Target)
		ret := []tree.Res{strlit.StrLit(res)}
		return ret, nil
	}

	return []tree.Res{strlit.StrLit("")}, nil
}
<|endoftext|>


<commit_before>package experiments

import (
	"github.com/yaricom/goNEAT/neat"
	"os"
	"fmt"
	"github.com/yaricom/goNEAT/neat/genetics"
	"math"
)

// The precision to use for XOR evaluation, i.e. one is x > 1 - precision and zero is x < precision
const precision = 0.5

// XOR is very simple and does not make a very interesting scientific experiment; however, it is a good way to
// check whether your system works.
// Make sure recurrency is disabled for the XOR test. If NEAT is able to add recurrent connections, it may solve XOR by
// memorizing the order of the training set. (Which is why you may even want to randomize order to be most safe) All
// documented experiments with XOR are without recurrent connections. Interestingly, XOR can be solved by a recurrent
// network with no hidden nodes.
//
// This method performs evolution on XOR for specified number of generations. It will read NEAT context configuration
// from contextPath, the start genome configuration from genomePath, and output results into outDirPath
func XOR(context_path, genome_path, out_dir_path string, generations int) (*genetics.Population, error) {

	// Load context configuration
	configFile, err := os.Open(context_path)
	if err != nil {
		fmt.Println("Failed to load context")
		return nil, err
	}
	context := neat.LoadContext(configFile)
	context.IsDebugEnabled = true

	// Load Genome
	fmt.Println("Loading start genome for XOR experiment")
	genomeFile, err := os.Open(genome_path)
	if err != nil {
		fmt.Println("Failed to open genome file")
		return nil, err
	}
	start_genome, err := genetics.ReadGenome(genomeFile, 1)
	if err != nil {
		fmt.Println("Failed to read start genome")
		return nil, err
	}
	fmt.Println(start_genome)

	// Check if output dir exists
	if _, err := os.Stat(out_dir_path); err == nil {
		// clear it
		os.RemoveAll(out_dir_path)
	}
	// create output dir
	err = os.MkdirAll(out_dir_path, os.ModePerm)
	if err != nil {
		return nil, err
	}

	// Holders of records for each run
	evals := make([]int, context.NumRuns)
	genes := make([]int, context.NumRuns)
	nodes := make([]int, context.NumRuns)

	var successful_pop *genetics.Population

	for exp_count := 0; exp_count < context.NumRuns; exp_count++ {
		fmt.Print("\n>>>>> Spawning new population ")
		pop, err := genetics.NewPopulation(start_genome, context)
		if err != nil {
			fmt.Println("Failed to spawn new population from start genome")
			return nil, err
		} else {
			fmt.Println("OK <<<<<")
		}
		fmt.Print(">>>>> Verifying spawned population ")
		_, err = pop.Verify()
		if err != nil {
			fmt.Println("\n!!!!! Population verification failed !!!!!")
			return nil, err
		} else {
			fmt.Println("OK <<<<<")
		}

		for gen := 1; gen <= generations; gen ++ {
			fmt.Printf(">>>>> Epoch: %d\n", gen)
			success, winner_num, winner_genes, winner_nodes, err := xor_epoch(pop, gen, out_dir_path, context)
			if err != nil {
				fmt.Println("!!!!! Epoch evaluation failed !!!!!")
				return nil, err
			}
			if success {
				// Collect Stats on end of experiment
				evals[exp_count] = context.PopSize * (gen - 1) + winner_num
				genes[exp_count] = winner_genes
				nodes[exp_count] = winner_nodes
				fmt.Println(">>>>> The winner organism found! <<<<<")
				break
			}
		}

	}

	// Average and print stats

	fmt.Print("\nNodes: ")
	total_nodes := 0
	for exp_count := 0; exp_count < context.NumRuns; exp_count++ {
		fmt.Printf("\t%d",nodes[exp_count])
		total_nodes += nodes[exp_count]
	}

	fmt.Print("\nGenes: ")
	total_genes := 0
	for exp_count := 0; exp_count < context.NumRuns; exp_count++ {
		fmt.Printf("\t%d", genes[exp_count])
		total_genes += genes[exp_count]
	}

	fmt.Print("\nEvals: ")
	total_evals := 0
	for exp_count := 0; exp_count < context.NumRuns; exp_count++ {
		fmt.Printf("\t%d", evals[exp_count])
		total_evals += evals[exp_count]
	}

	fmt.Printf("\n>>>>>\nAverage Nodes:\t%d\nAverage Genes:\t%d\nAverage Evals:\t%d\n",
		total_nodes / context.NumRuns, total_genes / context.NumRuns, total_evals / context.NumRuns)

	return successful_pop, nil
}

// This method evaluates one epoch for given population and prints results into specified directory if any.
func xor_epoch(pop *genetics.Population, generation int, out_dir_path string, context *neat.NeatContext) (success bool, winner_num, winner_genes, winner_nodes int, err error) {
	// The flag to indicate that we have winner organism
	success = false
	// Evaluate each organism on a test
	for _, org := range pop.Organisms {
		res, err := xor_evaluate(org)
		if err != nil {
			return false, -1, -1, -1, err
		}
		if res {
			success = true
			winner_num = org.GNome.Id
			winner_genes = org.GNome.Extrons()
			winner_nodes = len(org.GNome.Nodes)
			if (winner_nodes == 5) {
				// You could dump out optimal genomes here if desired
				opt_path := fmt.Sprintf("%s/%s", out_dir_path, "xor_optimal")
				file, err := os.Create(opt_path)
				if err != nil {
					fmt.Printf("Failed to dump optimal genome, reason: %s\n", err)
				} else {
					org.GNome.Write(file)
					fmt.Printf("Dumped optimal genome to: %s\n", opt_path)
				}
			}
			break // we have winner
		}
	}
	// Average and max their fitnesses for dumping to file and snapshot
	for _, curr_species := range pop.Species {
		// This experiment control routine issues commands to collect ave and max fitness, as opposed to having
		// the snapshot do it, because this allows flexibility in terms of what time to observe fitnesses at
		curr_species.ComputeAvgFitness()
		curr_species.ComputeMaxFitness()
	}

	// Only print to file every print_every generations
	if success || generation % context.PrintEvery == 0 {
		pop_path := fmt.Sprintf("%s/gen_%d", out_dir_path, generation)
		file, err := os.Create(pop_path)
		if err != nil {
			fmt.Printf("Failed to dump population, reason: %s\n", err)
		} else {
			pop.WriteBySpecies(file)
		}
	}

	if success {
		// print winner organism
		for _, org := range pop.Organisms {
			if org.IsWinner {
				// Prints the winner organism to file!
				org_path := fmt.Sprintf("%s/%s", out_dir_path, "xor_winner")
				file, err := os.Create(org_path)
				if err != nil {
					fmt.Printf("Failed to dump winner organism genome, reason: %s\n", err)
				} else {
					org.GNome.Write(file)
					fmt.Printf("Generation #%d winner dumped to: %s\n", generation, org_path)
				}
				break
			}
		}
	} else {
		// Move to the next epoch if failed to find winner
		fmt.Println(">>>>> start next generation")
		_, err = pop.Epoch(generation, context)
	}

	return success, winner_num, winner_genes, winner_nodes, err
}

// This methods evalueates provided organism
func xor_evaluate(org *genetics.Organism) (bool, error) {
	// The four possible input combinations to xor
	// The first number is for biasing
	in := [][]float64{
		{1.0, 0.0, 0.0},
		{1.0, 0.0, 1.0},
		{1.0, 1.0, 0.0},
		{1.0, 1.0, 1.0}}

	net_depth, err := org.Net.MaxDepth() // The max depth of the network to be activated
	if err != nil {
		fmt.Println("Failed to estimate maximal depth of the network")
		return false, err
	}
	fmt.Printf("Network depth: %d for organism: %d\n", net_depth, org.GNome.Id)

	success := false  // Check for successful activation
	out := make([]float64, 4) // The four outputs

	// Load and activate the network on each input
	for count := 0; count < 4; count++ {
		org.Net.LoadSensors(in[count])

		// Relax net and get output
		success, err = org.Net.Activate()
		if err != nil {
			fmt.Println("Failed to activate network")
			return false, err
		}

		// use depth to ensure relaxation
		for relax := 0; relax <= net_depth; relax++ {
			success, err = org.Net.Activate()
			if err != nil {
				fmt.Println("Failed to activate network")
				return false, err
			}
		}
		out[count] = org.Net.Outputs[0].Activation

		//fmt.Println(org.Net.Outputs)

		org.Net.Flush()
	}

	error_sum := 0.0
	if (success) {
		// Mean Squared Error
		error_sum = math.Abs(out[0]) + math.Abs(1.0 - out[1]) + math.Abs(1.0 - out[2]) + math.Abs(out[3])
		org.Fitness = math.Pow(4.0 - error_sum, 2.0)
		org.Error = error_sum
	} else {
		// The network is flawed (shouldn't happen)
		error_sum = 999.0
		org.Fitness = 0.001
	}

	if out[0] < precision && out[1] >= 1 - precision && out[2] >= 1 - precision && out[3] < precision {
		org.IsWinner = true
		fmt.Printf(">>>> Output activations: %e\n", out)

	} else {
		org.IsWinner = false
	}
	return org.IsWinner, nil
}
<commit_msg>Added additional debug output when network depth is ZERO<commit_after>package experiments

import (
	"github.com/yaricom/goNEAT/neat"
	"os"
	"fmt"
	"github.com/yaricom/goNEAT/neat/genetics"
	"math"
)

// The precision to use for XOR evaluation, i.e. one is x > 1 - precision and zero is x < precision
const precision = 0.5

// XOR is very simple and does not make a very interesting scientific experiment; however, it is a good way to
// check whether your system works.
// Make sure recurrency is disabled for the XOR test. If NEAT is able to add recurrent connections, it may solve XOR by
// memorizing the order of the training set. (Which is why you may even want to randomize order to be most safe) All
// documented experiments with XOR are without recurrent connections. Interestingly, XOR can be solved by a recurrent
// network with no hidden nodes.
//
// This method performs evolution on XOR for specified number of generations. It will read NEAT context configuration
// from contextPath, the start genome configuration from genomePath, and output results into outDirPath
func XOR(context_path, genome_path, out_dir_path string, generations int) (*genetics.Population, error) {

	// Load context configuration
	configFile, err := os.Open(context_path)
	if err != nil {
		fmt.Println("Failed to load context")
		return nil, err
	}
	context := neat.LoadContext(configFile)
	context.IsDebugEnabled = true

	// Load Genome
	fmt.Println("Loading start genome for XOR experiment")
	genomeFile, err := os.Open(genome_path)
	if err != nil {
		fmt.Println("Failed to open genome file")
		return nil, err
	}
	start_genome, err := genetics.ReadGenome(genomeFile, 1)
	if err != nil {
		fmt.Println("Failed to read start genome")
		return nil, err
	}
	fmt.Println(start_genome)

	// Check if output dir exists
	if _, err := os.Stat(out_dir_path); err == nil {
		// clear it
		os.RemoveAll(out_dir_path)
	}
	// create output dir
	err = os.MkdirAll(out_dir_path, os.ModePerm)
	if err != nil {
		return nil, err
	}

	// Holders of records for each run
	evals := make([]int, context.NumRuns)
	genes := make([]int, context.NumRuns)
	nodes := make([]int, context.NumRuns)

	var successful_pop *genetics.Population

	for exp_count := 0; exp_count < context.NumRuns; exp_count++ {
		fmt.Print("\n>>>>> Spawning new population ")
		pop, err := genetics.NewPopulation(start_genome, context)
		if err != nil {
			fmt.Println("Failed to spawn new population from start genome")
			return nil, err
		} else {
			fmt.Println("OK <<<<<")
		}
		fmt.Print(">>>>> Verifying spawned population ")
		_, err = pop.Verify()
		if err != nil {
			fmt.Println("\n!!!!! Population verification failed !!!!!")
			return nil, err
		} else {
			fmt.Println("OK <<<<<")
		}

		for gen := 1; gen <= generations; gen ++ {
			fmt.Printf(">>>>> Epoch: %d\n", gen)
			success, winner_num, winner_genes, winner_nodes, err := xor_epoch(pop, gen, out_dir_path, context)
			if err != nil {
				fmt.Println("!!!!! Epoch evaluation failed !!!!!")
				return nil, err
			}
			if success {
				// Collect Stats on end of experiment
				evals[exp_count] = context.PopSize * (gen - 1) + winner_num
				genes[exp_count] = winner_genes
				nodes[exp_count] = winner_nodes
				fmt.Println(">>>>> The winner organism found! <<<<<")
				break
			}
		}

	}

	// Average and print stats

	fmt.Print("\nNodes: ")
	total_nodes := 0
	for exp_count := 0; exp_count < context.NumRuns; exp_count++ {
		fmt.Printf("\t%d",nodes[exp_count])
		total_nodes += nodes[exp_count]
	}

	fmt.Print("\nGenes: ")
	total_genes := 0
	for exp_count := 0; exp_count < context.NumRuns; exp_count++ {
		fmt.Printf("\t%d", genes[exp_count])
		total_genes += genes[exp_count]
	}

	fmt.Print("\nEvals: ")
	total_evals := 0
	for exp_count := 0; exp_count < context.NumRuns; exp_count++ {
		fmt.Printf("\t%d", evals[exp_count])
		total_evals += evals[exp_count]
	}

	fmt.Printf("\n>>>>>\nAverage Nodes:\t%d\nAverage Genes:\t%d\nAverage Evals:\t%d\n",
		total_nodes / context.NumRuns, total_genes / context.NumRuns, total_evals / context.NumRuns)

	return successful_pop, nil
}

// This method evaluates one epoch for given population and prints results into specified directory if any.
func xor_epoch(pop *genetics.Population, generation int, out_dir_path string, context *neat.NeatContext) (success bool, winner_num, winner_genes, winner_nodes int, err error) {
	// The flag to indicate that we have winner organism
	success = false
	// Evaluate each organism on a test
	for _, org := range pop.Organisms {
		res, err := xor_evaluate(org)
		if err != nil {
			return false, -1, -1, -1, err
		}
		if res {
			success = true
			winner_num = org.GNome.Id
			winner_genes = org.GNome.Extrons()
			winner_nodes = len(org.GNome.Nodes)
			if (winner_nodes == 5) {
				// You could dump out optimal genomes here if desired
				opt_path := fmt.Sprintf("%s/%s", out_dir_path, "xor_optimal")
				file, err := os.Create(opt_path)
				if err != nil {
					fmt.Printf("Failed to dump optimal genome, reason: %s\n", err)
				} else {
					org.GNome.Write(file)
					fmt.Printf("Dumped optimal genome to: %s\n", opt_path)
				}
			}
			break // we have winner
		}
	}
	// Average and max their fitnesses for dumping to file and snapshot
	for _, curr_species := range pop.Species {
		// This experiment control routine issues commands to collect ave and max fitness, as opposed to having
		// the snapshot do it, because this allows flexibility in terms of what time to observe fitnesses at
		curr_species.ComputeAvgFitness()
		curr_species.ComputeMaxFitness()
	}

	// Only print to file every print_every generations
	if success || generation % context.PrintEvery == 0 {
		pop_path := fmt.Sprintf("%s/gen_%d", out_dir_path, generation)
		file, err := os.Create(pop_path)
		if err != nil {
			fmt.Printf("Failed to dump population, reason: %s\n", err)
		} else {
			pop.WriteBySpecies(file)
		}
	}

	if success {
		// print winner organism
		for _, org := range pop.Organisms {
			if org.IsWinner {
				// Prints the winner organism to file!
				org_path := fmt.Sprintf("%s/%s", out_dir_path, "xor_winner")
				file, err := os.Create(org_path)
				if err != nil {
					fmt.Printf("Failed to dump winner organism genome, reason: %s\n", err)
				} else {
					org.GNome.Write(file)
					fmt.Printf("Generation #%d winner dumped to: %s\n", generation, org_path)
				}
				break
			}
		}
	} else {
		// Move to the next epoch if failed to find winner
		fmt.Println(">>>>> start next generation")
		_, err = pop.Epoch(generation, context)
	}

	return success, winner_num, winner_genes, winner_nodes, err
}

// This methods evalueates provided organism
func xor_evaluate(org *genetics.Organism) (bool, error) {
	// The four possible input combinations to xor
	// The first number is for biasing
	in := [][]float64{
		{1.0, 0.0, 0.0},
		{1.0, 0.0, 1.0},
		{1.0, 1.0, 0.0},
		{1.0, 1.0, 1.0}}

	net_depth, err := org.Net.MaxDepth() // The max depth of the network to be activated
	if err != nil {
		fmt.Println("Failed to estimate maximal depth of the network")
		return false, err
	}
	fmt.Printf("Network depth: %d for organism: %d\n", net_depth, org.GNome.Id)
	if net_depth == 0 {
		fmt.Println(org.GNome)
	}

	success := false  // Check for successful activation
	out := make([]float64, 4) // The four outputs

	// Load and activate the network on each input
	for count := 0; count < 4; count++ {
		org.Net.LoadSensors(in[count])

		// Relax net and get output
		success, err = org.Net.Activate()
		if err != nil {
			fmt.Println("Failed to activate network")
			return false, err
		}

		// use depth to ensure relaxation
		for relax := 0; relax <= net_depth; relax++ {
			success, err = org.Net.Activate()
			if err != nil {
				fmt.Println("Failed to activate network")
				return false, err
			}
		}
		out[count] = org.Net.Outputs[0].Activation

		//fmt.Println(org.Net.Outputs)

		org.Net.Flush()
	}

	error_sum := 0.0
	if (success) {
		// Mean Squared Error
		error_sum = math.Abs(out[0]) + math.Abs(1.0 - out[1]) + math.Abs(1.0 - out[2]) + math.Abs(out[3])
		org.Fitness = math.Pow(4.0 - error_sum, 2.0)
		org.Error = error_sum
	} else {
		// The network is flawed (shouldn't happen)
		error_sum = 999.0
		org.Fitness = 0.001
	}

	if out[0] < precision && out[1] >= 1 - precision && out[2] >= 1 - precision && out[3] < precision {
		org.IsWinner = true
		fmt.Printf(">>>> Output activations: %e\n", out)

	} else {
		org.IsWinner = false
	}
	return org.IsWinner, nil
}
<|endoftext|>