<commit_before>package gateway

import (
	"time"

	"github.com/rivine/rivine/build"
)

const (
	// maxLocalOutbound is currently set to 3, meaning the gateway will not
	// consider a local node to be an outbound peer if the gateway already has
	// 3 outbound peers. Three is currently needed to handle situations where
	// the gateway is at high risk of connecting to itself (such as a low
	// number of total peers, especially such as in a testing environment).
	// Once the gateway has a proper way to figure out that it's trying to
	// connect to itself, this number can be reduced.
	maxLocalOutboundPeers = 3

	// EncodedVersionHeaderLength is the length of a session header encoded
	// with the encode package.
	EncodedVersionHeaderLength = 57

	// saveFrequency defines how often the gateway saves its persistence.
	saveFrequency = time.Minute * 2
)

var (
	// minAcceptableVersion is the version below which the gateway will refuse to
	// connect to peers and reject connection attempts
	minAcceptableVersion = build.NewVersion(0, 0, 1)

	// fastNodePurgeDelay defines the amount of time that is waited between each
	// iteration of the purge loop when the gateway has enough nodes to be
	// needing to purge quickly.
	fastNodePurgeDelay = build.Select(build.Var{
		Standard: 1 * time.Minute,
		Dev:      5 * time.Second,
		Testing:  200 * time.Millisecond,
	}).(time.Duration)

	// healthyNodeListLen defines the number of nodes that the gateway must
	// have in the node list before it will stop asking peers for more nodes.
	healthyNodeListLen = build.Select(build.Var{
		Standard: int(200),
		Dev:      int(30),
		Testing:  int(15),
	}).(int)

	// maxSharedNodes defines the number of nodes that will be shared between
	// peers when they are expanding their node lists.
	maxSharedNodes = build.Select(build.Var{
		Standard: uint64(10),
		Dev:      uint64(5),
		Testing:  uint64(3),
	}).(uint64)

	// nodePurgeDelay defines the amount of time that is waited between each
	// iteration of the node purge loop.
	nodePurgeDelay = build.Select(build.Var{
		Standard: 10 * time.Minute,
		Dev:      20 * time.Second,
		Testing:  500 * time.Millisecond,
	}).(time.Duration)

	// nodeListDelay defines the amount of time that is waited between each
	// iteration of the node list loop.
	nodeListDelay = build.Select(build.Var{
		Standard: 5 * time.Second,
		Dev:      3 * time.Second,
		Testing:  500 * time.Millisecond,
	}).(time.Duration)

	// peerRPCDelay defines the amount of time waited between each RPC accepted
	// from a peer. Without this delay, a peer can force us to spin up thousands
	// of goroutines per second.
	peerRPCDelay = build.Select(build.Var{
		Standard: 1 * time.Second,
		Dev:      1 * time.Second,
		Testing:  10 * time.Millisecond,
	}).(time.Duration)

	// pruneNodeListLen defines the number of nodes that the gateway must have
	// to be pruning nodes from the node list.
	pruneNodeListLen = build.Select(build.Var{
		Standard: int(50),
		Dev:      int(15),
		Testing:  int(10),
	}).(int)

	// quickPruneListLen defines the number of nodes that the gateway must have
	// to be pruning nodes quickly from the node list.
	quickPruneListLen = build.Select(build.Var{
		Standard: int(250),
		Dev:      int(40),
		Testing:  int(20),
	}).(int)
)

var (
	// The gateway will sleep this long between incoming connections. For
	// attack reasons, the acceptInterval should be longer than the
	// nodeListDelay. Right at startup, a node is vulnerable to being flooded
	// by Sybil attackers. The node's best defense is to wait until it has
	// filled out its nodelist somewhat from the bootstrap nodes. An attacker
	// needs to completely dominate the nodelist and the peerlist to be
	// successful, so just a few honest nodes from requests to the bootstraps
	// should be enough to fend from most attacks.
	acceptInterval = build.Select(build.Var{
		Standard: 6 * time.Second,
		Dev:      3 * time.Second,
		Testing:  100 * time.Millisecond,
	}).(time.Duration)

	// acquiringPeersDelay defines the amount of time that is waited between
	// iterations of the peer acquisition loop if the gateway is actively
	// forming new connections with peers.
	acquiringPeersDelay = build.Select(build.Var{
		Standard: 5 * time.Second,
		Dev:      3 * time.Second,
		Testing:  500 * time.Millisecond,
	}).(time.Duration)

	// fullyConnectedThreshold defines the number of peers that the gateway can
	// have before it stops accepting inbound connections.
	fullyConnectedThreshold = build.Select(build.Var{
		Standard: 128,
		Dev:      20,
		Testing:  10,
	}).(int)

	// maxConcurrentOutboundPeerRequests defines the maximum number of peer
	// connections that the gateway will try to form concurrently.
	maxConcurrentOutboundPeerRequests = build.Select(build.Var{
		Standard: 3,
		Dev:      2,
		Testing:  2,
	}).(int)

	// noNodesDelay defines the amount of time that is waited between
	// iterations of the peer acquisition loop if the gateway does not have any
	// nodes in the nodelist.
	noNodesDelay = build.Select(build.Var{
		Standard: 20 * time.Second,
		Dev:      10 * time.Second,
		Testing:  3 * time.Second,
	}).(time.Duration)

	// unwawntedLocalPeerDelay defines the amount of time that is waited
	// between iterations of the permanentPeerManager if the gateway has at
	// least a few outbound peers, but is not well connected, and the recently
	// selected peer was a local peer. The wait is mostly to prevent the
	// gateway from hogging the CPU in the event that all peers are local
	// peers.
	unwantedLocalPeerDelay = build.Select(build.Var{
		Standard: 2 * time.Second,
		Dev:      1 * time.Second,
		Testing:  100 * time.Millisecond,
	}).(time.Duration)

	// wellConnectedDelay defines the amount of time that is waited between
	// iterations of the peer acquisition loop if the gateway is well
	// connected.
	wellConnectedDelay = build.Select(build.Var{
		Standard: 5 * time.Minute,
		Dev:      1 * time.Minute,
		Testing:  3 * time.Second,
	}).(time.Duration)

	// wellConnectedThreshold is the number of outbound connections at which
	// the gateway will not attempt to make new outbound connections.
	wellConnectedThreshold = build.Select(build.Var{
		Standard: 8,
		Dev:      5,
		Testing:  4,
	}).(int)
)

var (
	// connStdDeadline defines the standard deadline that should be used for
	// all temporary connections to the gateway.
	connStdDeadline = build.Select(build.Var{
		Standard: 5 * time.Minute,
		Dev:      2 * time.Minute,
		Testing:  30 * time.Second,
	}).(time.Duration)

	// the gateway will abort a connection attempt after this long
	dialTimeout = build.Select(build.Var{
		Standard: 3 * time.Minute,
		Dev:      20 * time.Second,
		Testing:  500 * time.Millisecond,
	}).(time.Duration)

	// rpcStdDeadline defines the standard deadline that should be used for all
	// incoming RPC calls.
	rpcStdDeadline = build.Select(build.Var{
		Standard: 5 * time.Minute,
		Dev:      3 * time.Minute,
		Testing:  30 * time.Second,
	}).(time.Duration)
)
<commit_msg>increase the peerRPCDelay in standard<commit_after>package gateway

import (
	"time"

	"github.com/rivine/rivine/build"
)

const (
	// maxLocalOutbound is currently set to 3, meaning the gateway will not
	// consider a local node to be an outbound peer if the gateway already has
	// 3 outbound peers. Three is currently needed to handle situations where
	// the gateway is at high risk of connecting to itself (such as a low
	// number of total peers, especially such as in a testing environment).
	// Once the gateway has a proper way to figure out that it's trying to
	// connect to itself, this number can be reduced.
	maxLocalOutboundPeers = 3

	// EncodedVersionHeaderLength is the length of a session header encoded
	// with the encode package.
	EncodedVersionHeaderLength = 57

	// saveFrequency defines how often the gateway saves its persistence.
	saveFrequency = time.Minute * 2
)

var (
	// minAcceptableVersion is the version below which the gateway will refuse to
	// connect to peers and reject connection attempts
	minAcceptableVersion = build.NewVersion(0, 0, 1)

	// fastNodePurgeDelay defines the amount of time that is waited between each
	// iteration of the purge loop when the gateway has enough nodes to be
	// needing to purge quickly.
	fastNodePurgeDelay = build.Select(build.Var{
		Standard: 1 * time.Minute,
		Dev:      5 * time.Second,
		Testing:  200 * time.Millisecond,
	}).(time.Duration)

	// healthyNodeListLen defines the number of nodes that the gateway must
	// have in the node list before it will stop asking peers for more nodes.
	healthyNodeListLen = build.Select(build.Var{
		Standard: int(200),
		Dev:      int(30),
		Testing:  int(15),
	}).(int)

	// maxSharedNodes defines the number of nodes that will be shared between
	// peers when they are expanding their node lists.
	maxSharedNodes = build.Select(build.Var{
		Standard: uint64(10),
		Dev:      uint64(5),
		Testing:  uint64(3),
	}).(uint64)

	// nodePurgeDelay defines the amount of time that is waited between each
	// iteration of the node purge loop.
	nodePurgeDelay = build.Select(build.Var{
		Standard: 10 * time.Minute,
		Dev:      20 * time.Second,
		Testing:  500 * time.Millisecond,
	}).(time.Duration)

	// nodeListDelay defines the amount of time that is waited between each
	// iteration of the node list loop.
	nodeListDelay = build.Select(build.Var{
		Standard: 5 * time.Second,
		Dev:      3 * time.Second,
		Testing:  500 * time.Millisecond,
	}).(time.Duration)

	// peerRPCDelay defines the amount of time waited between each RPC accepted
	// from a peer. Without this delay, a peer can force us to spin up thousands
	// of goroutines per second.
	peerRPCDelay = build.Select(build.Var{
		Standard: 3 * time.Second,
		Dev:      1 * time.Second,
		Testing:  10 * time.Millisecond,
	}).(time.Duration)

	// pruneNodeListLen defines the number of nodes that the gateway must have
	// to be pruning nodes from the node list.
	pruneNodeListLen = build.Select(build.Var{
		Standard: int(50),
		Dev:      int(15),
		Testing:  int(10),
	}).(int)

	// quickPruneListLen defines the number of nodes that the gateway must have
	// to be pruning nodes quickly from the node list.
	quickPruneListLen = build.Select(build.Var{
		Standard: int(250),
		Dev:      int(40),
		Testing:  int(20),
	}).(int)
)

var (
	// The gateway will sleep this long between incoming connections. For
	// attack reasons, the acceptInterval should be longer than the
	// nodeListDelay. Right at startup, a node is vulnerable to being flooded
	// by Sybil attackers. The node's best defense is to wait until it has
	// filled out its nodelist somewhat from the bootstrap nodes. An attacker
	// needs to completely dominate the nodelist and the peerlist to be
	// successful, so just a few honest nodes from requests to the bootstraps
	// should be enough to fend from most attacks.
	acceptInterval = build.Select(build.Var{
		Standard: 6 * time.Second,
		Dev:      3 * time.Second,
		Testing:  100 * time.Millisecond,
	}).(time.Duration)

	// acquiringPeersDelay defines the amount of time that is waited between
	// iterations of the peer acquisition loop if the gateway is actively
	// forming new connections with peers.
	acquiringPeersDelay = build.Select(build.Var{
		Standard: 5 * time.Second,
		Dev:      3 * time.Second,
		Testing:  500 * time.Millisecond,
	}).(time.Duration)

	// fullyConnectedThreshold defines the number of peers that the gateway can
	// have before it stops accepting inbound connections.
	fullyConnectedThreshold = build.Select(build.Var{
		Standard: 128,
		Dev:      20,
		Testing:  10,
	}).(int)

	// maxConcurrentOutboundPeerRequests defines the maximum number of peer
	// connections that the gateway will try to form concurrently.
	maxConcurrentOutboundPeerRequests = build.Select(build.Var{
		Standard: 3,
		Dev:      2,
		Testing:  2,
	}).(int)

	// noNodesDelay defines the amount of time that is waited between
	// iterations of the peer acquisition loop if the gateway does not have any
	// nodes in the nodelist.
	noNodesDelay = build.Select(build.Var{
		Standard: 20 * time.Second,
		Dev:      10 * time.Second,
		Testing:  3 * time.Second,
	}).(time.Duration)

	// unwawntedLocalPeerDelay defines the amount of time that is waited
	// between iterations of the permanentPeerManager if the gateway has at
	// least a few outbound peers, but is not well connected, and the recently
	// selected peer was a local peer. The wait is mostly to prevent the
	// gateway from hogging the CPU in the event that all peers are local
	// peers.
	unwantedLocalPeerDelay = build.Select(build.Var{
		Standard: 2 * time.Second,
		Dev:      1 * time.Second,
		Testing:  100 * time.Millisecond,
	}).(time.Duration)

	// wellConnectedDelay defines the amount of time that is waited between
	// iterations of the peer acquisition loop if the gateway is well
	// connected.
	wellConnectedDelay = build.Select(build.Var{
		Standard: 5 * time.Minute,
		Dev:      1 * time.Minute,
		Testing:  3 * time.Second,
	}).(time.Duration)

	// wellConnectedThreshold is the number of outbound connections at which
	// the gateway will not attempt to make new outbound connections.
	wellConnectedThreshold = build.Select(build.Var{
		Standard: 8,
		Dev:      5,
		Testing:  4,
	}).(int)
)

var (
	// connStdDeadline defines the standard deadline that should be used for
	// all temporary connections to the gateway.
	connStdDeadline = build.Select(build.Var{
		Standard: 5 * time.Minute,
		Dev:      2 * time.Minute,
		Testing:  30 * time.Second,
	}).(time.Duration)

	// the gateway will abort a connection attempt after this long
	dialTimeout = build.Select(build.Var{
		Standard: 3 * time.Minute,
		Dev:      20 * time.Second,
		Testing:  500 * time.Millisecond,
	}).(time.Duration)

	// rpcStdDeadline defines the standard deadline that should be used for all
	// incoming RPC calls.
	rpcStdDeadline = build.Select(build.Var{
		Standard: 5 * time.Minute,
		Dev:      3 * time.Minute,
		Testing:  30 * time.Second,
	}).(time.Duration)
)
<|endoftext|>


<commit_before>package iplookup

import (
	_ "errors"
	_ "fmt"
	"github.com/oschwald/maxminddb-golang"
	"github.com/whosonfirst/go-whosonfirst-log"
	"net"
)

// See also: https://github.com/whosonfirst/p5-Whosonfirst-MaxMind-Writer/blob/master/lib/Whosonfirst/MaxMind/Types.pm
// see the way we're prefixing `whosonfirst` with maxmindb... yeah, I'm not sure either...

type Response interface {
	WOFId() int64
}

type WOFResponse struct {
	WhosonfirstId uint64 `maxminddb:"whosonfirst_id"`
}

func (rsp WOFResponse) WOFId() int64 {
	wofid := int64(rsp.WhosonfirstId)
	return wofid
}

type MaxMindResponse struct {
	Country struct {
		ISOCode       string `maxminddb:"iso_code"`
		GeonameId     uint64 `maxminddb:"geoname_id"`
		WhosonfirstId uint64 `maxminddb:"whosonfirst_id"`
	} `maxminddb:"country"`
	City struct {
		GeonameId     uint64 `maxminddb:"geoname_id"`
		WhosonfirstId uint64 `maxminddb:"whosonfirst_id"`
	} `maxminddb:"city"`
}

func (rsp MaxMindResponse) WOFId() int64 {

	candidate := rsp.City.WhosonfirstId

	if candidate == 0 {
		candidate = rsp.Country.WhosonfirstId
	}

	wofid := int64(candidate)
	return wofid
}

type IPLookup struct {
	mmdb   *maxminddb.Reader
	source string
	logger *log.WOFLogger
}

func NewIPLookup(db string, source string, logger *log.WOFLogger) (*IPLookup, error) {

	mmdb, err := maxminddb.Open(db)

	if err != nil {
		return nil, err
	}

	ip := IPLookup{
		mmdb:   mmdb,
		source: source,
		logger: logger,
	}

	return &ip, nil
}

func (ip *IPLookup) Query(addr net.IP) (int64, error) {

	rsp, err := ip.QueryRaw(addr)

	if err != nil {
		return 0, err
	}

	wofid := rsp.WOFId()
	return wofid, nil
}

func (ip *IPLookup) QueryRaw(addr net.IP) (Response, error) {

	var rsp Response
	var err error

	if ip.source == "wof" {
		rsp, err = ip.query_wof(addr)
	} else {
		rsp, err = ip.query_maxmind(addr)
	}

	return rsp, err
}

func (ip *IPLookup) query_wof(addr net.IP) (Response, error) {

	var rsp WOFResponse
	err := ip.mmdb.Lookup(addr, &rsp)

	if err != nil {
		return nil, err
	}

	return rsp, nil
}

func (ip *IPLookup) query_maxmind(addr net.IP) (Response, error) {

	var rsp MaxMindResponse
	err := ip.mmdb.Lookup(addr, &rsp)

	if err != nil {
		return nil, err
	}

	return rsp, nil
}
<commit_msg>stub out the notion of a WOFConcordanceResponse<commit_after>package iplookup

import (
	_ "errors"
	_ "fmt"
	"github.com/oschwald/maxminddb-golang"
	"github.com/whosonfirst/go-whosonfirst-log"
	"net"
)

// See also: https://github.com/whosonfirst/p5-Whosonfirst-MaxMind-Writer/blob/master/lib/Whosonfirst/MaxMind/Types.pm
// see the way we're prefixing `whosonfirst` with maxmindb... yeah, I'm not sure either...

type Response interface {
	WOFId() int64
}

type WOFResponse struct {
	WhosonfirstId uint64 `maxminddb:"whosonfirst_id"`
}

func (rsp WOFResponse) WOFId() int64 {
	wofid := int64(rsp.WhosonfirstId)
	return wofid
}

type WOFConcordanceResponse struct {
	Country struct {
		// ISOCode       string `maxminddb:"iso_code"`
		GeonameId uint64 `maxminddb:"geoname_id"`
	} `maxminddb:"country"`
	City struct {
		GeonameId uint64 `maxminddb:"geoname_id"`
	} `maxminddb:"city"`
}

func (rsp WOFConcordanceResponse) WOFId() int64 {
	return 0

	/*
	possible := make([]uint64, 0)

		 possible = append(possible, rsp.City.GeonameId)
		 possible = append(possible, rsp.Country.GeonameId)

		 ip.logger.Debug("possible matches for %v: %v", addr, possible)

		 for _, gnid := range possible {

		     if gnid == 0 {
		     	     continue
				}

					wofid, err := ip.ConcordifyGeonames(gnid)

					       if err != nil {
					       	      continue
							}

								return wofid, nil
								}
	*/
}

/*
func (rsp WOFConcordanceResponse) ConcordifyGeonames(gnid uint64) (int64, error) {

     ip.logger.Debug("concordify geonames %d", gnid)

     str_gnid := strconv.FormatUint(gnid, 10)

     rows, err := ip.concordances.Where("gn:id", str_gnid)

     if err != nil {
     	return -1, err
	}

	first := rows[0]
	others := first.AsMap()

	str_wofid, ok := others["wof:id"]

	if !ok {
	   return -1, errors.New("Unable to locate concordance")
	   }

	   wofid, err := strconv.ParseInt(str_wofid, 10, 64)

	   if err != nil {
	      return -1, err
	      }

	      ip.logger.Debug("geonames ID (%d) is WOF ID %d", gnid, wofid)
	      return wofid, nil
}
*/

type MaxMindResponse struct {
	Country struct {
		// ISOCode       string `maxminddb:"iso_code"`
		GeonameId     uint64 `maxminddb:"geoname_id"`
		WhosonfirstId uint64 `maxminddb:"whosonfirst_id"`
	} `maxminddb:"country"`
	City struct {
		GeonameId     uint64 `maxminddb:"geoname_id"`
		WhosonfirstId uint64 `maxminddb:"whosonfirst_id"`
	} `maxminddb:"city"`
}

func (rsp MaxMindResponse) WOFId() int64 {

	candidate := rsp.City.WhosonfirstId

	if candidate == 0 {
		candidate = rsp.Country.WhosonfirstId
	}

	wofid := int64(candidate)
	return wofid
}

type IPLookup struct {
	mmdb   *maxminddb.Reader
	source string
	logger *log.WOFLogger
}

func NewIPLookup(db string, source string, logger *log.WOFLogger) (*IPLookup, error) {

	mmdb, err := maxminddb.Open(db)

	if err != nil {
		return nil, err
	}

	ip := IPLookup{
		mmdb:   mmdb,
		source: source,
		logger: logger,
	}

	return &ip, nil
}

func (ip *IPLookup) Query(addr net.IP) (int64, error) {

	rsp, err := ip.QueryRaw(addr)

	if err != nil {
		return 0, err
	}

	wofid := rsp.WOFId()
	return wofid, nil
}

func (ip *IPLookup) QueryRaw(addr net.IP) (Response, error) {

	var rsp Response
	var err error

	if ip.source == "wof" {
		rsp, err = ip.query_wof(addr)
	} else {
		rsp, err = ip.query_maxmind(addr)
	}

	return rsp, err
}

func (ip *IPLookup) query_wof(addr net.IP) (Response, error) {

	var rsp WOFResponse
	err := ip.mmdb.Lookup(addr, &rsp)

	if err != nil {
		return nil, err
	}

	return rsp, nil
}

func (ip *IPLookup) query_maxmind(addr net.IP) (Response, error) {

	var rsp MaxMindResponse
	err := ip.mmdb.Lookup(addr, &rsp)

	if err != nil {
		return nil, err
	}

	return rsp, nil
}
<|endoftext|>


<commit_before>package irc

import (
	//	"github.com/thoj/go-ircevent"
	"crypto/tls"
	"testing"
	"time"
)

func TestParserCrash1(t *testing.T) {
	Fuzz([]byte("pRIVMSG \x01ACTION\x01"))
}

func TestConnectionEmtpyServer(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("")
	if err == nil {
		t.Fatal("emtpy server string not detected")
	}
}

func TestConnectionDoubleColon(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("::")
	if err == nil {
		t.Fatal("wrong number of ':' not detected")
	}
}

func TestConnectionMissingHost(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect(":6667")
	if err == nil {
		t.Fatal("missing host not detected")
	}
}

func TestConnectionMissingPort(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("chat.freenode.net:")
	if err == nil {
		t.Fatal("missing port not detected")
	}
}

func TestConnectionNegativePort(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("chat.freenode.net:-1")
	if err == nil {
		t.Fatal("negative port number not detected")
	}
}

func TestConnectionTooLargePort(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("chat.freenode.net:65536")
	if err == nil {
		t.Fatal("too large port number not detected")
	}
}

func TestConnectionMissingLog(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.Log = nil
	err := irccon.Connect("chat.freenode.net:6667")
	if err == nil {
		t.Fatal("missing 'Log' not detected")
	}
}

func TestConnectionEmptyUser(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	// user may be changed after creation
	irccon.user = ""
	err := irccon.Connect("chat.freenode.net:6667")
	if err == nil {
		t.Fatal("empty 'user' not detected")
	}
}

func TestConnectionEmptyNick(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	// nick may be changed after creation
	irccon.nick = ""
	err := irccon.Connect("chat.freenode.net:6667")
	if err == nil {
		t.Fatal("empty 'nick' not detected")
	}
}

func TestRemoveCallback(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.VerboseCallbackHandler = true
	irccon.Debug = true

	done := make(chan int, 10)

	irccon.AddCallback("TEST", func(e *Event) { done <- 1 })
	id := irccon.AddCallback("TEST", func(e *Event) { done <- 2 })
	irccon.AddCallback("TEST", func(e *Event) { done <- 3 })

	// Should remove callback at index 1
	irccon.RemoveCallback("TEST", id)

	irccon.RunCallbacks(&Event{
		Code: "TEST",
	})

	var results []int

	results = append(results, <-done)
	results = append(results, <-done)

	if len(results) != 2 || results[0] == 2 || results[1] == 2 {
		t.Error("Callback 2 not removed")
	}
}

func TestWildcardCallback(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.VerboseCallbackHandler = true
	irccon.Debug = true

	done := make(chan int, 10)

	irccon.AddCallback("TEST", func(e *Event) { done <- 1 })
	irccon.AddCallback("*", func(e *Event) { done <- 2 })

	irccon.RunCallbacks(&Event{
		Code: "TEST",
	})

	var results []int

	results = append(results, <-done)
	results = append(results, <-done)

	if len(results) != 2 || !(results[0] == 1 && results[1] == 2) {
		t.Error("Wildcard callback not called")
	}
}

func TestClearCallback(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.VerboseCallbackHandler = true
	irccon.Debug = true

	done := make(chan int, 10)

	irccon.AddCallback("TEST", func(e *Event) { done <- 0 })
	irccon.AddCallback("TEST", func(e *Event) { done <- 1 })
	irccon.ClearCallback("TEST")
	irccon.AddCallback("TEST", func(e *Event) { done <- 2 })
	irccon.AddCallback("TEST", func(e *Event) { done <- 3 })

	irccon.RunCallbacks(&Event{
		Code: "TEST",
	})

	var results []int

	results = append(results, <-done)
	results = append(results, <-done)

	if len(results) != 2 || !(results[0] == 2 && results[1] == 3) {
		t.Error("Callbacks not cleared")
	}
}

func TestIRCemptyNick(t *testing.T) {
	irccon := IRC("", "go-eventirc")
	irccon = nil
	if irccon != nil {
		t.Error("empty nick didn't result in error")
		t.Fail()
	}
}

func TestIRCemptyUser(t *testing.T) {
	irccon := IRC("go-eventirc", "")
	if irccon != nil {
		t.Error("empty user didn't result in error")
	}
}
func TestConnection(t *testing.T) {
	irccon1 := IRC("go-eventirc1", "go-eventirc1")
	irccon1.VerboseCallbackHandler = true
	irccon1.Debug = true
	irccon2 := IRC("go-eventirc2", "go-eventirc2")
	irccon2.VerboseCallbackHandler = true
	irccon2.Debug = true
	err := irccon1.Connect("irc.freenode.net:6667")
	if err != nil {
		t.Log(err.Error())
		t.Fatal("Can't connect to freenode.")
	}
	err = irccon2.Connect("irc.freenode.net:6667")
	if err != nil {
		t.Log(err.Error())
		t.Fatal("Can't connect to freenode.")
	}
	irccon1.AddCallback("001", func(e *Event) { irccon1.Join("#go-eventirc") })
	irccon2.AddCallback("001", func(e *Event) { irccon2.Join("#go-eventirc") })
	con2ok := false
	irccon1.AddCallback("366", func(e *Event) {
		t := time.NewTicker(1 * time.Second)
		i := 10
		for {
			<-t.C
			irccon1.Privmsgf("#go-eventirc", "Test Message%d\n", i)
			if con2ok {
				i -= 1
			}
			if i == 0 {
				t.Stop()
				irccon1.Quit()
			}
		}
	})

	irccon2.AddCallback("366", func(e *Event) {
		irccon2.Privmsg("#go-eventirc", "Test Message\n")
		con2ok = true
		irccon2.Nick("go-eventnewnick")
	})

	irccon2.AddCallback("PRIVMSG", func(e *Event) {
		t.Log(e.Message())
		if e.Message() == "Test Message5" {
			irccon2.Quit()
		}
	})

	irccon2.AddCallback("NICK", func(e *Event) {
		if irccon2.nickcurrent == "go-eventnewnick" {
			t.Fatal("Nick change did not work!")
		}
	})
	go irccon2.Loop()
	irccon1.Loop()
}

func TestConnectionSSL(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.VerboseCallbackHandler = true
	irccon.Debug = true
	irccon.UseTLS = true
	irccon.TLSConfig = &tls.Config{InsecureSkipVerify: true}
	err := irccon.Connect("irc.freenode.net:7000")
	if err != nil {
		t.Log(err.Error())
		t.Fatal("Can't connect to freenode.")
	}
	irccon.AddCallback("001", func(e *Event) { irccon.Join("#go-eventirc") })

	irccon.AddCallback("366", func(e *Event) {
		irccon.Privmsg("#go-eventirc", "Test Message\n")
		time.Sleep(2 * time.Second)
		irccon.Quit()
	})

	irccon.Loop()
}
<commit_msg>Remove fuzzing testcases<commit_after>package irc

import (
	"crypto/tls"
	"testing"
	"time"
)

func TestConnectionEmtpyServer(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("")
	if err == nil {
		t.Fatal("emtpy server string not detected")
	}
}

func TestConnectionDoubleColon(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("::")
	if err == nil {
		t.Fatal("wrong number of ':' not detected")
	}
}

func TestConnectionMissingHost(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect(":6667")
	if err == nil {
		t.Fatal("missing host not detected")
	}
}

func TestConnectionMissingPort(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("chat.freenode.net:")
	if err == nil {
		t.Fatal("missing port not detected")
	}
}

func TestConnectionNegativePort(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("chat.freenode.net:-1")
	if err == nil {
		t.Fatal("negative port number not detected")
	}
}

func TestConnectionTooLargePort(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	err := irccon.Connect("chat.freenode.net:65536")
	if err == nil {
		t.Fatal("too large port number not detected")
	}
}

func TestConnectionMissingLog(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.Log = nil
	err := irccon.Connect("chat.freenode.net:6667")
	if err == nil {
		t.Fatal("missing 'Log' not detected")
	}
}

func TestConnectionEmptyUser(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	// user may be changed after creation
	irccon.user = ""
	err := irccon.Connect("chat.freenode.net:6667")
	if err == nil {
		t.Fatal("empty 'user' not detected")
	}
}

func TestConnectionEmptyNick(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	// nick may be changed after creation
	irccon.nick = ""
	err := irccon.Connect("chat.freenode.net:6667")
	if err == nil {
		t.Fatal("empty 'nick' not detected")
	}
}

func TestRemoveCallback(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.VerboseCallbackHandler = true
	irccon.Debug = true

	done := make(chan int, 10)

	irccon.AddCallback("TEST", func(e *Event) { done <- 1 })
	id := irccon.AddCallback("TEST", func(e *Event) { done <- 2 })
	irccon.AddCallback("TEST", func(e *Event) { done <- 3 })

	// Should remove callback at index 1
	irccon.RemoveCallback("TEST", id)

	irccon.RunCallbacks(&Event{
		Code: "TEST",
	})

	var results []int

	results = append(results, <-done)
	results = append(results, <-done)

	if len(results) != 2 || results[0] == 2 || results[1] == 2 {
		t.Error("Callback 2 not removed")
	}
}

func TestWildcardCallback(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.VerboseCallbackHandler = true
	irccon.Debug = true

	done := make(chan int, 10)

	irccon.AddCallback("TEST", func(e *Event) { done <- 1 })
	irccon.AddCallback("*", func(e *Event) { done <- 2 })

	irccon.RunCallbacks(&Event{
		Code: "TEST",
	})

	var results []int

	results = append(results, <-done)
	results = append(results, <-done)

	if len(results) != 2 || !(results[0] == 1 && results[1] == 2) {
		t.Error("Wildcard callback not called")
	}
}

func TestClearCallback(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.VerboseCallbackHandler = true
	irccon.Debug = true

	done := make(chan int, 10)

	irccon.AddCallback("TEST", func(e *Event) { done <- 0 })
	irccon.AddCallback("TEST", func(e *Event) { done <- 1 })
	irccon.ClearCallback("TEST")
	irccon.AddCallback("TEST", func(e *Event) { done <- 2 })
	irccon.AddCallback("TEST", func(e *Event) { done <- 3 })

	irccon.RunCallbacks(&Event{
		Code: "TEST",
	})

	var results []int

	results = append(results, <-done)
	results = append(results, <-done)

	if len(results) != 2 || !(results[0] == 2 && results[1] == 3) {
		t.Error("Callbacks not cleared")
	}
}

func TestIRCemptyNick(t *testing.T) {
	irccon := IRC("", "go-eventirc")
	irccon = nil
	if irccon != nil {
		t.Error("empty nick didn't result in error")
		t.Fail()
	}
}

func TestIRCemptyUser(t *testing.T) {
	irccon := IRC("go-eventirc", "")
	if irccon != nil {
		t.Error("empty user didn't result in error")
	}
}
func TestConnection(t *testing.T) {
	irccon1 := IRC("go-eventirc1", "go-eventirc1")
	irccon1.VerboseCallbackHandler = true
	irccon1.Debug = true
	irccon2 := IRC("go-eventirc2", "go-eventirc2")
	irccon2.VerboseCallbackHandler = true
	irccon2.Debug = true
	err := irccon1.Connect("irc.freenode.net:6667")
	if err != nil {
		t.Log(err.Error())
		t.Fatal("Can't connect to freenode.")
	}
	err = irccon2.Connect("irc.freenode.net:6667")
	if err != nil {
		t.Log(err.Error())
		t.Fatal("Can't connect to freenode.")
	}
	irccon1.AddCallback("001", func(e *Event) { irccon1.Join("#go-eventirc") })
	irccon2.AddCallback("001", func(e *Event) { irccon2.Join("#go-eventirc") })
	con2ok := false
	irccon1.AddCallback("366", func(e *Event) {
		t := time.NewTicker(1 * time.Second)
		i := 10
		for {
			<-t.C
			irccon1.Privmsgf("#go-eventirc", "Test Message%d\n", i)
			if con2ok {
				i -= 1
			}
			if i == 0 {
				t.Stop()
				irccon1.Quit()
			}
		}
	})

	irccon2.AddCallback("366", func(e *Event) {
		irccon2.Privmsg("#go-eventirc", "Test Message\n")
		con2ok = true
		irccon2.Nick("go-eventnewnick")
	})

	irccon2.AddCallback("PRIVMSG", func(e *Event) {
		t.Log(e.Message())
		if e.Message() == "Test Message5" {
			irccon2.Quit()
		}
	})

	irccon2.AddCallback("NICK", func(e *Event) {
		if irccon2.nickcurrent == "go-eventnewnick" {
			t.Fatal("Nick change did not work!")
		}
	})
	go irccon2.Loop()
	irccon1.Loop()
}

func TestConnectionSSL(t *testing.T) {
	irccon := IRC("go-eventirc", "go-eventirc")
	irccon.VerboseCallbackHandler = true
	irccon.Debug = true
	irccon.UseTLS = true
	irccon.TLSConfig = &tls.Config{InsecureSkipVerify: true}
	err := irccon.Connect("irc.freenode.net:7000")
	if err != nil {
		t.Log(err.Error())
		t.Fatal("Can't connect to freenode.")
	}
	irccon.AddCallback("001", func(e *Event) { irccon.Join("#go-eventirc") })

	irccon.AddCallback("366", func(e *Event) {
		irccon.Privmsg("#go-eventirc", "Test Message\n")
		time.Sleep(2 * time.Second)
		irccon.Quit()
	})

	irccon.Loop()
}
<|endoftext|>


<commit_before>package piecerequest

import (
	"math/rand"
	"sort"
	"sync"
	"time"

	"code.uber.internal/infra/kraken/core"

	"github.com/andres-erbsen/clock"
	"github.com/willf/bitset"
)

// Status enumerates possible statuses of a Request.
type Status int

const (
	// StatusPending denotes a valid request which is still in-flight.
	StatusPending Status = iota

	// StatusExpired denotes an in-flight request which has timed out on our end.
	StatusExpired

	// StatusUnsent denotes an unsent request that is safe to retry to the same peer.
	StatusUnsent

	// StatusInvalid denotes a completed request that resulted in an invalid payload.
	StatusInvalid
)

// Request represents a piece request to peer.
type Request struct {
	Piece  int
	PeerID core.PeerID
	Status Status

	sentAt time.Time
}

// Manager encapsulates thread-safe piece request bookkeeping. It is not responsible
// for sending nor receiving pieces in any way.
type Manager struct {
	sync.RWMutex

	// requests and requestsByPeer holds the same data, just indexed differently.
	requests       map[int][]*Request
	requestsByPeer map[core.PeerID]map[int]*Request

	clock         clock.Clock
	timeout       time.Duration
	pipelineLimit int
}

// NewManager creates a new Manager.
func NewManager(clk clock.Clock, timeout time.Duration, pipelineLimit int) *Manager {
	return &Manager{
		requests:       make(map[int][]*Request),
		requestsByPeer: make(map[core.PeerID]map[int]*Request),
		clock:          clk,
		timeout:        timeout,
		pipelineLimit:  pipelineLimit,
	}
}

// ReservePieces selects the next piece(s) to be requested from given peer. If
// allowDuplicates is set, may return pieces which have already been reserved
// under other peers.
func (m *Manager) ReservePieces(
	peerID core.PeerID, candidates *bitset.BitSet, allowDuplicates bool) []int {

	m.Lock()
	defer m.Unlock()

	// TODO(codyg): Fix bug where we don't consider failed piece requests here.
	// If a request has a fail status, it'll still count towards the pipeline limit.
	if pm, ok := m.requestsByPeer[peerID]; ok && len(pm) > m.pipelineLimit {
		return nil
	}

	// Reservoir sampling.
	var pieces []int
	count := 0
	for i, e := candidates.NextSet(0); e; i, e = candidates.NextSet(i + 1) {
		if !m.validRequest(peerID, int(i), allowDuplicates) {
			continue
		}
		if count < m.pipelineLimit {
			pieces = append(pieces, int(i))
		} else if rand.Intn(count) < 1 {
			// Replace existing result.
			pieces[rand.Intn(m.pipelineLimit)] = int(i)
		}
		count++
	}

	// Set as pending in requests map.
	for _, i := range pieces {
		r := &Request{
			Piece:  i,
			PeerID: peerID,
			Status: StatusPending,
			sentAt: m.clock.Now(),
		}
		m.requests[i] = append(m.requests[i], r)
		if _, ok := m.requestsByPeer[peerID]; !ok {
			m.requestsByPeer[peerID] = make(map[int]*Request)
		}
		m.requestsByPeer[peerID][i] = r
	}

	return pieces
}

// MarkUnsent marks the piece request for piece i as unsent.
func (m *Manager) MarkUnsent(peerID core.PeerID, i int) {
	m.markStatus(peerID, i, StatusUnsent)
}

// MarkInvalid marks the piece request for piece i as invalid.
func (m *Manager) MarkInvalid(peerID core.PeerID, i int) {
	m.markStatus(peerID, i, StatusInvalid)
}

// Clear deletes the piece request for piece i. Should be used for freeing up
// unneeded request bookkeeping.
func (m *Manager) Clear(i int) {
	m.Lock()
	defer m.Unlock()

	delete(m.requests, i)

	for peerID, pm := range m.requestsByPeer {
		delete(pm, i)
		if len(pm) == 0 {
			delete(m.requestsByPeer, peerID)
		}
	}
}

// PendingPieces returns the pieces for all pending requests to peerID in sorted
// order. Intended primarily for testing purposes.
func (m *Manager) PendingPieces(peerID core.PeerID) []int {
	m.RLock()
	defer m.RUnlock()

	var pieces []int
	for i, r := range m.requestsByPeer[peerID] {
		if r.Status == StatusPending {
			pieces = append(pieces, i)
		}
	}
	sort.Ints(pieces)
	return pieces
}

// ClearPeer deletes all piece requests for peerID.
func (m *Manager) ClearPeer(peerID core.PeerID) {
	m.Lock()
	defer m.Unlock()

	delete(m.requestsByPeer, peerID)

	for i, rs := range m.requests {
		for j, r := range rs {
			if r.PeerID == peerID {
				// Eject request.
				rs[j] = rs[len(rs)-1]
				m.requests[i] = rs[:len(rs)-1]
				break
			}
		}
	}
}

// GetFailedRequests returns a copy of all failed piece requests.
func (m *Manager) GetFailedRequests() []Request {
	m.RLock()
	defer m.RUnlock()

	var failed []Request
	for _, rs := range m.requests {
		for _, r := range rs {
			status := r.Status
			if status == StatusPending && m.expired(r) {
				status = StatusExpired
			}
			if status != StatusPending {
				failed = append(failed, Request{
					Piece:  r.Piece,
					PeerID: r.PeerID,
					Status: status,
				})
			}
		}
	}
	return failed
}

func (m *Manager) validRequest(peerID core.PeerID, i int, allowDuplicates bool) bool {
	for _, r := range m.requests[i] {
		if r.Status == StatusPending && !m.expired(r) {
			if r.PeerID == peerID {
				return false
			}
			if !allowDuplicates {
				return false
			}
		}
	}
	return true
}

func (m *Manager) expired(r *Request) bool {
	expiresAt := r.sentAt.Add(m.timeout)
	return m.clock.Now().After(expiresAt)
}

func (m *Manager) markStatus(peerID core.PeerID, i int, s Status) {
	m.Lock()
	defer m.Unlock()

	for _, r := range m.requests[i] {
		if r.PeerID == peerID {
			r.Status = s
		}
	}
}
<commit_msg>kraken: Comments about piece selection logic.<commit_after>package piecerequest

import (
	"math/rand"
	"sort"
	"sync"
	"time"

	"code.uber.internal/infra/kraken/core"

	"github.com/andres-erbsen/clock"
	"github.com/willf/bitset"
)

// Status enumerates possible statuses of a Request.
type Status int

const (
	// StatusPending denotes a valid request which is still in-flight.
	StatusPending Status = iota

	// StatusExpired denotes an in-flight request which has timed out on our end.
	StatusExpired

	// StatusUnsent denotes an unsent request that is safe to retry to the same peer.
	StatusUnsent

	// StatusInvalid denotes a completed request that resulted in an invalid payload.
	StatusInvalid
)

// Request represents a piece request to peer.
type Request struct {
	Piece  int
	PeerID core.PeerID
	Status Status

	sentAt time.Time
}

// Manager encapsulates thread-safe piece request bookkeeping. It is not responsible
// for sending nor receiving pieces in any way.
type Manager struct {
	sync.RWMutex

	// requests and requestsByPeer holds the same data, just indexed differently.
	requests       map[int][]*Request
	requestsByPeer map[core.PeerID]map[int]*Request

	clock         clock.Clock
	timeout       time.Duration
	pipelineLimit int
}

// NewManager creates a new Manager.
func NewManager(clk clock.Clock, timeout time.Duration, pipelineLimit int) *Manager {
	return &Manager{
		requests:       make(map[int][]*Request),
		requestsByPeer: make(map[core.PeerID]map[int]*Request),
		clock:          clk,
		timeout:        timeout,
		pipelineLimit:  pipelineLimit,
	}
}

// ReservePieces selects the next piece(s) to be requested from given peer. If
// allowDuplicates is set, may return pieces which have already been reserved
// under other peers.
func (m *Manager) ReservePieces(
	peerID core.PeerID, candidates *bitset.BitSet, allowDuplicates bool) []int {

	m.Lock()
	defer m.Unlock()

	// TODO(codyg): Fix bug where we don't consider failed piece requests here.
	// If a request has a fail status, it'll still count towards the pipeline
	// limit.
	if pm, ok := m.requestsByPeer[peerID]; ok && len(pm) > m.pipelineLimit {
		return nil
	}

	// Reservoir sampling. There are faster sampling solutions, but most would
	// require converting the bitset to a list, which itself would take O(n).
	var pieces []int
	count := 0
	for i, e := candidates.NextSet(0); e; i, e = candidates.NextSet(i + 1) {
		if !m.validRequest(peerID, int(i), allowDuplicates) {
			continue
		}
		if count < m.pipelineLimit {
			pieces = append(pieces, int(i))
		} else if rand.Intn(count) < 1 {
			// Replace existing result.
			pieces[rand.Intn(m.pipelineLimit)] = int(i)
		}
		count++
	}

	// Set as pending in requests map.
	for _, i := range pieces {
		r := &Request{
			Piece:  i,
			PeerID: peerID,
			Status: StatusPending,
			sentAt: m.clock.Now(),
		}
		m.requests[i] = append(m.requests[i], r)
		if _, ok := m.requestsByPeer[peerID]; !ok {
			m.requestsByPeer[peerID] = make(map[int]*Request)
		}
		m.requestsByPeer[peerID][i] = r
	}

	return pieces
}

// MarkUnsent marks the piece request for piece i as unsent.
func (m *Manager) MarkUnsent(peerID core.PeerID, i int) {
	m.markStatus(peerID, i, StatusUnsent)
}

// MarkInvalid marks the piece request for piece i as invalid.
func (m *Manager) MarkInvalid(peerID core.PeerID, i int) {
	m.markStatus(peerID, i, StatusInvalid)
}

// Clear deletes the piece request for piece i. Should be used for freeing up
// unneeded request bookkeeping.
func (m *Manager) Clear(i int) {
	m.Lock()
	defer m.Unlock()

	delete(m.requests, i)

	for peerID, pm := range m.requestsByPeer {
		delete(pm, i)
		if len(pm) == 0 {
			delete(m.requestsByPeer, peerID)
		}
	}
}

// PendingPieces returns the pieces for all pending requests to peerID in sorted
// order. Intended primarily for testing purposes.
func (m *Manager) PendingPieces(peerID core.PeerID) []int {
	m.RLock()
	defer m.RUnlock()

	var pieces []int
	for i, r := range m.requestsByPeer[peerID] {
		if r.Status == StatusPending {
			pieces = append(pieces, i)
		}
	}
	sort.Ints(pieces)
	return pieces
}

// ClearPeer deletes all piece requests for peerID.
func (m *Manager) ClearPeer(peerID core.PeerID) {
	m.Lock()
	defer m.Unlock()

	delete(m.requestsByPeer, peerID)

	for i, rs := range m.requests {
		for j, r := range rs {
			if r.PeerID == peerID {
				// Eject request.
				rs[j] = rs[len(rs)-1]
				m.requests[i] = rs[:len(rs)-1]
				break
			}
		}
	}
}

// GetFailedRequests returns a copy of all failed piece requests.
func (m *Manager) GetFailedRequests() []Request {
	m.RLock()
	defer m.RUnlock()

	var failed []Request
	for _, rs := range m.requests {
		for _, r := range rs {
			status := r.Status
			if status == StatusPending && m.expired(r) {
				status = StatusExpired
			}
			if status != StatusPending {
				failed = append(failed, Request{
					Piece:  r.Piece,
					PeerID: r.PeerID,
					Status: status,
				})
			}
		}
	}
	return failed
}

func (m *Manager) validRequest(peerID core.PeerID, i int, allowDuplicates bool) bool {
	for _, r := range m.requests[i] {
		if r.Status == StatusPending && !m.expired(r) {
			if r.PeerID == peerID {
				return false
			}
			if !allowDuplicates {
				return false
			}
		}
	}
	return true
}

func (m *Manager) expired(r *Request) bool {
	expiresAt := r.sentAt.Add(m.timeout)
	return m.clock.Now().After(expiresAt)
}

func (m *Manager) markStatus(peerID core.PeerID, i int, s Status) {
	m.Lock()
	defer m.Unlock()

	for _, r := range m.requests[i] {
		if r.PeerID == peerID {
			r.Status = s
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/golang/glog"
	multierror "github.com/hashicorp/go-multierror"

	"istio.io/istio/tests/e2e/framework"
	"istio.io/istio/tests/e2e/util"
)

const (
	u1           = "normal-user"
	u2           = "test-user"
	create       = "create"
	bookinfoYaml = "samples/apps/bookinfo/bookinfo.yaml"
	modelDir     = "tests/apps/bookinfo/output"
	rulesDir     = "samples/apps/bookinfo"
	allRule      = "route-rule-all-v1.yaml"
	delayRule    = "route-rule-delay.yaml"
	fiftyRule    = "route-rule-reviews-50-v3.yaml"
	testRule     = "route-rule-reviews-test-v2.yaml"
)

var (
	tc *testConfig
)

type testConfig struct {
	*framework.CommonConfig
	gateway  string
	rulesDir string
}

func getWithCookie(url string, cookies []http.Cookie) (*http.Response, error) {
	// Declare http client
	client := &http.Client{}

	// Declare HTTP Method and Url
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	if cookies != nil {
		for _, c := range cookies {
			// Set cookie
			req.AddCookie(&c)
		}
	}
	return client.Do(req)
}

func closeResponseBody(r *http.Response) {
	if err := r.Body.Close(); err != nil {
		glog.Error(err)
	}
}

func (t *testConfig) Setup() error {
	t.gateway = "http://" + tc.Kube.Ingress
	//generate rule yaml files, replace with actual namespace and user
	for _, rule := range []string{allRule, delayRule, fiftyRule, testRule} {
		src := util.GetResourcePath(filepath.Join(rulesDir, rule))
		dest := filepath.Join(t.rulesDir, rule)
		ori, err := ioutil.ReadFile(src)
		if err != nil {
			glog.Errorf("Failed to read original rule file %s", src)
			return err
		}
		content := string(ori)
		content = strings.Replace(content, "default", t.Kube.Namespace, -1)
		content = strings.Replace(content, "jason", u2, -1)
		err = ioutil.WriteFile(dest, []byte(content), 0600)
		if err != nil {
			glog.Errorf("Failed to write into new rule file %s", dest)
			return err
		}

	}
	return setUpDefaultRouting()
}

func (t *testConfig) Teardown() error {
	return nil
}

func check(err error, msg string) {
	if err != nil {
		glog.Fatalf("%s. Error %s", msg, err)
	}
}

func inspect(err error, fMsg, sMsg string, t *testing.T) {
	if err != nil {
		glog.Errorf("%s. Error %s", fMsg, err)
		t.Error(err)
	} else if sMsg != "" {
		glog.Info(sMsg)
	}
}

func setUpDefaultRouting() error {
	standby := 0
	for i := 0; i <= 10; i++ {
		time.Sleep(time.Duration(standby) * time.Second)
		resp, err := http.Get(fmt.Sprintf("%s/productpage", tc.gateway))
		if err != nil {
			glog.Infof("Error talking to productpage: %s", err)
		} else {
			glog.Infof("Get from page: %d", resp.StatusCode)
			if resp.StatusCode == http.StatusOK {
				glog.Info("Get response from product page!")
				break
			}
			closeResponseBody(resp)
		}
		if i == 10 {
			return fmt.Errorf("unable to set default route")
		}
		standby += 5
		glog.Errorf("Couldn't get to the bookinfo product page, trying again in %d second", standby)
	}
	glog.Info("Success! Default route got expected response")
	return nil
}

func checkRoutingResponse(user, version, gateway, modelFile string) (int, error) {
	startT := time.Now()
	cookies := []http.Cookie{
		{
			Name:  "foo",
			Value: "bar",
		},
		{
			Name:  "user",
			Value: user,
		},
	}
	resp, err := getWithCookie(fmt.Sprintf("%s/productpage", gateway), cookies)
	if err != nil {
		return -1, err
	}
	if resp.StatusCode != http.StatusOK {
		return -1, fmt.Errorf("status code is %d", resp.StatusCode)
	}
	duration := int(time.Since(startT) / (time.Second / time.Nanosecond))
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return -1, err
	}

	if err = util.CompareToFile(body, modelFile); err != nil {
		glog.Errorf("Error: User %s in version %s didn't get expected response", user, version)
		duration = -1
	}
	closeResponseBody(resp)
	return duration, err
}

func deleteRules(ruleKeys []string) error {
	var err error
	for _, ruleKey := range ruleKeys {
		rule := filepath.Join(tc.rulesDir, ruleKey)
		if e := tc.Kube.Istioctl.DeleteRule(rule); e != nil {
			err = multierror.Append(err, err)
		}
	}
	glog.Info("Waiting for rule to be cleaned up...")
	time.Sleep(time.Duration(30) * time.Second)
	return err
}

func applyRules(ruleKeys []string, operation string) error {
	var fn func(string) error
	switch operation {
	case "create":
		fn = tc.Kube.Istioctl.CreateRule
	case "replace":
		fn = tc.Kube.Istioctl.ReplaceRule
	default:
		return fmt.Errorf("operation %s not supported", operation)
	}
	for _, ruleKey := range ruleKeys {
		rule := filepath.Join(tc.rulesDir, ruleKey)
		if err := fn(rule); err != nil {
			return err
		}
	}
	glog.Info("Waiting for rules to propagate...")
	time.Sleep(time.Duration(30) * time.Second)
	return nil
}

func TestVersionRouting(t *testing.T) {
	var err error
	var rules = []string{allRule, testRule}
	inspect(applyRules(rules, create), "failed to apply rules", "", t)
	defer func() {
		inspect(deleteRules(rules), "failed to delete rules", "", t)
	}()

	v1File := util.GetResourcePath(filepath.Join(modelDir, "productpage-normal-user-v1.html"))
	v2File := util.GetResourcePath(filepath.Join(modelDir, "productpage-test-user-v2.html"))

	_, err = checkRoutingResponse(u1, "v1", tc.gateway, v1File)
	inspect(
		err, fmt.Sprintf("Failed version routing! %s in v1", u1),
		fmt.Sprintf("Success! Response matches with expected! %s in v1", u1), t)
	_, err = checkRoutingResponse(u2, "v2", tc.gateway, v2File)
	inspect(
		err, fmt.Sprintf("Failed version routing! %s in v2", u2),
		fmt.Sprintf("Success! Response matches with expected! %s in v2", u2), t)
}

func TestFaultDelay(t *testing.T) {
	var rules = []string{allRule, testRule, delayRule}
	inspect(applyRules(rules, create), "failed to apply rules", "", t)
	defer func() {
		inspect(deleteRules(rules), "failed to delete rules", "", t)
	}()
	minDuration := 5
	maxDuration := 8
	standby := 10
	testModel := util.GetResourcePath(
		filepath.Join(modelDir, "productpage-test-user-v1-review-timeout.html"))
	for i := 0; i < 5; i++ {
		duration, err := checkRoutingResponse(
			u2, "v1-timeout", tc.gateway,
			testModel)
		glog.Infof("Get response in %d second", duration)
		if err == nil && duration >= minDuration && duration <= maxDuration {
			glog.Info("Success! Fault delay as expected")
			break
		}

		if i == 4 {
			t.Errorf("Fault delay failed! Delay in %ds while expected between %ds and %ds, %s",
				duration, minDuration, maxDuration, err)
			break
		}

		glog.Infof("Unexpected response, retry in %ds", standby)
		time.Sleep(time.Duration(standby) * time.Second)
	}
}

func TestVersionMigration(t *testing.T) {
	var rules = []string{fiftyRule}
	inspect(applyRules(rules, create), "failed to apply rules", "", t)
	defer func() {
		inspect(deleteRules(rules), fmt.Sprintf("failed to delete rules"), "", t)
	}()

	// Percentage moved to new version
	migrationRate := 0.5
	tolerance := 0.05
	totalShot := 100
	modelV1 := util.GetResourcePath(filepath.Join(modelDir, "productpage-normal-user-v1.html"))
	modelV3 := util.GetResourcePath(filepath.Join(modelDir, "productpage-normal-user-v3.html"))

	cookies := []http.Cookie{
		{
			Name:  "foo",
			Value: "bar",
		},
		{
			Name:  "user",
			Value: "normal-user",
		},
	}

	for i := 0; i < 5; i++ {
		c1, c3 := 0, 0
		for c := 0; c < totalShot; c++ {
			resp, err := getWithCookie(fmt.Sprintf("%s/productpage", tc.gateway), cookies)
			inspect(err, "Failed to record", "", t)
			if resp.StatusCode != http.StatusOK {
				t.Errorf("unexpected response status %d", resp.StatusCode)
				continue
			}
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				t.Error(err)
				continue
			}
			if err = util.CompareToFile(body, modelV1); err == nil {
				c1++
			} else if err = util.CompareToFile(body, modelV3); err == nil {
				c3++
			}
			closeResponseBody(resp)
		}
		c1Percent := int((migrationRate + tolerance) * float64(totalShot))
		c3Percent := int((migrationRate - tolerance) * float64(totalShot))
		if (c1 <= c1Percent) && (c3 >= c3Percent) {
			glog.Infof(
				"Success! Version migration acts as expected, "+
					"old version hit %d, new version hit %d", c1, c3)
			break
		}

		if i == 4 {
			t.Errorf("Failed version migration test, "+
				"old version hit %d, new version hit %d", c1, c3)
		}
	}
}

func setTestConfig() error {
	cc, err := framework.NewCommonConfig("demo_test")
	if err != nil {
		return err
	}
	tc = new(testConfig)
	tc.CommonConfig = cc
	tc.rulesDir, err = ioutil.TempDir(os.TempDir(), "demo_test")
	if err != nil {
		return err
	}
	demoApp := &framework.App{
		AppYaml:    util.GetResourcePath(bookinfoYaml),
		KubeInject: true,
	}
	tc.Kube.AppManager.AddApp(demoApp)
	return nil
}

func TestMain(m *testing.M) {
	flag.Parse()
	check(framework.InitGlog(), "cannot setup glog")
	check(setTestConfig(), "could not create TestConfig")
	tc.Cleanup.RegisterCleanable(tc)
	os.Exit(tc.RunTest(m))
}
<commit_msg>Make default rules use v1 (#562)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/golang/glog"
	multierror "github.com/hashicorp/go-multierror"

	"istio.io/istio/tests/e2e/framework"
	"istio.io/istio/tests/e2e/util"
)

const (
	u1           = "normal-user"
	u2           = "test-user"
	create       = "create"
	bookinfoYaml = "samples/apps/bookinfo/bookinfo.yaml"
	modelDir     = "tests/apps/bookinfo/output"
	rulesDir     = "samples/apps/bookinfo"
	allRule      = "route-rule-all-v1.yaml"
	delayRule    = "route-rule-delay.yaml"
	fiftyRule    = "route-rule-reviews-50-v3.yaml"
	testRule     = "route-rule-reviews-test-v2.yaml"
)

var (
	tc           *testConfig
	defaultRules = []string{allRule}
)

type testConfig struct {
	*framework.CommonConfig
	gateway  string
	rulesDir string
}

func getWithCookie(url string, cookies []http.Cookie) (*http.Response, error) {
	// Declare http client
	client := &http.Client{}

	// Declare HTTP Method and Url
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	if cookies != nil {
		for _, c := range cookies {
			// Set cookie
			req.AddCookie(&c)
		}
	}
	return client.Do(req)
}

func closeResponseBody(r *http.Response) {
	if err := r.Body.Close(); err != nil {
		glog.Error(err)
	}
}

func (t *testConfig) Setup() error {
	t.gateway = "http://" + tc.Kube.Ingress
	//generate rule yaml files, replace with actual namespace and user
	for _, rule := range []string{allRule, delayRule, fiftyRule, testRule} {
		src := util.GetResourcePath(filepath.Join(rulesDir, rule))
		dest := filepath.Join(t.rulesDir, rule)
		ori, err := ioutil.ReadFile(src)
		if err != nil {
			glog.Errorf("Failed to read original rule file %s", src)
			return err
		}
		content := string(ori)
		content = strings.Replace(content, "default", t.Kube.Namespace, -1)
		content = strings.Replace(content, "jason", u2, -1)
		err = ioutil.WriteFile(dest, []byte(content), 0600)
		if err != nil {
			glog.Errorf("Failed to write into new rule file %s", dest)
			return err
		}

	}
	return setUpDefaultRouting()
}

func (t *testConfig) Teardown() error {
	if err := deleteRules(defaultRules); err != nil {
		// don't report errors if the rule being deleted doesn't exist
		if notFound := strings.Contains(err.Error(), "not found"); notFound {
			return nil
		}
		return err
	}
	return nil
}

func check(err error, msg string) {
	if err != nil {
		glog.Fatalf("%s. Error %s", msg, err)
	}
}

func inspect(err error, fMsg, sMsg string, t *testing.T) {
	if err != nil {
		glog.Errorf("%s. Error %s", fMsg, err)
		t.Error(err)
	} else if sMsg != "" {
		glog.Info(sMsg)
	}
}

func setUpDefaultRouting() error {
	if err := applyRules(defaultRules, create); err != nil {
		return fmt.Errorf("could not apply rule '%s': %v", allRule, err)
	}
	standby := 0
	for i := 0; i <= 10; i++ {
		time.Sleep(time.Duration(standby) * time.Second)
		resp, err := http.Get(fmt.Sprintf("%s/productpage", tc.gateway))
		if err != nil {
			glog.Infof("Error talking to productpage: %s", err)
		} else {
			glog.Infof("Get from page: %d", resp.StatusCode)
			if resp.StatusCode == http.StatusOK {
				glog.Info("Get response from product page!")
				break
			}
			closeResponseBody(resp)
		}
		if i == 10 {
			return errors.New("unable to set default route")
		}
		standby += 5
		glog.Errorf("Couldn't get to the bookinfo product page, trying again in %d second", standby)
	}
	glog.Info("Success! Default route got expected response")
	return nil
}

func checkRoutingResponse(user, version, gateway, modelFile string) (int, error) {
	startT := time.Now()
	cookies := []http.Cookie{
		{
			Name:  "foo",
			Value: "bar",
		},
		{
			Name:  "user",
			Value: user,
		},
	}
	resp, err := getWithCookie(fmt.Sprintf("%s/productpage", gateway), cookies)
	if err != nil {
		return -1, err
	}
	if resp.StatusCode != http.StatusOK {
		return -1, fmt.Errorf("status code is %d", resp.StatusCode)
	}
	duration := int(time.Since(startT) / (time.Second / time.Nanosecond))
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return -1, err
	}

	if err = util.CompareToFile(body, modelFile); err != nil {
		glog.Errorf("Error: User %s in version %s didn't get expected response", user, version)
		duration = -1
	}
	closeResponseBody(resp)
	return duration, err
}

func deleteRules(ruleKeys []string) error {
	var err error
	for _, ruleKey := range ruleKeys {
		rule := filepath.Join(tc.rulesDir, ruleKey)
		if e := tc.Kube.Istioctl.DeleteRule(rule); e != nil {
			err = multierror.Append(err, e)
		}
	}
	glog.Info("Waiting for rule to be cleaned up...")
	time.Sleep(time.Duration(30) * time.Second)
	return err
}

func applyRules(ruleKeys []string, operation string) error {
	var fn func(string) error
	switch operation {
	case "create":
		fn = tc.Kube.Istioctl.CreateRule
	case "replace":
		fn = tc.Kube.Istioctl.ReplaceRule
	default:
		return fmt.Errorf("operation %s not supported", operation)
	}
	for _, ruleKey := range ruleKeys {
		rule := filepath.Join(tc.rulesDir, ruleKey)
		if err := fn(rule); err != nil {
			return err
		}
	}
	glog.Info("Waiting for rules to propagate...")
	time.Sleep(time.Duration(30) * time.Second)
	return nil
}

func TestVersionRouting(t *testing.T) {
	var err error
	var rules = []string{testRule}
	inspect(applyRules(rules, create), "failed to apply rules", "", t)
	defer func() {
		inspect(deleteRules(rules), "failed to delete rules", "", t)
	}()

	v1File := util.GetResourcePath(filepath.Join(modelDir, "productpage-normal-user-v1.html"))
	v2File := util.GetResourcePath(filepath.Join(modelDir, "productpage-test-user-v2.html"))

	_, err = checkRoutingResponse(u1, "v1", tc.gateway, v1File)
	inspect(
		err, fmt.Sprintf("Failed version routing! %s in v1", u1),
		fmt.Sprintf("Success! Response matches with expected! %s in v1", u1), t)
	_, err = checkRoutingResponse(u2, "v2", tc.gateway, v2File)
	inspect(
		err, fmt.Sprintf("Failed version routing! %s in v2", u2),
		fmt.Sprintf("Success! Response matches with expected! %s in v2", u2), t)
}

func TestFaultDelay(t *testing.T) {
	var rules = []string{testRule, delayRule}
	inspect(applyRules(rules, create), "failed to apply rules", "", t)
	defer func() {
		inspect(deleteRules(rules), "failed to delete rules", "", t)
	}()
	minDuration := 5
	maxDuration := 8
	standby := 10
	testModel := util.GetResourcePath(
		filepath.Join(modelDir, "productpage-test-user-v1-review-timeout.html"))
	for i := 0; i < 5; i++ {
		duration, err := checkRoutingResponse(
			u2, "v1-timeout", tc.gateway,
			testModel)
		glog.Infof("Get response in %d second", duration)
		if err == nil && duration >= minDuration && duration <= maxDuration {
			glog.Info("Success! Fault delay as expected")
			break
		}

		if i == 4 {
			t.Errorf("Fault delay failed! Delay in %ds while expected between %ds and %ds, %s",
				duration, minDuration, maxDuration, err)
			break
		}

		glog.Infof("Unexpected response, retry in %ds", standby)
		time.Sleep(time.Duration(standby) * time.Second)
	}
}

func TestVersionMigration(t *testing.T) {
	var rules = []string{fiftyRule}
	inspect(applyRules(rules, "replace"), "failed to apply rules", "", t)
	defer func() {
		inspect(deleteRules(rules), fmt.Sprintf("failed to delete rules"), "", t)
	}()

	// Percentage moved to new version
	migrationRate := 0.5
	tolerance := 0.05
	totalShot := 100
	modelV1 := util.GetResourcePath(filepath.Join(modelDir, "productpage-normal-user-v1.html"))
	modelV3 := util.GetResourcePath(filepath.Join(modelDir, "productpage-normal-user-v3.html"))

	cookies := []http.Cookie{
		{
			Name:  "foo",
			Value: "bar",
		},
		{
			Name:  "user",
			Value: "normal-user",
		},
	}

	for i := 0; i < 5; i++ {
		c1, c3 := 0, 0
		for c := 0; c < totalShot; c++ {
			resp, err := getWithCookie(fmt.Sprintf("%s/productpage", tc.gateway), cookies)
			inspect(err, "Failed to record", "", t)
			if resp.StatusCode != http.StatusOK {
				t.Errorf("unexpected response status %d", resp.StatusCode)
				continue
			}
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				t.Error(err)
				continue
			}
			if err = util.CompareToFile(body, modelV1); err == nil {
				c1++
			} else if err = util.CompareToFile(body, modelV3); err == nil {
				c3++
			}
			closeResponseBody(resp)
		}
		c1Percent := int((migrationRate + tolerance) * float64(totalShot))
		c3Percent := int((migrationRate - tolerance) * float64(totalShot))
		if (c1 <= c1Percent) && (c3 >= c3Percent) {
			glog.Infof(
				"Success! Version migration acts as expected, "+
					"old version hit %d, new version hit %d", c1, c3)
			break
		}

		if i == 4 {
			t.Errorf("Failed version migration test, "+
				"old version hit %d, new version hit %d", c1, c3)
		}
	}
}

func setTestConfig() error {
	cc, err := framework.NewCommonConfig("demo_test")
	if err != nil {
		return err
	}
	tc = new(testConfig)
	tc.CommonConfig = cc
	tc.rulesDir, err = ioutil.TempDir(os.TempDir(), "demo_test")
	if err != nil {
		return err
	}
	demoApp := &framework.App{
		AppYaml:    util.GetResourcePath(bookinfoYaml),
		KubeInject: true,
	}
	tc.Kube.AppManager.AddApp(demoApp)
	return nil
}

func TestMain(m *testing.M) {
	flag.Parse()
	check(framework.InitGlog(), "cannot setup glog")
	check(setTestConfig(), "could not create TestConfig")
	tc.Cleanup.RegisterCleanable(tc)
	os.Exit(tc.RunTest(m))
}
<|endoftext|>


<commit_before>package controllers

import (
    //"fmt"
    "github.com/astaxie/beego"
    "github.com/astaxie/beego/orm"
    "github.com/astaxie/beego/validation"
    models "github.com/mrsaints/AD-ACP/models"
)

type PlaceController struct {
    AdminController
}

func (this *PlaceController) Get() {
    this.Data["Title"] = "View Places"
    this.TplNames = "admin/pages/view_places.tpl"

    o := orm.NewOrm()
    var places []*models.Place
    num, err := o.QueryTable("Places").All(&places)

    if err != orm.ErrNoRows && num > 0 {
        this.Data["Places"] = places
    } else {
        flash := beego.NewFlash()
        flash.Notice("No places found. There may be an error on the server.")
        flash.Store(&this.Controller)
        beego.Error(err)
    }
}

func (this *PlaceController) AddPlace() {
    this.Data["Title"] = "Add Place"
    this.TplNames = "admin/pages/add_place.tpl"

    beego.ReadFromRequest(&this.Controller)

    if this.Ctx.Input.IsPost() {
        this._AddPlace()
    }
}

func (this *PlaceController) _AddPlace() {
    place := new(models.Place)

    if err := this.ParseForm(place); err != nil {
        beego.Error("Could not parse form values to struct: ", err)
    } else {
        valid := validation.Validation{}
        isValid, _ := valid.Valid(place)

        if !isValid {
            // Store form session
            this.Data["Place"] = place
            this.Data["Errors"] = valid.ErrorsMap
            beego.Debug(this.Data["Errors"])
        } else {
            flash := beego.NewFlash()
            o := orm.NewOrm()
            _, err := o.Insert(place)

            if err == nil {
                flash.Notice("The place has been successfully added.")
            } else {
                flash.Error("We are unable to add the place at this time due to a database error. Please try again later.")
                beego.Error(err)
            }

            flash.Store(&this.Controller)
            this.Redirect(this.UrlFor("PlaceController.AddPlace"), 302)
        }
    }
}<commit_msg>Added flash error for ParseForm failure<commit_after>package controllers

import (
    "fmt"
    "github.com/astaxie/beego"
    "github.com/astaxie/beego/orm"
    "github.com/astaxie/beego/validation"
    models "github.com/mrsaints/AD-ACP/models"
)

type PlaceController struct {
    AdminController
}

func (this *PlaceController) Get() {
    this.Data["Title"] = "View Places"
    this.TplNames = "admin/pages/view_places.tpl"

    o := orm.NewOrm()
    var places []*models.Place
    num, err := o.QueryTable("Places").All(&places)

    if err != orm.ErrNoRows && num > 0 {
        this.Data["Places"] = places
    } else {
        flash := beego.NewFlash()
        flash.Notice("No places found. This may be due to a server error.")
        flash.Store(&this.Controller)
        beego.Error(err)
    }
}

func (this *PlaceController) AddPlace() {
    this.Data["Title"] = "Add Place"
    this.TplNames = "admin/pages/add_place.tpl"

    beego.ReadFromRequest(&this.Controller)

    if this.Ctx.Input.IsPost() {
        this._AddPlace()
    }
}

func (this *PlaceController) _AddPlace() {
    flash := beego.NewFlash()
    serverErr := "We are unable to add the place at this time due to a server error. Please try again later."

    place := new(models.Place)
    if err := this.ParseForm(place); err != nil {
        flash.Error(serverErr)
        beego.Error("Could not parse form values to struct: ", err)
    } else {
        valid := validation.Validation{}
        isValid, _ := valid.Valid(place)

        if !isValid {
            // Store form session
            this.Data["Place"] = place
            this.Data["Errors"] = valid.ErrorsMap
            beego.Debug(this.Data["Errors"])
        } else {
            o := orm.NewOrm()
            id, err := o.Insert(place)

            if err == nil {
                flash.Notice("The place has been successfully added.")
                beego.Debug(fmt.Sprintf("Added a new place with ID: ", id))
            } else {
                flash.Error(serverErr)
                beego.Error(err)
            }

            flash.Store(&this.Controller)
            this.Redirect(this.UrlFor("PlaceController.AddPlace"), 302)
        }
    }
}<|endoftext|>


<commit_before>// Integration test
// This needs some env variables to run the tests
//      APP_TOKEN
//      APP_SECRET
//      ACCESS_TOKEN
//      ACCESS_SECRET
package copy

import (
	"io"
	"log"
	"net/url"
	"os"
	"testing"
)

const (
	appTokenEnv     = "APP_TOKEN"
	appSecretEnv    = "APP_SECRET"
	accessTokenEnv  = "ACCESS_TOKEN"
	accessSecretEnv = "ACCESS_SECRET"
)

var (
	appToken     string
	appSecret    string
	accessToken  string
	accessSecret string
	session      Session
)

func setup() {
	appToken = os.Getenv(appTokenEnv)
	appSecret = os.Getenv(appSecretEnv)
	accessToken = os.Getenv(accessTokenEnv)
	accessSecret = os.Getenv(accessSecretEnv)

	session = NewSession(
		AppToken{
			Token: appToken,
			Key:   appSecret,
		},
		AccessToken{
			Token: accessToken,
			Key:   accessSecret,
		},
	)
}

func tearDown() {

}

// Checks if the credentials for the integration tests are set in the env vars
func TestCredentialData(t *testing.T) {
	setup()
	defer tearDown()

	if appToken == "" {
		t.Error("Expected", appTokenEnv, "env var")
	}

	if appSecret == "" {
		t.Error("Expected", appSecretEnv, "env var")
	}

	if accessToken == "" {
		t.Error("Expected", accessTokenEnv, "env var")
	}

	if accessSecret == "" {
		t.Error("Expected", accessSecretEnv, "env var")
	}
}

// Check the GET request in a valid copy resource
func TestGetRequest(t *testing.T) {
	setup()
	defer tearDown()

	resp, err := session.Get("https://api.copy.com/rest/user", nil)

	if err != nil {
		t.Error("Expected no error in GET request")
	}

	if resp.StatusCode != 200 {
		t.Errorf("Response status error shouldn't be: %v", resp.StatusCode)
	}
}

// Check the GET request in an invalid copy resource with valid credentials
func TestGetRequestWrongResource(t *testing.T) {
	setup()
	defer tearDown()

	resp, _ := session.Get("https://api.copy.com/rest/userfail", nil)

	if resp.StatusCode != 400 {
		t.Errorf("Response status error should be: %v", resp.StatusCode)
	}
}

// Check the GET request in a valid copy resource with wrong credentials
func TestGetRequestWrongCredentials(t *testing.T) {
	setup()
	defer tearDown()

	session.TokenCreds.Secret = "You shall not pass!"

	resp, _ := session.Get("https://api.copy.com/rest/user", nil)

	if resp.StatusCode != 400 {
		t.Errorf("Response status error should be: %v", resp.StatusCode)
	}
}

// Check the PUT request in a valid copy resource
func TestPutRequest(t *testing.T) {
	setup()
	defer tearDown()

	values := url.Values{
		"first_name": {"Test"},
		"last_name":  {"Test2"},
	}

	resp, err := session.Put("https://api.copy.com/rest/user", values)

	defer resp.Body.Close()
	if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
		log.Fatal(err)
	}

	if err != nil {
		t.Error("Expected no error in POST request")
	}

	if resp.StatusCode != 200 {
		t.Errorf("Response status error shouldn't be: %v", resp.StatusCode)
	}
}
<commit_msg>Finish Put test properly<commit_after>// Integration test
// This needs some env variables to run the tests
//      APP_TOKEN
//      APP_SECRET
//      ACCESS_TOKEN
//      ACCESS_SECRET
package copy

import (
	"bytes"
	"fmt"
	"math/rand"
	"net/url"
	"os"
	"strings"
	"testing"
	"time"
)

const (
	appTokenEnv     = "APP_TOKEN"
	appSecretEnv    = "APP_SECRET"
	accessTokenEnv  = "ACCESS_TOKEN"
	accessSecretEnv = "ACCESS_SECRET"
)

var (
	appToken     string
	appSecret    string
	accessToken  string
	accessSecret string
	session      Session
)

func setup() {
	appToken = os.Getenv(appTokenEnv)
	appSecret = os.Getenv(appSecretEnv)
	accessToken = os.Getenv(accessTokenEnv)
	accessSecret = os.Getenv(accessSecretEnv)

	session = NewSession(
		AppToken{
			Token: appToken,
			Key:   appSecret,
		},
		AccessToken{
			Token: accessToken,
			Key:   accessSecret,
		},
	)
}

func tearDown() {

}

// Checks if the credentials for the integration tests are set in the env vars
func TestCredentialData(t *testing.T) {
	setup()
	defer tearDown()

	if appToken == "" {
		t.Error("Expected", appTokenEnv, "env var")
	}

	if appSecret == "" {
		t.Error("Expected", appSecretEnv, "env var")
	}

	if accessToken == "" {
		t.Error("Expected", accessTokenEnv, "env var")
	}

	if accessSecret == "" {
		t.Error("Expected", accessSecretEnv, "env var")
	}
}

// Check the GET request in a valid copy resource
func TestGetRequest(t *testing.T) {
	setup()
	defer tearDown()

	resp, err := session.Get("https://api.copy.com/rest/user", nil)

	if err != nil {
		t.Error("Expected no error in GET request")
	}

	if resp.StatusCode != 200 {
		t.Errorf("Response status error shouldn't be: %v", resp.StatusCode)
	}
}

// Check the GET request in an invalid copy resource with valid credentials
func TestGetRequestWrongResource(t *testing.T) {
	setup()
	defer tearDown()

	resp, _ := session.Get("https://api.copy.com/rest/userfail", nil)

	if resp.StatusCode != 400 {
		t.Errorf("Response status error should be: %v", resp.StatusCode)
	}
}

// Check the GET request in a valid copy resource with wrong credentials
func TestGetRequestWrongCredentials(t *testing.T) {
	setup()
	defer tearDown()

	session.TokenCreds.Secret = "You shall not pass!"

	resp, _ := session.Get("https://api.copy.com/rest/user", nil)

	if resp.StatusCode != 400 {
		t.Errorf("Response status error should be: %v", resp.StatusCode)
	}
}

// Check the PUT request in a valid copy resource
func TestPutRequest(t *testing.T) {
	setup()
	defer tearDown()

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	values := url.Values{
		"first_name": {fmt.Sprintf("TestName %d", r.Intn(100))},
		"last_name":  {fmt.Sprintf("TestSurname %d", r.Intn(100))},
	}

	resp, err := session.Put("https://api.copy.com/rest/user", values)

	defer resp.Body.Close()

	if err != nil {
		t.Error("Expected no error in POST request")
	}

	if resp.StatusCode != 200 {
		t.Errorf("Response status error shouldn't be: %v", resp.StatusCode)
	}

	// Get the response body to check the content
	buf := new(bytes.Buffer)
	_, err = buf.ReadFrom(resp.Body)

	if err != nil {
		t.Errorf("Error reading the response body")
	}
	respBody := buf.String()

	if !strings.Contains(respBody, values["first_name"][0]) ||
		!strings.Contains(respBody, values["last_name"][0]) {
		t.Errorf("Not updated content with the REST API")
	}
}
<|endoftext|>


<commit_before>package core

import (
	"errors"

	"github.com/livepeer/go-livepeer/drivers"
	"github.com/livepeer/go-livepeer/net"
	"github.com/livepeer/lpms/ffmpeg"
)

type Capability int
type CapabilityString []uint64
type Constraints struct{}
type Capabilities struct {
	bitstring   CapabilityString
	mandatories CapabilityString
	constraints Constraints
}

// Do not rearrange these values! Only append.
const (
	Capability_Invalid Capability = iota - 2
	Capability_Unused
	Capability_H264
	Capability_MPEGTS
	Capability_MP4
	Capability_FractionalFramerates
	Capability_StorageDirect
	Capability_StorageS3
	Capability_StorageGCS
	Capability_ProfileH264Baseline
	Capability_ProfileH264Main
	Capability_ProfileH264High
	Capability_ProfileH264ConstrainedHigh
	Capability_GOP
	Capability_AuthToken
	Capability_SceneClassification
)

var capFormatConv = errors.New("capability: unknown format")
var capStorageConv = errors.New("capability: unknown storage")
var capProfileConv = errors.New("capability: unknown profile")

func DefaultCapabilities() []Capability {
	// Add to this list as new features are added.
	return []Capability{
		Capability_H264,
		Capability_MPEGTS,
		Capability_MP4,
		Capability_FractionalFramerates,
		Capability_StorageDirect,
		Capability_StorageS3,
		Capability_StorageGCS,
		Capability_ProfileH264Baseline,
		Capability_ProfileH264Main,
		Capability_ProfileH264High,
		Capability_ProfileH264ConstrainedHigh,
		Capability_GOP,
		Capability_AuthToken,
	}
}

func ExperimentalCapabilities() []Capability {
	// Add experimental capabilities if enabled during build
	return experimentalCapabilities
}

func MandatoryCapabilities() []Capability {
	// Add to this list as certain features become mandatory.
	// Use sparingly, as adding to this is a hard break with older nodes
	return []Capability{
		Capability_AuthToken,
	}
}

func NewCapabilityString(caps []Capability) CapabilityString {
	capStr := []uint64{}
	for _, v := range caps {
		if v <= Capability_Unused {
			continue
		}
		int_index := int(v) / 64 // floors automatically
		bit_index := int(v) % 64
		// grow capStr until it's of length int_index
		for len(capStr) <= int_index {
			capStr = append(capStr, 0)
		}
		capStr[int_index] |= uint64(1 << bit_index)
	}
	return capStr

}

func (c1 CapabilityString) CompatibleWith(c2 CapabilityString) bool {
	// checks: ( c1 AND c2 ) == c1
	if len(c1) > len(c2) {
		return false
	}
	for i := range c1 {
		if (c1[i] & c2[i]) != c1[i] {
			return false
		}
	}
	return true
}

func JobCapabilities(params *StreamParameters) (*Capabilities, error) {
	caps := make(map[Capability]bool)

	// Define any default capabilities (especially ones that may be mandatory)
	caps[Capability_H264] = true
	caps[Capability_AuthToken] = true

	// capabilities based on requested output
	for _, v := range params.Profiles {
		// set format
		c, err := formatToCapability(v.Format)
		if err != nil {
			return nil, err
		}
		caps[c] = true

		// fractional framerates
		if v.FramerateDen > 0 {
			caps[Capability_FractionalFramerates] = true
		}

		// set profiles
		c, err = profileToCapability(v.Profile)
		if err != nil {
			return nil, err
		}
		caps[c] = true

		// gop
		if v.GOP != 0 {
			caps[Capability_GOP] = true
		}
	}

	// capabilities based on broadacster or stream properties

	// set expected storage
	storageCap, err := storageToCapability(params.OS)
	if err != nil {
		return nil, err
	}
	caps[storageCap] = true

	// capabilities based on detector profiles
	for _, profile := range params.Detection.Profiles {
		switch profile.Type() {
		case ffmpeg.SceneClassification:
			caps[Capability_SceneClassification] = true
		}
	}

	// generate bitstring
	capList := []Capability{}
	for k := range caps {
		capList = append(capList, k)
	}

	return &Capabilities{bitstring: NewCapabilityString(capList)}, nil
}

func (bcast *Capabilities) CompatibleWith(orch *net.Capabilities) bool {
	// Ensure bcast and orch are compatible with one another.

	if bcast == nil {
		// Weird golang behavior: interface value can evaluate to non-nil
		// even if the underlying concrete type is nil.
		// cf. common.CapabilityComparator
		return false
	}

	// For now, check this:
	// ( orch.mandatories  AND bcast.bitstring ) == orch.mandatories &&
	// ( bcast.bitstring   AND orch.bitstring  ) == bcast.bitstring

	// TODO Can simplify later on to:
	// ( bcast.bitstring AND orch.bitstring ) == ( bcast.bistring OR orch.mandatories )

	if !CapabilityString(orch.Mandatories).CompatibleWith(bcast.bitstring) {
		return false
	}

	return bcast.bitstring.CompatibleWith(orch.Bitstring)
}

func (c *Capabilities) ToNetCapabilities() *net.Capabilities {
	if c == nil {
		return nil
	}
	return &net.Capabilities{Bitstring: c.bitstring, Mandatories: c.mandatories}
}

func CapabilitiesFromNetCapabilities(caps *net.Capabilities) *Capabilities {
	if caps == nil {
		return nil
	}
	return &Capabilities{
		bitstring:   caps.Bitstring,
		mandatories: caps.Mandatories,
	}
}

func NewCapabilities(caps []Capability, m []Capability) *Capabilities {
	c := &Capabilities{}
	if caps != nil && len(caps) > 0 {
		c.bitstring = NewCapabilityString(caps)
	}
	if m != nil && len(m) > 0 {
		c.mandatories = NewCapabilityString(m)
	}
	return c
}

func formatToCapability(format ffmpeg.Format) (Capability, error) {
	switch format {
	case ffmpeg.FormatNone:
		return Capability_MPEGTS, nil
	case ffmpeg.FormatMPEGTS:
		return Capability_MPEGTS, nil
	case ffmpeg.FormatMP4:
		return Capability_MP4, nil
	}
	return Capability_Invalid, capFormatConv
}

func storageToCapability(os drivers.OSSession) (Capability, error) {
	if os == nil || os.GetInfo() == nil {
		return Capability_Unused, nil // unused
	}
	switch os.GetInfo().StorageType {
	case net.OSInfo_S3:
		return Capability_StorageS3, nil
	case net.OSInfo_GOOGLE:
		return Capability_StorageGCS, nil
	case net.OSInfo_DIRECT:
		return Capability_StorageDirect, nil
	}
	return Capability_Invalid, capStorageConv
}

func profileToCapability(profile ffmpeg.Profile) (Capability, error) {
	switch profile {
	case ffmpeg.ProfileNone:
		return Capability_Unused, nil
	case ffmpeg.ProfileH264Baseline:
		return Capability_ProfileH264Baseline, nil
	case ffmpeg.ProfileH264Main:
		return Capability_ProfileH264Main, nil
	case ffmpeg.ProfileH264High:
		return Capability_ProfileH264High, nil
	case ffmpeg.ProfileH264ConstrainedHigh:
		return Capability_ProfileH264ConstrainedHigh, nil
	}
	return Capability_Invalid, capProfileConv
}

// Fixed forever - don't change this list unless removing interoperability
// with nodes that don't support capability discovery
// (in which case, just remove everything)
var legacyCapabilities = []Capability{
	Capability_H264,
	Capability_MPEGTS,
	Capability_MP4,
	Capability_FractionalFramerates,
	Capability_StorageDirect,
	Capability_StorageS3,
	Capability_StorageGCS,
	Capability_ProfileH264Baseline,
	Capability_ProfileH264Main,
	Capability_ProfileH264High,
	Capability_ProfileH264ConstrainedHigh,
	Capability_GOP,
}
var legacyCapabilityString = NewCapabilityString(legacyCapabilities)

func (bcast *Capabilities) LegacyOnly() bool {
	if bcast == nil {
		// Weird golang behavior: interface value can evaluate to non-nil
		// even if the underlying concrete type is nil.
		// cf. common.CapabilityComparator
		return false
	}
	return bcast.bitstring.CompatibleWith(legacyCapabilityString)
}
<commit_msg>core/capabilities: add MPEG7 signature capability<commit_after>package core

import (
	"errors"

	"github.com/livepeer/go-livepeer/drivers"
	"github.com/livepeer/go-livepeer/net"
	"github.com/livepeer/lpms/ffmpeg"
)

type Capability int
type CapabilityString []uint64
type Constraints struct{}
type Capabilities struct {
	bitstring   CapabilityString
	mandatories CapabilityString
	constraints Constraints
}

// Do not rearrange these values! Only append.
const (
	Capability_Invalid Capability = iota - 2
	Capability_Unused
	Capability_H264
	Capability_MPEGTS
	Capability_MP4
	Capability_FractionalFramerates
	Capability_StorageDirect
	Capability_StorageS3
	Capability_StorageGCS
	Capability_ProfileH264Baseline
	Capability_ProfileH264Main
	Capability_ProfileH264High
	Capability_ProfileH264ConstrainedHigh
	Capability_GOP
	Capability_AuthToken
	Capability_SceneClassification
	Capability_MPEG7VideoSignature
)

var capFormatConv = errors.New("capability: unknown format")
var capStorageConv = errors.New("capability: unknown storage")
var capProfileConv = errors.New("capability: unknown profile")

func DefaultCapabilities() []Capability {
	// Add to this list as new features are added.
	return []Capability{
		Capability_H264,
		Capability_MPEGTS,
		Capability_MP4,
		Capability_FractionalFramerates,
		Capability_StorageDirect,
		Capability_StorageS3,
		Capability_StorageGCS,
		Capability_ProfileH264Baseline,
		Capability_ProfileH264Main,
		Capability_ProfileH264High,
		Capability_ProfileH264ConstrainedHigh,
		Capability_GOP,
		Capability_AuthToken,
		Capability_MPEG7VideoSignature,
	}
}

func ExperimentalCapabilities() []Capability {
	// Add experimental capabilities if enabled during build
	return experimentalCapabilities
}

func MandatoryCapabilities() []Capability {
	// Add to this list as certain features become mandatory.
	// Use sparingly, as adding to this is a hard break with older nodes
	return []Capability{
		Capability_AuthToken,
	}
}

func NewCapabilityString(caps []Capability) CapabilityString {
	capStr := []uint64{}
	for _, v := range caps {
		if v <= Capability_Unused {
			continue
		}
		int_index := int(v) / 64 // floors automatically
		bit_index := int(v) % 64
		// grow capStr until it's of length int_index
		for len(capStr) <= int_index {
			capStr = append(capStr, 0)
		}
		capStr[int_index] |= uint64(1 << bit_index)
	}
	return capStr

}

func (c1 CapabilityString) CompatibleWith(c2 CapabilityString) bool {
	// checks: ( c1 AND c2 ) == c1
	if len(c1) > len(c2) {
		return false
	}
	for i := range c1 {
		if (c1[i] & c2[i]) != c1[i] {
			return false
		}
	}
	return true
}

func JobCapabilities(params *StreamParameters) (*Capabilities, error) {
	caps := make(map[Capability]bool)

	// Define any default capabilities (especially ones that may be mandatory)
	caps[Capability_H264] = true
	caps[Capability_AuthToken] = true

	// capabilities based on requested output
	for _, v := range params.Profiles {
		// set format
		c, err := formatToCapability(v.Format)
		if err != nil {
			return nil, err
		}
		caps[c] = true

		// fractional framerates
		if v.FramerateDen > 0 {
			caps[Capability_FractionalFramerates] = true
		}

		// set profiles
		c, err = profileToCapability(v.Profile)
		if err != nil {
			return nil, err
		}
		caps[c] = true

		// gop
		if v.GOP != 0 {
			caps[Capability_GOP] = true
		}
	}

	// capabilities based on broadacster or stream properties

	// set expected storage
	storageCap, err := storageToCapability(params.OS)
	if err != nil {
		return nil, err
	}
	caps[storageCap] = true

	// capabilities based on detector profiles
	for _, profile := range params.Detection.Profiles {
		switch profile.Type() {
		case ffmpeg.SceneClassification:
			caps[Capability_SceneClassification] = true
		}
	}

	// generate bitstring
	capList := []Capability{}
	for k := range caps {
		capList = append(capList, k)
	}

	return &Capabilities{bitstring: NewCapabilityString(capList)}, nil
}

func (bcast *Capabilities) CompatibleWith(orch *net.Capabilities) bool {
	// Ensure bcast and orch are compatible with one another.

	if bcast == nil {
		// Weird golang behavior: interface value can evaluate to non-nil
		// even if the underlying concrete type is nil.
		// cf. common.CapabilityComparator
		return false
	}

	// For now, check this:
	// ( orch.mandatories  AND bcast.bitstring ) == orch.mandatories &&
	// ( bcast.bitstring   AND orch.bitstring  ) == bcast.bitstring

	// TODO Can simplify later on to:
	// ( bcast.bitstring AND orch.bitstring ) == ( bcast.bistring OR orch.mandatories )

	if !CapabilityString(orch.Mandatories).CompatibleWith(bcast.bitstring) {
		return false
	}

	return bcast.bitstring.CompatibleWith(orch.Bitstring)
}

func (c *Capabilities) ToNetCapabilities() *net.Capabilities {
	if c == nil {
		return nil
	}
	return &net.Capabilities{Bitstring: c.bitstring, Mandatories: c.mandatories}
}

func CapabilitiesFromNetCapabilities(caps *net.Capabilities) *Capabilities {
	if caps == nil {
		return nil
	}
	return &Capabilities{
		bitstring:   caps.Bitstring,
		mandatories: caps.Mandatories,
	}
}

func NewCapabilities(caps []Capability, m []Capability) *Capabilities {
	c := &Capabilities{}
	if caps != nil && len(caps) > 0 {
		c.bitstring = NewCapabilityString(caps)
	}
	if m != nil && len(m) > 0 {
		c.mandatories = NewCapabilityString(m)
	}
	return c
}

func formatToCapability(format ffmpeg.Format) (Capability, error) {
	switch format {
	case ffmpeg.FormatNone:
		return Capability_MPEGTS, nil
	case ffmpeg.FormatMPEGTS:
		return Capability_MPEGTS, nil
	case ffmpeg.FormatMP4:
		return Capability_MP4, nil
	}
	return Capability_Invalid, capFormatConv
}

func storageToCapability(os drivers.OSSession) (Capability, error) {
	if os == nil || os.GetInfo() == nil {
		return Capability_Unused, nil // unused
	}
	switch os.GetInfo().StorageType {
	case net.OSInfo_S3:
		return Capability_StorageS3, nil
	case net.OSInfo_GOOGLE:
		return Capability_StorageGCS, nil
	case net.OSInfo_DIRECT:
		return Capability_StorageDirect, nil
	}
	return Capability_Invalid, capStorageConv
}

func profileToCapability(profile ffmpeg.Profile) (Capability, error) {
	switch profile {
	case ffmpeg.ProfileNone:
		return Capability_Unused, nil
	case ffmpeg.ProfileH264Baseline:
		return Capability_ProfileH264Baseline, nil
	case ffmpeg.ProfileH264Main:
		return Capability_ProfileH264Main, nil
	case ffmpeg.ProfileH264High:
		return Capability_ProfileH264High, nil
	case ffmpeg.ProfileH264ConstrainedHigh:
		return Capability_ProfileH264ConstrainedHigh, nil
	}
	return Capability_Invalid, capProfileConv
}

// Fixed forever - don't change this list unless removing interoperability
// with nodes that don't support capability discovery
// (in which case, just remove everything)
var legacyCapabilities = []Capability{
	Capability_H264,
	Capability_MPEGTS,
	Capability_MP4,
	Capability_FractionalFramerates,
	Capability_StorageDirect,
	Capability_StorageS3,
	Capability_StorageGCS,
	Capability_ProfileH264Baseline,
	Capability_ProfileH264Main,
	Capability_ProfileH264High,
	Capability_ProfileH264ConstrainedHigh,
	Capability_GOP,
}
var legacyCapabilityString = NewCapabilityString(legacyCapabilities)

func (bcast *Capabilities) LegacyOnly() bool {
	if bcast == nil {
		// Weird golang behavior: interface value can evaluate to non-nil
		// even if the underlying concrete type is nil.
		// cf. common.CapabilityComparator
		return false
	}
	return bcast.bitstring.CompatibleWith(legacyCapabilityString)
}
<|endoftext|>


<commit_before>package rbuf

// copyright (c) 2016, Jason E. Aten
// license: MIT

import "io"

// Float64RingBuf:
//
//  a fixed-size circular ring buffer of float64
//
type Float64RingBuf struct {
	A        []float64
	N        int // MaxView, the total size of A, whether or not in use.
	Beg      int // start of in-use data in A
	Readable int // number of pointers available in A (in use)
}

// constructor. NewFloat64RingBuf will allocate internally
// a slice of size maxViewInBytes.
func NewFloat64RingBuf(maxViewInBytes int) *Float64RingBuf {
	n := maxViewInBytes
	r := &Float64RingBuf{
		N:        n,
		Beg:      0,
		Readable: 0,
	}
	r.A = make([]float64, n, n)

	return r
}

// TwoContig returns all readable pointers, but in two separate slices,
// to avoid copying. The two slices are from the same buffer, but
// are not contiguous. Either or both may be empty slices.
func (b *Float64RingBuf) TwoContig(makeCopy bool) (first []float64, second []float64) {

	extent := b.Beg + b.Readable
	if extent <= b.N {
		// we fit contiguously in this buffer without wrapping to the other.
		// Let second stay an empty slice.
		return b.A[b.Beg:(b.Beg + b.Readable)], second
	}

	return b.A[b.Beg:b.N], b.A[0:(extent % b.N)]
}

// Earliest returns the earliest written value v. ok will be
// true unless the ring is empty, in which case ok will be false,
// and v will be zero.
func (b *Float64RingBuf) Earliest() (v float64, ok bool) {
	if b.N == 0 {
		return
	}

	extent := b.Beg + b.Readable
	if extent <= b.N {
		return b.A[(b.Beg+b.Readable)-1], true
	}

	// INVAR: extent > b.N, so extent%b.N > 0
	return b.A[(extent%b.N)-1], true
}

// ReadFloat64():
//
// from bytes.Buffer.Read(): Read reads the next len(p) float64
// pointers from the buffer or until the buffer is drained. The return
// value n is the number of bytes read. If the buffer has no data
// to return, err is io.EOF (unless len(p) is zero); otherwise it is nil.
func (b *Float64RingBuf) ReadFloat64(p []float64) (n int, err error) {
	return b.readAndMaybeAdvance(p, true)
}

// ReadWithoutAdvance(): if you want to Read the data and leave
// it in the buffer, so as to peek ahead for example.
func (b *Float64RingBuf) ReadWithoutAdvance(p []float64) (n int, err error) {
	return b.readAndMaybeAdvance(p, false)
}

func (b *Float64RingBuf) readAndMaybeAdvance(p []float64, doAdvance bool) (n int, err error) {
	if len(p) == 0 {
		return 0, nil
	}
	if b.Readable == 0 {
		return 0, io.EOF
	}
	extent := b.Beg + b.Readable
	if extent <= b.N {
		n += copy(p, b.A[b.Beg:extent])
	} else {
		n += copy(p, b.A[b.Beg:b.N])
		if n < len(p) {
			n += copy(p[n:], b.A[0:(extent%b.N)])
		}
	}
	if doAdvance {
		b.Advance(n)
	}
	return
}

//
// WriteAndMaybeOverwriteOldestData always consumes the full
// buffer p, even if that means blowing away the oldest
// unread bytes in the ring to make room. In reality, only the last
// min(len(p),b.N) bytes of p will end up being written to the ring.
//
// This allows the ring to act as a record of the most recent
// b.N bytes of data -- a kind of temporal LRU cache, so the
// speak. The linux kernel's dmesg ring buffer is similar.
//
func (b *Float64RingBuf) WriteAndMaybeOverwriteOldestData(p []float64) (n int, err error) {
	writeCapacity := b.N - b.Readable
	if len(p) > writeCapacity {
		b.Advance(len(p) - writeCapacity)
	}
	startPos := 0
	if len(p) > b.N {
		startPos = len(p) - b.N
	}
	n, err = b.Write(p[startPos:])
	if err != nil {
		return n, err
	}
	return len(p), nil
}

//
// Write writes len(p) float64 values from p to
// the underlying data stream.
// It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// Write must return a non-nil error if it returns n < len(p).
//
func (b *Float64RingBuf) Write(p []float64) (n int, err error) {
	for {
		if len(p) == 0 {
			// nothing (left) to copy in; notice we shorten our
			// local copy p (below) as we read from it.
			return
		}

		writeCapacity := b.N - b.Readable
		if writeCapacity <= 0 {
			// we are all full up already.
			return n, io.ErrShortWrite
		}
		if len(p) > writeCapacity {
			err = io.ErrShortWrite
			// leave err set and
			// keep going, write what we can.
		}

		writeStart := (b.Beg + b.Readable) % b.N

		upperLim := intMin(writeStart+writeCapacity, b.N)

		k := copy(b.A[writeStart:upperLim], p)

		n += k
		b.Readable += k
		p = p[k:]

		// we can fill from b.A[0:something] from
		// p's remainder, so loop
	}
}

// Reset quickly forgets any data stored in the ring buffer. The
// data is still there, but the ring buffer will ignore it and
// overwrite those buffers as new data comes in.
func (b *Float64RingBuf) Reset() {
	b.Beg = 0
	b.Readable = 0
}

// Advance(): non-standard, but better than Next(),
// because we don't have to unwrap our buffer and pay the cpu time
// for the copy that unwrapping may need.
// Useful in conjuction/after ReadWithoutAdvance() above.
func (b *Float64RingBuf) Advance(n int) {
	if n <= 0 {
		return
	}
	if n > b.Readable {
		n = b.Readable
	}
	b.Readable -= n
	b.Beg = (b.Beg + n) % b.N
}

// Adopt(): non-standard.
//
// For efficiency's sake, (possibly) take ownership of
// already allocated slice offered in me.
//
// If me is large we will adopt it, and we will potentially then
// write to the me buffer.
// If we already have a bigger buffer, copy me into the existing
// buffer instead.
func (b *Float64RingBuf) Adopt(me []float64) {
	n := len(me)
	if n > b.N {
		b.A = me
		b.N = n
		b.Beg = 0
		b.Readable = n
	} else {
		// we already have a larger buffer, reuse it.
		copy(b.A, me)
		b.Beg = 0
		b.Readable = n
	}
}
<commit_msg>fix impl of Earliest<commit_after>package rbuf

// copyright (c) 2016, Jason E. Aten
// license: MIT

import (
	"io"
)

// Float64RingBuf:
//
//  a fixed-size circular ring buffer of float64
//
type Float64RingBuf struct {
	A        []float64
	N        int // MaxView, the total size of A, whether or not in use.
	Beg      int // start of in-use data in A
	Readable int // number of pointers available in A (in use)
}

// constructor. NewFloat64RingBuf will allocate internally
// a slice of size maxViewInBytes.
func NewFloat64RingBuf(maxViewInBytes int) *Float64RingBuf {
	n := maxViewInBytes
	r := &Float64RingBuf{
		N:        n,
		Beg:      0,
		Readable: 0,
	}
	r.A = make([]float64, n, n)

	return r
}

// TwoContig returns all readable pointers, but in two separate slices,
// to avoid copying. The two slices are from the same buffer, but
// are not contiguous. Either or both may be empty slices.
func (b *Float64RingBuf) TwoContig(makeCopy bool) (first []float64, second []float64) {

	extent := b.Beg + b.Readable
	if extent <= b.N {
		// we fit contiguously in this buffer without wrapping to the other.
		// Let second stay an empty slice.
		return b.A[b.Beg:(b.Beg + b.Readable)], second
	}

	return b.A[b.Beg:b.N], b.A[0:(extent % b.N)]
}

// Earliest returns the earliest written value v. ok will be
// true unless the ring is empty, in which case ok will be false,
// and v will be zero.
func (b *Float64RingBuf) Earliest() (v float64, ok bool) {
	if b.Readable == 0 {
		return
	}

	return b.A[b.Beg], true
}

// ReadFloat64():
//
// from bytes.Buffer.Read(): Read reads the next len(p) float64
// pointers from the buffer or until the buffer is drained. The return
// value n is the number of bytes read. If the buffer has no data
// to return, err is io.EOF (unless len(p) is zero); otherwise it is nil.
func (b *Float64RingBuf) ReadFloat64(p []float64) (n int, err error) {
	return b.readAndMaybeAdvance(p, true)
}

// ReadWithoutAdvance(): if you want to Read the data and leave
// it in the buffer, so as to peek ahead for example.
func (b *Float64RingBuf) ReadWithoutAdvance(p []float64) (n int, err error) {
	return b.readAndMaybeAdvance(p, false)
}

func (b *Float64RingBuf) readAndMaybeAdvance(p []float64, doAdvance bool) (n int, err error) {
	if len(p) == 0 {
		return 0, nil
	}
	if b.Readable == 0 {
		return 0, io.EOF
	}
	extent := b.Beg + b.Readable
	if extent <= b.N {
		n += copy(p, b.A[b.Beg:extent])
	} else {
		n += copy(p, b.A[b.Beg:b.N])
		if n < len(p) {
			n += copy(p[n:], b.A[0:(extent%b.N)])
		}
	}
	if doAdvance {
		b.Advance(n)
	}
	return
}

//
// WriteAndMaybeOverwriteOldestData always consumes the full
// buffer p, even if that means blowing away the oldest
// unread bytes in the ring to make room. In reality, only the last
// min(len(p),b.N) bytes of p will end up being written to the ring.
//
// This allows the ring to act as a record of the most recent
// b.N bytes of data -- a kind of temporal LRU cache, so the
// speak. The linux kernel's dmesg ring buffer is similar.
//
func (b *Float64RingBuf) WriteAndMaybeOverwriteOldestData(p []float64) (n int, err error) {
	writeCapacity := b.N - b.Readable
	if len(p) > writeCapacity {
		b.Advance(len(p) - writeCapacity)
	}
	startPos := 0
	if len(p) > b.N {
		startPos = len(p) - b.N
	}
	n, err = b.Write(p[startPos:])
	if err != nil {
		return n, err
	}
	return len(p), nil
}

//
// Write writes len(p) float64 values from p to
// the underlying data stream.
// It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// Write must return a non-nil error if it returns n < len(p).
//
func (b *Float64RingBuf) Write(p []float64) (n int, err error) {
	for {
		if len(p) == 0 {
			// nothing (left) to copy in; notice we shorten our
			// local copy p (below) as we read from it.
			return
		}

		writeCapacity := b.N - b.Readable
		if writeCapacity <= 0 {
			// we are all full up already.
			return n, io.ErrShortWrite
		}
		if len(p) > writeCapacity {
			err = io.ErrShortWrite
			// leave err set and
			// keep going, write what we can.
		}

		writeStart := (b.Beg + b.Readable) % b.N

		upperLim := intMin(writeStart+writeCapacity, b.N)

		k := copy(b.A[writeStart:upperLim], p)

		n += k
		b.Readable += k
		p = p[k:]

		// we can fill from b.A[0:something] from
		// p's remainder, so loop
	}
}

// Reset quickly forgets any data stored in the ring buffer. The
// data is still there, but the ring buffer will ignore it and
// overwrite those buffers as new data comes in.
func (b *Float64RingBuf) Reset() {
	b.Beg = 0
	b.Readable = 0
}

// Advance(): non-standard, but better than Next(),
// because we don't have to unwrap our buffer and pay the cpu time
// for the copy that unwrapping may need.
// Useful in conjuction/after ReadWithoutAdvance() above.
func (b *Float64RingBuf) Advance(n int) {
	if n <= 0 {
		return
	}
	if n > b.Readable {
		n = b.Readable
	}
	b.Readable -= n
	b.Beg = (b.Beg + n) % b.N
}

// Adopt(): non-standard.
//
// For efficiency's sake, (possibly) take ownership of
// already allocated slice offered in me.
//
// If me is large we will adopt it, and we will potentially then
// write to the me buffer.
// If we already have a bigger buffer, copy me into the existing
// buffer instead.
func (b *Float64RingBuf) Adopt(me []float64) {
	n := len(me)
	if n > b.N {
		b.A = me
		b.N = n
		b.Beg = 0
		b.Readable = n
	} else {
		// we already have a larger buffer, reuse it.
		copy(b.A, me)
		b.Beg = 0
		b.Readable = n
	}
}
<|endoftext|>