<commit_before>package prometheus

import (
	"runtime"
	"runtime/debug"
	"time"
)

type goCollector struct {
	goroutines   Gauge
	gcDesc       *Desc
	alloc        Gauge
	totalAlloc   Counter
	sys          Counter
	lookups      Counter
	mallocs      Counter
	frees        Counter
	heapAlloc    Gauge
	heapSys      Gauge
	heapIdle     Gauge
	heapInuse    Gauge
	heapReleased Gauge
	heapObjects  Gauge
}

// NewGoCollector returns a collector which exports metrics about the current
// go process.
func NewGoCollector() *goCollector {
	return &goCollector{
		goroutines: NewGauge(GaugeOpts{
			Name: "go_goroutines",
			Help: "Number of goroutines that currently exist.",
		}),
		gcDesc: NewDesc(
			"go_gc_duration_seconds",
			"A summary of the GC invocation durations.",
			nil, nil),
		alloc: NewGauge(GaugeOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "alloc_bytes",
			Help:      "Number of bytes allocated and still in use.",
		}),
		totalAlloc: NewCounter(CounterOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "alloc_bytes_total",
			Help:      "Total number of bytes allocated, even if freed.",
		}),
		sys: NewGauge(GaugeOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "sys_bytes",
			Help:      "Number of bytes obtained from system",
		}),
		lookups: NewCounter(CounterOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "lookups_total",
			Help:      "Total number of pointer lookups.",
		}),
		mallocs: NewCounter(CounterOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "mallocs_total",
			Help:      "Total number of mallocs.",
		}),
		frees: NewCounter(CounterOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "frees_total",
			Help:      "Total number of frees.",
		}),
		heapAlloc: NewGauge(GaugeOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "heap_alloc_bytes",
			Help:      "Number heap bytes allocated and still in use.",
		}),
		heapSys: NewGauge(GaugeOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "heap_sys_bytes",
			Help:      "Total bytes in heap obtained from system.",
		}),
		heapIdle: NewGauge(GaugeOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "heap_idle_bytes",
			Help:      "Number bytes in heap waiting to be used.",
		}),
		heapInuse: NewGauge(GaugeOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "heap_inuse_bytes",
			Help:      "Number of bytes in heap that are in use.",
		}),
		heapReleased: NewGauge(GaugeOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "heap_released_bytes",
			Help:      "Number of bytes in heap released to OS.",
		}),
		heapObjects: NewGauge(GaugeOpts{
			Namespace: "go",
			Subsystem: "memstats",
			Name:      "heap_objects",
			Help:      "Number of allocated objects.",
		}),
	}
}

// Describe returns all descriptions of the collector.
func (c *goCollector) Describe(ch chan<- *Desc) {
	ch <- c.goroutines.Desc()
	ch <- c.gcDesc
	ch <- c.alloc.Desc()
	ch <- c.totalAlloc.Desc()
	ch <- c.sys.Desc()
	ch <- c.lookups.Desc()
	ch <- c.mallocs.Desc()
	ch <- c.frees.Desc()
	ch <- c.heapAlloc.Desc()
	ch <- c.heapSys.Desc()
	ch <- c.heapIdle.Desc()
	ch <- c.heapInuse.Desc()
	ch <- c.heapReleased.Desc()
	ch <- c.heapObjects.Desc()
}

// Collect returns the current state of all metrics of the collector.
func (c *goCollector) Collect(ch chan<- Metric) {
	c.goroutines.Set(float64(runtime.NumGoroutine()))
	ch <- c.goroutines

	var stats debug.GCStats
	stats.PauseQuantiles = make([]time.Duration, 5)
	debug.ReadGCStats(&stats)

	quantiles := make(map[float64]float64)
	for idx, pq := range stats.PauseQuantiles[1:] {
		quantiles[float64(idx+1)/float64(len(stats.PauseQuantiles)-1)] = pq.Seconds()
	}
	quantiles[0.0] = stats.PauseQuantiles[0].Seconds()
	ch <- MustNewConstSummary(c.gcDesc, uint64(stats.NumGC), float64(stats.PauseTotal.Seconds()), quantiles)

	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)

	c.alloc.Set(float64(ms.Alloc))
	ch <- c.alloc
	c.totalAlloc.Set(float64(ms.TotalAlloc))
	ch <- c.totalAlloc
	c.sys.Set(float64(ms.Sys))
	ch <- c.sys
	c.lookups.Set(float64(ms.Lookups))
	ch <- c.lookups
	c.mallocs.Set(float64(ms.Mallocs))
	ch <- c.mallocs
	c.frees.Set(float64(ms.Frees))
	ch <- c.frees
	c.heapAlloc.Set(float64(ms.HeapAlloc))
	ch <- c.heapAlloc
	c.heapSys.Set(float64(ms.HeapSys))
	ch <- c.heapSys
	c.heapIdle.Set(float64(ms.HeapIdle))
	ch <- c.heapIdle
	c.heapInuse.Set(float64(ms.HeapInuse))
	ch <- c.heapInuse
	c.heapReleased.Set(float64(ms.HeapReleased))
	ch <- c.heapReleased
	c.heapObjects.Set(float64(ms.HeapObjects))
	ch <- c.heapObjects
}
<commit_msg>use metrics struct. include more statistics<commit_after>package prometheus

import (
	"fmt"
	"runtime"
	"runtime/debug"
	"time"
)

type goCollector struct {
	goroutines Gauge
	gcDesc     *Desc

	memstats *memStatCollector
}

// NewGoCollector returns a collector which exports metrics about the current
// go process.
func NewGoCollector() *goCollector {
	return &goCollector{
		goroutines: NewGauge(GaugeOpts{
			Namespace: "go",
			Name:      "goroutines",
			Help:      "Number of goroutines that currently exist.",
		}),
		gcDesc: NewDesc(
			"go_gc_duration_seconds",
			"A summary of the GC invocation durations.",
			nil, nil),
		memstats: &memStatCollector{
			ms: new(runtime.MemStats),
			metrics: memStatsMetrics{
				{
					desc: NewDesc(
						memstatNamespace("alloc_bytes"),
						"Number of bytes allocated and still in use.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("alloc_bytes_total"),
						"Total number of bytes allocated, even if freed.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) },
					valType: CounterValue,
				}, {
					desc: NewDesc(
						memstatNamespace("sys_bytes"),
						"Number of bytes obtained from system",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Sys) },
					valType: CounterValue,
				}, {
					desc: NewDesc(
						memstatNamespace("lookups_total"),
						"Total number of pointer lookups.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Lookups) },
					valType: CounterValue,
				}, {
					desc: NewDesc(
						memstatNamespace("mallocs_total"),
						"Total number of mallocs.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) },
					valType: CounterValue,
				}, {
					desc: NewDesc(
						memstatNamespace("frees_total"),
						"Total number of frees.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Frees) },
					valType: CounterValue,
				}, {
					desc: NewDesc(
						memstatNamespace("heap_alloc_bytes"),
						"Number heap bytes allocated and still in use.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("heap_sys_bytes"),
						"Total bytes in heap obtained from system.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("heap_idle_bytes"),
						"Number bytes in heap waiting to be used.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("heap_inuse_bytes"),
						"Number of bytes in heap that are in use.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("heap_released_bytes"),
						"Number of bytes in heap released to OS.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("heap_objects"),
						"Number of allocated objects.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("stack_bytes_inuse"),
						"Number of bytes in use by the stack allocator.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("stack_sys_bytes"),
						"Number of bytes in obtained from system for stack allocator.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("mspan_inuse"),
						"Number of mspan structures in use.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("mspan_sys"),
						"Number of mspan structures obtained from system.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("mcache_inuse"),
						"Number of mcache structures in use.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("mcache_sys"),
						"Number of mcache structures obtained from system.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("buck_hash_sys"),
						"Profiling bucket hash table.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("gc_metadata"),
						"GC metadata.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("other_sys"),
						"Other system allocations.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("next_gc"),
						"Next collection will happen when HeapAlloc ≥ this amount.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) },
					valType: GaugeValue,
				}, {
					desc: NewDesc(
						memstatNamespace("last_gc"),
						"End time of last garbage collection (nanoseconds since 1970).",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.LastGC) },
					valType: CounterValue,
				}, {
					desc: NewDesc(
						memstatNamespace("pause_total"),
						"Total garbage collection pauses for all collections.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.PauseTotalNs) },
					valType: CounterValue,
				}, {
					desc: NewDesc(
						memstatNamespace("gc_total"),
						"Number of garbage collection.",
						nil, nil,
					),
					eval:    func(ms *runtime.MemStats) float64 { return float64(ms.NumGC) },
					valType: CounterValue,
				},
			},
		},
	}
}

func memstatNamespace(s string) string {
	return fmt.Sprintf("go_memstats_%s", s)
}

// Describe returns all descriptions of the collector.
func (c *goCollector) Describe(ch chan<- *Desc) {
	ch <- c.goroutines.Desc()
	ch <- c.gcDesc

	c.memstats.Describe(ch)
}

// Collect returns the current state of all metrics of the collector.
func (c *goCollector) Collect(ch chan<- Metric) {
	c.goroutines.Set(float64(runtime.NumGoroutine()))
	ch <- c.goroutines

	var stats debug.GCStats
	stats.PauseQuantiles = make([]time.Duration, 5)
	debug.ReadGCStats(&stats)

	quantiles := make(map[float64]float64)
	for idx, pq := range stats.PauseQuantiles[1:] {
		quantiles[float64(idx+1)/float64(len(stats.PauseQuantiles)-1)] = pq.Seconds()
	}
	quantiles[0.0] = stats.PauseQuantiles[0].Seconds()
	ch <- MustNewConstSummary(c.gcDesc, uint64(stats.NumGC), float64(stats.PauseTotal.Seconds()), quantiles)

	c.memstats.Collect(ch)
}

// metrics that provide description, value, and value type for memstat metrics
type memStatsMetrics []struct {
	desc    *Desc
	eval    func(*runtime.MemStats) float64
	valType ValueType
}

type memStatCollector struct {
	// memstats object to reuse
	ms *runtime.MemStats
	// metrics to describe and collect
	metrics memStatsMetrics
}

func (c *memStatCollector) Describe(ch chan<- *Desc) {
	for _, i := range c.metrics {
		ch <- i.desc
	}
}

func (c *memStatCollector) Collect(ch chan<- Metric) {
	runtime.ReadMemStats(c.ms)
	for _, i := range c.metrics {
		ch <- MustNewConstMetric(i.desc, i.valType, i.eval(c.ms))
	}
}
<|endoftext|>


<commit_before>package network

import (
	"github.com/yaricom/goNEAT/neat"
	"io"
	"fmt"
	"errors"
)

// A NODE is either a NEURON or a SENSOR.
//   - If it's a sensor, it can be loaded with a value for output
//   - If it's a neuron, it has a list of its incoming input signals ([]*Link is used)
// Use an activation count to avoid flushing
type NNode interface {

	// Return the ID of the node
	NodeId() int32
	// Returns the placement of the node in the network layers (INPUT, HIDDEN, OUTPUT)
	GenNodeLabel() int32
	// Sets new trait to the node
	SetTrait(t *Trait)
	// Returns number of activations for current node
	ActivationCount() int32

	// Return activation for current step
	GetActiveOut() float64
	// Return activation from PREVIOUS (time-delayed) time step, if there is one
	GetActiveOutTd() float64

	// Returns the type of the node (NEURON or SENSOR)
	GetType() int32
	// Allows alteration between NEURON and SENSOR.  Returns its argument
	SetType(ntype int32)

	// If the node is a SENSOR, returns TRUE and loads the value
	SensorLoad(load float64) bool

	// Adds a NONRECURRENT Link to a new NNode with specified weight in the incoming List
	AddIncoming(in *NNode, weight float64)
	// Adds a Link to a new NNode in the incoming List
	AddIncomingRecurrent(in *NNode, weight float64, recur bool);

	// Recursively deactivate backwards through the network including this NNode and reccurencies
	Flushback()

	// Write this node into writer
	WriteNode(w *io.Writer)

	// Find the greatest depth starting from this neuron at depth d
	Depth(d int32, mynet *Network) int32

	// Verify flushing for debug
	FlushbackCheck(seenlist []*NNode) error

}

// Creates new node with specified type (NEURON or SENSOR) and ID
func NewNNode(ntype, nodeid int) NNode {
	n := newNode()
	n.ntype = ntype
	n.node_id = nodeid
	return n
}

// Creates new node with specified type (NEURON or SENSOR), ID and in the specified
// layer (INPUT, HIDDEN, OUTPUT)
func NewNNodeInPlace(ntype, nodeid, placement int) NNode {
	n := newNode()
	n.ntype = ntype
	n.node_id = nodeid
	n.gen_node_label = placement
	return n
}

// Construct a NNode off another NNode with given trait for genome purposes
func NewNNodeCopy(n *NNode, t *Trait) NNode {
	node := newNode()
	node.ntype = (*n).GetType()
	node.node_id = (*n).NodeId()
	node.gen_node_label = (*n).GenNodeLabel()
	node.SetTrait(t)
	return node
}

// Read a NNode from specified Reader (r) and applies corresponding trait to it from a list of traits provided
func ReadNNode(r *io.Reader, traits []*Trait) {
	n := newNode()
	var trait_id int32
	fmt.Fscanf(r, "%d %d %d %d", &n.node_id, &trait_id, &n.ntype, &n.gen_node_label)
	if trait_id != 0 && traits != nil {
		// find corresponding node trait from list
		for _, t := range traits {
			if trait_id == (*t).TraitId() {
				n.nodetrait = t
				break
			}
		}
	}
	return n
}

// private structure to hold values
type nnode struct {
	// The activation function type is either SIGMOID ..or others that can be added
	ftype int32
	// The NN node type is either NEURON or SENSOR
	ntype int32
	// The incoming activity before being processed
	activesum float64
	// The total activation entering the NNode
	activation float64
	// To make sure outputs are active (allows to disable this node)
	active_flag bool

	// The following parameters are for use in neurons that learn through habituation,
	// sensitization, or Hebbian-type processes
	params []float64

	// Keeps track of which activation the node is currently in
	activation_count int32
	// Activation value of node at time t-1; Holds the previous step's activation for recurrency
	last_activation float64
	// Activation value of node at time t-2 Holds the activation before  the previous step's
	// This is necessary for a special recurrent case when the innode of a recurrent link is one time step ahead of the outnode.
	// The innode then needs to send from TWO time steps ago
	last_activation2 float64

	//Points to a trait of parameters
	nodetrait *Trait

	// Is a reference to a Node; It's used to generate and point from a genetic node (genotype)
	// to a real node (fenotype) during 'genesis' process (Gene decoding)
	analogue *NNode
	// Is a  temporary reference to a Node; It's used to generate a new genome during duplicate phase of genotype.
	dup *NNode

	// A list of pointers to incoming weighted signals from other nodes
	incoming []*Link
	// A list of pointers to links carrying this node's signal
	outgoing []*Link

	// A node can be given an identification number for saving in files
	node_id int32
	// Used for genetic marking of nodes
	gen_node_label int32
}

// The private default constructor
func newNode() nnode {
	return nnode{
		ftype:SIGMOID,
		params:make([]float64, neat.Num_trait_params),
		incoming:make([]*Link, 0),
		outgoing:make([]*Link, 0),
		gen_node_label:HIDDEN,
	}
}

// The NNode interface implementation
func (n *nnode) ActivationCount() {
	return n.activation_count
}
func (n *nnode) NodeType() int32 {
	return n.ntype
}
func (n *nnode) NodeId() int32  {
	return n.node_id
}
func (n *nnode) GenNodeLabel() int32  {
	return n.gen_node_label
}
func (n *nnode) SetTrait(t *Trait) {
	n.nodetrait = t
}
func (n *nnode) GetActiveOut() float64 {
	if n.activation_count > 0 {
		return n.activation
	} else {
		return 0.0
	}
}
func (n *nnode) GetActiveOutTd() float64 {
	if n.activation_count > 1 {
		return n.last_activation
	} else {
		return 0.0
	}
}
func (n *nnode) GetType() int32 {
	return n.ntype
}
func (n *nnode) SetType(ntype int32) {
	n.ntype = ntype
}
func (n *nnode) SensorLoad(load float64) bool {
	if n.ntype == SENSOR {
		// Time delay memory
		n.last_activation2 = n.last_activation
		n.last_activation = n.activation
		// Puts sensor into next time-step
		n.activation_count += 1
		n.activation = load
		return true
	} else {
		return false
	}
}
func (n *nnode) AddIncoming(in *NNode, weight float64) {
	newLink := NewLink(weight, in, n, false)
	n.incoming = append(n.incoming, newLink)
}
func (n *nnode) AddIncomingRecurrent(in *NNode, weight float64, recur bool) {
	newLink := NewLink(weight, in, n, recur)
	n.incoming = append(n.incoming, newLink)
}
func (n *nnode) Flushback() {
	n.activation_count = 0
	n.activation = 0
	n.last_activation = 0
	n.last_activation2 = 0

	if n.ntype == NEURON {
		//Flush back recursively
		for _, l := range n.incoming {
			(*l).SetAddedWeight(0)
			if (*l).InNode().ActivationCount() > 0 {
				(*l).InNode().Flushback()
			}
		}

	}
}
func (n *nnode) FlushbackCheck() error {
	if n.activation_count > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has activation count %d", n, n.activation_count))
	}
	if n.activation > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has activation %f", n, n.activation))
	}
	if n.last_activation > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has last_activation %f", n, n.last_activation))
	}
	if n.last_activation2 > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has last_activation2 %f", n, n.last_activation2))
	}

	if n.ntype == NEURON {
		// Flush back check recursively
		for _, l := range n.incoming {
			err := (*l).InNode().FlushbackCheck()
			if err != nil {
				return err
			}
		}

	}
	return nil
}
func (n *nnode) WriteNode(w *io.Writer) {
	trait_id := 0
	if n.nodetrait != nil {
		trait_id = (*n.nodetrait).TraitId()
	}
	fmt.Fprintf(w, "%d %d %d %d", n.node_id, trait_id, n.ntype, n.gen_node_label)
}
func (n *nnode) Depth(d int32, mynet *Network) int32 {
	cur_depth := 0 //The depth of the current node
	max := d //The max depth

	if d > 100 {
		fmt.Println("** DEPTH NOT DETERMINED FOR NETWORK WITH LOOP")
		return 10;
	}
	// Base Case
	if n.ntype == SENSOR {
		return d
	} else {
		// recursion
		for _, l := range n.incoming {
			cur_depth = (*l).InNode().Depth(d + 1, mynet)
			if cur_depth > max {
				max = cur_depth
			}
		}
		return max
	}

}

func (n nnode) String() string {
	if n.ntype == SENSOR {
		return fmt.Sprintf("(S %d, step %d : %f)", n.node_id, n.activation_count, n.activation)
	} else {
		return fmt.Sprintf("(N %d, step %d : %f)", n.node_id, n.activation_count, n.activation)
	}
}

<commit_msg>Optimized NNode flushback<commit_after>package network

import (
	"github.com/yaricom/goNEAT/neat"
	"io"
	"fmt"
	"errors"
)

// A NODE is either a NEURON or a SENSOR.
//   - If it's a sensor, it can be loaded with a value for output
//   - If it's a neuron, it has a list of its incoming input signals ([]*Link is used)
// Use an activation count to avoid flushing
type NNode interface {

	// Return the ID of the node
	NodeId() int32
	// Returns the placement of the node in the network layers (INPUT, HIDDEN, OUTPUT)
	GenNodeLabel() int32
	// Sets new trait to the node
	SetTrait(t *Trait)
	// Returns number of activations for current node
	ActivationCount() int32

	// Return activation for current step
	GetActiveOut() float64
	// Return activation from PREVIOUS (time-delayed) time step, if there is one
	GetActiveOutTd() float64

	// Returns the type of the node (NEURON or SENSOR)
	GetType() int32
	// Allows alteration between NEURON and SENSOR.  Returns its argument
	SetType(ntype int32)

	// If the node is a SENSOR, returns TRUE and loads the value
	SensorLoad(load float64) bool

	// Adds a NONRECURRENT Link to a new NNode with specified weight in the incoming List
	AddIncoming(in *NNode, weight float64)
	// Adds a Link to a new NNode in the incoming List
	AddIncomingRecurrent(in *NNode, weight float64, recur bool);

	// Recursively deactivate backwards through the network including this NNode and reccurencies
	Flushback()

	// Write this node into writer
	WriteNode(w *io.Writer)

	// Find the greatest depth starting from this neuron at depth d
	Depth(d int32, mynet *Network) int32

	// Verify flushing for debug
	FlushbackCheck() error

}

// Creates new node with specified type (NEURON or SENSOR) and ID
func NewNNode(ntype, nodeid int) NNode {
	n := newNode()
	n.ntype = ntype
	n.node_id = nodeid
	return n
}

// Creates new node with specified type (NEURON or SENSOR), ID and in the specified
// layer (INPUT, HIDDEN, OUTPUT)
func NewNNodeInPlace(ntype, nodeid, placement int) NNode {
	n := newNode()
	n.ntype = ntype
	n.node_id = nodeid
	n.gen_node_label = placement
	return n
}

// Construct a NNode off another NNode with given trait for genome purposes
func NewNNodeCopy(n *NNode, t *Trait) NNode {
	node := newNode()
	node.ntype = (*n).GetType()
	node.node_id = (*n).NodeId()
	node.gen_node_label = (*n).GenNodeLabel()
	node.SetTrait(t)
	return node
}

// Read a NNode from specified Reader (r) and applies corresponding trait to it from a list of traits provided
func ReadNNode(r *io.Reader, traits []*Trait) {
	n := newNode()
	var trait_id int32
	fmt.Fscanf(r, "%d %d %d %d", &n.node_id, &trait_id, &n.ntype, &n.gen_node_label)
	if trait_id != 0 && traits != nil {
		// find corresponding node trait from list
		for _, t := range traits {
			if trait_id == (*t).TraitId() {
				n.nodetrait = t
				break
			}
		}
	}
	return n
}

// private structure to hold values
type nnode struct {
	// The activation function type is either SIGMOID ..or others that can be added
	ftype int32
	// The NN node type is either NEURON or SENSOR
	ntype int32
	// The incoming activity before being processed
	activesum float64
	// The total activation entering the NNode
	activation float64
	// To make sure outputs are active (allows to disable this node)
	active_flag bool

	// The following parameters are for use in neurons that learn through habituation,
	// sensitization, or Hebbian-type processes
	params []float64

	// Keeps track of which activation the node is currently in
	activation_count int32
	// Activation value of node at time t-1; Holds the previous step's activation for recurrency
	last_activation float64
	// Activation value of node at time t-2 Holds the activation before  the previous step's
	// This is necessary for a special recurrent case when the innode of a recurrent link is one time step ahead of the outnode.
	// The innode then needs to send from TWO time steps ago
	last_activation2 float64

	//Points to a trait of parameters
	nodetrait *Trait

	// Is a reference to a Node; It's used to generate and point from a genetic node (genotype)
	// to a real node (fenotype) during 'genesis' process (Gene decoding)
	analogue *NNode
	// Is a  temporary reference to a Node; It's used to generate a new genome during duplicate phase of genotype.
	dup *NNode

	// A list of pointers to incoming weighted signals from other nodes
	incoming []*Link
	// A list of pointers to links carrying this node's signal
	outgoing []*Link

	// A node can be given an identification number for saving in files
	node_id int32
	// Used for genetic marking of nodes
	gen_node_label int32
}

// The private default constructor
func newNode() nnode {
	return nnode{
		ftype:SIGMOID,
		params:make([]float64, neat.Num_trait_params),
		incoming:make([]*Link, 0),
		outgoing:make([]*Link, 0),
		gen_node_label:HIDDEN,
	}
}

// The NNode interface implementation
func (n *nnode) ActivationCount() {
	return n.activation_count
}
func (n *nnode) NodeType() int32 {
	return n.ntype
}
func (n *nnode) NodeId() int32  {
	return n.node_id
}
func (n *nnode) GenNodeLabel() int32  {
	return n.gen_node_label
}
func (n *nnode) SetTrait(t *Trait) {
	n.nodetrait = t
}
func (n *nnode) GetActiveOut() float64 {
	if n.activation_count > 0 {
		return n.activation
	} else {
		return 0.0
	}
}
func (n *nnode) GetActiveOutTd() float64 {
	if n.activation_count > 1 {
		return n.last_activation
	} else {
		return 0.0
	}
}
func (n *nnode) GetType() int32 {
	return n.ntype
}
func (n *nnode) SetType(ntype int32) {
	n.ntype = ntype
}
func (n *nnode) SensorLoad(load float64) bool {
	if n.ntype == SENSOR {
		// Time delay memory
		n.last_activation2 = n.last_activation
		n.last_activation = n.activation
		// Puts sensor into next time-step
		n.activation_count += 1
		n.activation = load
		return true
	} else {
		return false
	}
}
func (n *nnode) AddIncoming(in *NNode, weight float64) {
	newLink := NewLink(weight, in, n, false)
	n.incoming = append(n.incoming, newLink)
}
func (n *nnode) AddIncomingRecurrent(in *NNode, weight float64, recur bool) {
	newLink := NewLink(weight, in, n, recur)
	n.incoming = append(n.incoming, newLink)
}
func (n *nnode) Flushback() {
	n.activation_count = 0
	n.activation = 0
	n.last_activation = 0
	n.last_activation2 = 0

	//if n.ntype == NEURON {
	//	// Flush back recursively
	//	for _, l := range n.incoming {
	//		(*l).SetAddedWeight(0)
	//		if (*l).InNode().ActivationCount() > 0 {
	//			(*l).InNode().Flushback()
	//		}
	//	}
	//}
}
func (n *nnode) FlushbackCheck() error {
	if n.activation_count > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has activation count %d", n, n.activation_count))
	}
	if n.activation > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has activation %f", n, n.activation))
	}
	if n.last_activation > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has last_activation %f", n, n.last_activation))
	}
	if n.last_activation2 > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has last_activation2 %f", n, n.last_activation2))
	}

	//if n.ntype == NEURON {
	//	// Flush back check recursively
	//	for _, l := range n.incoming {
	//		err := (*l).InNode().FlushbackCheck()
	//		if err != nil {
	//			return err
	//		}
	//	}
	//
	//}
	return nil
}
func (n *nnode) WriteNode(w *io.Writer) {
	trait_id := 0
	if n.nodetrait != nil {
		trait_id = (*n.nodetrait).TraitId()
	}
	fmt.Fprintf(w, "%d %d %d %d", n.node_id, trait_id, n.ntype, n.gen_node_label)
}
func (n *nnode) Depth(d int32, mynet *Network) int32 {
	cur_depth := 0 //The depth of the current node
	max := d //The max depth

	if d > 100 {
		fmt.Println("** DEPTH NOT DETERMINED FOR NETWORK WITH LOOP")
		return 10;
	}
	// Base Case
	if n.ntype == SENSOR {
		return d
	} else {
		// recursion
		for _, l := range n.incoming {
			cur_depth = (*l).InNode().Depth(d + 1, mynet)
			if cur_depth > max {
				max = cur_depth
			}
		}
		return max
	}

}

func (n nnode) String() string {
	if n.ntype == SENSOR {
		return fmt.Sprintf("(S %d, step %d : %f)", n.node_id, n.activation_count, n.activation)
	} else {
		return fmt.Sprintf("(N %d, step %d : %f)", n.node_id, n.activation_count, n.activation)
	}
}

<|endoftext|>


<commit_before>package alidns

import (
	"fmt"
	"os"
	"strings"

	"github.com/Sirupsen/logrus"
	api "github.com/denverdino/aliyungo/dns"
	"github.com/rancher/external-dns/providers"
	"github.com/rancher/external-dns/utils"
)

type AlidnsProvider struct {
	client         *api.Client
	rootDomainName string
}

func init() {
	providers.RegisterProvider("alidns", &AlidnsProvider{})
}

func (a *AlidnsProvider) Init(rootDomainName string) error {
	accessKey := os.Getenv("ALICLOUD_ACCESS_KEY_ID")
	if len(accessKey) == 0 {
		return fmt.Errorf("ALICLOUD_ACCESS_KEY_ID is not set")
	}

	secretKey := os.Getenv("ALICLOUD_ACCESS_KEY_SECRET")
	if len(secretKey) == 0 {
		return fmt.Errorf("ALICLOUD_ACCESS_KEY_SECRET is not set")
	}

	a.client = api.NewClient(accessKey, secretKey)
	a.rootDomainName = utils.UnFqdn(rootDomainName)

	if _, err := a.client.DescribeDomainInfo(&api.DescribeDomainInfoArgs{
		DomainName: a.rootDomainName,
	}); err != nil {
		return fmt.Errorf("Failed to describe root domain name for '%s': %v", a.rootDomainName, err)
	}

	logrus.Infof("Configured %s with zone '%s'", a.GetName(), a.rootDomainName)
	return nil
}

func (a *AlidnsProvider) GetName() string {
	return "AliDNS"
}

func (a *AlidnsProvider) HealthCheck() error {
	_, err := a.client.DescribeDomainInfo(&api.DescribeDomainInfoArgs{
		DomainName: a.rootDomainName,
	})
	return err
}

func (a *AlidnsProvider) AddRecord(record utils.DnsRecord) error {
	for _, rec := range record.Records {
		r := a.prepareRecord(record, rec)
		if _, err := a.client.AddDomainRecord(r); err != nil {
			return fmt.Errorf("Alibaba Cloud API call has failed: %v", err)
		}
	}

	return nil
}

func (a *AlidnsProvider) UpdateRecord(record utils.DnsRecord) error {
	if err := a.RemoveRecord(record); err != nil {
		return err
	}

	return a.AddRecord(record)
}

func (a *AlidnsProvider) RemoveRecord(record utils.DnsRecord) error {
	records, err := a.findRecords(record)
	if err != nil {
		return err
	}

	for _, rec := range records {
		if _, err := a.client.DeleteDomainRecord(&api.DeleteDomainRecordArgs{
			RecordId: rec.RecordId,
		}); err != nil {
			return fmt.Errorf("Alibaba Cloud API call has failed: %v", err)
		}
	}

	return nil
}

func (a *AlidnsProvider) GetRecords() ([]utils.DnsRecord, error) {
	var records []utils.DnsRecord
	result, err := a.client.DescribeDomainRecords(&api.DescribeDomainRecordsArgs{
		DomainName: a.rootDomainName,
	})
	if err != nil {
		return records, fmt.Errorf("Alibaba Cloud API call has failed: %v", err)
	}

	recordMap := map[string]map[string][]string{}
	recordTTLs := map[string]map[string]int{}

	for _, rec := range result.DomainRecords.Record {
		var fqdn string
		if rec.RR == "" {
			fqdn = a.rootDomainName + "."
		} else {
			fqdn = fmt.Sprintf("%s.%s.", rec.RR, a.rootDomainName)
		}

		recordTTLs[fqdn] = map[string]int{}
		recordTTLs[fqdn][rec.Type] = int(rec.TTL)
		recordSet, exists := recordMap[fqdn]
		if exists {
			recordSlice, sliceExists := recordSet[rec.Type]
			if sliceExists {
				recordSlice = append(recordSlice, rec.Value)
				recordSet[rec.Type] = recordSlice
			} else {
				recordSet[rec.Type] = []string{rec.Value}
			}
		} else {
			recordMap[fqdn] = map[string][]string{}
			recordMap[fqdn][rec.Type] = []string{rec.Value}
		}
	}

	for fqdn, recordSet := range recordMap {
		for recordType, recordSlice := range recordSet {
			ttl := recordTTLs[fqdn][recordType]
			record := utils.DnsRecord{Fqdn: fqdn, Records: recordSlice, Type: recordType, TTL: ttl}
			records = append(records, record)
		}
	}

	return records, nil
}

func (a *AlidnsProvider) parseName(record utils.DnsRecord) string {
	return strings.TrimSuffix(record.Fqdn, fmt.Sprintf(".%s.", a.rootDomainName))
}

func (a *AlidnsProvider) prepareRecord(record utils.DnsRecord, rec string) *api.AddDomainRecordArgs {
	return &api.AddDomainRecordArgs{
		DomainName: a.rootDomainName,
		RR:         a.parseName(record),
		Type:       record.Type,
		Value:      rec,
		TTL:        int32(record.TTL),
	}
}

func (a *AlidnsProvider) findRecords(record utils.DnsRecord) ([]api.RecordType, error) {
	var records []api.RecordType
	result, err := a.client.DescribeDomainRecords(&api.DescribeDomainRecordsArgs{
		DomainName: a.rootDomainName,
	})
	if err != nil {
		return records, fmt.Errorf("Alibaba Cloud API call has failed: %v", err)
	}

	name := a.parseName(record)
	for _, rec := range result.DomainRecords.Record {
		if rec.RR == name && rec.Type == record.Type {
			records = append(records, rec)
		}
	}

	return records, nil
}
<commit_msg>use new Aliyun DNS query API<commit_after>package alidns

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	api "github.com/denverdino/aliyungo/dns"
	"github.com/rancher/external-dns/providers"
	"github.com/rancher/external-dns/utils"
)

type AlidnsProvider struct {
	client         *api.Client
	rootDomainName string
}

func init() {
	providers.RegisterProvider("alidns", &AlidnsProvider{})
}

func (a *AlidnsProvider) Init(rootDomainName string) error {
	accessKey := os.Getenv("ALICLOUD_ACCESS_KEY_ID")
	if len(accessKey) == 0 {
		return fmt.Errorf("ALICLOUD_ACCESS_KEY_ID is not set")
	}

	secretKey := os.Getenv("ALICLOUD_ACCESS_KEY_SECRET")
	if len(secretKey) == 0 {
		return fmt.Errorf("ALICLOUD_ACCESS_KEY_SECRET is not set")
	}

	a.client = api.NewClient(accessKey, secretKey)
	a.rootDomainName = utils.UnFqdn(rootDomainName)

	if _, err := a.client.DescribeDomainInfo(&api.DescribeDomainInfoArgs{
		DomainName: a.rootDomainName,
	}); err != nil {
		return fmt.Errorf("Failed to describe root domain name for '%s': %v", a.rootDomainName, err)
	}

	logrus.Infof("Configured %s with zone '%s'", a.GetName(), a.rootDomainName)
	return nil
}

func (a *AlidnsProvider) GetName() string {
	return "AliDNS"
}

func (a *AlidnsProvider) HealthCheck() error {
	_, err := a.client.DescribeDomainInfo(&api.DescribeDomainInfoArgs{
		DomainName: a.rootDomainName,
	})
	return err
}

func (a *AlidnsProvider) AddRecord(record utils.DnsRecord) error {
	for _, rec := range record.Records {
		r := a.prepareRecord(record, rec)
		if _, err := a.client.AddDomainRecord(r); err != nil {
			return fmt.Errorf("Alibaba Cloud API call has failed: %v", err)
		}
	}

	return nil
}

func (a *AlidnsProvider) UpdateRecord(record utils.DnsRecord) error {
	if err := a.RemoveRecord(record); err != nil {
		return err
	}

	return a.AddRecord(record)
}

func (a *AlidnsProvider) RemoveRecord(record utils.DnsRecord) error {
	records, err := a.findRecords(record)
	if err != nil {
		return err
	}

	for _, rec := range records {
		if _, err := a.client.DeleteDomainRecord(&api.DeleteDomainRecordArgs{
			RecordId: rec.RecordId,
		}); err != nil {
			return fmt.Errorf("Alibaba Cloud API call has failed: %v", err)
		}
	}

	return nil
}

func (a *AlidnsProvider) GetRecords() ([]utils.DnsRecord, error) {
	var records []utils.DnsRecord
	result, err := a.client.DescribeDomainRecordsNew(&api.DescribeDomainRecordsNewArgs{
		DomainName: a.rootDomainName,
	})
	if err != nil {
		return records, fmt.Errorf("Alibaba Cloud API call has failed: %v", err)
	}

	recordMap := map[string]map[string][]string{}
	recordTTLs := map[string]map[string]int{}

	for _, rec := range result.DomainRecords.Record {
		var fqdn string
		if rec.RR == "" {
			fqdn = a.rootDomainName + "."
		} else {
			fqdn = fmt.Sprintf("%s.%s.", rec.RR, a.rootDomainName)
		}

		recordTTLs[fqdn] = map[string]int{}
		if recordTTLs[fqdn][rec.Type], err = strconv.Atoi(rec.TTL); err != nil {
			return records, fmt.Errorf("Failed to convert TTL from '%s' to int: %v", rec.TTL, err)
		}

		recordSet, exists := recordMap[fqdn]
		if exists {
			recordSlice, sliceExists := recordSet[rec.Type]
			if sliceExists {
				recordSlice = append(recordSlice, rec.Value)
				recordSet[rec.Type] = recordSlice
			} else {
				recordSet[rec.Type] = []string{rec.Value}
			}
		} else {
			recordMap[fqdn] = map[string][]string{}
			recordMap[fqdn][rec.Type] = []string{rec.Value}
		}
	}

	for fqdn, recordSet := range recordMap {
		for recordType, recordSlice := range recordSet {
			ttl := recordTTLs[fqdn][recordType]
			record := utils.DnsRecord{Fqdn: fqdn, Records: recordSlice, Type: recordType, TTL: ttl}
			records = append(records, record)
		}
	}

	return records, nil
}

func (a *AlidnsProvider) parseName(record utils.DnsRecord) string {
	return strings.TrimSuffix(record.Fqdn, fmt.Sprintf(".%s.", a.rootDomainName))
}

func (a *AlidnsProvider) prepareRecord(record utils.DnsRecord, rec string) *api.AddDomainRecordArgs {
	return &api.AddDomainRecordArgs{
		DomainName: a.rootDomainName,
		RR:         a.parseName(record),
		Type:       record.Type,
		Value:      rec,
		TTL:        int32(record.TTL),
	}
}

func (a *AlidnsProvider) findRecords(record utils.DnsRecord) ([]api.RecordTypeNew, error) {
	var records []api.RecordTypeNew
	result, err := a.client.DescribeDomainRecordsNew(&api.DescribeDomainRecordsNewArgs{
		DomainName: a.rootDomainName,
	})
	if err != nil {
		return records, fmt.Errorf("Alibaba Cloud API call has failed: %v", err)
	}

	name := a.parseName(record)
	for _, rec := range result.DomainRecords.Record {
		if rec.RR == name && rec.Type == record.Type {
			records = append(records, rec)
		}
	}

	return records, nil
}
<|endoftext|>


<commit_before>
<commit_msg>Delete aci.go<commit_after><|endoftext|>


<commit_before>package tests

import (
	"fmt"
	"strings"
	"testing"
	"time"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/reporters"
	. "github.com/onsi/gomega"
	"github.com/portworx/torpedo/drivers/scheduler"
	. "github.com/portworx/torpedo/tests"
)

var (
	defaultCoolDownPeriod      = int64(60)
	pxVolumeUsagePercent       = "100 * (px_volume_usage_bytes / px_volume_capacity_bytes)"
	pxVolumeCapacityPercent    = "px_volume_capacity_bytes / 1000000000"
	labelSelectorOpGt          = "Gt"
	labelSelectorOpLt          = "Lt"
	specActionName             = "openstorage.io.action.volume/resize"
	ruleActionsScalePercentage = "scalepercentage"
	ruleActionsMaxSize         = "maxsize"
)

var (
	testNameSuite            = "AutopilotVolumeResize"
	timeout                  = 30 * time.Minute
	scaleTimeout             = 2 * time.Hour
	workloadTimeout          = 2 * time.Hour
	retryInterval            = 30 * time.Second
	unscheduledResizeTimeout = 10 * time.Minute
)

var ruleResizeBy50IfPvcUsageMoreThan50 = scheduler.AutopilotRuleParameters{
	// Resize PVC by 50% until volume usage is more than 50Gb
	ActionsCoolDownPeriod: defaultCoolDownPeriod,
	RuleConditionExpressions: []scheduler.AutopilotRuleConditionExpressions{
		{
			Key:      pxVolumeUsagePercent,
			Operator: labelSelectorOpGt,
			Values:   []string{"50"},
		},
	},
	RuleActions: []scheduler.AutopilotRuleActions{
		{
			Name: specActionName,
			Params: map[string]string{
				ruleActionsScalePercentage: "50",
			},
		},
	},
	ExpectedPVCSize: 27179089920,
}

var ruleResizeBy50IfPvcCapacityLessThan10 = scheduler.AutopilotRuleParameters{
	// Resize PVC by 50% until volume is less than 10Gb
	ActionsCoolDownPeriod: defaultCoolDownPeriod,
	RuleConditionExpressions: []scheduler.AutopilotRuleConditionExpressions{
		{
			Key:      pxVolumeCapacityPercent,
			Operator: labelSelectorOpLt,
			Values:   []string{"10"},
		},
	},
	RuleActions: []scheduler.AutopilotRuleActions{
		{
			Name: specActionName,
			Params: map[string]string{
				ruleActionsScalePercentage: "50",
			},
		},
	},
	ExpectedPVCSize: 12079595520,
}

var ruleResizeBy50UntilPvcMaxSize20 = scheduler.AutopilotRuleParameters{
	// Resize PVC by 50% until volume size is 20Gb
	ActionsCoolDownPeriod: defaultCoolDownPeriod,
	RuleConditionExpressions: []scheduler.AutopilotRuleConditionExpressions{
		{
			Key:      pxVolumeUsagePercent,
			Operator: labelSelectorOpGt,
			Values:   []string{"50"},
		},
	},
	RuleActions: []scheduler.AutopilotRuleActions{
		{
			Name: specActionName,
			Params: map[string]string{
				ruleActionsScalePercentage: "50",
				ruleActionsMaxSize:         "20Gi",
			},
		},
	},
	ExpectedPVCSize: 21474836480,
}

var autopilotruleBasicTestCases = []scheduler.AutopilotRuleParameters{
	ruleResizeBy50IfPvcUsageMoreThan50,
	ruleResizeBy50UntilPvcMaxSize20,
}

var autopilotruleScaleTestCases = []scheduler.AutopilotRuleParameters{
	ruleResizeBy50IfPvcCapacityLessThan10,
}

func TestAutoPilot(t *testing.T) {
	RegisterFailHandler(Fail)

	var specReporters []Reporter
	junitReporter := reporters.NewJUnitReporter("/testresults/junit_autopilot.xml")
	specReporters = append(specReporters, junitReporter)
	RunSpecsWithDefaultAndCustomReporters(t, "Torpedo : Autopilot", specReporters)
}

var _ = BeforeSuite(func() {
	InitInstance()
})

// This testsuite is used for performing basic scenarios with Autopilot rules where it
// schedules apps and wait until workload is completed on the volumes and then validates
// PVC sizes of the volumes
var _ = Describe(fmt.Sprintf("{%sWaitForWorkload}", testNameSuite), func() {
	It("has to fill up the volume completely, resize the volume, validate and teardown apps", func() {
		var contexts []*scheduler.Context
		var err error
		testName := strings.ToLower(fmt.Sprintf("%sWaitForWorkload", testNameSuite))

		for _, apRule := range autopilotruleBasicTestCases {
			apParameters := &scheduler.AutopilotParameters{
				Enabled:                 true,
				Name:                    testName,
				AutopilotRuleParameters: apRule,
			}

			Step("schedule applications", func() {
				for i := 0; i < Inst().ScaleFactor; i++ {
					taskName := fmt.Sprintf("%s-%v", fmt.Sprintf("%s-%d", testName, i), Inst().InstanceID)
					context, err := Inst().S.Schedule(taskName, scheduler.ScheduleOptions{
						AppKeys:             Inst().AppList,
						StorageProvisioner:  Inst().Provisioner,
						AutopilotParameters: apParameters,
					})
					Expect(err).NotTo(HaveOccurred())
					Expect(context).NotTo(BeEmpty())
					contexts = append(contexts, context...)
				}
			})

			Step("wait until workload completes on volume", func() {
				for _, ctx := range contexts {
					err = Inst().S.WaitForRunning(ctx, workloadTimeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, timeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step(fmt.Sprintf("wait for unscheduled resize of volume (%s)", unscheduledResizeTimeout), func() {
				time.Sleep(unscheduledResizeTimeout)
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, timeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step("destroy apps", func() {
				opts := make(map[string]bool)
				opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
				for _, ctx := range contexts {
					TearDownContext(ctx, opts)
				}
			})

			contexts = nil
		}
	})
})

// This testsuite is used for performing basic scenarios with Autopilot rules where it
// schedules apps and doesn't wait until workload is completed on the volumes and then
// validates PVC sizes of the volumes
var _ = Describe(fmt.Sprintf("{%sDoesNotWaitForWorkload}", testNameSuite), func() {
	It("will resize the volume until the max size of the volume", func() {
		var contexts []*scheduler.Context
		var err error
		testName := strings.ToLower(fmt.Sprintf("%sDoesNotWaitForWorkload", testNameSuite))

		for _, apRule := range autopilotruleScaleTestCases {
			apParameters := &scheduler.AutopilotParameters{
				Enabled:                 true,
				Name:                    testName,
				AutopilotRuleParameters: apRule,
			}

			Step("schedule applications", func() {
				for i := 0; i < Inst().ScaleFactor; i++ {
					taskName := fmt.Sprintf("%s-%v", fmt.Sprintf("%s-%d", testName, i), Inst().InstanceID)
					context, err := Inst().S.Schedule(taskName, scheduler.ScheduleOptions{
						AppKeys:             Inst().AppList,
						StorageProvisioner:  Inst().Provisioner,
						AutopilotParameters: apParameters,
					})
					Expect(err).NotTo(HaveOccurred())
					Expect(context).NotTo(BeEmpty())
					contexts = append(contexts, context...)
				}
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, scaleTimeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step(fmt.Sprintf("wait for unscheduled resize of volume (%s)", unscheduledResizeTimeout), func() {
				time.Sleep(unscheduledResizeTimeout)
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, scaleTimeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step("destroy apps", func() {
				opts := make(map[string]bool)
				opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
				for _, ctx := range contexts {
					TearDownContext(ctx, opts)
				}
			})
		}
	})
})

var _ = AfterSuite(func() {
	PerformSystemCheck()
	ValidateCleanup()
})

func init() {
	ParseFlags()
}
<commit_msg>PTX-1932 Added VolumeDriverDown test during resize<commit_after>package tests

import (
	"fmt"
	"strings"
	"testing"
	"time"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/reporters"
	. "github.com/onsi/gomega"
	"github.com/portworx/torpedo/drivers/node"
	"github.com/portworx/torpedo/drivers/scheduler"
	. "github.com/portworx/torpedo/tests"
)

var (
	defaultCoolDownPeriod      = int64(60)
	pxVolumeUsagePercent       = "100 * (px_volume_usage_bytes / px_volume_capacity_bytes)"
	pxVolumeCapacityPercent    = "px_volume_capacity_bytes / 1000000000"
	labelSelectorOpGt          = "Gt"
	labelSelectorOpLt          = "Lt"
	specActionName             = "openstorage.io.action.volume/resize"
	ruleActionsScalePercentage = "scalepercentage"
	ruleActionsMaxSize         = "maxsize"
)

var (
	testNameSuite            = "AutopilotVolumeResize"
	timeout                  = 30 * time.Minute
	scaleTimeout             = 2 * time.Hour
	workloadTimeout          = 2 * time.Hour
	retryInterval            = 30 * time.Second
	unscheduledResizeTimeout = 10 * time.Minute
)

var ruleResizeBy50IfPvcUsageMoreThan50 = scheduler.AutopilotRuleParameters{
	// Resize PVC by 50% until volume usage is more than 50Gb
	ActionsCoolDownPeriod: defaultCoolDownPeriod,
	RuleConditionExpressions: []scheduler.AutopilotRuleConditionExpressions{
		{
			Key:      pxVolumeUsagePercent,
			Operator: labelSelectorOpGt,
			Values:   []string{"50"},
		},
	},
	RuleActions: []scheduler.AutopilotRuleActions{
		{
			Name: specActionName,
			Params: map[string]string{
				ruleActionsScalePercentage: "50",
			},
		},
	},
	ExpectedPVCSize: 27179089920,
}

var ruleResizeBy50IfPvcCapacityLessThan10 = scheduler.AutopilotRuleParameters{
	// Resize PVC by 50% until volume is less than 10Gb
	ActionsCoolDownPeriod: defaultCoolDownPeriod,
	RuleConditionExpressions: []scheduler.AutopilotRuleConditionExpressions{
		{
			Key:      pxVolumeCapacityPercent,
			Operator: labelSelectorOpLt,
			Values:   []string{"10"},
		},
	},
	RuleActions: []scheduler.AutopilotRuleActions{
		{
			Name: specActionName,
			Params: map[string]string{
				ruleActionsScalePercentage: "50",
			},
		},
	},
	ExpectedPVCSize: 12079595520,
}

var ruleResizeBy50UntilPvcMaxSize20 = scheduler.AutopilotRuleParameters{
	// Resize PVC by 50% until volume size is 20Gb
	ActionsCoolDownPeriod: defaultCoolDownPeriod,
	RuleConditionExpressions: []scheduler.AutopilotRuleConditionExpressions{
		{
			Key:      pxVolumeUsagePercent,
			Operator: labelSelectorOpGt,
			Values:   []string{"50"},
		},
	},
	RuleActions: []scheduler.AutopilotRuleActions{
		{
			Name: specActionName,
			Params: map[string]string{
				ruleActionsScalePercentage: "50",
				ruleActionsMaxSize:         "20Gi",
			},
		},
	},
	ExpectedPVCSize: 21474836480,
}

var autopilotruleBasicTestCases = []scheduler.AutopilotRuleParameters{
	ruleResizeBy50IfPvcUsageMoreThan50,
	ruleResizeBy50UntilPvcMaxSize20,
}

var autopilotruleScaleTestCases = []scheduler.AutopilotRuleParameters{
	ruleResizeBy50IfPvcCapacityLessThan10,
}

func TestAutoPilot(t *testing.T) {
	RegisterFailHandler(Fail)

	var specReporters []Reporter
	junitReporter := reporters.NewJUnitReporter("/testresults/junit_autopilot.xml")
	specReporters = append(specReporters, junitReporter)
	RunSpecsWithDefaultAndCustomReporters(t, "Torpedo : Autopilot", specReporters)
}

var _ = BeforeSuite(func() {
	InitInstance()
})

// This testsuite is used for performing basic scenarios with Autopilot rules where it
// schedules apps and wait until workload is completed on the volumes and then validates
// PVC sizes of the volumes
var _ = Describe(fmt.Sprintf("{%sWaitForWorkload}", testNameSuite), func() {
	It("has to fill up the volume completely, resize the volume, validate and teardown apps", func() {
		var contexts []*scheduler.Context
		var err error
		testName := strings.ToLower(fmt.Sprintf("%sWaitForWorkload", testNameSuite))

		for _, apRule := range autopilotruleBasicTestCases {
			apParameters := &scheduler.AutopilotParameters{
				Enabled:                 true,
				Name:                    testName,
				AutopilotRuleParameters: apRule,
			}

			Step("schedule applications", func() {
				for i := 0; i < Inst().ScaleFactor; i++ {
					taskName := fmt.Sprintf("%s-%v", fmt.Sprintf("%s-%d", testName, i), Inst().InstanceID)
					context, err := Inst().S.Schedule(taskName, scheduler.ScheduleOptions{
						AppKeys:             Inst().AppList,
						StorageProvisioner:  Inst().Provisioner,
						AutopilotParameters: apParameters,
					})
					Expect(err).NotTo(HaveOccurred())
					Expect(context).NotTo(BeEmpty())
					contexts = append(contexts, context...)
				}
			})

			Step("wait until workload completes on volume", func() {
				for _, ctx := range contexts {
					err = Inst().S.WaitForRunning(ctx, workloadTimeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, timeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step(fmt.Sprintf("wait for unscheduled resize of volume (%s)", unscheduledResizeTimeout), func() {
				time.Sleep(unscheduledResizeTimeout)
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, timeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step("destroy apps", func() {
				opts := make(map[string]bool)
				opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
				for _, ctx := range contexts {
					TearDownContext(ctx, opts)
				}
			})

			contexts = nil
		}
	})
})

// This testsuite is used for performing basic scenarios with Autopilot rules where it
// schedules apps and doesn't wait until workload is completed on the volumes and then
// validates PVC sizes of the volumes
var _ = Describe(fmt.Sprintf("{%sDoesNotWaitForWorkload}", testNameSuite), func() {
	It("will resize the volume until the max size of the volume", func() {
		var contexts []*scheduler.Context
		var err error
		testName := strings.ToLower(fmt.Sprintf("%sDoesNotWaitForWorkload", testNameSuite))

		for _, apRule := range autopilotruleScaleTestCases {
			apParameters := &scheduler.AutopilotParameters{
				Enabled:                 true,
				Name:                    testName,
				AutopilotRuleParameters: apRule,
			}

			Step("schedule applications", func() {
				for i := 0; i < Inst().ScaleFactor; i++ {
					taskName := fmt.Sprintf("%s-%v", fmt.Sprintf("%s-%d", testName, i), Inst().InstanceID)
					context, err := Inst().S.Schedule(taskName, scheduler.ScheduleOptions{
						AppKeys:             Inst().AppList,
						StorageProvisioner:  Inst().Provisioner,
						AutopilotParameters: apParameters,
					})
					Expect(err).NotTo(HaveOccurred())
					Expect(context).NotTo(BeEmpty())
					contexts = append(contexts, context...)
				}
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, scaleTimeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step(fmt.Sprintf("wait for unscheduled resize of volume (%s)", unscheduledResizeTimeout), func() {
				time.Sleep(unscheduledResizeTimeout)
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, scaleTimeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step("destroy apps", func() {
				opts := make(map[string]bool)
				opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
				for _, ctx := range contexts {
					TearDownContext(ctx, opts)
				}
			})
		}
	})
})

// This testsuite is used for performing basic scenarios with Autopilot rules where it
// schedules apps and wait until workload is completed on the volumes. Restarts volume
// driver and validates PVC sizes of the volumes
var _ = Describe(fmt.Sprintf("{%sVolumeDriverDown}", testNameSuite), func() {
	It("has to fill up the volume completely, resize the volume, validate and teardown apps", func() {
		var contexts []*scheduler.Context
		var err error
		testName := strings.ToLower(fmt.Sprintf("%sVolumeDriverDown", testNameSuite))

		for _, apRule := range autopilotruleBasicTestCases {
			apParameters := &scheduler.AutopilotParameters{
				Enabled:                 true,
				Name:                    testName,
				AutopilotRuleParameters: apRule,
			}

			Step("schedule applications", func() {
				for i := 0; i < Inst().ScaleFactor; i++ {
					taskName := fmt.Sprintf("%s-%v", fmt.Sprintf("%s-%d", testName, i), Inst().InstanceID)
					context, err := Inst().S.Schedule(taskName, scheduler.ScheduleOptions{
						AppKeys:             Inst().AppList,
						StorageProvisioner:  Inst().Provisioner,
						AutopilotParameters: apParameters,
					})
					Expect(err).NotTo(HaveOccurred())
					Expect(context).NotTo(BeEmpty())
					contexts = append(contexts, context...)
				}
			})

			Step("wait until workload completes on volume", func() {
				for _, ctx := range contexts {
					err = Inst().S.WaitForRunning(ctx, workloadTimeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step("get nodes bounce volume driver", func() {
				for _, appNode := range node.GetStorageDriverNodes() {
					Step(
						fmt.Sprintf("stop volume driver %s on node: %s",
							Inst().V.String(), appNode.Name),
						func() {
							StopVolDriverAndWait([]node.Node{appNode})
						})

					Step(
						fmt.Sprintf("starting volume %s driver on node %s",
							Inst().V.String(), appNode.Name),
						func() {
							StartVolDriverAndWait([]node.Node{appNode})
						})

					Step("Giving few seconds for volume driver to stabilize", func() {
						time.Sleep(20 * time.Second)
					})
				}
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, timeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step(fmt.Sprintf("wait for unscheduled resize of volume (%s)", unscheduledResizeTimeout), func() {
				time.Sleep(unscheduledResizeTimeout)
			})

			Step("validating volumes and verifying size of volumes", func() {
				for _, ctx := range contexts {
					err = Inst().S.InspectVolumes(ctx, timeout, retryInterval)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			Step("destroy apps", func() {
				opts := make(map[string]bool)
				opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
				for _, ctx := range contexts {
					TearDownContext(ctx, opts)
				}
			})
		}
	})
})

var _ = AfterSuite(func() {
	PerformSystemCheck()
	ValidateCleanup()
})

func init() {
	ParseFlags()
}
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"encoding/hex"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/pingcap/errors"
	pb "github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/tidb/store/tikv/client"
	tikverr "github.com/pingcap/tidb/store/tikv/error"
	"github.com/pingcap/tidb/store/tikv/logutil"
	"github.com/pingcap/tidb/store/tikv/metrics"
	"github.com/pingcap/tidb/store/tikv/retry"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/prometheus/client_golang/prometheus"
	"go.uber.org/zap"
)

type actionCommit struct{ retry bool }

var _ twoPhaseCommitAction = actionCommit{}

func (actionCommit) String() string {
	return "commit"
}

func (actionCommit) tiKVTxnRegionsNumHistogram() prometheus.Observer {
	return metrics.TxnRegionsNumHistogramCommit
}

func (actionCommit) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) (err error) {
	keys := batch.mutations.GetKeys()
	req := tikvrpc.NewRequest(tikvrpc.CmdCommit, &pb.CommitRequest{
		StartVersion:  c.startTS,
		Keys:          keys,
		CommitVersion: c.commitTS,
	}, pb.Context{Priority: c.priority, SyncLog: c.syncLog, ResourceGroupTag: c.resourceGroupTag})

	tBegin := time.Now()
	attempts := 0

	sender := NewRegionRequestSender(c.store.regionCache, c.store.GetTiKVClient())
	defer func() {
		if err != nil {
			// If we fail to receive response for the request that commits primary key, it will be undetermined whether this
			// transaction has been successfully committed.
			// Under this circumstance, we can not declare the commit is complete (may lead to data lost), nor can we throw
			// an error (may lead to the duplicated key error when upper level restarts the transaction). Currently the best
			// solution is to populate this error and let upper layer drop the connection to the corresponding mysql client.
			if batch.isPrimary && sender.rpcError != nil && !c.isAsyncCommit() {
				c.setUndeterminedErr(errors.Trace(sender.rpcError))
			}
		}
	}()
	for {
		attempts++
		if time.Since(tBegin) > slowRequestThreshold {
			logutil.BgLogger().Warn("slow commit request", zap.Uint64("startTS", c.startTS), zap.Stringer("region", &batch.region), zap.Int("attempts", attempts))
			tBegin = time.Now()
		}

		resp, err := sender.SendReq(bo, req, batch.region, client.ReadTimeoutShort)
		// Unexpected error occurs, return it.
		if err != nil {
			return errors.Trace(err)
		}

		regionErr, err := resp.GetRegionError()
		if err != nil {
			return errors.Trace(err)
		}
		if regionErr != nil {
			// For other region error and the fake region error, backoff because
			// there's something wrong.
			// For the real EpochNotMatch error, don't backoff.
			if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) {
				err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String()))
				if err != nil {
					return errors.Trace(err)
				}
			}
			same, err := batch.relocate(bo, c.store.regionCache)
			if err != nil {
				return errors.Trace(err)
			}
			if same {
				continue
			}
			err = c.doActionOnMutations(bo, actionCommit{true}, batch.mutations)
			return errors.Trace(err)
		}

		if resp.Resp == nil {
			return errors.Trace(tikverr.ErrBodyMissing)
		}
		commitResp := resp.Resp.(*pb.CommitResponse)
		if keyErr := commitResp.GetError(); keyErr != nil {
			if rejected := keyErr.GetCommitTsExpired(); rejected != nil {
				logutil.Logger(bo.GetCtx()).Info("2PC commitTS rejected by TiKV, retry with a newer commitTS",
					zap.Uint64("txnStartTS", c.startTS),
					zap.Stringer("info", logutil.Hex(rejected)))

				// Do not retry for a txn which has a too large MinCommitTs
				// 3600000 << 18 = 943718400000
				if rejected.MinCommitTs-rejected.AttemptedCommitTs > 943718400000 {
					err := errors.Errorf("2PC MinCommitTS is too large, we got MinCommitTS: %d, and AttemptedCommitTS: %d",
						rejected.MinCommitTs, rejected.AttemptedCommitTs)
					return errors.Trace(err)
				}

				// Update commit ts and retry.
				commitTS, err := c.store.getTimestampWithRetry(bo, c.txn.GetScope())
				if err != nil {
					logutil.Logger(bo.GetCtx()).Warn("2PC get commitTS failed",
						zap.Error(err),
						zap.Uint64("txnStartTS", c.startTS))
					return errors.Trace(err)
				}

				c.mu.Lock()
				c.commitTS = commitTS
				c.mu.Unlock()
				// Update the commitTS of the request and retry.
				req.Commit().CommitVersion = commitTS
				continue
			}

			c.mu.RLock()
			defer c.mu.RUnlock()
			err = extractKeyErr(keyErr)
			if c.mu.committed {
				// No secondary key could be rolled back after it's primary key is committed.
				// There must be a serious bug somewhere.
				hexBatchKeys := func(keys [][]byte) []string {
					var res []string
					for _, k := range keys {
						res = append(res, hex.EncodeToString(k))
					}
					return res
				}
				logutil.Logger(bo.GetCtx()).Error("2PC failed commit key after primary key committed",
					zap.Error(err),
					zap.Uint64("txnStartTS", c.startTS),
					zap.Uint64("commitTS", c.commitTS),
					zap.Strings("keys", hexBatchKeys(keys)))
				return errors.Trace(err)
			}
			// The transaction maybe rolled back by concurrent transactions.
			logutil.Logger(bo.GetCtx()).Debug("2PC failed commit primary key",
				zap.Error(err),
				zap.Uint64("txnStartTS", c.startTS))
			return err
		}
		break
	}

	c.mu.Lock()
	defer c.mu.Unlock()
	// Group that contains primary key is always the first.
	// We mark transaction's status committed when we receive the first success response.
	c.mu.committed = true
	return nil
}

func (c *twoPhaseCommitter) commitMutations(bo *Backoffer, mutations CommitterMutations) error {
	if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("twoPhaseCommitter.commitMutations", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1))
	}

	return c.doActionOnMutations(bo, actionCommit{}, mutations)
}
<commit_msg>store/tikv: better handle undetermined error for committing primary key (#25115)<commit_after>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"encoding/hex"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/pingcap/errors"
	pb "github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/tidb/store/tikv/client"
	tikverr "github.com/pingcap/tidb/store/tikv/error"
	"github.com/pingcap/tidb/store/tikv/logutil"
	"github.com/pingcap/tidb/store/tikv/metrics"
	"github.com/pingcap/tidb/store/tikv/retry"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/prometheus/client_golang/prometheus"
	"go.uber.org/zap"
)

type actionCommit struct{ retry bool }

var _ twoPhaseCommitAction = actionCommit{}

func (actionCommit) String() string {
	return "commit"
}

func (actionCommit) tiKVTxnRegionsNumHistogram() prometheus.Observer {
	return metrics.TxnRegionsNumHistogramCommit
}

func (actionCommit) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) error {
	keys := batch.mutations.GetKeys()
	req := tikvrpc.NewRequest(tikvrpc.CmdCommit, &pb.CommitRequest{
		StartVersion:  c.startTS,
		Keys:          keys,
		CommitVersion: c.commitTS,
	}, pb.Context{Priority: c.priority, SyncLog: c.syncLog, ResourceGroupTag: c.resourceGroupTag})

	tBegin := time.Now()
	attempts := 0

	sender := NewRegionRequestSender(c.store.regionCache, c.store.GetTiKVClient())
	for {
		attempts++
		if time.Since(tBegin) > slowRequestThreshold {
			logutil.BgLogger().Warn("slow commit request", zap.Uint64("startTS", c.startTS), zap.Stringer("region", &batch.region), zap.Int("attempts", attempts))
			tBegin = time.Now()
		}

		resp, err := sender.SendReq(bo, req, batch.region, client.ReadTimeoutShort)
		// If we fail to receive response for the request that commits primary key, it will be undetermined whether this
		// transaction has been successfully committed.
		// Under this circumstance, we can not declare the commit is complete (may lead to data lost), nor can we throw
		// an error (may lead to the duplicated key error when upper level restarts the transaction). Currently the best
		// solution is to populate this error and let upper layer drop the connection to the corresponding mysql client.
		if batch.isPrimary && sender.rpcError != nil && !c.isAsyncCommit() {
			c.setUndeterminedErr(errors.Trace(sender.rpcError))
		}

		// Unexpected error occurs, return it.
		if err != nil {
			return errors.Trace(err)
		}

		regionErr, err := resp.GetRegionError()
		if err != nil {
			return errors.Trace(err)
		}
		if regionErr != nil {
			// For other region error and the fake region error, backoff because
			// there's something wrong.
			// For the real EpochNotMatch error, don't backoff.
			if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) {
				err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String()))
				if err != nil {
					return errors.Trace(err)
				}
			}
			same, err := batch.relocate(bo, c.store.regionCache)
			if err != nil {
				return errors.Trace(err)
			}
			if same {
				continue
			}
			err = c.doActionOnMutations(bo, actionCommit{true}, batch.mutations)
			return errors.Trace(err)
		}

		if resp.Resp == nil {
			return errors.Trace(tikverr.ErrBodyMissing)
		}
		commitResp := resp.Resp.(*pb.CommitResponse)
		// Here we can make sure tikv has processed the commit primary key request. So
		// we can clean undetermined error.
		if batch.isPrimary && !c.isAsyncCommit() {
			c.setUndeterminedErr(nil)
		}
		if keyErr := commitResp.GetError(); keyErr != nil {
			if rejected := keyErr.GetCommitTsExpired(); rejected != nil {
				logutil.Logger(bo.GetCtx()).Info("2PC commitTS rejected by TiKV, retry with a newer commitTS",
					zap.Uint64("txnStartTS", c.startTS),
					zap.Stringer("info", logutil.Hex(rejected)))

				// Do not retry for a txn which has a too large MinCommitTs
				// 3600000 << 18 = 943718400000
				if rejected.MinCommitTs-rejected.AttemptedCommitTs > 943718400000 {
					err := errors.Errorf("2PC MinCommitTS is too large, we got MinCommitTS: %d, and AttemptedCommitTS: %d",
						rejected.MinCommitTs, rejected.AttemptedCommitTs)
					return errors.Trace(err)
				}

				// Update commit ts and retry.
				commitTS, err := c.store.getTimestampWithRetry(bo, c.txn.GetScope())
				if err != nil {
					logutil.Logger(bo.GetCtx()).Warn("2PC get commitTS failed",
						zap.Error(err),
						zap.Uint64("txnStartTS", c.startTS))
					return errors.Trace(err)
				}

				c.mu.Lock()
				c.commitTS = commitTS
				c.mu.Unlock()
				// Update the commitTS of the request and retry.
				req.Commit().CommitVersion = commitTS
				continue
			}

			c.mu.RLock()
			defer c.mu.RUnlock()
			err = extractKeyErr(keyErr)
			if c.mu.committed {
				// No secondary key could be rolled back after it's primary key is committed.
				// There must be a serious bug somewhere.
				hexBatchKeys := func(keys [][]byte) []string {
					var res []string
					for _, k := range keys {
						res = append(res, hex.EncodeToString(k))
					}
					return res
				}
				logutil.Logger(bo.GetCtx()).Error("2PC failed commit key after primary key committed",
					zap.Error(err),
					zap.Uint64("txnStartTS", c.startTS),
					zap.Uint64("commitTS", c.commitTS),
					zap.Strings("keys", hexBatchKeys(keys)))
				return errors.Trace(err)
			}
			// The transaction maybe rolled back by concurrent transactions.
			logutil.Logger(bo.GetCtx()).Debug("2PC failed commit primary key",
				zap.Error(err),
				zap.Uint64("txnStartTS", c.startTS))
			return err
		}
		break
	}

	c.mu.Lock()
	defer c.mu.Unlock()
	// Group that contains primary key is always the first.
	// We mark transaction's status committed when we receive the first success response.
	c.mu.committed = true
	return nil
}

func (c *twoPhaseCommitter) commitMutations(bo *Backoffer, mutations CommitterMutations) error {
	if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("twoPhaseCommitter.commitMutations", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1))
	}

	return c.doActionOnMutations(bo, actionCommit{}, mutations)
}
<|endoftext|>


<commit_before>// robustsession represents a RobustIRC session and handles all communication
// to the RobustIRC network.
package robustsession

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"hash/fnv"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/robustirc/robustirc/types"

	"github.com/sorcix/irc"
)

const (
	pathCreateSession = "/robustirc/v1/session"
	pathDeleteSession = "/robustirc/v1/%s"
	pathPostMessage   = "/robustirc/v1/%s/message"
	pathGetMessages   = "/robustirc/v1/%s/messages?lastseen=%s"
)

var (
	NoSuchSession = errors.New("No such RobustIRC session (killed by the network?)")

	networks   = make(map[string]*network)
	networksMu sync.Mutex
)

type backoffState struct {
	exp  float64
	next time.Time
}

type network struct {
	servers []string
	mu      sync.RWMutex
	backoff map[string]backoffState
}

func newNetwork(networkname string) (*network, error) {
	var servers []string

	parts := strings.Split(networkname, ",")
	if len(parts) > 1 {
		log.Printf("Interpreting %q as list of servers instead of network name\n", networkname)
		servers = parts
	} else {
		// Try to resolve the DNS name up to 5 times. This is to be nice to
		// people in environments with flaky network connections at boot, who,
		// for some reason, don’t run this program under systemd with
		// Restart=on-failure.
		try := 0
		for {
			_, addrs, err := net.LookupSRV("robustirc", "tcp", networkname)
			if err != nil {
				log.Println(err)
				if try < 4 {
					time.Sleep(time.Duration(int64(math.Pow(2, float64(try)))) * time.Second)
				} else {
					return nil, fmt.Errorf("DNS lookup of %q failed 5 times", networkname)
				}
				try++
				continue
			}
			// Randomly shuffle the addresses.
			for i := range addrs {
				j := rand.Intn(i + 1)
				addrs[i], addrs[j] = addrs[j], addrs[i]
			}
			for _, addr := range addrs {
				target := addr.Target
				if target[len(target)-1] == '.' {
					target = target[:len(target)-1]
				}
				servers = append(servers, fmt.Sprintf("%s:%d", target, addr.Port))
			}
			break
		}
	}

	return &network{
		servers: servers,
		backoff: make(map[string]backoffState),
	}, nil
}

// server (eventually) returns the host:port to which we should connect to. In
// case back-off prevents us from connecting anywhere right now, the function
// blocks until back-off is over.
func (n *network) server() string {
	n.mu.RLock()
	defer n.mu.RUnlock()

	for {
		soonest := time.Duration(math.MaxInt64)
		for _, server := range n.servers {
			wait := n.backoff[server].next.Sub(time.Now())
			if wait <= 0 {
				return server
			}
			if wait < soonest {
				soonest = wait
			}
		}

		time.Sleep(soonest)
	}
}

func (n *network) setServers(servers []string) {
	n.mu.Lock()
	defer n.mu.Unlock()

	// TODO(secure): we should clean up n.backoff from servers which no longer exist
	n.servers = servers
}

// prefer adds the specified server to the front of the servers list, thereby
// trying to prefer it over other servers for the next request. Note that
// exponential backoff overrides this, so this is only a hint, not a guarantee.
func (n *network) prefer(server string) {
	n.mu.Lock()
	defer n.mu.Unlock()

	n.servers = append([]string{server}, n.servers...)
}

func (n *network) failed(server string) {
	n.mu.Lock()
	defer n.mu.Unlock()

	b := n.backoff[server]
	b.exp++
	b.next = time.Now().Add(time.Duration(math.Pow(2, b.exp)) * time.Second)
	n.backoff[server] = b
}

func (n *network) succeeded(server string) {
	n.mu.Lock()
	defer n.mu.Unlock()

	delete(n.backoff, server)
}

func discardResponse(resp *http.Response) {
	// We need to read the entire body, otherwise net/http will not
	// re-use this connection.
	ioutil.ReadAll(resp.Body)
	resp.Body.Close()
}

type RobustSession struct {
	IrcPrefix *irc.Prefix
	Messages  chan string
	Errors    chan error

	sessionId   string
	sessionAuth string
	deleted     bool
	done        chan bool
	network     *network
	client      *http.Client
}

func (s *RobustSession) sendRequest(method, path string, data []byte) (string, *http.Response, error) {
	for !s.deleted {
		target := s.network.server()
		req, err := http.NewRequest(method, fmt.Sprintf("https://%s%s", target, path), bytes.NewBuffer(data))
		if err != nil {
			return "", nil, err
		}
		req.Header.Set("X-Session-Auth", s.sessionAuth)
		req.Header.Set("Content-Type", "application/json")
		resp, err := s.client.Do(req)
		if err != nil {
			s.network.failed(target)
			log.Printf("sendRequest(%q) failed: %v\n", path, err)
			continue
		}
		if resp.StatusCode == http.StatusNotFound {
			s.network.failed(target)
			return "", nil, NoSuchSession
		}
		// Server errors, temporary.
		if resp.StatusCode >= 500 && resp.StatusCode < 600 {
			s.network.failed(target)
			log.Printf("sendRequest(%q) failed with %v (retrying)\n", path, resp.Status)
			continue
		}
		// Client errors and anything unexpected.
		if resp.StatusCode != http.StatusOK {
			message, _ := ioutil.ReadAll(resp.Body)
			resp.Body.Close()
			s.network.failed(target)
			log.Printf("sendRequest(%q) failed with %v: %q\n", path, resp.Status, message)
			continue
		}
		return target, resp, nil
	}

	return "", nil, NoSuchSession
}

// Create creates a new RobustIRC session. It resolves the given network name
// (e.g. "robustirc.net") to a set of servers by querying the
// _robustirc._tcp.<network> SRV record and sends the CreateSession request.
//
// When err == nil, the caller MUST read the RobustSession.Messages and
// RobustSession.Errors channels.
func Create(network string, tlsCAFile string) (*RobustSession, error) {
	networksMu.Lock()
	n, ok := networks[network]
	if !ok {
		var err error
		n, err = newNetwork(network)
		if err != nil {
			return nil, err
		}
		networks[network] = n
	}
	networksMu.Unlock()

	var client *http.Client

	if tlsCAFile != "" {
		roots := x509.NewCertPool()
		contents, err := ioutil.ReadFile(tlsCAFile)
		if err != nil {
			log.Fatalf("Could not read cert.pem: %v", err)
		}
		if !roots.AppendCertsFromPEM(contents) {
			log.Fatalf("Could not parse %q", tlsCAFile)
		}

		client = &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{RootCAs: roots},
			},
		}
	} else {
		client = http.DefaultClient
	}

	s := &RobustSession{
		Messages: make(chan string),
		Errors:   make(chan error),
		done:     make(chan bool, 1),
		network:  n,
		client:   client,
	}

	_, resp, err := s.sendRequest("POST", pathCreateSession, nil)
	if err != nil {
		return nil, err
	}
	defer discardResponse(resp)

	var createSessionReply struct {
		Sessionid   string
		Sessionauth string
		Prefix      string
	}

	if err := json.NewDecoder(resp.Body).Decode(&createSessionReply); err != nil {
		return nil, err
	}

	if cl := resp.Header.Get("Content-Location"); cl != "" {
		if location, err := url.Parse(cl); err == nil {
			log.Printf("Preferring %q (current leader)\n", location.Host)
			s.network.prefer(location.Host)
		}
	}

	s.sessionId = createSessionReply.Sessionid
	s.sessionAuth = createSessionReply.Sessionauth
	s.IrcPrefix = &irc.Prefix{Name: createSessionReply.Prefix}

	go s.getMessages()

	return s, nil
}

func (s *RobustSession) getMessages() {
	var lastseen types.RobustId

	for !s.deleted {
		target, resp, err := s.sendRequest("GET", fmt.Sprintf(pathGetMessages, s.sessionId, lastseen.String()), nil)
		if err != nil {
			s.Errors <- err
			return
		}

		msgchan := make(chan types.RobustMessage, 1)
		errchan := make(chan error)
		go func() {
			dec := json.NewDecoder(resp.Body)
			for {
				var msg types.RobustMessage
				if err := dec.Decode(&msg); err != nil {
					errchan <- err
					return
				}
				msgchan <- msg
			}
		}()

	ReadLoop:
		for !s.deleted {
			select {
			case msg := <-msgchan:
				if msg.Type == types.RobustPing {
					s.network.setServers(msg.Servers)
				} else if msg.Type == types.RobustIRCToClient {
					// TODO: remove/debug
					log.Printf("<-robustirc: %q\n", msg.Data)
					s.Messages <- msg.Data
					lastseen = msg.Id
				}

			case err := <-errchan:
				log.Printf("Protocol error on %q: Could not decode response chunk as JSON: %v\n", target, err)
				s.network.failed(target)
				break ReadLoop

			case <-time.After(1 * time.Minute):
				log.Printf("Timeout (60s) on GetMessages, reconnecting…\n")
				s.network.failed(target)
				break ReadLoop

			case <-s.done:
				break ReadLoop
			}
		}
		resp.Body.Close()

		// Delay reconnecting for somewhere in between [250, 500) ms to avoid
		// overloading the remaining servers from many clients at once when one
		// server fails.
		time.Sleep(time.Duration(250+rand.Int63n(250)) * time.Millisecond)
	}
}

// PostMessage posts the given IRC message.
func (s *RobustSession) PostMessage(message string) error {
	type postMessageRequest struct {
		Data            string
		ClientMessageId uint64
	}

	h := fnv.New32()
	h.Write([]byte(message))
	// The message id should be unique across separate instances of the bridge,
	// even if they were attached to the same session. A collision in this case
	// means one bridge instance (with the same session) is unable to send a
	// message because the message id is equal to the one the other bridge
	// instance just sent. With the hash of the message itself, such a
	// collision can only occur when both instances try to send exactly the
	// same message _and_ the random value is the same for both instances.
	msgid := (uint64(h.Sum32()) << 32) | uint64(rand.Int31n(math.MaxInt32))

	b, err := json.Marshal(postMessageRequest{
		Data:            message,
		ClientMessageId: msgid,
	})
	if err != nil {
		return fmt.Errorf("Message could not be encoded as JSON: %v\n", err)
	}

	target, resp, err := s.sendRequest("POST", fmt.Sprintf(pathPostMessage, s.sessionId), b)
	if err != nil {
		return err
	}
	discardResponse(resp)
	s.network.succeeded(target)
	return nil
}

// Delete sends a delete request for this session on the server.
//
// This session MUST not be used after this method returns. Even if the delete
// request did not succeed, the session is deleted from the client’s point of
// view.
func (s *RobustSession) Delete(quitmessage string) error {
	defer func() {
		s.deleted = true
		s.done <- true
	}()

	b, err := json.Marshal(struct{ Quitmessage string }{quitmessage})
	if err != nil {
		return err
	}
	_, resp, err := s.sendRequest("DELETE", fmt.Sprintf(pathDeleteSession, s.sessionId), b)
	if err != nil {
		return err
	}
	discardResponse(resp)
	return nil
}
<commit_msg>bugfix: bridge: close resp.Body in all error cases<commit_after>// robustsession represents a RobustIRC session and handles all communication
// to the RobustIRC network.
package robustsession

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"hash/fnv"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/robustirc/robustirc/types"

	"github.com/sorcix/irc"
)

const (
	pathCreateSession = "/robustirc/v1/session"
	pathDeleteSession = "/robustirc/v1/%s"
	pathPostMessage   = "/robustirc/v1/%s/message"
	pathGetMessages   = "/robustirc/v1/%s/messages?lastseen=%s"
)

var (
	NoSuchSession = errors.New("No such RobustIRC session (killed by the network?)")

	networks   = make(map[string]*network)
	networksMu sync.Mutex
)

type backoffState struct {
	exp  float64
	next time.Time
}

type network struct {
	servers []string
	mu      sync.RWMutex
	backoff map[string]backoffState
}

func newNetwork(networkname string) (*network, error) {
	var servers []string

	parts := strings.Split(networkname, ",")
	if len(parts) > 1 {
		log.Printf("Interpreting %q as list of servers instead of network name\n", networkname)
		servers = parts
	} else {
		// Try to resolve the DNS name up to 5 times. This is to be nice to
		// people in environments with flaky network connections at boot, who,
		// for some reason, don’t run this program under systemd with
		// Restart=on-failure.
		try := 0
		for {
			_, addrs, err := net.LookupSRV("robustirc", "tcp", networkname)
			if err != nil {
				log.Println(err)
				if try < 4 {
					time.Sleep(time.Duration(int64(math.Pow(2, float64(try)))) * time.Second)
				} else {
					return nil, fmt.Errorf("DNS lookup of %q failed 5 times", networkname)
				}
				try++
				continue
			}
			// Randomly shuffle the addresses.
			for i := range addrs {
				j := rand.Intn(i + 1)
				addrs[i], addrs[j] = addrs[j], addrs[i]
			}
			for _, addr := range addrs {
				target := addr.Target
				if target[len(target)-1] == '.' {
					target = target[:len(target)-1]
				}
				servers = append(servers, fmt.Sprintf("%s:%d", target, addr.Port))
			}
			break
		}
	}

	return &network{
		servers: servers,
		backoff: make(map[string]backoffState),
	}, nil
}

// server (eventually) returns the host:port to which we should connect to. In
// case back-off prevents us from connecting anywhere right now, the function
// blocks until back-off is over.
func (n *network) server() string {
	n.mu.RLock()
	defer n.mu.RUnlock()

	for {
		soonest := time.Duration(math.MaxInt64)
		for _, server := range n.servers {
			wait := n.backoff[server].next.Sub(time.Now())
			if wait <= 0 {
				return server
			}
			if wait < soonest {
				soonest = wait
			}
		}

		time.Sleep(soonest)
	}
}

func (n *network) setServers(servers []string) {
	n.mu.Lock()
	defer n.mu.Unlock()

	// TODO(secure): we should clean up n.backoff from servers which no longer exist
	n.servers = servers
}

// prefer adds the specified server to the front of the servers list, thereby
// trying to prefer it over other servers for the next request. Note that
// exponential backoff overrides this, so this is only a hint, not a guarantee.
func (n *network) prefer(server string) {
	n.mu.Lock()
	defer n.mu.Unlock()

	n.servers = append([]string{server}, n.servers...)
}

func (n *network) failed(server string) {
	n.mu.Lock()
	defer n.mu.Unlock()

	b := n.backoff[server]
	b.exp++
	b.next = time.Now().Add(time.Duration(math.Pow(2, b.exp)) * time.Second)
	n.backoff[server] = b
}

func (n *network) succeeded(server string) {
	n.mu.Lock()
	defer n.mu.Unlock()

	delete(n.backoff, server)
}

func discardResponse(resp *http.Response) {
	// We need to read the entire body, otherwise net/http will not
	// re-use this connection.
	ioutil.ReadAll(resp.Body)
	resp.Body.Close()
}

type RobustSession struct {
	IrcPrefix *irc.Prefix
	Messages  chan string
	Errors    chan error

	sessionId   string
	sessionAuth string
	deleted     bool
	done        chan bool
	network     *network
	client      *http.Client
}

func (s *RobustSession) sendRequest(method, path string, data []byte) (string, *http.Response, error) {
	for !s.deleted {
		target := s.network.server()
		req, err := http.NewRequest(method, fmt.Sprintf("https://%s%s", target, path), bytes.NewBuffer(data))
		if err != nil {
			return "", nil, err
		}
		req.Header.Set("X-Session-Auth", s.sessionAuth)
		req.Header.Set("Content-Type", "application/json")
		resp, err := s.client.Do(req)
		if err != nil {
			s.network.failed(target)
			log.Printf("sendRequest(%q) failed: %v\n", path, err)
			continue
		}
		if resp.StatusCode == http.StatusOK {
			return target, resp, nil
		}
		message, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		s.network.failed(target)
		if resp.StatusCode == http.StatusNotFound {
			return "", nil, NoSuchSession
		}
		// Server errors, temporary.
		if resp.StatusCode >= 500 && resp.StatusCode < 600 {
			log.Printf("sendRequest(%q) failed with %v: %q (retrying)\n", path, resp.Status, message)
			continue
		}
		// Client errors and anything unexpected, assumed to be permanent.
		return "", nil, fmt.Errorf("sendRequest(%q) failed with %v: %q\n", path, resp.Status, message)
	}

	return "", nil, NoSuchSession
}

// Create creates a new RobustIRC session. It resolves the given network name
// (e.g. "robustirc.net") to a set of servers by querying the
// _robustirc._tcp.<network> SRV record and sends the CreateSession request.
//
// When err == nil, the caller MUST read the RobustSession.Messages and
// RobustSession.Errors channels.
func Create(network string, tlsCAFile string) (*RobustSession, error) {
	networksMu.Lock()
	n, ok := networks[network]
	if !ok {
		var err error
		n, err = newNetwork(network)
		if err != nil {
			return nil, err
		}
		networks[network] = n
	}
	networksMu.Unlock()

	var client *http.Client

	if tlsCAFile != "" {
		roots := x509.NewCertPool()
		contents, err := ioutil.ReadFile(tlsCAFile)
		if err != nil {
			log.Fatalf("Could not read cert.pem: %v", err)
		}
		if !roots.AppendCertsFromPEM(contents) {
			log.Fatalf("Could not parse %q", tlsCAFile)
		}

		client = &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{RootCAs: roots},
			},
		}
	} else {
		client = http.DefaultClient
	}

	s := &RobustSession{
		Messages: make(chan string),
		Errors:   make(chan error),
		done:     make(chan bool, 1),
		network:  n,
		client:   client,
	}

	_, resp, err := s.sendRequest("POST", pathCreateSession, nil)
	if err != nil {
		return nil, err
	}
	defer discardResponse(resp)

	var createSessionReply struct {
		Sessionid   string
		Sessionauth string
		Prefix      string
	}

	if err := json.NewDecoder(resp.Body).Decode(&createSessionReply); err != nil {
		return nil, err
	}

	if cl := resp.Header.Get("Content-Location"); cl != "" {
		if location, err := url.Parse(cl); err == nil {
			log.Printf("Preferring %q (current leader)\n", location.Host)
			s.network.prefer(location.Host)
		}
	}

	s.sessionId = createSessionReply.Sessionid
	s.sessionAuth = createSessionReply.Sessionauth
	s.IrcPrefix = &irc.Prefix{Name: createSessionReply.Prefix}

	go s.getMessages()

	return s, nil
}

func (s *RobustSession) getMessages() {
	var lastseen types.RobustId

	for !s.deleted {
		target, resp, err := s.sendRequest("GET", fmt.Sprintf(pathGetMessages, s.sessionId, lastseen.String()), nil)
		if err != nil {
			s.Errors <- err
			return
		}

		msgchan := make(chan types.RobustMessage, 1)
		errchan := make(chan error)
		go func() {
			dec := json.NewDecoder(resp.Body)
			for {
				var msg types.RobustMessage
				if err := dec.Decode(&msg); err != nil {
					errchan <- err
					return
				}
				msgchan <- msg
			}
		}()

	ReadLoop:
		for !s.deleted {
			select {
			case msg := <-msgchan:
				if msg.Type == types.RobustPing {
					s.network.setServers(msg.Servers)
				} else if msg.Type == types.RobustIRCToClient {
					// TODO: remove/debug
					log.Printf("<-robustirc: %q\n", msg.Data)
					s.Messages <- msg.Data
					lastseen = msg.Id
				}

			case err := <-errchan:
				log.Printf("Protocol error on %q: Could not decode response chunk as JSON: %v\n", target, err)
				s.network.failed(target)
				break ReadLoop

			case <-time.After(1 * time.Minute):
				log.Printf("Timeout (60s) on GetMessages, reconnecting…\n")
				s.network.failed(target)
				break ReadLoop

			case <-s.done:
				break ReadLoop
			}
		}
		resp.Body.Close()

		// Delay reconnecting for somewhere in between [250, 500) ms to avoid
		// overloading the remaining servers from many clients at once when one
		// server fails.
		time.Sleep(time.Duration(250+rand.Int63n(250)) * time.Millisecond)
	}
}

// PostMessage posts the given IRC message.
func (s *RobustSession) PostMessage(message string) error {
	type postMessageRequest struct {
		Data            string
		ClientMessageId uint64
	}

	h := fnv.New32()
	h.Write([]byte(message))
	// The message id should be unique across separate instances of the bridge,
	// even if they were attached to the same session. A collision in this case
	// means one bridge instance (with the same session) is unable to send a
	// message because the message id is equal to the one the other bridge
	// instance just sent. With the hash of the message itself, such a
	// collision can only occur when both instances try to send exactly the
	// same message _and_ the random value is the same for both instances.
	msgid := (uint64(h.Sum32()) << 32) | uint64(rand.Int31n(math.MaxInt32))

	b, err := json.Marshal(postMessageRequest{
		Data:            message,
		ClientMessageId: msgid,
	})
	if err != nil {
		return fmt.Errorf("Message could not be encoded as JSON: %v\n", err)
	}

	target, resp, err := s.sendRequest("POST", fmt.Sprintf(pathPostMessage, s.sessionId), b)
	if err != nil {
		return err
	}
	discardResponse(resp)
	s.network.succeeded(target)
	return nil
}

// Delete sends a delete request for this session on the server.
//
// This session MUST not be used after this method returns. Even if the delete
// request did not succeed, the session is deleted from the client’s point of
// view.
func (s *RobustSession) Delete(quitmessage string) error {
	defer func() {
		s.deleted = true
		s.done <- true
	}()

	b, err := json.Marshal(struct{ Quitmessage string }{quitmessage})
	if err != nil {
		return err
	}
	_, resp, err := s.sendRequest("DELETE", fmt.Sprintf(pathDeleteSession, s.sessionId), b)
	if err != nil {
		return err
	}
	discardResponse(resp)
	return nil
}
<|endoftext|>