<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/cilium/cilium/api/v1/health/models"
	ciliumModels "github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/health/client"
	"github.com/cilium/cilium/pkg/health/defaults"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"github.com/servak/go-fastping"
	"github.com/sirupsen/logrus"
)

// healthReport is a snapshot of the health of the cluster.
type healthReport struct {
	startTime time.Time
	nodes     []*models.NodeStatus
}

type prober struct {
	*fastping.Pinger
	server *Server

	// 'stop' is closed upon a call to prober.Stop(). When the stopping is
	// finished, then prober.Done() will be notified.
	stop         chan bool
	proberExited chan bool
	done         chan bool

	// The lock protects multiple requests attempting to update the status
	// at the same time - ie, serialize updates between the periodic prober
	// and probes initiated via "GET /status/probe". It is also used to
	// co-ordinate updates of the ICMP responses and the HTTP responses.
	lock.RWMutex

	// start is the start time for the current probe cycle.
	start   time.Time
	results map[ipString]*models.PathStatus
	nodes   nodeMap

	// TODO: If nodes leave the cluster, we will never clear out their
	//       entries in the 'results' map.
}

// copyResultRLocked makes a copy of the path status for the specified IP.
func (p *prober) copyResultRLocked(ip string) *models.PathStatus {
	status := p.results[ipString(ip)]
	if status == nil {
		return nil
	}

	result := &models.PathStatus{
		IP: ip,
	}
	paths := map[**models.ConnectivityStatus]*models.ConnectivityStatus{
		&result.Icmp: status.Icmp,
		&result.HTTP: status.HTTP,
	}
	for res, value := range paths {
		if value != nil {
			*res = &*value
		}
	}
	return result
}

// getResults gathers a copy of all of the results for nodes currently in the
// cluster.
func (p *prober) getResults() *healthReport {
	p.RLock()
	defer p.RUnlock()

	// De-duplicate IPs in 'p.nodes' by building a map based on node.Name.
	resultMap := map[string]*models.NodeStatus{}
	for _, node := range p.nodes {
		if resultMap[node.Name] != nil {
			continue
		}
		primaryIP := node.PrimaryIP()
		healthIP := node.HealthIP()
		status := &models.NodeStatus{
			Name: node.Name,
			Host: &models.HostStatus{
				PrimaryAddress: p.copyResultRLocked(primaryIP),
			},
		}
		if healthIP != "" {
			status.Endpoint = p.copyResultRLocked(healthIP)
		}
		secondaryResults := []*models.PathStatus{}
		for _, addr := range node.SecondaryAddresses {
			if addr.Enabled {
				secondaryStatus := p.copyResultRLocked(addr.IP)
				secondaryResults = append(secondaryResults, secondaryStatus)
			}
		}
		status.Host.SecondaryAddresses = secondaryResults
		resultMap[node.Name] = status
	}

	result := &healthReport{startTime: p.start}
	for _, res := range resultMap {
		result.nodes = append(result.nodes, res)
	}
	return result
}

func isIPv4(ip string) bool {
	netIP := net.ParseIP(ip)
	return netIP != nil && !strings.Contains(ip, ":")
}

func skipAddress(elem *ciliumModels.NodeAddressingElement) bool {
	return elem == nil || !elem.Enabled || elem.IP == "<nil>"
}

// resolveIP attempts to sanitize 'node' and 'ip', and if successful, returns
// the name of the node and the IP address specified in the addressing element.
// If validation fails or this IP should not be pinged, 'ip' is returned as nil.
func resolveIP(n *healthNode, addr *ciliumModels.NodeAddressingElement, proto string, primary bool) (string, *net.IPAddr) {
	node := n.NodeElement
	network := "ip6:icmp"
	if isIPv4(addr.IP) {
		network = "ip4:icmp"
	}
	scopedLog := log.WithFields(logrus.Fields{
		logfields.NodeName: node.Name,
		logfields.IPAddr:   addr.IP,
		"primary":          primary,
	})

	if skipAddress(addr) {
		scopedLog.Debug("Skipping probe for address")
		return "", nil
	}

	ra, err := net.ResolveIPAddr(network, addr.IP)
	if err != nil {
		scopedLog.Debug("Unable to resolve address")
		return "", nil
	}

	scopedLog.WithField("protocol", proto).Debug("Probing for connectivity to node")
	return node.Name, ra
}

// setNodes sets the list of nodes for the prober, and updates the pinger to
// start sending pings to all of the nodes.
// setNodes will steal references to nodes referenced from 'nodes', so the
// caller should not modify them after a call to setNodes.
func (p *prober) setNodes(nodes nodeMap) {
	p.Lock()
	defer p.Unlock()

	for _, n := range nodes {
		for elem, primary := range n.Addresses() {
			_, addr := resolveIP(&n, elem, "icmp", primary)

			ip := ipString(elem.IP)
			result := &models.ConnectivityStatus{}
			if addr == nil {
				result.Status = "Failed to resolve IP"
			} else {
				result.Status = "Connection timed out"
				p.AddIPAddr(addr)
				p.nodes[ip] = n
			}

			if p.results[ip] == nil {
				p.results[ip] = &models.PathStatus{
					IP: elem.IP,
				}
			}
			p.results[ip].Icmp = result
		}
	}
}

func (p *prober) httpProbe(node string, ip string, port int) *models.ConnectivityStatus {
	result := &models.ConnectivityStatus{}

	host := fmt.Sprintf("http://%s:%d", ip, port)
	scopedLog := log.WithFields(logrus.Fields{
		logfields.NodeName: node,
		logfields.IPAddr:   ip,
		"host":             host,
		"path":             PortToPaths[port],
	})

	client, err := client.NewClient(host)
	if err == nil {
		scopedLog.Debug("Greeting host")
		start := time.Now()
		_, err = client.Restapi.GetHello(nil)
		rtt := time.Since(start)
		if err == nil {
			scopedLog.WithField("rtt", rtt).Debug("Greeting successful")
			result.Status = ""
			result.Latency = rtt.Nanoseconds()
		} else {
			scopedLog.WithError(err).Debug("Greeting snubbed")
			result.Status = "Connection timed out"
		}
	} else {
		scopedLog.WithError(err).Info("Failed to express greeting to host")
		result.Status = err.Error()
	}

	return result
}

func (p *prober) runHTTPProbe() {
	startTime := time.Now()
	p.Lock()
	p.start = startTime
	p.Unlock()

	// p.nodes is mapped from all known IPs -> nodes in N:M configuration,
	// so multiple IPs could refer to the same node. To ensure we only
	// ping each node once, deduplicate nodes into map of nodeName -> []IP.
	// When probing below, we won't hold the lock on 'p.nodes' so take
	// a copy of all of the IPs we need to reference.
	nodes := make(map[string][]*net.IPAddr)
	p.RLock()
	for _, node := range p.nodes {
		if nodes[node.Name] != nil {
			// Already handled this node.
			continue
		}
		nodes[node.Name] = []*net.IPAddr{}
		for elem, primary := range node.Addresses() {
			if _, addr := resolveIP(&node, elem, "http", primary); addr != nil {
				nodes[node.Name] = append(nodes[node.Name], addr)
			}
		}
	}
	p.RUnlock()

	for name, ips := range nodes {
		for _, ip := range ips {
			scopedLog := log.WithFields(logrus.Fields{
				logfields.NodeName: name,
				logfields.IPAddr:   ip.String(),
			})

			status := &models.PathStatus{}
			ports := map[int]**models.ConnectivityStatus{
				defaults.HTTPPathPort: &status.HTTP,
			}
			for port, result := range ports {
				*result = p.httpProbe(name, ip.String(), port)
				if status.HTTP.Status != "" {
					scopedLog.WithFields(logrus.Fields{
						logfields.Port: port,
					}).Debugf("Failed to probe: %s", status.HTTP.Status)
				}
			}

			peer := ipString(ip.String())
			p.Lock()
			if _, ok := p.results[peer]; ok {
				p.results[peer].HTTP = status.HTTP
			} else {
				// While we weren't holding the lock, the
				// pinger's OnIdle() callback fired and updated
				// the set of nodes to remove this node.
				scopedLog.Debug("Node disappeared before result written")
			}
			p.Unlock()
		}
	}
}

// Done returns a channel that is closed when RunLoop() is stopped by an error.
// It must be called after the RunLoop() call.
func (p *prober) Done() <-chan bool {
	return p.done
}

// Run sends a single probes out to all of the other cilium nodes to gather
// connectivity status for the cluster.
func (p *prober) Run() error {
	err := p.Pinger.Run()
	p.runHTTPProbe()
	return err
}

// Stop disrupts the currently running RunLoop(). This may only be called after
// a call to RunLoop().
func (p *prober) Stop() {
	p.Pinger.Stop()
	close(p.stop)
	<-p.proberExited
	close(p.done)
}

// RunLoop periodically sends probes out to all of the other cilium nodes to
// gather connectivity status for the cluster.
//
// This is a non-blocking method so it immediately returns. If you want to
// stop sending packets, call Stop().
func (p *prober) RunLoop() {
	// FIXME: Spread the probes out across the probing interval
	p.Pinger.RunLoop()

	go func() {
		tick := time.NewTicker(p.server.ProbeInterval)
	loop:
		for {
			select {
			case <-p.stop:
				break loop
			case <-tick.C:
				p.runHTTPProbe()
				continue
			}
		}
		tick.Stop()
		close(p.proberExited)
	}()
}

// newPinger prepares a prober. The caller may invoke one the Run* methods of
// the prober to populate its 'results' map.
func newProber(s *Server, nodes nodeMap) *prober {
	prober := &prober{
		Pinger:       fastping.NewPinger(),
		server:       s,
		done:         make(chan bool),
		proberExited: make(chan bool),
		stop:         make(chan bool),
		results:      make(map[ipString]*models.PathStatus),
		nodes:        make(nodeMap),
	}
	prober.MaxRTT = s.ProbeDeadline

	prober.setNodes(nodes)
	prober.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
		prober.Lock()
		defer prober.Unlock()
		node, exists := prober.nodes[ipString(addr.String())]

		scopedLog := log.WithFields(logrus.Fields{
			logfields.IPAddr: addr,
			"rtt":            rtt,
		})
		if !exists {
			scopedLog.Debugf("Node disappeared, skip result")
			return
		}

		prober.results[ipString(addr.String())].Icmp = &models.ConnectivityStatus{
			Latency: rtt.Nanoseconds(),
			Status:  "",
		}
		scopedLog.WithFields(logrus.Fields{
			logfields.NodeName: node.Name,
		}).Debugf("Probe successful")
	}

	return prober
}
<commit_msg>health: Mark and sweep nodes<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/cilium/cilium/api/v1/health/models"
	ciliumModels "github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/health/client"
	"github.com/cilium/cilium/pkg/health/defaults"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"github.com/servak/go-fastping"
	"github.com/sirupsen/logrus"
)

// healthReport is a snapshot of the health of the cluster.
type healthReport struct {
	startTime time.Time
	nodes     []*models.NodeStatus
}

type prober struct {
	*fastping.Pinger
	server *Server

	// 'stop' is closed upon a call to prober.Stop(). When the stopping is
	// finished, then prober.Done() will be notified.
	stop         chan bool
	proberExited chan bool
	done         chan bool

	// The lock protects multiple requests attempting to update the status
	// at the same time - ie, serialize updates between the periodic prober
	// and probes initiated via "GET /status/probe". It is also used to
	// co-ordinate updates of the ICMP responses and the HTTP responses.
	lock.RWMutex

	// start is the start time for the current probe cycle.
	start   time.Time
	results map[ipString]*models.PathStatus
	nodes   nodeMap
}

// copyResultRLocked makes a copy of the path status for the specified IP.
func (p *prober) copyResultRLocked(ip string) *models.PathStatus {
	status := p.results[ipString(ip)]
	if status == nil {
		return nil
	}

	result := &models.PathStatus{
		IP: ip,
	}
	paths := map[**models.ConnectivityStatus]*models.ConnectivityStatus{
		&result.Icmp: status.Icmp,
		&result.HTTP: status.HTTP,
	}
	for res, value := range paths {
		if value != nil {
			*res = &*value
		}
	}
	return result
}

// getResults gathers a copy of all of the results for nodes currently in the
// cluster.
func (p *prober) getResults() *healthReport {
	p.RLock()
	defer p.RUnlock()

	// De-duplicate IPs in 'p.nodes' by building a map based on node.Name.
	resultMap := map[string]*models.NodeStatus{}
	for _, node := range p.nodes {
		if resultMap[node.Name] != nil {
			continue
		}
		primaryIP := node.PrimaryIP()
		healthIP := node.HealthIP()
		status := &models.NodeStatus{
			Name: node.Name,
			Host: &models.HostStatus{
				PrimaryAddress: p.copyResultRLocked(primaryIP),
			},
		}
		if healthIP != "" {
			status.Endpoint = p.copyResultRLocked(healthIP)
		}
		secondaryResults := []*models.PathStatus{}
		for _, addr := range node.SecondaryAddresses {
			if addr.Enabled {
				secondaryStatus := p.copyResultRLocked(addr.IP)
				secondaryResults = append(secondaryResults, secondaryStatus)
			}
		}
		status.Host.SecondaryAddresses = secondaryResults
		resultMap[node.Name] = status
	}

	result := &healthReport{startTime: p.start}
	for _, res := range resultMap {
		result.nodes = append(result.nodes, res)
	}
	return result
}

func isIPv4(ip string) bool {
	netIP := net.ParseIP(ip)
	return netIP != nil && !strings.Contains(ip, ":")
}

func skipAddress(elem *ciliumModels.NodeAddressingElement) bool {
	return elem == nil || !elem.Enabled || elem.IP == "<nil>"
}

// resolveIP attempts to sanitize 'node' and 'ip', and if successful, returns
// the name of the node and the IP address specified in the addressing element.
// If validation fails or this IP should not be pinged, 'ip' is returned as nil.
func resolveIP(n *healthNode, addr *ciliumModels.NodeAddressingElement, proto string, primary bool) (string, *net.IPAddr) {
	node := n.NodeElement
	network := "ip6:icmp"
	if isIPv4(addr.IP) {
		network = "ip4:icmp"
	}
	scopedLog := log.WithFields(logrus.Fields{
		logfields.NodeName: node.Name,
		logfields.IPAddr:   addr.IP,
		"primary":          primary,
	})

	if skipAddress(addr) {
		scopedLog.Debug("Skipping probe for address")
		return "", nil
	}

	ra, err := net.ResolveIPAddr(network, addr.IP)
	if err != nil {
		scopedLog.Debug("Unable to resolve address")
		return "", nil
	}

	scopedLog.WithField("protocol", proto).Debug("Probing for connectivity to node")
	return node.Name, ra
}

// markIPsLocked marks all nodes in the prober for deletion.
func (p *prober) markIPsLocked() {
	for ip, node := range p.nodes {
		node.deletionMark = true
		p.nodes[ip] = node
	}
}

// sweepIPsLocked iterates through nodes in the prober and removes nodes which
// are marked for deletion.
func (p *prober) sweepIPsLocked() {
	for ip, node := range p.nodes {
		if node.deletionMark {
			// Remove deleted nodes from:
			// * Results (accessed from ICMP pinger or TCP prober)
			// * ICMP pinger
			// * TCP prober
			for elem := range node.Addresses() {
				delete(p.results, ipString(elem.IP))
				p.RemoveIP(elem.IP) // ICMP pinger
			}
			delete(p.nodes, ip) // TCP prober
		}
	}
}

// setNodes sets the list of nodes for the prober, and updates the pinger to
// start sending pings to all of the nodes.
// setNodes will steal references to nodes referenced from 'nodes', so the
// caller should not modify them after a call to setNodes.
func (p *prober) setNodes(nodes nodeMap) {
	p.Lock()
	defer p.Unlock()

	// Mark all nodes for deletion, insert nodes that should not be deleted
	// then at the end of the function, sweep all nodes that remain as
	// "to be deleted".
	p.markIPsLocked()

	for _, n := range nodes {
		for elem, primary := range n.Addresses() {
			_, addr := resolveIP(&n, elem, "icmp", primary)

			ip := ipString(elem.IP)
			result := &models.ConnectivityStatus{}
			if addr == nil {
				result.Status = "Failed to resolve IP"
			} else {
				result.Status = "Connection timed out"
				p.AddIPAddr(addr)
				p.nodes[ip] = n
			}

			if p.results[ip] == nil {
				p.results[ip] = &models.PathStatus{
					IP: elem.IP,
				}
			}
			p.results[ip].Icmp = result
		}
	}

	p.sweepIPsLocked()
}

func (p *prober) httpProbe(node string, ip string, port int) *models.ConnectivityStatus {
	result := &models.ConnectivityStatus{}

	host := fmt.Sprintf("http://%s:%d", ip, port)
	scopedLog := log.WithFields(logrus.Fields{
		logfields.NodeName: node,
		logfields.IPAddr:   ip,
		"host":             host,
		"path":             PortToPaths[port],
	})

	client, err := client.NewClient(host)
	if err == nil {
		scopedLog.Debug("Greeting host")
		start := time.Now()
		_, err = client.Restapi.GetHello(nil)
		rtt := time.Since(start)
		if err == nil {
			scopedLog.WithField("rtt", rtt).Debug("Greeting successful")
			result.Status = ""
			result.Latency = rtt.Nanoseconds()
		} else {
			scopedLog.WithError(err).Debug("Greeting snubbed")
			result.Status = "Connection timed out"
		}
	} else {
		scopedLog.WithError(err).Info("Failed to express greeting to host")
		result.Status = err.Error()
	}

	return result
}

func (p *prober) runHTTPProbe() {
	startTime := time.Now()
	p.Lock()
	p.start = startTime
	p.Unlock()

	// p.nodes is mapped from all known IPs -> nodes in N:M configuration,
	// so multiple IPs could refer to the same node. To ensure we only
	// ping each node once, deduplicate nodes into map of nodeName -> []IP.
	// When probing below, we won't hold the lock on 'p.nodes' so take
	// a copy of all of the IPs we need to reference.
	nodes := make(map[string][]*net.IPAddr)
	p.RLock()
	for _, node := range p.nodes {
		if nodes[node.Name] != nil {
			// Already handled this node.
			continue
		}
		nodes[node.Name] = []*net.IPAddr{}
		for elem, primary := range node.Addresses() {
			if _, addr := resolveIP(&node, elem, "http", primary); addr != nil {
				nodes[node.Name] = append(nodes[node.Name], addr)
			}
		}
	}
	p.RUnlock()

	for name, ips := range nodes {
		for _, ip := range ips {
			scopedLog := log.WithFields(logrus.Fields{
				logfields.NodeName: name,
				logfields.IPAddr:   ip.String(),
			})

			status := &models.PathStatus{}
			ports := map[int]**models.ConnectivityStatus{
				defaults.HTTPPathPort: &status.HTTP,
			}
			for port, result := range ports {
				*result = p.httpProbe(name, ip.String(), port)
				if status.HTTP.Status != "" {
					scopedLog.WithFields(logrus.Fields{
						logfields.Port: port,
					}).Debugf("Failed to probe: %s", status.HTTP.Status)
				}
			}

			peer := ipString(ip.String())
			p.Lock()
			if _, ok := p.results[peer]; ok {
				p.results[peer].HTTP = status.HTTP
			} else {
				// While we weren't holding the lock, the
				// pinger's OnIdle() callback fired and updated
				// the set of nodes to remove this node.
				scopedLog.Debug("Node disappeared before result written")
			}
			p.Unlock()
		}
	}
}

// Done returns a channel that is closed when RunLoop() is stopped by an error.
// It must be called after the RunLoop() call.
func (p *prober) Done() <-chan bool {
	return p.done
}

// Run sends a single probes out to all of the other cilium nodes to gather
// connectivity status for the cluster.
func (p *prober) Run() error {
	err := p.Pinger.Run()
	p.runHTTPProbe()
	return err
}

// Stop disrupts the currently running RunLoop(). This may only be called after
// a call to RunLoop().
func (p *prober) Stop() {
	p.Pinger.Stop()
	close(p.stop)
	<-p.proberExited
	close(p.done)
}

// RunLoop periodically sends probes out to all of the other cilium nodes to
// gather connectivity status for the cluster.
//
// This is a non-blocking method so it immediately returns. If you want to
// stop sending packets, call Stop().
func (p *prober) RunLoop() {
	// FIXME: Spread the probes out across the probing interval
	p.Pinger.RunLoop()

	go func() {
		tick := time.NewTicker(p.server.ProbeInterval)
	loop:
		for {
			select {
			case <-p.stop:
				break loop
			case <-tick.C:
				p.runHTTPProbe()
				continue
			}
		}
		tick.Stop()
		close(p.proberExited)
	}()
}

// newPinger prepares a prober. The caller may invoke one the Run* methods of
// the prober to populate its 'results' map.
func newProber(s *Server, nodes nodeMap) *prober {
	prober := &prober{
		Pinger:       fastping.NewPinger(),
		server:       s,
		done:         make(chan bool),
		proberExited: make(chan bool),
		stop:         make(chan bool),
		results:      make(map[ipString]*models.PathStatus),
		nodes:        make(nodeMap),
	}
	prober.MaxRTT = s.ProbeDeadline

	prober.setNodes(nodes)
	prober.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
		prober.Lock()
		defer prober.Unlock()
		node, exists := prober.nodes[ipString(addr.String())]

		scopedLog := log.WithFields(logrus.Fields{
			logfields.IPAddr: addr,
			"rtt":            rtt,
		})
		if !exists {
			scopedLog.Debugf("Node disappeared, skip result")
			return
		}

		prober.results[ipString(addr.String())].Icmp = &models.ConnectivityStatus{
			Latency: rtt.Nanoseconds(),
			Status:  "",
		}
		scopedLog.WithFields(logrus.Fields{
			logfields.NodeName: node.Name,
		}).Debugf("Probe successful")
	}

	return prober
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pleg

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/kubelet/metrics"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/pkg/util/wait"
)

// GenericPLEG is an extremely simple generic PLEG that relies solely on
// periodic listing to discover container changes. It should be be used
// as temporary replacement for container runtimes do not support a proper
// event generator yet.
//
// Note that GenericPLEG assumes that a container would not be created,
// terminated, and garbage collected within one relist period. If such an
// incident happens, GenenricPLEG would miss all events regarding this
// container. In the case of relisting failure, the window may become longer.
// Note that this assumption is not unique -- many kubelet internal components
// rely on terminated containers as tombstones for bookkeeping purposes. The
// garbage collector is implemented to work with such situtations. However, to
// guarantee that kubelet can handle missing container events, it is
// recommended to set the relist period short and have an auxiliary, longer
// periodic sync in kubelet as the safety net.
type GenericPLEG struct {
	// The period for relisting.
	relistPeriod time.Duration
	// The container runtime.
	runtime kubecontainer.Runtime
	// The channel from which the subscriber listens events.
	eventChannel chan *PodLifecycleEvent
	// The internal cache for pod/container information.
	podRecords podRecords
	// Time of the last relisting.
	lastRelistTime time.Time
	// Cache for storing the runtime states required for syncing pods.
	cache kubecontainer.Cache
}

// plegContainerState has a one-to-one mapping to the
// kubecontainer.ContainerState except for the non-existent state. This state
// is introduced here to complete the state transition scenarios.
type plegContainerState string

const (
	plegContainerRunning     plegContainerState = "running"
	plegContainerExited      plegContainerState = "exited"
	plegContainerUnknown     plegContainerState = "unknown"
	plegContainerNonExistent plegContainerState = "non-existent"
)

func convertState(state kubecontainer.ContainerState) plegContainerState {
	switch state {
	case kubecontainer.ContainerStateRunning:
		return plegContainerRunning
	case kubecontainer.ContainerStateExited:
		return plegContainerExited
	case kubecontainer.ContainerStateUnknown:
		return plegContainerUnknown
	default:
		panic(fmt.Sprintf("unrecognized container state: %v", state))
	}
}

type podRecord struct {
	old     *kubecontainer.Pod
	current *kubecontainer.Pod
}

type podRecords map[types.UID]*podRecord

func NewGenericPLEG(runtime kubecontainer.Runtime, channelCapacity int,
	relistPeriod time.Duration, cache kubecontainer.Cache) PodLifecycleEventGenerator {
	return &GenericPLEG{
		relistPeriod: relistPeriod,
		runtime:      runtime,
		eventChannel: make(chan *PodLifecycleEvent, channelCapacity),
		podRecords:   make(podRecords),
		cache:        cache,
	}
}

// Returns a channel from which the subscriber can receive PodLifecycleEvent
// events.
// TODO: support multiple subscribers.
func (g *GenericPLEG) Watch() chan *PodLifecycleEvent {
	return g.eventChannel
}

// Start spawns a goroutine to relist periodically.
func (g *GenericPLEG) Start() {
	go wait.Until(g.relist, g.relistPeriod, wait.NeverStop)
}

func generateEvent(podID types.UID, cid string, oldState, newState plegContainerState) *PodLifecycleEvent {
	if newState == oldState {
		return nil
	}
	glog.V(4).Infof("GenericPLEG: %v/%v: %v -> %v", podID, cid, oldState, newState)
	switch newState {
	case plegContainerRunning:
		return &PodLifecycleEvent{ID: podID, Type: ContainerStarted, Data: cid}
	case plegContainerExited:
		return &PodLifecycleEvent{ID: podID, Type: ContainerDied, Data: cid}
	case plegContainerUnknown:
		return &PodLifecycleEvent{ID: podID, Type: ContainerChanged, Data: cid}
	case plegContainerNonExistent:
		// We report "ContainerDied" when container was stopped OR removed. We
		// may want to distinguish the two cases in the future.
		switch oldState {
		case plegContainerExited:
			// We already reported that the container died before.
			return &PodLifecycleEvent{ID: podID, Type: ContainerRemoved, Data: cid}
		default:
			// TODO: We may want to generate a ContainerRemoved event as well.
			// It's ok now because no one relies on the ContainerRemoved event.
			return &PodLifecycleEvent{ID: podID, Type: ContainerDied, Data: cid}
		}
	default:
		panic(fmt.Sprintf("unrecognized container state: %v", newState))
	}
	return nil
}

// relist queries the container runtime for list of pods/containers, compare
// with the internal pods/containers, and generats events accordingly.
func (g *GenericPLEG) relist() {
	glog.V(5).Infof("GenericPLEG: Relisting")
	timestamp := time.Now()

	if !g.lastRelistTime.IsZero() {
		metrics.PLEGRelistInterval.Observe(metrics.SinceInMicroseconds(g.lastRelistTime))
	}
	defer func() {
		// Update the relist time.
		g.lastRelistTime = timestamp
		metrics.PLEGRelistLatency.Observe(metrics.SinceInMicroseconds(timestamp))
	}()

	// Get all the pods.
	podList, err := g.runtime.GetPods(true)
	if err != nil {
		glog.Errorf("GenericPLEG: Unable to retrieve pods: %v", err)
		return
	}
	pods := kubecontainer.Pods(podList)
	for _, pod := range pods {
		g.podRecords.setCurrent(pod)
	}

	// Compare the old and the current pods, and generate events.
	eventsByPodID := map[types.UID][]*PodLifecycleEvent{}
	for pid := range g.podRecords {
		oldPod := g.podRecords.getOld(pid)
		pod := g.podRecords.getCurrent(pid)
		// Get all containers in the old and the new pod.
		allContainers := getContainersFromPods(oldPod, pod)
		for _, container := range allContainers {
			e := computeEvent(oldPod, pod, &container.ID)
			updateEvents(eventsByPodID, e)
		}
	}

	// If there are events associated with a pod, we should update the
	// podCache.
	for pid, events := range eventsByPodID {
		pod := g.podRecords.getCurrent(pid)
		if g.cacheEnabled() {
			// updateCache() will inspect the pod and update the cache. If an
			// error occurs during the inspection, we want PLEG to retry again
			// in the next relist. To achieve this, we do not update the
			// associated podRecord of the pod, so that the change will be
			// detect again in the next relist.
			// TODO: If many pods changed during the same relist period,
			// inspecting the pod and getting the PodStatus to update the cache
			// serially may take a while. We should be aware of this and
			// parallelize if needed.
			if err := g.updateCache(pod, pid); err != nil {
				glog.Errorf("PLEG: Ignoring events for pod %s/%s: %v", pod.Name, pod.Namespace, err)
				continue
			}
		}
		// Update the internal storage and send out the events.
		g.podRecords.update(pid)
		for i := range events {
			// Filter out events that are not reliable and no other components use yet.
			if events[i].Type == ContainerChanged || events[i].Type == ContainerRemoved {
				continue
			}
			g.eventChannel <- events[i]
		}
	}

	if g.cacheEnabled() {
		// Update the cache timestamp.  This needs to happen *after*
		// all pods have been properly updated in the cache.
		g.cache.UpdateTime(timestamp)
	}
}

func getContainersFromPods(pods ...*kubecontainer.Pod) []*kubecontainer.Container {
	cidSet := sets.NewString()
	var containers []*kubecontainer.Container
	for _, p := range pods {
		if p == nil {
			continue
		}
		for _, c := range p.Containers {
			cid := string(c.ID.ID)
			if cidSet.Has(cid) {
				continue
			}
			cidSet.Insert(cid)
			containers = append(containers, c)
		}
	}
	return containers
}

func computeEvent(oldPod, newPod *kubecontainer.Pod, cid *kubecontainer.ContainerID) *PodLifecycleEvent {
	var pid types.UID
	if oldPod != nil {
		pid = oldPod.ID
	} else if newPod != nil {
		pid = newPod.ID
	}
	oldState := getContainerState(oldPod, cid)
	newState := getContainerState(newPod, cid)
	return generateEvent(pid, cid.ID, oldState, newState)
}

func (g *GenericPLEG) cacheEnabled() bool {
	return g.cache != nil
}

func (g *GenericPLEG) updateCache(pod *kubecontainer.Pod, pid types.UID) error {
	if pod == nil {
		// The pod is missing in the current relist. This means that
		// the pod has no visible (active or inactive) containers.
		glog.V(4).Infof("PLEG: Delete status for pod %q", string(pid))
		g.cache.Delete(pid)
		return nil
	}
	timestamp := time.Now()
	// TODO: Consider adding a new runtime method
	// GetPodStatus(pod *kubecontainer.Pod) so that Docker can avoid listing
	// all containers again.
	status, err := g.runtime.GetPodStatus(pod.ID, pod.Name, pod.Namespace)
	glog.V(4).Infof("PLEG: Write status for %s/%s: %+v (err: %v)", pod.Name, pod.Namespace, status, err)
	g.cache.Set(pod.ID, status, err, timestamp)
	return err
}

func updateEvents(eventsByPodID map[types.UID][]*PodLifecycleEvent, e *PodLifecycleEvent) {
	if e == nil {
		return
	}
	eventsByPodID[e.ID] = append(eventsByPodID[e.ID], e)
}

func getContainerState(pod *kubecontainer.Pod, cid *kubecontainer.ContainerID) plegContainerState {
	// Default to the non-existent state.
	state := plegContainerNonExistent
	if pod == nil {
		return state
	}
	container := pod.FindContainerByID(*cid)
	if container == nil {
		return state
	}
	return convertState(container.State)
}

func (pr podRecords) getOld(id types.UID) *kubecontainer.Pod {
	r, ok := pr[id]
	if !ok {
		return nil
	}
	return r.old
}

func (pr podRecords) getCurrent(id types.UID) *kubecontainer.Pod {
	r, ok := pr[id]
	if !ok {
		return nil
	}
	return r.current
}

func (pr podRecords) setCurrent(pod *kubecontainer.Pod) {
	if r, ok := pr[pod.ID]; ok {
		r.current = pod
		return
	}
	pr[pod.ID] = &podRecord{current: pod}
}

func (pr podRecords) update(id types.UID) {
	r, ok := pr[id]
	if !ok {
		return
	}
	pr.updateInternal(id, r)
}

func (pr podRecords) updateInternal(id types.UID, r *podRecord) {
	if r.current == nil {
		// Pod no longer exists; delete the entry.
		delete(pr, id)
		return
	}
	r.old = r.current
	r.current = nil
}
<commit_msg>kubelet: clear current pod records before relist<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pleg

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/kubelet/metrics"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/pkg/util/wait"
)

// GenericPLEG is an extremely simple generic PLEG that relies solely on
// periodic listing to discover container changes. It should be be used
// as temporary replacement for container runtimes do not support a proper
// event generator yet.
//
// Note that GenericPLEG assumes that a container would not be created,
// terminated, and garbage collected within one relist period. If such an
// incident happens, GenenricPLEG would miss all events regarding this
// container. In the case of relisting failure, the window may become longer.
// Note that this assumption is not unique -- many kubelet internal components
// rely on terminated containers as tombstones for bookkeeping purposes. The
// garbage collector is implemented to work with such situtations. However, to
// guarantee that kubelet can handle missing container events, it is
// recommended to set the relist period short and have an auxiliary, longer
// periodic sync in kubelet as the safety net.
type GenericPLEG struct {
	// The period for relisting.
	relistPeriod time.Duration
	// The container runtime.
	runtime kubecontainer.Runtime
	// The channel from which the subscriber listens events.
	eventChannel chan *PodLifecycleEvent
	// The internal cache for pod/container information.
	podRecords podRecords
	// Time of the last relisting.
	lastRelistTime time.Time
	// Cache for storing the runtime states required for syncing pods.
	cache kubecontainer.Cache
}

// plegContainerState has a one-to-one mapping to the
// kubecontainer.ContainerState except for the non-existent state. This state
// is introduced here to complete the state transition scenarios.
type plegContainerState string

const (
	plegContainerRunning     plegContainerState = "running"
	plegContainerExited      plegContainerState = "exited"
	plegContainerUnknown     plegContainerState = "unknown"
	plegContainerNonExistent plegContainerState = "non-existent"
)

func convertState(state kubecontainer.ContainerState) plegContainerState {
	switch state {
	case kubecontainer.ContainerStateRunning:
		return plegContainerRunning
	case kubecontainer.ContainerStateExited:
		return plegContainerExited
	case kubecontainer.ContainerStateUnknown:
		return plegContainerUnknown
	default:
		panic(fmt.Sprintf("unrecognized container state: %v", state))
	}
}

type podRecord struct {
	old     *kubecontainer.Pod
	current *kubecontainer.Pod
}

type podRecords map[types.UID]*podRecord

func NewGenericPLEG(runtime kubecontainer.Runtime, channelCapacity int,
	relistPeriod time.Duration, cache kubecontainer.Cache) PodLifecycleEventGenerator {
	return &GenericPLEG{
		relistPeriod: relistPeriod,
		runtime:      runtime,
		eventChannel: make(chan *PodLifecycleEvent, channelCapacity),
		podRecords:   make(podRecords),
		cache:        cache,
	}
}

// Returns a channel from which the subscriber can receive PodLifecycleEvent
// events.
// TODO: support multiple subscribers.
func (g *GenericPLEG) Watch() chan *PodLifecycleEvent {
	return g.eventChannel
}

// Start spawns a goroutine to relist periodically.
func (g *GenericPLEG) Start() {
	go wait.Until(g.relist, g.relistPeriod, wait.NeverStop)
}

func generateEvent(podID types.UID, cid string, oldState, newState plegContainerState) *PodLifecycleEvent {
	if newState == oldState {
		return nil
	}
	glog.V(4).Infof("GenericPLEG: %v/%v: %v -> %v", podID, cid, oldState, newState)
	switch newState {
	case plegContainerRunning:
		return &PodLifecycleEvent{ID: podID, Type: ContainerStarted, Data: cid}
	case plegContainerExited:
		return &PodLifecycleEvent{ID: podID, Type: ContainerDied, Data: cid}
	case plegContainerUnknown:
		return &PodLifecycleEvent{ID: podID, Type: ContainerChanged, Data: cid}
	case plegContainerNonExistent:
		// We report "ContainerDied" when container was stopped OR removed. We
		// may want to distinguish the two cases in the future.
		switch oldState {
		case plegContainerExited:
			// We already reported that the container died before.
			return &PodLifecycleEvent{ID: podID, Type: ContainerRemoved, Data: cid}
		default:
			// TODO: We may want to generate a ContainerRemoved event as well.
			// It's ok now because no one relies on the ContainerRemoved event.
			return &PodLifecycleEvent{ID: podID, Type: ContainerDied, Data: cid}
		}
	default:
		panic(fmt.Sprintf("unrecognized container state: %v", newState))
	}
	return nil
}

// relist queries the container runtime for list of pods/containers, compare
// with the internal pods/containers, and generats events accordingly.
func (g *GenericPLEG) relist() {
	glog.V(5).Infof("GenericPLEG: Relisting")
	timestamp := time.Now()

	if !g.lastRelistTime.IsZero() {
		metrics.PLEGRelistInterval.Observe(metrics.SinceInMicroseconds(g.lastRelistTime))
	}
	defer func() {
		// Update the relist time.
		g.lastRelistTime = timestamp
		metrics.PLEGRelistLatency.Observe(metrics.SinceInMicroseconds(timestamp))
	}()

	// Get all the pods.
	podList, err := g.runtime.GetPods(true)
	if err != nil {
		glog.Errorf("GenericPLEG: Unable to retrieve pods: %v", err)
		return
	}
	pods := kubecontainer.Pods(podList)
	g.podRecords.setCurrent(pods)

	// Compare the old and the current pods, and generate events.
	eventsByPodID := map[types.UID][]*PodLifecycleEvent{}
	for pid := range g.podRecords {
		oldPod := g.podRecords.getOld(pid)
		pod := g.podRecords.getCurrent(pid)
		// Get all containers in the old and the new pod.
		allContainers := getContainersFromPods(oldPod, pod)
		for _, container := range allContainers {
			e := computeEvent(oldPod, pod, &container.ID)
			updateEvents(eventsByPodID, e)
		}
	}

	// If there are events associated with a pod, we should update the
	// podCache.
	for pid, events := range eventsByPodID {
		pod := g.podRecords.getCurrent(pid)
		if g.cacheEnabled() {
			// updateCache() will inspect the pod and update the cache. If an
			// error occurs during the inspection, we want PLEG to retry again
			// in the next relist. To achieve this, we do not update the
			// associated podRecord of the pod, so that the change will be
			// detect again in the next relist.
			// TODO: If many pods changed during the same relist period,
			// inspecting the pod and getting the PodStatus to update the cache
			// serially may take a while. We should be aware of this and
			// parallelize if needed.
			if err := g.updateCache(pod, pid); err != nil {
				glog.Errorf("PLEG: Ignoring events for pod %s/%s: %v", pod.Name, pod.Namespace, err)
				continue
			}
		}
		// Update the internal storage and send out the events.
		g.podRecords.update(pid)
		for i := range events {
			// Filter out events that are not reliable and no other components use yet.
			if events[i].Type == ContainerChanged || events[i].Type == ContainerRemoved {
				continue
			}
			g.eventChannel <- events[i]
		}
	}

	if g.cacheEnabled() {
		// Update the cache timestamp.  This needs to happen *after*
		// all pods have been properly updated in the cache.
		g.cache.UpdateTime(timestamp)
	}
}

func getContainersFromPods(pods ...*kubecontainer.Pod) []*kubecontainer.Container {
	cidSet := sets.NewString()
	var containers []*kubecontainer.Container
	for _, p := range pods {
		if p == nil {
			continue
		}
		for _, c := range p.Containers {
			cid := string(c.ID.ID)
			if cidSet.Has(cid) {
				continue
			}
			cidSet.Insert(cid)
			containers = append(containers, c)
		}
	}
	return containers
}

func computeEvent(oldPod, newPod *kubecontainer.Pod, cid *kubecontainer.ContainerID) *PodLifecycleEvent {
	var pid types.UID
	if oldPod != nil {
		pid = oldPod.ID
	} else if newPod != nil {
		pid = newPod.ID
	}
	oldState := getContainerState(oldPod, cid)
	newState := getContainerState(newPod, cid)
	return generateEvent(pid, cid.ID, oldState, newState)
}

func (g *GenericPLEG) cacheEnabled() bool {
	return g.cache != nil
}

func (g *GenericPLEG) updateCache(pod *kubecontainer.Pod, pid types.UID) error {
	if pod == nil {
		// The pod is missing in the current relist. This means that
		// the pod has no visible (active or inactive) containers.
		glog.V(4).Infof("PLEG: Delete status for pod %q", string(pid))
		g.cache.Delete(pid)
		return nil
	}
	timestamp := time.Now()
	// TODO: Consider adding a new runtime method
	// GetPodStatus(pod *kubecontainer.Pod) so that Docker can avoid listing
	// all containers again.
	status, err := g.runtime.GetPodStatus(pod.ID, pod.Name, pod.Namespace)
	glog.V(4).Infof("PLEG: Write status for %s/%s: %+v (err: %v)", pod.Name, pod.Namespace, status, err)
	g.cache.Set(pod.ID, status, err, timestamp)
	return err
}

func updateEvents(eventsByPodID map[types.UID][]*PodLifecycleEvent, e *PodLifecycleEvent) {
	if e == nil {
		return
	}
	eventsByPodID[e.ID] = append(eventsByPodID[e.ID], e)
}

func getContainerState(pod *kubecontainer.Pod, cid *kubecontainer.ContainerID) plegContainerState {
	// Default to the non-existent state.
	state := plegContainerNonExistent
	if pod == nil {
		return state
	}
	container := pod.FindContainerByID(*cid)
	if container == nil {
		return state
	}
	return convertState(container.State)
}

func (pr podRecords) getOld(id types.UID) *kubecontainer.Pod {
	r, ok := pr[id]
	if !ok {
		return nil
	}
	return r.old
}

func (pr podRecords) getCurrent(id types.UID) *kubecontainer.Pod {
	r, ok := pr[id]
	if !ok {
		return nil
	}
	return r.current
}

func (pr podRecords) setCurrent(pods []*kubecontainer.Pod) {
	for i := range pr {
		pr[i].current = nil
	}
	for _, pod := range pods {
		if r, ok := pr[pod.ID]; ok {
			r.current = pod
		} else {
			pr[pod.ID] = &podRecord{current: pod}
		}
	}
}

func (pr podRecords) update(id types.UID) {
	r, ok := pr[id]
	if !ok {
		return
	}
	pr.updateInternal(id, r)
}

func (pr podRecords) updateInternal(id types.UID, r *podRecord) {
	if r.current == nil {
		// Pod no longer exists; delete the entry.
		delete(pr, id)
		return
	}
	r.old = r.current
	r.current = nil
}
<|endoftext|>


<commit_before>package redis

import (
	"sync"

	"github.com/go-redis/redis/internal/pool"
)

type pipelineExecer func([]Cmder) error

type Pipeliner interface {
	StatefulCmdable
	Process(cmd Cmder) error
	Close() error
	Discard() error
	Exec() ([]Cmder, error)
}

var _ Pipeliner = (*Pipeline)(nil)

// Pipeline implements pipelining as described in
// http://redis.io/topics/pipelining. It's safe for concurrent use
// by multiple goroutines.
type Pipeline struct {
	statefulCmdable

	exec pipelineExecer

	mu     sync.Mutex
	cmds   []Cmder
	closed bool
}

func (c *Pipeline) Do(args ...interface{}) *Cmd {
	cmd := NewCmd(args...)
	_ = c.Process(cmd)
	return cmd
}

// Process queues the cmd for later execution.
func (c *Pipeline) Process(cmd Cmder) error {
	c.mu.Lock()
	c.cmds = append(c.cmds, cmd)
	c.mu.Unlock()
	return nil
}

// Close closes the pipeline, releasing any open resources.
func (c *Pipeline) Close() error {
	c.mu.Lock()
	c.discard()
	c.closed = true
	c.mu.Unlock()
	return nil
}

// Discard resets the pipeline and discards queued commands.
func (c *Pipeline) Discard() error {
	c.mu.Lock()
	err := c.discard()
	c.mu.Unlock()
	return err
}

func (c *Pipeline) discard() error {
	if c.closed {
		return pool.ErrClosed
	}
	c.cmds = c.cmds[:0]
	return nil
}

// Exec executes all previously queued commands using one
// client-server roundtrip.
//
// Exec always returns list of commands and error of the first failed
// command if any.
func (c *Pipeline) Exec() ([]Cmder, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.closed {
		return nil, pool.ErrClosed
	}

	if len(c.cmds) == 0 {
		return nil, nil
	}

	cmds := c.cmds
	c.cmds = nil

	return cmds, c.exec(cmds)
}

func (c *Pipeline) pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
	if err := fn(c); err != nil {
		return nil, err
	}
	cmds, err := c.Exec()
	_ = c.Close()
	return cmds, err
}

func (c *Pipeline) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
	return c.pipelined(fn)
}

func (c *Pipeline) Pipeline() Pipeliner {
	return c
}

func (c *Pipeline) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
	return c.pipelined(fn)
}

func (c *Pipeline) TxPipeline() Pipeliner {
	return c
}
<commit_msg>Add Pipeliner.Do<commit_after>package redis

import (
	"sync"

	"github.com/go-redis/redis/internal/pool"
)

type pipelineExecer func([]Cmder) error

type Pipeliner interface {
	StatefulCmdable
	Do(args ...interface{}) *Cmd
	Process(cmd Cmder) error
	Close() error
	Discard() error
	Exec() ([]Cmder, error)
}

var _ Pipeliner = (*Pipeline)(nil)

// Pipeline implements pipelining as described in
// http://redis.io/topics/pipelining. It's safe for concurrent use
// by multiple goroutines.
type Pipeline struct {
	statefulCmdable

	exec pipelineExecer

	mu     sync.Mutex
	cmds   []Cmder
	closed bool
}

func (c *Pipeline) Do(args ...interface{}) *Cmd {
	cmd := NewCmd(args...)
	_ = c.Process(cmd)
	return cmd
}

// Process queues the cmd for later execution.
func (c *Pipeline) Process(cmd Cmder) error {
	c.mu.Lock()
	c.cmds = append(c.cmds, cmd)
	c.mu.Unlock()
	return nil
}

// Close closes the pipeline, releasing any open resources.
func (c *Pipeline) Close() error {
	c.mu.Lock()
	c.discard()
	c.closed = true
	c.mu.Unlock()
	return nil
}

// Discard resets the pipeline and discards queued commands.
func (c *Pipeline) Discard() error {
	c.mu.Lock()
	err := c.discard()
	c.mu.Unlock()
	return err
}

func (c *Pipeline) discard() error {
	if c.closed {
		return pool.ErrClosed
	}
	c.cmds = c.cmds[:0]
	return nil
}

// Exec executes all previously queued commands using one
// client-server roundtrip.
//
// Exec always returns list of commands and error of the first failed
// command if any.
func (c *Pipeline) Exec() ([]Cmder, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.closed {
		return nil, pool.ErrClosed
	}

	if len(c.cmds) == 0 {
		return nil, nil
	}

	cmds := c.cmds
	c.cmds = nil

	return cmds, c.exec(cmds)
}

func (c *Pipeline) pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
	if err := fn(c); err != nil {
		return nil, err
	}
	cmds, err := c.Exec()
	_ = c.Close()
	return cmds, err
}

func (c *Pipeline) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
	return c.pipelined(fn)
}

func (c *Pipeline) Pipeline() Pipeliner {
	return c
}

func (c *Pipeline) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
	return c.pipelined(fn)
}

func (c *Pipeline) TxPipeline() Pipeliner {
	return c
}
<|endoftext|>


<commit_before>package modules

import (
	"encoding/json"
	"log"
	"time"

	twitch "github.com/gempir/go-twitch-irc"
	"github.com/pajlada/pajbot2/filter"
	"github.com/pkg/errors"
)

func maxpenis(a, b int) int {
	if a > b {
		return a
	}

	return b
}

type UnicodeRange struct {
	Start rune
	End   rune
}

type LatinFilter struct {
	server *server

	transparentList  *filter.TransparentList
	unicodeWhitelist []UnicodeRange
}

func NewLatinFilter() *LatinFilter {
	return &LatinFilter{
		server: &_server,

		transparentList: filter.NewTransparentList(),
	}
}

func (m *LatinFilter) addToWhitelist(start, end rune) {
	m.unicodeWhitelist = append(m.unicodeWhitelist, UnicodeRange{start, end})
}

func (m *LatinFilter) Register() error {
	m.transparentList.Add("(/ﾟДﾟ)/")
	m.transparentList.Add("(╯°□°）╯︵ ┻━┻")
	m.transparentList.Add("(╯°Д°）╯︵/(.□ . )")
	m.transparentList.Add("(ノಠ益ಠ)ノ彡┻━┻")
	m.transparentList.Add("୧༼ಠ益ಠ༽୨")
	m.transparentList.Add("༼ ºل͟º ༽")
	m.transparentList.Add("༼つಠ益ಠ༽つ")
	m.transparentList.Add("( ° ͜ʖ͡°)╭∩╮")
	m.transparentList.Add("ᕙ༼ຈل͜ຈ༽ᕗ")
	m.transparentList.Add("ʕ•ᴥ•ʔ")
	m.transparentList.Add("༼▀̿ Ĺ̯▀̿༽")
	m.transparentList.Add("( ͡° ͜🔴 ͡°)")

	err := m.transparentList.Build()
	if err != nil {
		return errors.Wrap(err, "Failed to build transparent list")
	}

	m.addToWhitelist(0x20, 0x7e)       // Basic latin
	m.addToWhitelist(0x1f600, 0x1f64f) // Emojis
	m.addToWhitelist(0x1f300, 0x1f5ff) // "Miscellaneous symbols and pictographs". Includes some emojis like 100
	m.addToWhitelist(0x1f44c, 0x1f44c) // Chatterino?
	m.addToWhitelist(0x206d, 0x206d)   // Chatterino?
	m.addToWhitelist(0x2660, 0x2765)   // Chatterino?

	m.addToWhitelist(0x1f171, 0x1f171) // B emoji
	m.addToWhitelist(0x1f900, 0x1f9ff) // More emojis

	// Rain
	m.addToWhitelist(0x30fd, 0x30fd)
	m.addToWhitelist(0xff40, 0xff40)
	m.addToWhitelist(0x3001, 0x3001)
	m.addToWhitelist(0x2602, 0x2602)

	// From Karl
	m.addToWhitelist(0x1d100, 0x1d1ff)
	m.addToWhitelist(0x1f680, 0x1f6ff)
	m.addToWhitelist(0x2600, 0x26ff)
	m.addToWhitelist(0xfe00, 0xfe0f) // Emoji variation selector 1 to 16
	m.addToWhitelist(0x2012, 0x2015) // Various dashes
	m.addToWhitelist(0x3010, 0x3011) // 【 and 】

	return nil
}

func (m LatinFilter) OnMessage(channel string, user twitch.User, message twitch.Message) error {
	if user.UserType == "" || true {
		lol := struct {
			FullMessage   string
			Message       string
			BadCharacters []rune
			Username      string
			Channel       string
			Timestamp     time.Time
		}{
			FullMessage: message.Text,
			Username:    user.Username,
			Channel:     channel,
			Timestamp:   time.Now().UTC(),
		}
		messageRunes := []rune(message.Text)
		transparentSkipRange := m.transparentList.Find(messageRunes)
		messageLength := len(messageRunes)
		for i := 0; i < messageLength; {
			if skipLength := transparentSkipRange.ShouldSkip(i); skipLength > 0 {
				i = i + skipLength
				continue
			}

			r := messageRunes[i]
			allowed := false

			for _, allowedRange := range m.unicodeWhitelist {
				if r >= allowedRange.Start && r <= allowedRange.End {
					allowed = true
					break
				}
			}

			if !allowed {
				if lol.Message == "" {
					lol.Message = message.Text[maxpenis(0, i-2):len(message.Text)]
				}

				alreadySet := false
				for _, bc := range lol.BadCharacters {
					if bc == r {
						alreadySet = true
						break
					}
				}

				if !alreadySet {
					lol.BadCharacters = append(lol.BadCharacters, r)
				}

			}
			i++
		}

		if lol.Message != "" {
			c := m.server.redis.Pool.Get()
			bytes, _ := json.Marshal(&lol)
			c.Do("LPUSH", "karl_kons", bytes)
			c.Close()
			log.Printf("First bad character: 0x%0x message '%s' from '%s' in '#%s' is disallowed due to our whitelist\n", lol.BadCharacters[0], message.Text, user.Username, channel)
		}
	}

	return nil
}
<commit_msg>Add scuffed ' to latin filter whitelist<commit_after>package modules

import (
	"encoding/json"
	"log"
	"time"

	twitch "github.com/gempir/go-twitch-irc"
	"github.com/pajlada/pajbot2/filter"
	"github.com/pkg/errors"
)

func maxpenis(a, b int) int {
	if a > b {
		return a
	}

	return b
}

type UnicodeRange struct {
	Start rune
	End   rune
}

type LatinFilter struct {
	server *server

	transparentList  *filter.TransparentList
	unicodeWhitelist []UnicodeRange
}

func NewLatinFilter() *LatinFilter {
	return &LatinFilter{
		server: &_server,

		transparentList: filter.NewTransparentList(),
	}
}

func (m *LatinFilter) addToWhitelist(start, end rune) {
	m.unicodeWhitelist = append(m.unicodeWhitelist, UnicodeRange{start, end})
}

func (m *LatinFilter) Register() error {
	m.transparentList.Add("(/ﾟДﾟ)/")
	m.transparentList.Add("(╯°□°）╯︵ ┻━┻")
	m.transparentList.Add("(╯°Д°）╯︵/(.□ . )")
	m.transparentList.Add("(ノಠ益ಠ)ノ彡┻━┻")
	m.transparentList.Add("୧༼ಠ益ಠ༽୨")
	m.transparentList.Add("༼ ºل͟º ༽")
	m.transparentList.Add("༼つಠ益ಠ༽つ")
	m.transparentList.Add("( ° ͜ʖ͡°)╭∩╮")
	m.transparentList.Add("ᕙ༼ຈل͜ຈ༽ᕗ")
	m.transparentList.Add("ʕ•ᴥ•ʔ")
	m.transparentList.Add("༼▀̿ Ĺ̯▀̿༽")
	m.transparentList.Add("( ͡° ͜🔴 ͡°)")

	err := m.transparentList.Build()
	if err != nil {
		return errors.Wrap(err, "Failed to build transparent list")
	}

	m.addToWhitelist(0x20, 0x7e)       // Basic latin
	m.addToWhitelist(0x1f600, 0x1f64f) // Emojis
	m.addToWhitelist(0x1f300, 0x1f5ff) // "Miscellaneous symbols and pictographs". Includes some emojis like 100
	m.addToWhitelist(0x1f44c, 0x1f44c) // Chatterino?
	m.addToWhitelist(0x206d, 0x206d)   // Chatterino?
	m.addToWhitelist(0x2660, 0x2765)   // Chatterino?

	m.addToWhitelist(0x1f171, 0x1f171) // B emoji
	m.addToWhitelist(0x1f900, 0x1f9ff) // More emojis

	m.addToWhitelist(0x2019, 0x2019) // Scuffed '

	// Rain
	m.addToWhitelist(0x30fd, 0x30fd)
	m.addToWhitelist(0xff40, 0xff40)
	m.addToWhitelist(0x3001, 0x3001)
	m.addToWhitelist(0x2602, 0x2602)

	// From Karl
	m.addToWhitelist(0x1d100, 0x1d1ff)
	m.addToWhitelist(0x1f680, 0x1f6ff)
	m.addToWhitelist(0x2600, 0x26ff)
	m.addToWhitelist(0xfe00, 0xfe0f) // Emoji variation selector 1 to 16
	m.addToWhitelist(0x2012, 0x2015) // Various dashes
	m.addToWhitelist(0x3010, 0x3011) // 【 and 】

	return nil
}

func (m LatinFilter) OnMessage(channel string, user twitch.User, message twitch.Message) error {
	if user.UserType == "" || true {
		lol := struct {
			FullMessage   string
			Message       string
			BadCharacters []rune
			Username      string
			Channel       string
			Timestamp     time.Time
		}{
			FullMessage: message.Text,
			Username:    user.Username,
			Channel:     channel,
			Timestamp:   time.Now().UTC(),
		}
		messageRunes := []rune(message.Text)
		transparentSkipRange := m.transparentList.Find(messageRunes)
		messageLength := len(messageRunes)
		for i := 0; i < messageLength; {
			if skipLength := transparentSkipRange.ShouldSkip(i); skipLength > 0 {
				i = i + skipLength
				continue
			}

			r := messageRunes[i]
			allowed := false

			for _, allowedRange := range m.unicodeWhitelist {
				if r >= allowedRange.Start && r <= allowedRange.End {
					allowed = true
					break
				}
			}

			if !allowed {
				if lol.Message == "" {
					lol.Message = message.Text[maxpenis(0, i-2):len(message.Text)]
				}

				alreadySet := false
				for _, bc := range lol.BadCharacters {
					if bc == r {
						alreadySet = true
						break
					}
				}

				if !alreadySet {
					lol.BadCharacters = append(lol.BadCharacters, r)
				}

			}
			i++
		}

		if lol.Message != "" {
			c := m.server.redis.Pool.Get()
			bytes, _ := json.Marshal(&lol)
			c.Do("LPUSH", "karl_kons", bytes)
			c.Close()
			log.Printf("First bad character: 0x%0x message '%s' from '%s' in '#%s' is disallowed due to our whitelist\n", lol.BadCharacters[0], message.Text, user.Username, channel)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package network

import (
	"reflect"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"nimona.io/internal/fixtures"
	"nimona.io/internal/net"
	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
	"nimona.io/pkg/tilde"
)

func TestNetwork_SimpleConnection(t *testing.T) {
	k1, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	k2, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	n1 := New(context.Background(), net.New(k1), k1)
	n2 := New(context.Background(), net.New(k2), k2)

	l1, err := n1.Listen(context.Background(), "127.0.0.1:0", ListenOnLocalIPs)
	require.NoError(t, err)
	defer l1.Close()

	l2, err := n2.Listen(context.Background(), "127.0.0.1:0", ListenOnLocalIPs)
	require.NoError(t, err)
	defer l2.Close()

	testObj := &object.Object{
		Type: "foo",
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}

	// subscribe to objects of type "foo" coming to n2
	s2 := n2.Subscribe(
		FilterByObjectType("foo"),
	)
	require.NotNil(t, s2)

	// send from p1 to p2
	err = n1.Send(
		context.Background(),
		testObj,
		n2.GetPeerKey().PublicKey(),
		SendWithConnectionInfo(
			&peer.ConnectionInfo{
				PublicKey: n2.GetPeerKey().PublicKey(),
				Addresses: n2.GetAddresses(),
			},
		),
	)
	require.NoError(t, err)

	// wait for event from n1 to arrive
	env, err := s2.Next()
	require.NoError(t, err)
	assert.Equal(t, testObj, env.Payload)

	// subscribe to all objects coming to n1
	s1 := n1.Subscribe()

	// send from p2 to p1
	err = n2.Send(
		context.Background(),
		testObj,
		n1.GetPeerKey().PublicKey(),
		SendWithConnectionInfo(
			&peer.ConnectionInfo{
				PublicKey: n1.GetPeerKey().PublicKey(),
				Addresses: n1.GetAddresses(),
			},
		),
	)
	require.NoError(t, err)

	// next object should be our foo
	env, err = s1.Next()
	require.NoError(t, err)
	require.NotNil(t, s1)
	assert.Equal(t, testObj, env.Payload)

	t.Run("re-establish broken connections", func(t *testing.T) {
		// close p2's connection to p1
		c, err := n2.(*network).net.Dial(
			context.New(),
			&peer.ConnectionInfo{
				PublicKey: n1.GetPeerKey().PublicKey(),
			},
		)
		require.NoError(t, err)
		err = c.Close()
		require.NoError(t, err)
		// try to send something from p1 to p2
		err = n1.Send(
			context.Background(),
			testObj,
			n2.GetPeerKey().PublicKey(),
			SendWithConnectionInfo(
				&peer.ConnectionInfo{
					PublicKey: n2.GetPeerKey().PublicKey(),
					Addresses: n2.GetAddresses(),
				},
			),
		)
		require.NoError(t, err)
	})

	t.Run("wait for response", func(t *testing.T) {
		req := &fixtures.TestRequest{
			RequestID: "1",
			Foo:       "bar",
		}
		res := &fixtures.TestResponse{
			RequestID: "1",
			Foo:       "bar",
		}
		// sub for p2 based on rID
		gotRes := &fixtures.TestResponse{}
		reqSub := n2.Subscribe(
			FilterByRequestID("1"),
		)
		// send request from p1 to p2 in a go routine
		sendErr := make(chan error)
		go func() {
			reqo, err := object.Marshal(req)
			require.NoError(t, err)
			sendErr <- n1.Send(
				context.Background(),
				reqo,
				n2.GetPeerKey().PublicKey(),
				SendWithResponse(gotRes, 0),
			)
		}()
		// wait for p2 to get the req
		gotReq := <-reqSub.Channel()
		assert.Equal(t, "1", string(gotReq.Payload.Data["requestID"].(tilde.String)))
		// send response from p2 to p1
		reso, err := object.Marshal(res)
		require.NoError(t, err)
		require.NoError(t, err)
		// nolint: errcheck
		n2.Send(
			context.Background(),
			reso,
			n1.GetPeerKey().PublicKey(),
			SendWithConnectionInfo(
				&peer.ConnectionInfo{
					PublicKey: n1.GetPeerKey().PublicKey(),
					Addresses: n1.GetAddresses(),
				},
			),
		)
		// check response
		err = <-sendErr
		require.NoError(t, err)
		assert.Equal(t, res, gotRes)
	})
}

func TestNetwork_Relay(t *testing.T) {
	k0, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	k1, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	k2, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	n0 := New(context.Background(), net.New(k0), k0)
	n1 := New(context.Background(), net.New(k1), k1)
	n2 := New(context.Background(), net.New(k2), k2)

	l0, err := n0.Listen(context.Background(), "127.0.0.1:0", ListenOnLocalIPs)
	require.NoError(t, err)
	defer l0.Close()

	p0 := &peer.ConnectionInfo{
		PublicKey: n0.GetPeerKey().PublicKey(),
		Addresses: n0.GetAddresses(),
	}

	p1 := &peer.ConnectionInfo{
		PublicKey: n1.GetPeerKey().PublicKey(),
		Addresses: n1.GetAddresses(),
		Relays: []*peer.ConnectionInfo{
			p0,
		},
	}

	p2 := &peer.ConnectionInfo{
		PublicKey: n2.GetPeerKey().PublicKey(),
		Addresses: n2.GetAddresses(),
		Relays: []*peer.ConnectionInfo{
			p0,
		},
	}

	testObj := &object.Object{
		Type: "foo",
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}

	testObjFromP1 := &object.Object{
		Type: "foo",
		Metadata: object.Metadata{
			Owner: n1.GetPeerKey().PublicKey().DID(),
		},
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}

	testObjFromP2 := &object.Object{
		Type: "foo",
		Metadata: object.Metadata{
			Owner: n2.GetPeerKey().PublicKey().DID(),
		},
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}

	// send from p1 to p0
	err = n1.Send(
		context.Background(),
		testObj,
		p0.PublicKey,
		SendWithConnectionInfo(p0),
	)
	require.NoError(t, err)

	// send from p2 to p0
	err = n2.Send(
		context.Background(),
		testObj,
		p0.PublicKey,
		SendWithConnectionInfo(p0),
	)
	require.NoError(t, err)

	// now we should be able to send from p1 to p2
	sub := n2.Subscribe(FilterByObjectType("foo"))
	err = n1.Send(
		context.Background(),
		testObjFromP1,
		p2.PublicKey,
		SendWithConnectionInfo(p2),
	)
	require.NoError(t, err)

	env, err := sub.Next()
	require.NoError(t, err)

	require.NotNil(t, sub)
	assert.Equal(t,
		testObjFromP1.Metadata.Signature,
		env.Payload.Metadata.Signature,
	)

	// send from p2 to p1
	sub = n1.Subscribe(FilterByObjectType("foo"))

	err = n2.Send(
		context.Background(),
		testObjFromP2,
		p1.PublicKey,
		SendWithConnectionInfo(p1),
	)
	require.NoError(t, err)

	env, err = sub.Next()
	require.NoError(t, err)

	require.NotNil(t, sub)
	assert.Equal(t,
		testObjFromP2.Metadata.Signature,
		env.Payload.Metadata.Signature,
	)
}

func Test_network_lookup(t *testing.T) {
	p0, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	fooConnInfo := &peer.ConnectionInfo{
		Version:   1,
		PublicKey: p0.PublicKey(),
		Addresses: []string{"a", "b"},
	}
	type fields struct {
		resolvers []Resolver
	}
	type args struct {
		publicKey crypto.PublicKey
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		want    *peer.ConnectionInfo
		wantErr bool
	}{{
		name: "one resolver, returns, should pass",
		fields: fields{
			resolvers: []Resolver{
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{
						fooConnInfo.PublicKey.String(): fooConnInfo,
					},
				},
			},
		},
		args: args{
			publicKey: fooConnInfo.PublicKey,
		},
		want: fooConnInfo,
	}, {
		name: "two resolver, second returns, should pass",
		fields: fields{
			resolvers: []Resolver{
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{},
				},
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{
						fooConnInfo.PublicKey.String(): fooConnInfo,
					},
				},
			},
		},
		args: args{
			publicKey: fooConnInfo.PublicKey,
		},
		want: fooConnInfo,
	}, {
		name: "two resolver, none returns, should fail",
		fields: fields{
			resolvers: []Resolver{
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{},
				},
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{},
				},
			},
		},
		args: args{
			publicKey: fooConnInfo.PublicKey,
		},
		want:    nil,
		wantErr: true,
	}}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			k, err := crypto.NewEd25519PrivateKey()
			require.NoError(t, err)
			w := New(
				context.Background(),
				net.New(k),
				k,
			).(*network)
			for _, r := range tt.fields.resolvers {
				w.RegisterResolver(r)
			}
			got, err := w.lookup(context.Background(), tt.args.publicKey)
			if (err != nil) != tt.wantErr {
				t.Errorf("error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("got %v, want %v", got, tt.want)
			}
		})
	}
}

func BenchmarkNetworkSendToSinglePeer(b *testing.B) {
	k1, err := crypto.NewEd25519PrivateKey()
	require.NoError(b, err)
	n1 := New(context.Background(), net.New(k1), k1).(*network)

	l1, err := n1.Listen(context.Background(), "127.0.0.1:0", ListenOnLocalIPs)
	require.NoError(b, err)
	defer l1.Close()

	n1s := n1.Subscribe(FilterByObjectType("foo")).Channel()

	for n := 0; n < b.N; n++ {
		k2, err := crypto.NewEd25519PrivateKey()
		require.NoError(b, err)
		n2 := New(context.Background(), net.New(k2), k2).(*network)
		err = n2.Send(
			context.Background(),
			&object.Object{
				Type: "foo",
				Data: tilde.Map{
					"foo": tilde.String("bar"),
				},
			},
			k1.PublicKey(),
			SendWithConnectionInfo(
				&peer.ConnectionInfo{
					PublicKey: k1.PublicKey(),
					Addresses: n1.GetAddresses(),
				},
			),
		)
		require.NoError(b, err)
		select {
		case env := <-n1s:
			require.NotNil(b, env)
		case <-time.After(time.Second * 2):
			b.Fatal("timeout")
		}
		err = n2.Close()
		require.NoError(b, err)
	}
}

type testResolver struct {
	peers map[string]*peer.ConnectionInfo
}

func (r *testResolver) LookupPeer(
	ctx context.Context,
	publicKey crypto.PublicKey,
) (*peer.ConnectionInfo, error) {
	c, ok := r.peers[publicKey.String()]
	if !ok || c == nil {
		return nil, errors.Error("not found")
	}
	return c, nil
}
<commit_msg>fix(network): fix network test for GOMAXPROCS=1<commit_after>package network

import (
	"reflect"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"nimona.io/internal/fixtures"
	"nimona.io/internal/net"
	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
	"nimona.io/pkg/tilde"
)

func TestNetwork_SimpleConnection(t *testing.T) {
	k1, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	k2, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	n1 := New(context.Background(), net.New(k1), k1)
	n2 := New(context.Background(), net.New(k2), k2)

	l1, err := n1.Listen(context.Background(), "127.0.0.1:0", ListenOnLocalIPs)
	require.NoError(t, err)
	defer l1.Close()

	l2, err := n2.Listen(context.Background(), "127.0.0.1:0", ListenOnLocalIPs)
	require.NoError(t, err)
	defer l2.Close()

	testObj := &object.Object{
		Type: "foo",
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}

	// subscribe to objects of type "foo" coming to n2
	s2 := n2.Subscribe(
		FilterByObjectType("foo"),
	)
	require.NotNil(t, s2)

	// send from p1 to p2
	err = n1.Send(
		context.Background(),
		testObj,
		n2.GetPeerKey().PublicKey(),
		SendWithConnectionInfo(
			&peer.ConnectionInfo{
				PublicKey: n2.GetPeerKey().PublicKey(),
				Addresses: n2.GetAddresses(),
			},
		),
	)
	require.NoError(t, err)

	// wait for event from n1 to arrive
	env, err := s2.Next()
	require.NoError(t, err)
	assert.Equal(t, testObj, env.Payload)

	// subscribe to all objects coming to n1
	s1 := n1.Subscribe()

	// send from p2 to p1
	err = n2.Send(
		context.Background(),
		testObj,
		n1.GetPeerKey().PublicKey(),
		SendWithConnectionInfo(
			&peer.ConnectionInfo{
				PublicKey: n1.GetPeerKey().PublicKey(),
				Addresses: n1.GetAddresses(),
			},
		),
	)
	require.NoError(t, err)

	// next object should be our foo
	env, err = s1.Next()
	require.NoError(t, err)
	require.NotNil(t, s1)
	assert.Equal(t, testObj, env.Payload)

	t.Run("re-establish broken connections", func(t *testing.T) {
		// close p2's connection to p1
		c, err := n2.(*network).net.Dial(
			context.New(),
			&peer.ConnectionInfo{
				PublicKey: n1.GetPeerKey().PublicKey(),
			},
		)
		require.NoError(t, err)
		err = c.Close()
		require.NoError(t, err)
		// try to send something from p1 to p2
		err = n1.Send(
			context.Background(),
			testObj,
			n2.GetPeerKey().PublicKey(),
			SendWithConnectionInfo(
				&peer.ConnectionInfo{
					PublicKey: n2.GetPeerKey().PublicKey(),
					Addresses: n2.GetAddresses(),
				},
			),
		)
		require.NoError(t, err)
		// try to send something from p2 to p1
		err = n2.Send(
			context.Background(),
			testObj,
			k1.PublicKey(),
			SendWithConnectionInfo(
				&peer.ConnectionInfo{
					PublicKey: k1.PublicKey(),
					Addresses: n1.GetAddresses(),
				},
			),
		)
		require.NoError(t, err)
	})

	t.Run("wait for response", func(t *testing.T) {
		req := &fixtures.TestRequest{
			RequestID: "1",
			Foo:       "bar",
		}
		res := &fixtures.TestResponse{
			RequestID: "1",
			Foo:       "bar",
		}
		// sub for p2 based on rID
		gotRes := &fixtures.TestResponse{}
		reqSub := n2.Subscribe(
			FilterByRequestID("1"),
		)
		// send request from p1 to p2 in a go routine
		sendErr := make(chan error)
		go func() {
			reqo, err := object.Marshal(req)
			require.NoError(t, err)
			err = n1.Send(
				context.Background(),
				reqo,
				n2.GetPeerKey().PublicKey(),
				SendWithConnectionInfo(
					&peer.ConnectionInfo{
						PublicKey: n2.GetPeerKey().PublicKey(),
						Addresses: n2.GetAddresses(),
					},
				),
				SendWithResponse(gotRes, 0),
			)
			require.NoError(t, err)
			sendErr <- err
		}()
		// wait for p2 to get the req
		select {
		case gotReq := <-reqSub.Channel():
			v := string(gotReq.Payload.Data["requestID"].(tilde.String))
			assert.Equal(t, "1", v)
		case <-time.After(time.Second * 2):
			t.Fatal("timed out waiting for request")
		}
		// send response from p2 to p1
		reso, err := object.Marshal(res)
		require.NoError(t, err)
		require.NoError(t, err)
		// nolint: errcheck
		n2.Send(
			context.Background(),
			reso,
			n1.GetPeerKey().PublicKey(),
			SendWithConnectionInfo(
				&peer.ConnectionInfo{
					PublicKey: n1.GetPeerKey().PublicKey(),
					Addresses: n1.GetAddresses(),
				},
			),
		)
		// check response
		select {
		case err := <-sendErr:
			require.NoError(t, err)
			assert.Equal(t, res, gotRes)
		case <-time.After(time.Second * 2):
			t.Fatal("timeout waiting for response")
		}
	})
}

func TestNetwork_Relay(t *testing.T) {
	k0, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	k1, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	k2, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	n0 := New(context.Background(), net.New(k0), k0)
	n1 := New(context.Background(), net.New(k1), k1)
	n2 := New(context.Background(), net.New(k2), k2)

	l0, err := n0.Listen(context.Background(), "127.0.0.1:0", ListenOnLocalIPs)
	require.NoError(t, err)
	defer l0.Close()

	p0 := &peer.ConnectionInfo{
		PublicKey: n0.GetPeerKey().PublicKey(),
		Addresses: n0.GetAddresses(),
	}

	p1 := &peer.ConnectionInfo{
		PublicKey: n1.GetPeerKey().PublicKey(),
		Addresses: n1.GetAddresses(),
		Relays: []*peer.ConnectionInfo{
			p0,
		},
	}

	p2 := &peer.ConnectionInfo{
		PublicKey: n2.GetPeerKey().PublicKey(),
		Addresses: n2.GetAddresses(),
		Relays: []*peer.ConnectionInfo{
			p0,
		},
	}

	testObj := &object.Object{
		Type: "foo",
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}

	testObjFromP1 := &object.Object{
		Type: "foo",
		Metadata: object.Metadata{
			Owner: n1.GetPeerKey().PublicKey().DID(),
		},
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}

	testObjFromP2 := &object.Object{
		Type: "foo",
		Metadata: object.Metadata{
			Owner: n2.GetPeerKey().PublicKey().DID(),
		},
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}

	// send from p1 to p0
	err = n1.Send(
		context.Background(),
		testObj,
		p0.PublicKey,
		SendWithConnectionInfo(p0),
	)
	require.NoError(t, err)

	// send from p2 to p0
	err = n2.Send(
		context.Background(),
		testObj,
		p0.PublicKey,
		SendWithConnectionInfo(p0),
	)
	require.NoError(t, err)

	// now we should be able to send from p1 to p2
	sub := n2.Subscribe(FilterByObjectType("foo"))
	err = n1.Send(
		context.Background(),
		testObjFromP1,
		p2.PublicKey,
		SendWithConnectionInfo(p2),
	)
	require.NoError(t, err)

	env, err := sub.Next()
	require.NoError(t, err)

	require.NotNil(t, sub)
	assert.Equal(t,
		testObjFromP1.Metadata.Signature,
		env.Payload.Metadata.Signature,
	)

	// send from p2 to p1
	sub = n1.Subscribe(FilterByObjectType("foo"))

	err = n2.Send(
		context.Background(),
		testObjFromP2,
		p1.PublicKey,
		SendWithConnectionInfo(p1),
	)
	require.NoError(t, err)

	env, err = sub.Next()
	require.NoError(t, err)

	require.NotNil(t, sub)
	assert.Equal(t,
		testObjFromP2.Metadata.Signature,
		env.Payload.Metadata.Signature,
	)
}

func Test_network_lookup(t *testing.T) {
	p0, err := crypto.NewEd25519PrivateKey()
	require.NoError(t, err)

	fooConnInfo := &peer.ConnectionInfo{
		Version:   1,
		PublicKey: p0.PublicKey(),
		Addresses: []string{"a", "b"},
	}
	type fields struct {
		resolvers []Resolver
	}
	type args struct {
		publicKey crypto.PublicKey
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		want    *peer.ConnectionInfo
		wantErr bool
	}{{
		name: "one resolver, returns, should pass",
		fields: fields{
			resolvers: []Resolver{
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{
						fooConnInfo.PublicKey.String(): fooConnInfo,
					},
				},
			},
		},
		args: args{
			publicKey: fooConnInfo.PublicKey,
		},
		want: fooConnInfo,
	}, {
		name: "two resolver, second returns, should pass",
		fields: fields{
			resolvers: []Resolver{
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{},
				},
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{
						fooConnInfo.PublicKey.String(): fooConnInfo,
					},
				},
			},
		},
		args: args{
			publicKey: fooConnInfo.PublicKey,
		},
		want: fooConnInfo,
	}, {
		name: "two resolver, none returns, should fail",
		fields: fields{
			resolvers: []Resolver{
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{},
				},
				&testResolver{
					peers: map[string]*peer.ConnectionInfo{},
				},
			},
		},
		args: args{
			publicKey: fooConnInfo.PublicKey,
		},
		want:    nil,
		wantErr: true,
	}}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			k, err := crypto.NewEd25519PrivateKey()
			require.NoError(t, err)
			w := New(
				context.Background(),
				net.New(k),
				k,
			).(*network)
			for _, r := range tt.fields.resolvers {
				w.RegisterResolver(r)
			}
			got, err := w.lookup(context.Background(), tt.args.publicKey)
			if (err != nil) != tt.wantErr {
				t.Errorf("error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("got %v, want %v", got, tt.want)
			}
		})
	}
}

func BenchmarkNetworkSendToSinglePeer(b *testing.B) {
	k1, err := crypto.NewEd25519PrivateKey()
	require.NoError(b, err)
	n1 := New(context.Background(), net.New(k1), k1).(*network)

	l1, err := n1.Listen(context.Background(), "127.0.0.1:0", ListenOnLocalIPs)
	require.NoError(b, err)
	defer l1.Close()

	n1s := n1.Subscribe(FilterByObjectType("foo")).Channel()

	for n := 0; n < b.N; n++ {
		k2, err := crypto.NewEd25519PrivateKey()
		require.NoError(b, err)
		n2 := New(context.Background(), net.New(k2), k2).(*network)
		err = n2.Send(
			context.Background(),
			&object.Object{
				Type: "foo",
				Data: tilde.Map{
					"foo": tilde.String("bar"),
				},
			},
			k1.PublicKey(),
			SendWithConnectionInfo(
				&peer.ConnectionInfo{
					PublicKey: k1.PublicKey(),
					Addresses: n1.GetAddresses(),
				},
			),
		)
		require.NoError(b, err)
		select {
		case env := <-n1s:
			require.NotNil(b, env)
		case <-time.After(time.Second * 2):
			b.Fatal("timeout")
		}
		err = n2.Close()
		require.NoError(b, err)
	}
}

type testResolver struct {
	peers map[string]*peer.ConnectionInfo
}

func (r *testResolver) LookupPeer(
	ctx context.Context,
	publicKey crypto.PublicKey,
) (*peer.ConnectionInfo, error) {
	c, ok := r.peers[publicKey.String()]
	if !ok || c == nil {
		return nil, errors.Error("not found")
	}
	return c, nil
}
<|endoftext|>


<commit_before>package rabbitmqpool

import (
	"fmt"
	"strings"

	"github.com/rxwen/resourcepool"
	"github.com/rxwen/srvresolver"
	"github.com/streadway/amqp"
)

// CreateRabbitmqConnectionPool function creates a connection for specified Rabbitmq service.
func CreateRabbitmqConnectionPool(rabbitmqService string, poolSize int, timeoutSecond int) (*resourcepool.ResourcePool, error) {
	if rabbitmqService[len(rabbitmqService)-1] == '/' {
		rabbitmqService = rabbitmqService[0 : len(rabbitmqService)-1]
	}
	RabbitmqPool, err := resourcepool.NewResourcePool("", "", func(host, port string) (interface{}, error) {
		s := strings.Split(rabbitmqService, "@")
		server, port, err := srvresolver.ResolveSRV(s[1])
		if err != nil {
			return nil, err
		}
		url := fmt.Sprintf("%s@%s:%s/", s[0], server, port)
		c, err := amqp.Dial(url)
		return c, err
	}, func(c interface{}) error {
		c.(*amqp.Connection).Close()
		return nil
	}, poolSize, timeoutSecond)
	return RabbitmqPool, err
}
<commit_msg>check rabbitmq service endpoint format<commit_after>package rabbitmqpool

import (
	"errors"
	"fmt"
	"strings"

	"github.com/rxwen/resourcepool"
	"github.com/rxwen/srvresolver"
	"github.com/streadway/amqp"
)

// CreateRabbitmqConnectionPool function creates a connection for specified Rabbitmq service.
func CreateRabbitmqConnectionPool(rabbitmqService string, poolSize int, timeoutSecond int) (*resourcepool.ResourcePool, error) {
	if rabbitmqService[len(rabbitmqService)-1] == '/' {
		rabbitmqService = rabbitmqService[0 : len(rabbitmqService)-1]
	}
	RabbitmqPool, err := resourcepool.NewResourcePool("", "", func(host, port string) (interface{}, error) {
		s := strings.Split(rabbitmqService, "@")
		if len(s) != 2 {
			return nil, errors.New("invalid rabbitmq service endpoint, should be amqp://user:pass@host:port")
		}
		server, port, err := srvresolver.ResolveSRV(s[1])
		if err != nil {
			return nil, err
		}
		url := fmt.Sprintf("%s@%s:%s/", s[0], server, port)
		c, err := amqp.Dial(url)
		return c, err
	}, func(c interface{}) error {
		c.(*amqp.Connection).Close()
		return nil
	}, poolSize, timeoutSecond)
	return RabbitmqPool, err
}
<|endoftext|>


<commit_before>// Package gotest is a standard Go test output parser.
package gotest

import (
	"bufio"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jstemmer/go-junit-report/v2/pkg/gtr"
)

var (
	// regexBenchmark captures 3-5 groups: benchmark name, number of times ran, ns/op (with or without decimal), MB/sec (optional), B/op (optional), and allocs/op (optional).
	regexBenchmark = regexp.MustCompile(`^(Benchmark[^ -]+)(?:-\d+\s+|\s+)(\d+)\s+(\d+|\d+\.\d+)\sns\/op(?:\s+(\d+|\d+\.\d+)\sMB\/s)?(?:\s+(\d+)\sB\/op)?(?:\s+(\d+)\sallocs/op)?`)
	regexCoverage  = regexp.MustCompile(`^coverage:\s+(\d+|\d+\.\d+)%\s+of\s+statements(?:\sin\s(.+))?$`)
	regexEndTest   = regexp.MustCompile(`((?:    )*)--- (PASS|FAIL|SKIP): ([^ ]+) \((\d+\.\d+)(?: seconds|s)\)`)
	regexStatus    = regexp.MustCompile(`^(PASS|FAIL|SKIP)$`)
	regexSummary   = regexp.MustCompile(`^(ok|FAIL)\s+([^ ]+)\s+(?:(\d+\.\d+)s|(\(cached\)|\[\w+ failed]))(?:\s+coverage:\s+(\d+\.\d+)%\sof\sstatements(?:\sin\s(.+))?)?$`)
)

// Parse parses Go test output from the given io.Reader r.
func Parse(r io.Reader) ([]gtr.Event, error) {
	p := &parser{}

	s := bufio.NewScanner(r)
	for s.Scan() {
		p.parseLine(s.Text())
	}
	return p.events, s.Err()
}

type parser struct {
	events []gtr.Event
}

func (p *parser) parseLine(line string) {
	if strings.HasPrefix(line, "=== RUN ") {
		p.runTest(strings.TrimSpace(line[8:]))
	} else if strings.HasPrefix(line, "=== PAUSE ") {
		p.pauseTest(strings.TrimSpace(line[10:]))
	} else if strings.HasPrefix(line, "=== CONT ") {
		p.contTest(strings.TrimSpace(line[9:]))
	} else if matches := regexEndTest.FindStringSubmatch(line); len(matches) == 5 {
		p.endTest(line, matches[1], matches[2], matches[3], matches[4])
	} else if matches := regexStatus.FindStringSubmatch(line); len(matches) == 2 {
		p.status(matches[1])
	} else if matches := regexSummary.FindStringSubmatch(line); len(matches) == 7 {
		p.summary(matches[1], matches[2], matches[3], matches[4], matches[5], matches[6])
	} else if matches := regexCoverage.FindStringSubmatch(line); len(matches) == 3 {
		p.coverage(matches[1], matches[2])
	} else if matches := regexBenchmark.FindStringSubmatch(line); len(matches) == 7 {
		p.benchmark(matches[1], matches[2], matches[3], matches[4], matches[5], matches[6])
	} else if strings.HasPrefix(line, "# ") {
		fields := strings.Fields(strings.TrimPrefix(line, "# "))
		if len(fields) == 1 || len(fields) == 2 {
			p.buildOutput(fields[0])
		} else {
			p.output(line)
		}
	} else {
		p.output(line)
	}
}

func (p *parser) add(event gtr.Event) {
	p.events = append(p.events, event)
}

func (p *parser) runTest(name string) {
	p.add(gtr.Event{Type: "run_test", Name: name})
}

func (p *parser) pauseTest(name string) {
	p.add(gtr.Event{Type: "pause_test", Name: name})
}

func (p *parser) contTest(name string) {
	p.add(gtr.Event{Type: "cont_test", Name: name})
}

func (p *parser) endTest(line, indent, result, name, duration string) {
	if idx := strings.Index(line, fmt.Sprintf("%s--- %s:", indent, result)); idx > 0 {
		p.output(line[:idx])
	}
	_, n := stripIndent(indent)
	p.add(gtr.Event{
		Type:     "end_test",
		Name:     name,
		Result:   result,
		Indent:   n,
		Duration: parseSeconds(duration),
	})
}

func (p *parser) status(result string) {
	p.add(gtr.Event{Type: "status", Result: result})
}

func (p *parser) summary(result, name, duration, data, covpct, packages string) {
	p.add(gtr.Event{
		Type:        "summary",
		Result:      result,
		Name:        name,
		Duration:    parseSeconds(duration),
		Data:        data,
		CovPct:      parseFloat(covpct),
		CovPackages: parsePackages(packages),
	})
}

func (p *parser) coverage(percent, packages string) {
	p.add(gtr.Event{
		Type:        "coverage",
		CovPct:      parseFloat(percent),
		CovPackages: parsePackages(packages),
	})
}

func (p *parser) benchmark(name, iterations, nsPerOp, mbPerSec, bytesPerOp, allocsPerOp string) {
	p.add(gtr.Event{
		Type:        "benchmark",
		Name:        name,
		Iterations:  parseInt(iterations),
		NsPerOp:     parseFloat(nsPerOp),
		MBPerSec:    parseFloat(mbPerSec),
		BytesPerOp:  parseInt(bytesPerOp),
		AllocsPerOp: parseInt(allocsPerOp),
	})
}

func (p *parser) buildOutput(packageName string) {
	p.add(gtr.Event{
		Type: "build_output",
		Name: packageName,
	})
}

func (p *parser) output(line string) {
	p.add(gtr.Event{Type: "output", Data: line})
}

func parseSeconds(s string) time.Duration {
	if s == "" {
		return time.Duration(0)
	}
	// ignore error
	d, _ := time.ParseDuration(s + "s")
	return d
}

func parseFloat(s string) float64 {
	if s == "" {
		return 0
	}
	// ignore error
	pct, _ := strconv.ParseFloat(s, 64)
	return pct
}

func parsePackages(pkgList string) []string {
	if len(pkgList) == 0 {
		return nil
	}
	return strings.Split(pkgList, ", ")
}

func parseInt(s string) int64 {
	// ignore error
	n, _ := strconv.ParseInt(s, 10, 64)
	return n
}

func stripIndent(line string) (string, int) {
	var indent int
	for indent = 0; strings.HasPrefix(line, "    "); indent++ {
		line = line[4:]
	}
	return line, indent
}
<commit_msg>parser/gotest: Add TODO to remove the build_output type<commit_after>// Package gotest is a standard Go test output parser.
package gotest

import (
	"bufio"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jstemmer/go-junit-report/v2/pkg/gtr"
)

var (
	// regexBenchmark captures 3-5 groups: benchmark name, number of times ran, ns/op (with or without decimal), MB/sec (optional), B/op (optional), and allocs/op (optional).
	regexBenchmark = regexp.MustCompile(`^(Benchmark[^ -]+)(?:-\d+\s+|\s+)(\d+)\s+(\d+|\d+\.\d+)\sns\/op(?:\s+(\d+|\d+\.\d+)\sMB\/s)?(?:\s+(\d+)\sB\/op)?(?:\s+(\d+)\sallocs/op)?`)
	regexCoverage  = regexp.MustCompile(`^coverage:\s+(\d+|\d+\.\d+)%\s+of\s+statements(?:\sin\s(.+))?$`)
	regexEndTest   = regexp.MustCompile(`((?:    )*)--- (PASS|FAIL|SKIP): ([^ ]+) \((\d+\.\d+)(?: seconds|s)\)`)
	regexStatus    = regexp.MustCompile(`^(PASS|FAIL|SKIP)$`)
	regexSummary   = regexp.MustCompile(`^(ok|FAIL)\s+([^ ]+)\s+(?:(\d+\.\d+)s|(\(cached\)|\[\w+ failed]))(?:\s+coverage:\s+(\d+\.\d+)%\sof\sstatements(?:\sin\s(.+))?)?$`)
)

// Parse parses Go test output from the given io.Reader r.
func Parse(r io.Reader) ([]gtr.Event, error) {
	p := &parser{}

	s := bufio.NewScanner(r)
	for s.Scan() {
		p.parseLine(s.Text())
	}
	return p.events, s.Err()
}

type parser struct {
	events []gtr.Event
}

func (p *parser) parseLine(line string) {
	if strings.HasPrefix(line, "=== RUN ") {
		p.runTest(strings.TrimSpace(line[8:]))
	} else if strings.HasPrefix(line, "=== PAUSE ") {
		p.pauseTest(strings.TrimSpace(line[10:]))
	} else if strings.HasPrefix(line, "=== CONT ") {
		p.contTest(strings.TrimSpace(line[9:]))
	} else if matches := regexEndTest.FindStringSubmatch(line); len(matches) == 5 {
		p.endTest(line, matches[1], matches[2], matches[3], matches[4])
	} else if matches := regexStatus.FindStringSubmatch(line); len(matches) == 2 {
		p.status(matches[1])
	} else if matches := regexSummary.FindStringSubmatch(line); len(matches) == 7 {
		p.summary(matches[1], matches[2], matches[3], matches[4], matches[5], matches[6])
	} else if matches := regexCoverage.FindStringSubmatch(line); len(matches) == 3 {
		p.coverage(matches[1], matches[2])
	} else if matches := regexBenchmark.FindStringSubmatch(line); len(matches) == 7 {
		p.benchmark(matches[1], matches[2], matches[3], matches[4], matches[5], matches[6])
	} else if strings.HasPrefix(line, "# ") {
		// TODO(jstemmer): this should just be output; we should detect build output when building report
		fields := strings.Fields(strings.TrimPrefix(line, "# "))
		if len(fields) == 1 || len(fields) == 2 {
			p.buildOutput(fields[0])
		} else {
			p.output(line)
		}
	} else {
		p.output(line)
	}
}

func (p *parser) add(event gtr.Event) {
	p.events = append(p.events, event)
}

func (p *parser) runTest(name string) {
	p.add(gtr.Event{Type: "run_test", Name: name})
}

func (p *parser) pauseTest(name string) {
	p.add(gtr.Event{Type: "pause_test", Name: name})
}

func (p *parser) contTest(name string) {
	p.add(gtr.Event{Type: "cont_test", Name: name})
}

func (p *parser) endTest(line, indent, result, name, duration string) {
	if idx := strings.Index(line, fmt.Sprintf("%s--- %s:", indent, result)); idx > 0 {
		p.output(line[:idx])
	}
	_, n := stripIndent(indent)
	p.add(gtr.Event{
		Type:     "end_test",
		Name:     name,
		Result:   result,
		Indent:   n,
		Duration: parseSeconds(duration),
	})
}

func (p *parser) status(result string) {
	p.add(gtr.Event{Type: "status", Result: result})
}

func (p *parser) summary(result, name, duration, data, covpct, packages string) {
	p.add(gtr.Event{
		Type:        "summary",
		Result:      result,
		Name:        name,
		Duration:    parseSeconds(duration),
		Data:        data,
		CovPct:      parseFloat(covpct),
		CovPackages: parsePackages(packages),
	})
}

func (p *parser) coverage(percent, packages string) {
	p.add(gtr.Event{
		Type:        "coverage",
		CovPct:      parseFloat(percent),
		CovPackages: parsePackages(packages),
	})
}

func (p *parser) benchmark(name, iterations, nsPerOp, mbPerSec, bytesPerOp, allocsPerOp string) {
	p.add(gtr.Event{
		Type:        "benchmark",
		Name:        name,
		Iterations:  parseInt(iterations),
		NsPerOp:     parseFloat(nsPerOp),
		MBPerSec:    parseFloat(mbPerSec),
		BytesPerOp:  parseInt(bytesPerOp),
		AllocsPerOp: parseInt(allocsPerOp),
	})
}

func (p *parser) buildOutput(packageName string) {
	p.add(gtr.Event{
		Type: "build_output",
		Name: packageName,
	})
}

func (p *parser) output(line string) {
	p.add(gtr.Event{Type: "output", Data: line})
}

func parseSeconds(s string) time.Duration {
	if s == "" {
		return time.Duration(0)
	}
	// ignore error
	d, _ := time.ParseDuration(s + "s")
	return d
}

func parseFloat(s string) float64 {
	if s == "" {
		return 0
	}
	// ignore error
	pct, _ := strconv.ParseFloat(s, 64)
	return pct
}

func parsePackages(pkgList string) []string {
	if len(pkgList) == 0 {
		return nil
	}
	return strings.Split(pkgList, ", ")
}

func parseInt(s string) int64 {
	// ignore error
	n, _ := strconv.ParseInt(s, 10, 64)
	return n
}

func stripIndent(line string) (string, int) {
	var indent int
	for indent = 0; strings.HasPrefix(line, "    "); indent++ {
		line = line[4:]
	}
	return line, indent
}
<|endoftext|>