<commit_before>package velox

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"strconv"
	"sync"
	"time"

	"github.com/mattbaird/jsonpatch"
)

var (
	//15ms is approximately highest resolution on the JS eventloop
	MinThrottle     = 15 * time.Millisecond
	DefaultThrottle = 200 * time.Millisecond
)

//State must be embedded into a struct to make it syncable.
type State struct {
	//configuration
	Throttle time.Duration `json:"-"`
	//internal state
	initMut  sync.Mutex
	initd    bool
	gostruct interface{}
	dataMut  sync.RWMutex //protects bytes/delta/version
	bytes    []byte
	delta    []byte
	version  int64
	connMut  sync.Mutex
	conns    map[string]*conn
	push     struct {
		mut    sync.Mutex
		ing    bool
		queued bool
		start  time.Time
		wg     sync.WaitGroup
	}
}

func (s *State) init(gostruct interface{}) error {
	if s.Throttle < MinThrottle {
		s.Throttle = DefaultThrottle
	}
	//initial JSON bytes
	if b, err := json.Marshal(gostruct); err != nil {
		return fmt.Errorf("JSON marshalling failed: %s", err)
	} else {
		s.bytes = b
	}
	s.gostruct = gostruct
	s.version = 1
	s.conns = map[string]*conn{}
	s.initd = true
	return nil
}

func (s *State) sync(gostruct interface{}) (*State, error) {
	s.initMut.Lock()
	defer s.initMut.Unlock()
	if !s.initd {
		if err := s.init(gostruct); err != nil {
			return nil, err
		}
	} else if s.gostruct != gostruct {
		return nil, errors.New("A different struct is already synced")
	}
	return s, nil
}

func (s *State) subscribe(conn *conn) {
	//subscribe
	s.connMut.Lock()
	s.conns[conn.id] = conn
	s.connMut.Unlock()
	//and then unsubscribe on close
	go func() {
		conn.Wait()
		s.connMut.Lock()
		delete(s.conns, conn.id)
		s.connMut.Unlock()
	}()
}

func (s *State) NumConnections() int {
	s.connMut.Lock()
	n := len(s.conns)
	s.connMut.Unlock()
	return n
}

//Send the changes from this object to all connected clients.
//Push is thread-safe and is throttled so it can be called
//with abandon.
func (s *State) Push() {
	go s.gopush()
}

//non-blocking push
func (s *State) gopush() {
	s.push.mut.Lock()
	if s.push.ing {
		s.push.queued = true
		s.push.mut.Unlock()
		return
	}
	s.push.ing = true
	s.push.start = time.Now()
	//queue cleanup
	defer func() {
		//measure time passed, ensure we wait at least Throttle time
		tdelta := time.Now().Sub(s.push.start)
		if t := s.Throttle - tdelta; t > 0 {
			time.Sleep(t)
		}
		//cleanup
		s.push.ing = false
		if s.push.queued {
			s.push.queued = false
			s.push.mut.Unlock()
			s.Push() //auto-push
		} else {
			s.push.mut.Unlock()
		}
	}()
	//calculate new json state
	newBytes, err := json.Marshal(s.gostruct)
	if err != nil {
		log.Printf("velox: marshal failed: %s", err)
		return
	}
	//calculate change set from last version
	ops, _ := jsonpatch.CreatePatch(s.bytes, newBytes)
	if len(s.bytes) > 0 && len(ops) > 0 {
		//changes! bump version
		s.dataMut.Lock()
		s.delta, _ = json.Marshal(ops)
		s.bytes = newBytes
		s.version++
		s.dataMut.Unlock()
	}
	//send this new change to each subscriber
	s.connMut.Lock()
	for _, c := range s.conns {
		if c.version != s.version {
			s.push.wg.Add(1)
			go func(c *conn) {
				s.pushTo(c)
				s.push.wg.Done()
			}(c)
		}
	}
	s.connMut.Unlock()
	//wait for all connection pushes
	s.push.wg.Wait()
	//cleanup()
}

func (s *State) pushTo(c *conn) {
	if c.version == s.version {
		return
	}
	update := &update{Version: s.version}
	s.dataMut.RLock()
	//choose optimal update (send the smallest)
	if s.delta != nil && c.version == (s.version-1) && len(s.bytes) > 0 && len(s.delta) < len(s.bytes) {
		update.Delta = true
		update.Body = s.delta
	} else {
		update.Delta = false
		update.Body = s.bytes
	}
	//send update
	log.Printf("[%s] sending version %d (delta: %v)", c.id, s.version, update.Delta)
	if err := c.send(update); err == nil {
		c.version = s.version //sent! mark this version
	}
	s.dataMut.RUnlock()
}

//A single update. May contain compression flags in future.
type update struct {
	Ping    bool            `json:"ping,omitempty"`
	Delta   bool            `json:"delta,omitempty"`
	Version int64           `json:"version,omitempty"` //53 usable bits
	Body    json.RawMessage `json:"body,omitempty"`
}

//implement eventsource.Event interface
func (u *update) Id() string    { return strconv.FormatInt(u.Version, 10) }
func (u *update) Event() string { return "" }
func (u *update) Data() string {
	b, _ := json.Marshal(u)
	return string(b)
}
<commit_msg>remove rogue print statement<commit_after>package velox

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"strconv"
	"sync"
	"time"

	"github.com/mattbaird/jsonpatch"
)

var (
	//15ms is approximately highest resolution on the JS eventloop
	MinThrottle     = 15 * time.Millisecond
	DefaultThrottle = 200 * time.Millisecond
)

//State must be embedded into a struct to make it syncable.
type State struct {
	//configuration
	Throttle time.Duration `json:"-"`
	//internal state
	initMut  sync.Mutex
	initd    bool
	gostruct interface{}
	dataMut  sync.RWMutex //protects bytes/delta/version
	bytes    []byte
	delta    []byte
	version  int64
	connMut  sync.Mutex
	conns    map[string]*conn
	push     struct {
		mut    sync.Mutex
		ing    bool
		queued bool
		start  time.Time
		wg     sync.WaitGroup
	}
}

func (s *State) init(gostruct interface{}) error {
	if s.Throttle < MinThrottle {
		s.Throttle = DefaultThrottle
	}
	//initial JSON bytes
	if b, err := json.Marshal(gostruct); err != nil {
		return fmt.Errorf("JSON marshalling failed: %s", err)
	} else {
		s.bytes = b
	}
	s.gostruct = gostruct
	s.version = 1
	s.conns = map[string]*conn{}
	s.initd = true
	return nil
}

func (s *State) sync(gostruct interface{}) (*State, error) {
	s.initMut.Lock()
	defer s.initMut.Unlock()
	if !s.initd {
		if err := s.init(gostruct); err != nil {
			return nil, err
		}
	} else if s.gostruct != gostruct {
		return nil, errors.New("A different struct is already synced")
	}
	return s, nil
}

func (s *State) subscribe(conn *conn) {
	//subscribe
	s.connMut.Lock()
	s.conns[conn.id] = conn
	s.connMut.Unlock()
	//and then unsubscribe on close
	go func() {
		conn.Wait()
		s.connMut.Lock()
		delete(s.conns, conn.id)
		s.connMut.Unlock()
	}()
}

func (s *State) NumConnections() int {
	s.connMut.Lock()
	n := len(s.conns)
	s.connMut.Unlock()
	return n
}

//Send the changes from this object to all connected clients.
//Push is thread-safe and is throttled so it can be called
//with abandon.
func (s *State) Push() {
	go s.gopush()
}

//non-blocking push
func (s *State) gopush() {
	s.push.mut.Lock()
	if s.push.ing {
		s.push.queued = true
		s.push.mut.Unlock()
		return
	}
	s.push.ing = true
	s.push.start = time.Now()
	//queue cleanup
	defer func() {
		//measure time passed, ensure we wait at least Throttle time
		tdelta := time.Now().Sub(s.push.start)
		if t := s.Throttle - tdelta; t > 0 {
			time.Sleep(t)
		}
		//cleanup
		s.push.ing = false
		if s.push.queued {
			s.push.queued = false
			s.push.mut.Unlock()
			s.Push() //auto-push
		} else {
			s.push.mut.Unlock()
		}
	}()
	//calculate new json state
	newBytes, err := json.Marshal(s.gostruct)
	if err != nil {
		log.Printf("velox: marshal failed: %s", err)
		return
	}
	//calculate change set from last version
	ops, _ := jsonpatch.CreatePatch(s.bytes, newBytes)
	if len(s.bytes) > 0 && len(ops) > 0 {
		//changes! bump version
		s.dataMut.Lock()
		s.delta, _ = json.Marshal(ops)
		s.bytes = newBytes
		s.version++
		s.dataMut.Unlock()
	}
	//send this new change to each subscriber
	s.connMut.Lock()
	for _, c := range s.conns {
		if c.version != s.version {
			s.push.wg.Add(1)
			go func(c *conn) {
				s.pushTo(c)
				s.push.wg.Done()
			}(c)
		}
	}
	s.connMut.Unlock()
	//wait for all connection pushes
	s.push.wg.Wait()
	//cleanup()
}

func (s *State) pushTo(c *conn) {
	if c.version == s.version {
		return
	}
	update := &update{Version: s.version}
	s.dataMut.RLock()
	//choose optimal update (send the smallest)
	if s.delta != nil && c.version == (s.version-1) && len(s.bytes) > 0 && len(s.delta) < len(s.bytes) {
		update.Delta = true
		update.Body = s.delta
	} else {
		update.Delta = false
		update.Body = s.bytes
	}
	//send update
	if err := c.send(update); err == nil {
		c.version = s.version //sent! mark this version
	}
	s.dataMut.RUnlock()
}

//A single update. May contain compression flags in future.
type update struct {
	Ping    bool            `json:"ping,omitempty"`
	Delta   bool            `json:"delta,omitempty"`
	Version int64           `json:"version,omitempty"` //53 usable bits
	Body    json.RawMessage `json:"body,omitempty"`
}

//implement eventsource.Event interface
func (u *update) Id() string    { return strconv.FormatInt(u.Version, 10) }
func (u *update) Event() string { return "" }
func (u *update) Data() string {
	b, _ := json.Marshal(u)
	return string(b)
}
<|endoftext|>


<commit_before>package boomer

import (
	"time"
)

type RequestStats struct {
	Entries              map[string]*StatsEntry
	Errors               map[string]*StatsError
	NumRequests          int64
	NumFailures          int64
	MaxRequests          int64
	LastRequestTimestamp int64
	StartTime            int64
}

func (this *RequestStats) get(name string, method string) (entry *StatsEntry) {
	entry, ok := this.Entries[name+method]
	if !ok {
		entry = &StatsEntry{
			Stats:         this,
			Name:          name,
			Method:        method,
			NumReqsPerSec: make(map[int64]int64),
			ResponseTimes: make(map[float64]int64),
		}
		entry.reset()
		this.Entries[name+method] = entry
	}
	return this.Entries[name+method]
}

func (this *RequestStats) clearAll() {
	this.NumRequests = 0
	this.NumFailures = 0
	this.Entries = make(map[string]*StatsEntry)
	this.Errors = make(map[string]*StatsError)
	this.MaxRequests = 0
	this.LastRequestTimestamp = 0
	this.StartTime = 0
}

type StatsEntry struct {
	Stats                *RequestStats
	Name                 string
	Method               string
	NumRequests          int64
	NumFailures          int64
	TotalResponseTime    float64
	MinResponseTime      float64
	MaxResponseTime      float64
	NumReqsPerSec        map[int64]int64
	ResponseTimes        map[float64]int64
	TotalContentLength   int64
	StartTime            int64
	LastRequestTimestamp int64
}

func (this *StatsEntry) reset() {
	this.StartTime = int64(time.Now().Unix())
	this.NumRequests = 0
	this.NumFailures = 0
	this.TotalResponseTime = 0
	this.ResponseTimes = make(map[float64]int64)
	this.MinResponseTime = 0
	this.MaxResponseTime = 0
	this.LastRequestTimestamp = int64(time.Now().Unix())
	this.NumReqsPerSec = make(map[int64]int64)
	this.TotalContentLength = 0
}

func (this *StatsEntry) log(responseTime float64, contentLength int64) {

	this.NumRequests += 1

	this.logTimeOfRequest()
	this.logResponseTime(responseTime)

	this.TotalContentLength += contentLength

}

func (this *StatsEntry) logTimeOfRequest() {

	now := int64(time.Now().Unix())

	_, ok := this.NumReqsPerSec[now]
	if !ok {
		this.NumReqsPerSec[now] = 0
	} else {
		this.NumReqsPerSec[now] += 1
	}

	this.LastRequestTimestamp = now

}

func (this *StatsEntry) logResponseTime(responseTime float64) {
	this.TotalResponseTime += responseTime

	if this.MinResponseTime == 0 {
		this.MinResponseTime = responseTime
	}

	if responseTime < this.MinResponseTime {
		this.MinResponseTime = responseTime
	}

	if responseTime > this.MaxResponseTime {
		this.MaxResponseTime = responseTime
	}

	roundedResponseTime := float64(0)

	if responseTime < 100 {
		roundedResponseTime = responseTime
	} else if responseTime < 1000 {
		roundedResponseTime = float64(Round(responseTime, .5, -1))
	} else if responseTime < 10000 {
		roundedResponseTime = float64(Round(responseTime, .5, -2))
	} else {
		roundedResponseTime = float64(Round(responseTime, .5, -3))
	}

	_, ok := this.ResponseTimes[roundedResponseTime]
	if !ok {
		this.ResponseTimes[roundedResponseTime] = 0
	} else {
		this.ResponseTimes[roundedResponseTime] += 1
	}

}

func (this *StatsEntry) logError(err string) {
	this.NumFailures += 1
	key := MD5(this.Method, this.Name, err)
	entry, ok := this.Stats.Errors[key]
	if !ok {
		entry = &StatsError{
			Name:   this.Name,
			Method: this.Method,
			Error:  err,
		}
		this.Stats.Errors[key] = entry
	}
	entry.occured()
}

func (this *StatsEntry) serialize() map[string]interface{} {
	result := make(map[string]interface{})
	result["name"] = this.Name
	result["method"] = this.Method
	result["last_request_timestamp"] = this.LastRequestTimestamp
	result["start_time"] = this.StartTime
	result["num_requests"] = this.NumRequests
	result["num_failures"] = this.NumFailures
	result["total_response_time"] = this.TotalResponseTime
	result["max_response_time"] = this.MaxResponseTime
	result["min_response_time"] = this.MinResponseTime
	result["total_content_length"] = this.TotalContentLength
	result["response_times"] = this.ResponseTimes
	result["num_reqs_per_sec"] = this.NumReqsPerSec
	return result
}

func (this *StatsEntry) getStrippedReport() map[string]interface{} {
	report := this.serialize()
	this.reset()
	return report
}

type StatsError struct {
	Name       string
	Method     string
	Error      string
	Occurences int64
}

func (this *StatsError) occured() {
	this.Occurences += 1
}

func (this *StatsError) toMap() map[string]interface{} {
	m := make(map[string]interface{})
	m["method"] = this.Method
	m["name"] = this.Name
	m["error"] = this.Error
	m["occurences"] = this.Occurences
	return m
}

func reportToMasterHandler(data *map[string]interface{}) {
	entries := make([]interface{}, 0, len(stats.Entries))
	for _, v := range stats.Entries {
		if !(v.NumRequests == 0 && v.NumFailures == 0) {
			entries = append(entries, v.getStrippedReport())
		}
	}

	errors := make(map[string]map[string]interface{})
	for k, v := range stats.Errors {
		errors[k] = v.toMap()
	}

	(*data)["stats"] = entries
	(*data)["errors"] = errors
	stats.Entries = make(map[string]*StatsEntry)
	stats.Errors = make(map[string]*StatsError)
}

type RequestSuccess struct {
	requestType    string
	name           string
	responseTime   float64
	responseLength int64
}

type RequestFailure struct {
	requestType  string
	name         string
	responseTime float64
	error        string
}

var stats = new(RequestStats)
var RequestSuccessChannel = make(chan *RequestSuccess, 100)
var RequestFailureChannel = make(chan *RequestFailure, 100)
var ClearStatsChannel = make(chan bool)

func init() {
	Events.Subscribe("boomer:report_to_master", reportToMasterHandler)
	stats.Entries = make(map[string]*StatsEntry)
	stats.Errors = make(map[string]*StatsError)
	go func() {
		for {
			select {
			case m := <-RequestSuccessChannel:
				stats.get(m.name, m.requestType).log(m.responseTime, m.responseLength)
			case n := <-RequestFailureChannel:
				stats.get(n.name, n.requestType).logError(n.error)
			case <-ClearStatsChannel:
				stats.clearAll()
			}
		}
	}()
}
<commit_msg>double check<commit_after>package boomer

import (
	"time"
	"log"
)

type RequestStats struct {
	Entries              map[string]*StatsEntry
	Errors               map[string]*StatsError
	NumRequests          int64
	NumFailures          int64
	MaxRequests          int64
	LastRequestTimestamp int64
	StartTime            int64
}

func (this *RequestStats) get(name string, method string) (entry *StatsEntry) {
	entry, ok := this.Entries[name+method]
	if !ok {
		entry = &StatsEntry{
			Stats:         this,
			Name:          name,
			Method:        method,
			NumReqsPerSec: make(map[int64]int64),
			ResponseTimes: make(map[float64]int64),
		}
		entry.reset()
		this.Entries[name+method] = entry
	}
	entry, ok = this.Entries[name+method]
	if !ok {
		log.Fatal(ok, "entry is null, why?")
	}
	return entry
}

func (this *RequestStats) clearAll() {
	this.NumRequests = 0
	this.NumFailures = 0
	this.Entries = make(map[string]*StatsEntry)
	this.Errors = make(map[string]*StatsError)
	this.MaxRequests = 0
	this.LastRequestTimestamp = 0
	this.StartTime = 0
}

type StatsEntry struct {
	Stats                *RequestStats
	Name                 string
	Method               string
	NumRequests          int64
	NumFailures          int64
	TotalResponseTime    float64
	MinResponseTime      float64
	MaxResponseTime      float64
	NumReqsPerSec        map[int64]int64
	ResponseTimes        map[float64]int64
	TotalContentLength   int64
	StartTime            int64
	LastRequestTimestamp int64
}

func (this *StatsEntry) reset() {
	this.StartTime = int64(time.Now().Unix())
	this.NumRequests = 0
	this.NumFailures = 0
	this.TotalResponseTime = 0
	this.ResponseTimes = make(map[float64]int64)
	this.MinResponseTime = 0
	this.MaxResponseTime = 0
	this.LastRequestTimestamp = int64(time.Now().Unix())
	this.NumReqsPerSec = make(map[int64]int64)
	this.TotalContentLength = 0
}

func (this *StatsEntry) log(responseTime float64, contentLength int64) {

	this.NumRequests += 1

	this.logTimeOfRequest()
	this.logResponseTime(responseTime)

	this.TotalContentLength += contentLength

}

func (this *StatsEntry) logTimeOfRequest() {

	now := int64(time.Now().Unix())

	_, ok := this.NumReqsPerSec[now]
	if !ok {
		this.NumReqsPerSec[now] = 0
	} else {
		this.NumReqsPerSec[now] += 1
	}

	this.LastRequestTimestamp = now

}

func (this *StatsEntry) logResponseTime(responseTime float64) {
	this.TotalResponseTime += responseTime

	if this.MinResponseTime == 0 {
		this.MinResponseTime = responseTime
	}

	if responseTime < this.MinResponseTime {
		this.MinResponseTime = responseTime
	}

	if responseTime > this.MaxResponseTime {
		this.MaxResponseTime = responseTime
	}

	roundedResponseTime := float64(0)

	if responseTime < 100 {
		roundedResponseTime = responseTime
	} else if responseTime < 1000 {
		roundedResponseTime = float64(Round(responseTime, .5, -1))
	} else if responseTime < 10000 {
		roundedResponseTime = float64(Round(responseTime, .5, -2))
	} else {
		roundedResponseTime = float64(Round(responseTime, .5, -3))
	}

	_, ok := this.ResponseTimes[roundedResponseTime]
	if !ok {
		this.ResponseTimes[roundedResponseTime] = 0
	} else {
		this.ResponseTimes[roundedResponseTime] += 1
	}

}

func (this *StatsEntry) logError(err string) {
	this.NumFailures += 1
	key := MD5(this.Method, this.Name, err)
	entry, ok := this.Stats.Errors[key]
	if !ok {
		entry = &StatsError{
			Name:   this.Name,
			Method: this.Method,
			Error:  err,
		}
		this.Stats.Errors[key] = entry
	}
	entry.occured()
}

func (this *StatsEntry) serialize() map[string]interface{} {
	result := make(map[string]interface{})
	result["name"] = this.Name
	result["method"] = this.Method
	result["last_request_timestamp"] = this.LastRequestTimestamp
	result["start_time"] = this.StartTime
	result["num_requests"] = this.NumRequests
	result["num_failures"] = this.NumFailures
	result["total_response_time"] = this.TotalResponseTime
	result["max_response_time"] = this.MaxResponseTime
	result["min_response_time"] = this.MinResponseTime
	result["total_content_length"] = this.TotalContentLength
	result["response_times"] = this.ResponseTimes
	result["num_reqs_per_sec"] = this.NumReqsPerSec
	return result
}

func (this *StatsEntry) getStrippedReport() map[string]interface{} {
	report := this.serialize()
	this.reset()
	return report
}

type StatsError struct {
	Name       string
	Method     string
	Error      string
	Occurences int64
}

func (this *StatsError) occured() {
	this.Occurences += 1
}

func (this *StatsError) toMap() map[string]interface{} {
	m := make(map[string]interface{})
	m["method"] = this.Method
	m["name"] = this.Name
	m["error"] = this.Error
	m["occurences"] = this.Occurences
	return m
}

func reportToMasterHandler(data *map[string]interface{}) {
	entries := make([]interface{}, 0, len(stats.Entries))
	for _, v := range stats.Entries {
		if !(v.NumRequests == 0 && v.NumFailures == 0) {
			entries = append(entries, v.getStrippedReport())
		}
	}

	errors := make(map[string]map[string]interface{})
	for k, v := range stats.Errors {
		errors[k] = v.toMap()
	}

	(*data)["stats"] = entries
	(*data)["errors"] = errors
	stats.Entries = make(map[string]*StatsEntry)
	stats.Errors = make(map[string]*StatsError)
}

type RequestSuccess struct {
	requestType    string
	name           string
	responseTime   float64
	responseLength int64
}

type RequestFailure struct {
	requestType  string
	name         string
	responseTime float64
	error        string
}

var stats = new(RequestStats)
var RequestSuccessChannel = make(chan *RequestSuccess, 100)
var RequestFailureChannel = make(chan *RequestFailure, 100)
var ClearStatsChannel = make(chan bool)

func init() {
	Events.Subscribe("boomer:report_to_master", reportToMasterHandler)
	stats.Entries = make(map[string]*StatsEntry)
	stats.Errors = make(map[string]*StatsError)
	go func() {
		for {
			select {
			case m := <-RequestSuccessChannel:
				stats.get(m.name, m.requestType).log(m.responseTime, m.responseLength)
			case n := <-RequestFailureChannel:
				stats.get(n.name, n.requestType).logError(n.error)
			case <-ClearStatsChannel:
				stats.clearAll()
			}
		}
	}()
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"runtime/debug"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/micro/go-log"
	"github.com/micro/go-micro/broker"
	"github.com/micro/go-micro/codec"
	"github.com/micro/go-micro/metadata"
	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/transport"

	"github.com/micro/misc/lib/addr"

	"golang.org/x/net/context"
)

type rpcServer struct {
	rpc  *server
	exit chan chan error

	sync.RWMutex
	opts        Options
	handlers    map[string]Handler
	subscribers map[*subscriber][]broker.Subscriber
	// used for first registration
	registered bool
	// graceful exit
	wg sync.WaitGroup
}

func newRpcServer(opts ...Option) Server {
	options := newOptions(opts...)
	return &rpcServer{
		opts: options,
		rpc: &server{
			name:         options.Name,
			serviceMap:   make(map[string]*service),
			hdlrWrappers: options.HdlrWrappers,
		},
		handlers:    make(map[string]Handler),
		subscribers: make(map[*subscriber][]broker.Subscriber),
		exit:        make(chan chan error),
	}
}

func (s *rpcServer) accept(sock transport.Socket) {
	defer func() {
		// close socket
		sock.Close()

		if r := recover(); r != nil {
			log.Log("panic recovered: ", r)
			log.Log(string(debug.Stack()))
		}
	}()

	for {
		var msg transport.Message
		if err := sock.Recv(&msg); err != nil {
			return
		}

		// we use this Timeout header to set a server deadline
		to := msg.Header["Timeout"]
		// we use this Content-Type header to identify the codec needed
		ct := msg.Header["Content-Type"]

		cf, err := s.newCodec(ct)
		// TODO: needs better error handling
		if err != nil {
			sock.Send(&transport.Message{
				Header: map[string]string{
					"Content-Type": "text/plain",
				},
				Body: []byte(err.Error()),
			})
			return
		}

		codec := newRpcPlusCodec(&msg, sock, cf)

		// strip our headers
		hdr := make(map[string]string)
		for k, v := range msg.Header {
			hdr[k] = v
		}
		delete(hdr, "Content-Type")
		delete(hdr, "Timeout")

		ctx := metadata.NewContext(context.Background(), hdr)

		// set the timeout if we have it
		if len(to) > 0 {
			if n, err := strconv.ParseUint(to, 10, 64); err == nil {
				ctx, _ = context.WithTimeout(ctx, time.Duration(n))
			}
		}

		// add to wait group
		s.wg.Add(1)
		defer s.wg.Done()

		// TODO: needs better error handling
		if err := s.rpc.serveRequest(ctx, codec, ct); err != nil {
			log.Logf("Unexpected error serving request, closing socket: %v", err)
			return
		}
	}
}

func (s *rpcServer) newCodec(contentType string) (codec.NewCodec, error) {
	if cf, ok := s.opts.Codecs[contentType]; ok {
		return cf, nil
	}
	if cf, ok := defaultCodecs[contentType]; ok {
		return cf, nil
	}
	return nil, fmt.Errorf("Unsupported Content-Type: %s", contentType)
}

func (s *rpcServer) Options() Options {
	s.RLock()
	opts := s.opts
	s.RUnlock()
	return opts
}

func (s *rpcServer) Init(opts ...Option) error {
	s.Lock()
	for _, opt := range opts {
		opt(&s.opts)
	}
	// update internal server
	s.rpc = &server{
		name:         s.opts.Name,
		serviceMap:   s.rpc.serviceMap,
		hdlrWrappers: s.opts.HdlrWrappers,
	}
	s.Unlock()
	return nil
}

func (s *rpcServer) NewHandler(h interface{}, opts ...HandlerOption) Handler {
	return newRpcHandler(h, opts...)
}

func (s *rpcServer) Handle(h Handler) error {
	s.Lock()
	defer s.Unlock()

	if err := s.rpc.register(h.Handler()); err != nil {
		return err
	}

	s.handlers[h.Name()] = h

	return nil
}

func (s *rpcServer) NewSubscriber(topic string, sb interface{}, opts ...SubscriberOption) Subscriber {
	return newSubscriber(topic, sb, opts...)
}

func (s *rpcServer) Subscribe(sb Subscriber) error {
	sub, ok := sb.(*subscriber)
	if !ok {
		return fmt.Errorf("invalid subscriber: expected *subscriber")
	}
	if len(sub.handlers) == 0 {
		return fmt.Errorf("invalid subscriber: no handler functions")
	}

	if err := validateSubscriber(sb); err != nil {
		return err
	}

	s.Lock()
	_, ok = s.subscribers[sub]
	if ok {
		return fmt.Errorf("subscriber %v already exists", s)
	}
	s.subscribers[sub] = nil
	s.Unlock()
	return nil
}

func (s *rpcServer) Register() error {
	// parse address for host, port
	config := s.Options()
	var advt, host string
	var port int

	// check the advertise address first
	// if it exists then use it, otherwise
	// use the address
	if len(config.Advertise) > 0 {
		advt = config.Advertise
	} else {
		advt = config.Address
	}

	parts := strings.Split(advt, ":")
	if len(parts) > 1 {
		host = strings.Join(parts[:len(parts)-1], ":")
		port, _ = strconv.Atoi(parts[len(parts)-1])
	} else {
		host = parts[0]
	}

	addr, err := addr.Extract(host)
	if err != nil {
		return err
	}

	// register service
	node := &registry.Node{
		Id:       config.Name + "-" + config.Id,
		Address:  addr,
		Port:     port,
		Metadata: config.Metadata,
	}

	node.Metadata["transport"] = config.Transport.String()
	node.Metadata["broker"] = config.Broker.String()
	node.Metadata["server"] = s.String()
	node.Metadata["registry"] = config.Registry.String()

	s.RLock()
	// Maps are ordered randomly, sort the keys for consistency
	var handlerList []string
	for n, e := range s.handlers {
		// Only advertise non internal handlers
		if !e.Options().Internal {
			handlerList = append(handlerList, n)
		}
	}
	sort.Strings(handlerList)

	var subscriberList []*subscriber
	for e := range s.subscribers {
		// Only advertise non internal subscribers
		if !e.Options().Internal {
			subscriberList = append(subscriberList, e)
		}
	}
	sort.Slice(subscriberList, func(i, j int) bool {
		return subscriberList[i].topic > subscriberList[j].topic
	})

	var endpoints []*registry.Endpoint
	for _, n := range handlerList {
		endpoints = append(endpoints, s.handlers[n].Endpoints()...)
	}
	for _, e := range subscriberList {
		endpoints = append(endpoints, e.Endpoints()...)
	}
	s.RUnlock()

	service := &registry.Service{
		Name:      config.Name,
		Version:   config.Version,
		Nodes:     []*registry.Node{node},
		Endpoints: endpoints,
	}

	s.Lock()
	registered := s.registered
	s.Unlock()

	if !registered {
		log.Logf("Registering node: %s", node.Id)
	}

	// create registry options
	rOpts := []registry.RegisterOption{registry.RegisterTTL(config.RegisterTTL)}

	if err := config.Registry.Register(service, rOpts...); err != nil {
		return err
	}

	// already registered? don't need to register subscribers
	if registered {
		return nil
	}

	s.Lock()
	defer s.Unlock()

	s.registered = true

	for sb, _ := range s.subscribers {
		handler := s.createSubHandler(sb, s.opts)
		var opts []broker.SubscribeOption
		if queue := sb.Options().Queue; len(queue) > 0 {
			opts = append(opts, broker.Queue(queue))
		}
		sub, err := config.Broker.Subscribe(sb.Topic(), handler, opts...)
		if err != nil {
			return err
		}
		s.subscribers[sb] = []broker.Subscriber{sub}
	}

	return nil
}

func (s *rpcServer) Deregister() error {
	config := s.Options()
	var advt, host string
	var port int

	// check the advertise address first
	// if it exists then use it, otherwise
	// use the address
	if len(config.Advertise) > 0 {
		advt = config.Advertise
	} else {
		advt = config.Address
	}

	parts := strings.Split(advt, ":")
	if len(parts) > 1 {
		host = strings.Join(parts[:len(parts)-1], ":")
		port, _ = strconv.Atoi(parts[len(parts)-1])
	} else {
		host = parts[0]
	}

	addr, err := addr.Extract(host)
	if err != nil {
		return err
	}

	node := &registry.Node{
		Id:      config.Name + "-" + config.Id,
		Address: addr,
		Port:    port,
	}

	service := &registry.Service{
		Name:    config.Name,
		Version: config.Version,
		Nodes:   []*registry.Node{node},
	}

	log.Logf("Deregistering node: %s", node.Id)
	if err := config.Registry.Deregister(service); err != nil {
		return err
	}

	s.Lock()

	if !s.registered {
		s.Unlock()
		return nil
	}

	s.registered = false

	for sb, subs := range s.subscribers {
		for _, sub := range subs {
			log.Logf("Unsubscribing from topic: %s", sub.Topic())
			sub.Unsubscribe()
		}
		s.subscribers[sb] = nil
	}

	s.Unlock()
	return nil
}

func (s *rpcServer) Start() error {
	registerDebugHandler(s)
	config := s.Options()

	ts, err := config.Transport.Listen(config.Address)
	if err != nil {
		return err
	}

	log.Logf("Listening on %s", ts.Addr())
	s.Lock()
	s.opts.Address = ts.Addr()
	s.Unlock()

	go ts.Accept(s.accept)

	go func() {
		// wait for exit
		ch := <-s.exit

		// wait for requests to finish
		if wait(s.opts.Context) {
			s.wg.Wait()
		}

		// close transport listener
		ch <- ts.Close()

		// disconnect the broker
		config.Broker.Disconnect()
	}()

	// TODO: subscribe to cruft
	return config.Broker.Connect()
}

func (s *rpcServer) Stop() error {
	ch := make(chan error)
	s.exit <- ch
	return <-ch
}

func (s *rpcServer) String() string {
	return "rpc"
}
<commit_msg>fix possible deadlock since code can return without unlocking the Mutex<commit_after>package server

import (
	"fmt"
	"runtime/debug"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/micro/go-log"
	"github.com/micro/go-micro/broker"
	"github.com/micro/go-micro/codec"
	"github.com/micro/go-micro/metadata"
	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/transport"

	"github.com/micro/misc/lib/addr"

	"golang.org/x/net/context"
)

type rpcServer struct {
	rpc  *server
	exit chan chan error

	sync.RWMutex
	opts        Options
	handlers    map[string]Handler
	subscribers map[*subscriber][]broker.Subscriber
	// used for first registration
	registered bool
	// graceful exit
	wg sync.WaitGroup
}

func newRpcServer(opts ...Option) Server {
	options := newOptions(opts...)
	return &rpcServer{
		opts: options,
		rpc: &server{
			name:         options.Name,
			serviceMap:   make(map[string]*service),
			hdlrWrappers: options.HdlrWrappers,
		},
		handlers:    make(map[string]Handler),
		subscribers: make(map[*subscriber][]broker.Subscriber),
		exit:        make(chan chan error),
	}
}

func (s *rpcServer) accept(sock transport.Socket) {
	defer func() {
		// close socket
		sock.Close()

		if r := recover(); r != nil {
			log.Log("panic recovered: ", r)
			log.Log(string(debug.Stack()))
		}
	}()

	for {
		var msg transport.Message
		if err := sock.Recv(&msg); err != nil {
			return
		}

		// we use this Timeout header to set a server deadline
		to := msg.Header["Timeout"]
		// we use this Content-Type header to identify the codec needed
		ct := msg.Header["Content-Type"]

		cf, err := s.newCodec(ct)
		// TODO: needs better error handling
		if err != nil {
			sock.Send(&transport.Message{
				Header: map[string]string{
					"Content-Type": "text/plain",
				},
				Body: []byte(err.Error()),
			})
			return
		}

		codec := newRpcPlusCodec(&msg, sock, cf)

		// strip our headers
		hdr := make(map[string]string)
		for k, v := range msg.Header {
			hdr[k] = v
		}
		delete(hdr, "Content-Type")
		delete(hdr, "Timeout")

		ctx := metadata.NewContext(context.Background(), hdr)

		// set the timeout if we have it
		if len(to) > 0 {
			if n, err := strconv.ParseUint(to, 10, 64); err == nil {
				ctx, _ = context.WithTimeout(ctx, time.Duration(n))
			}
		}

		// add to wait group
		s.wg.Add(1)
		defer s.wg.Done()

		// TODO: needs better error handling
		if err := s.rpc.serveRequest(ctx, codec, ct); err != nil {
			log.Logf("Unexpected error serving request, closing socket: %v", err)
			return
		}
	}
}

func (s *rpcServer) newCodec(contentType string) (codec.NewCodec, error) {
	if cf, ok := s.opts.Codecs[contentType]; ok {
		return cf, nil
	}
	if cf, ok := defaultCodecs[contentType]; ok {
		return cf, nil
	}
	return nil, fmt.Errorf("Unsupported Content-Type: %s", contentType)
}

func (s *rpcServer) Options() Options {
	s.RLock()
	opts := s.opts
	s.RUnlock()
	return opts
}

func (s *rpcServer) Init(opts ...Option) error {
	s.Lock()
	for _, opt := range opts {
		opt(&s.opts)
	}
	// update internal server
	s.rpc = &server{
		name:         s.opts.Name,
		serviceMap:   s.rpc.serviceMap,
		hdlrWrappers: s.opts.HdlrWrappers,
	}
	s.Unlock()
	return nil
}

func (s *rpcServer) NewHandler(h interface{}, opts ...HandlerOption) Handler {
	return newRpcHandler(h, opts...)
}

func (s *rpcServer) Handle(h Handler) error {
	s.Lock()
	defer s.Unlock()

	if err := s.rpc.register(h.Handler()); err != nil {
		return err
	}

	s.handlers[h.Name()] = h

	return nil
}

func (s *rpcServer) NewSubscriber(topic string, sb interface{}, opts ...SubscriberOption) Subscriber {
	return newSubscriber(topic, sb, opts...)
}

func (s *rpcServer) Subscribe(sb Subscriber) error {
	sub, ok := sb.(*subscriber)
	if !ok {
		return fmt.Errorf("invalid subscriber: expected *subscriber")
	}
	if len(sub.handlers) == 0 {
		return fmt.Errorf("invalid subscriber: no handler functions")
	}

	if err := validateSubscriber(sb); err != nil {
		return err
	}

	s.Lock()
	defer s.Unlock()
	_, ok = s.subscribers[sub]
	if ok {
		return fmt.Errorf("subscriber %v already exists", s)
	}
	s.subscribers[sub] = nil
	return nil
}

func (s *rpcServer) Register() error {
	// parse address for host, port
	config := s.Options()
	var advt, host string
	var port int

	// check the advertise address first
	// if it exists then use it, otherwise
	// use the address
	if len(config.Advertise) > 0 {
		advt = config.Advertise
	} else {
		advt = config.Address
	}

	parts := strings.Split(advt, ":")
	if len(parts) > 1 {
		host = strings.Join(parts[:len(parts)-1], ":")
		port, _ = strconv.Atoi(parts[len(parts)-1])
	} else {
		host = parts[0]
	}

	addr, err := addr.Extract(host)
	if err != nil {
		return err
	}

	// register service
	node := &registry.Node{
		Id:       config.Name + "-" + config.Id,
		Address:  addr,
		Port:     port,
		Metadata: config.Metadata,
	}

	node.Metadata["transport"] = config.Transport.String()
	node.Metadata["broker"] = config.Broker.String()
	node.Metadata["server"] = s.String()
	node.Metadata["registry"] = config.Registry.String()

	s.RLock()
	// Maps are ordered randomly, sort the keys for consistency
	var handlerList []string
	for n, e := range s.handlers {
		// Only advertise non internal handlers
		if !e.Options().Internal {
			handlerList = append(handlerList, n)
		}
	}
	sort.Strings(handlerList)

	var subscriberList []*subscriber
	for e := range s.subscribers {
		// Only advertise non internal subscribers
		if !e.Options().Internal {
			subscriberList = append(subscriberList, e)
		}
	}
	sort.Slice(subscriberList, func(i, j int) bool {
		return subscriberList[i].topic > subscriberList[j].topic
	})

	var endpoints []*registry.Endpoint
	for _, n := range handlerList {
		endpoints = append(endpoints, s.handlers[n].Endpoints()...)
	}
	for _, e := range subscriberList {
		endpoints = append(endpoints, e.Endpoints()...)
	}
	s.RUnlock()

	service := &registry.Service{
		Name:      config.Name,
		Version:   config.Version,
		Nodes:     []*registry.Node{node},
		Endpoints: endpoints,
	}

	s.Lock()
	registered := s.registered
	s.Unlock()

	if !registered {
		log.Logf("Registering node: %s", node.Id)
	}

	// create registry options
	rOpts := []registry.RegisterOption{registry.RegisterTTL(config.RegisterTTL)}

	if err := config.Registry.Register(service, rOpts...); err != nil {
		return err
	}

	// already registered? don't need to register subscribers
	if registered {
		return nil
	}

	s.Lock()
	defer s.Unlock()

	s.registered = true

	for sb, _ := range s.subscribers {
		handler := s.createSubHandler(sb, s.opts)
		var opts []broker.SubscribeOption
		if queue := sb.Options().Queue; len(queue) > 0 {
			opts = append(opts, broker.Queue(queue))
		}
		sub, err := config.Broker.Subscribe(sb.Topic(), handler, opts...)
		if err != nil {
			return err
		}
		s.subscribers[sb] = []broker.Subscriber{sub}
	}

	return nil
}

func (s *rpcServer) Deregister() error {
	config := s.Options()
	var advt, host string
	var port int

	// check the advertise address first
	// if it exists then use it, otherwise
	// use the address
	if len(config.Advertise) > 0 {
		advt = config.Advertise
	} else {
		advt = config.Address
	}

	parts := strings.Split(advt, ":")
	if len(parts) > 1 {
		host = strings.Join(parts[:len(parts)-1], ":")
		port, _ = strconv.Atoi(parts[len(parts)-1])
	} else {
		host = parts[0]
	}

	addr, err := addr.Extract(host)
	if err != nil {
		return err
	}

	node := &registry.Node{
		Id:      config.Name + "-" + config.Id,
		Address: addr,
		Port:    port,
	}

	service := &registry.Service{
		Name:    config.Name,
		Version: config.Version,
		Nodes:   []*registry.Node{node},
	}

	log.Logf("Deregistering node: %s", node.Id)
	if err := config.Registry.Deregister(service); err != nil {
		return err
	}

	s.Lock()

	if !s.registered {
		s.Unlock()
		return nil
	}

	s.registered = false

	for sb, subs := range s.subscribers {
		for _, sub := range subs {
			log.Logf("Unsubscribing from topic: %s", sub.Topic())
			sub.Unsubscribe()
		}
		s.subscribers[sb] = nil
	}

	s.Unlock()
	return nil
}

func (s *rpcServer) Start() error {
	registerDebugHandler(s)
	config := s.Options()

	ts, err := config.Transport.Listen(config.Address)
	if err != nil {
		return err
	}

	log.Logf("Listening on %s", ts.Addr())
	s.Lock()
	s.opts.Address = ts.Addr()
	s.Unlock()

	go ts.Accept(s.accept)

	go func() {
		// wait for exit
		ch := <-s.exit

		// wait for requests to finish
		if wait(s.opts.Context) {
			s.wg.Wait()
		}

		// close transport listener
		ch <- ts.Close()

		// disconnect the broker
		config.Broker.Disconnect()
	}()

	// TODO: subscribe to cruft
	return config.Broker.Connect()
}

func (s *rpcServer) Stop() error {
	ch := make(chan error)
	s.exit <- ch
	return <-ch
}

func (s *rpcServer) String() string {
	return "rpc"
}
<|endoftext|>


<commit_before>// Copyright (C) 2019 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package adb

import (
	"bufio"
	"container/ring"
	"context"
	"encoding/base64"
	"errors"
	"io"
	"regexp"
	"strings"

	perfetto_pb "perfetto/config"

	"github.com/golang/protobuf/proto"
	"github.com/google/gapid/core/app/crash"
	"github.com/google/gapid/core/event/task"
	"github.com/google/gapid/core/log"
)

// StartPerfettoTrace starts a perfetto trace on this device.
func (b *binding) StartPerfettoTrace(ctx context.Context, config *perfetto_pb.TraceConfig, out string, stop task.Signal) error {
	// Silently attempt to delete the old trace in case the user has changed
	// If the user has changed from shell => root, this is fine. For root => shell,
	// the root user will have still have to manually delete the old trace if it exists
	b.RemoveFile(ctx, out)

	reader, stdout := io.Pipe()
	logRing := ring.New(10)
	data, err := proto.Marshal(config)
	if err != nil {
		return err
	}

	fail := make(chan error, 1)
	crash.Go(func() {
		buf := bufio.NewReader(reader)
		for {
			line, e := buf.ReadString('\n')
			logRing.Value = line
			logRing = logRing.Next()
			switch e {
			default:
				log.E(ctx, "[perfetto] Read error %v", e)
				fail <- e
				return
			case io.EOF:
				fail <- nil
				return
			case nil:
				log.I(ctx, "[perfetto] %s", strings.TrimSuffix(line, "\n"))
			}
		}
	})

	process, err := b.Shell("base64", "-d", "|", "perfetto", "-c", "-", "-o", out).
		Read(strings.NewReader(base64.StdEncoding.EncodeToString(data))).
		Capture(stdout, stdout).
		Start(ctx)
	if err != nil {
		stdout.Close()
		return err
	}

	wait := make(chan error, 1)
	crash.Go(func() {
		wait <- process.Wait(ctx)
	})

	select {
	case err = <-fail:
		return err
	case err = <-wait:
		// Use perfetto's error messaging instead of the default
		if err != nil {
			msg := ""
			logRing.Do(func(p interface{}) {
				if p != nil && len(p.(string)) > 0 {
					msg += p.(string) + "\n"
				}
			})
			// Remove ANSI color codes from perfetto output
			ansiRegex := regexp.MustCompile("\u001b\\[\\d+m")
			msg = ansiRegex.ReplaceAllString(msg, "")
			err = errors.New(msg)
		}
	case <-stop:
		// TODO: figure out why "killall -2 perfetto" doesn't work.
		var pid string
		if pid, err = b.Shell("pidof perfetto").Call(ctx); err != nil {
			break
		}
		if err = b.Shell("kill -2 " + pid).Run(ctx); err != nil {
			break
		}
		err = <-wait
	}

	stdout.Close()
	if err != nil {
		return err
	}
	return <-fail
}
<commit_msg>Small fix to the Perfetto error handling.<commit_after>// Copyright (C) 2019 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package adb

import (
	"bufio"
	"container/ring"
	"context"
	"encoding/base64"
	"errors"
	"io"
	"regexp"
	"strings"

	perfetto_pb "perfetto/config"

	"github.com/golang/protobuf/proto"
	"github.com/google/gapid/core/app/crash"
	"github.com/google/gapid/core/event/task"
	"github.com/google/gapid/core/log"
)

var (
	ansiRegex = regexp.MustCompile("\u001b\\[\\d+m")
)

// StartPerfettoTrace starts a perfetto trace on this device.
func (b *binding) StartPerfettoTrace(ctx context.Context, config *perfetto_pb.TraceConfig, out string, stop task.Signal) error {
	// Silently attempt to delete the old trace in case the user has changed
	// If the user has changed from shell => root, this is fine. For root => shell,
	// the root user will have still have to manually delete the old trace if it exists
	b.RemoveFile(ctx, out)

	reader, stdout := io.Pipe()
	logRing := ring.New(10)
	data, err := proto.Marshal(config)
	if err != nil {
		return err
	}

	fail := make(chan error, 1)
	crash.Go(func() {
		buf := bufio.NewReader(reader)
		for {
			line, e := buf.ReadString('\n')
			logRing.Value = line
			logRing = logRing.Next()
			switch e {
			default:
				log.E(ctx, "[perfetto] Read error %v", e)
				fail <- e
				return
			case io.EOF:
				fail <- nil
				return
			case nil:
				log.I(ctx, "[perfetto] %s", strings.TrimSuffix(line, "\n"))
			}
		}
	})

	process, err := b.Shell("base64", "-d", "|", "perfetto", "-c", "-", "-o", out).
		Read(strings.NewReader(base64.StdEncoding.EncodeToString(data))).
		Capture(stdout, stdout).
		Start(ctx)
	if err != nil {
		stdout.Close()
		return err
	}

	wait := make(chan error, 1)
	crash.Go(func() {
		wait <- process.Wait(ctx)
	})

	select {
	case err = <-fail:
		return err
	case err = <-wait:
		// Use perfetto's error messaging instead of the default
		if err != nil {
			msg := ""
			logRing.Do(func(p interface{}) {
				if s, ok := p.(string); ok && len(s) > 0 {
					msg += "\n" + s
				}
			})
			// Remove ANSI color codes from perfetto output
			msg = ansiRegex.ReplaceAllString(msg, "")

			if msg != "" {
				err = errors.New(err.Error() + "\nPerfetto Output:" + msg)
			}
		}
	case <-stop:
		// TODO: figure out why "killall -2 perfetto" doesn't work.
		var pid string
		if pid, err = b.Shell("pidof perfetto").Call(ctx); err != nil {
			break
		}
		if err = b.Shell("kill -2 " + pid).Run(ctx); err != nil {
			break
		}
		err = <-wait
	}

	stdout.Close()
	if err != nil {
		return err
	}
	return <-fail
}
<|endoftext|>


<commit_before>package stats

import (
	"math"
	"sort"
)

// Min finds the lowest number in a slice
func Min(input []float64) (min float64) {

	// Get the initial value
	// @todo add error if no length
	if len(input) > 0 {
		min = input[0]
	}

	// Iterate until done checking for a lower value
	for i := 1; i < len(input); i++ {
		if input[i] < min {
			min = input[i]
		}
	}
	return min
}

// Max finds the highest number in a slice
func Max(input []float64) (max float64) {
	if len(input) > 0 {
		max = input[0]
	}
	for i := 1; i < len(input); i++ {
		if input[i] > max {
			max = input[i]
		}
	}
	return max
}

// Sum adds all the numbers of a slice together
func Sum(input []float64) (sum float64) {
	for _, n := range input {
		sum += float64(n)
	}
	return sum
}

// Mean gets the average of a slice of numbers
func Mean(input []float64) (mean float64) {
	if len(input) == 0 {
		return 0.0
	}

	sum := Sum(input)

	return sum / float64(len(input))
}

// Median gets the median number in a slice of numbers
func Median(input []float64) (median float64) {

	// Start by sorting the slice
	sort.Float64s(input)

	l := len(input)

	// No math is needed if there are no numbers
	// For even numbers we add the two middle numbers
	// and divide by two using the mean function above
	// For odd numbers we just use the middle number
	if l == 0 {
		return 0.0
	} else if l%2 == 0 {
		median = Mean(input[l/2-1 : l/2+1])
	} else {
		median = float64(input[l/2])
	}

	return median
}

// Mode gets the mode of a slice of numbers
func Mode(input []float64) (mode []float64) {

	// Create a map with the counts for each number
	m := make(map[float64]int)
	for _, v := range input {
		m[v]++
	}

	// Find the highest counts to return as a slice
	// of ints to accomodate duplicate counts
	var current int
	for k, v := range m {

		// Depending if the count is lower, higher
		// or equal to the current numbers count
		// we return nothing, start a new mode or
		// append to the current mode
		switch {
		case v < current:
		case v > current:
			current = v
			mode = append(mode[:0], k)
		default:
			mode = append(mode, k)
		}
	}

	// Finally we check to see if there actually was
	// a mode by checking the length of the input and
	// mode against eachother
	l := len(input)
	lm := len(mode)
	if l == lm {
		return []float64{}
	}
	return mode
}

// Variance finds the variance for both population and sample data
func Variance(input []float64, sample int) (variance float64) {
	if len(input) == 0 {
		return 0.0
	}

	// Sum the square of the mean subtracted from each number
	m := Mean(input)
	for _, n := range input {
		variance += (float64(n) - m) * (float64(n) - m)
	}

	// When getting the mean of the squared differences
	// "sample" will allow us to know if it's a sample
	// or population and wether to subtract by one or not
	return variance / float64((len(input) - (1 * sample)))
}

// VarP finds the amount of variance within a population
func VarP(input []float64) (sdev float64) {
	return Variance(input, 0)
}

// VarS finds the amount of variance within a sample
func VarS(input []float64) (sdev float64) {
	return Variance(input, 1)
}

// StdDevP finds the amount of variation from the population
func StdDevP(input []float64) (sdev float64) {
	if len(input) == 0 {
		return 0.0
	}

	// Get the population variance
	m := VarP(input)

	// Return the population standard deviation
	return math.Pow(m, 0.5)
}

// StdDevS finds the amount of variation from a sample
func StdDevS(input []float64) (sdev float64) {
	if len(input) == 0 {
		return 0.0
	}

	// Get the sample variance
	m := VarS(input)

	// Return the sample standard deviation
	return math.Pow(m, 0.5)
}

// Round a float to a specific decimal place or precision
func Round(input float64, places int) (rounded float64) {

	// If the float is not a number we just return it
	// @todo add an error
	if math.IsNaN(input) {
		return input
	}

	// Find out the actual sign and correct the input for later
	sign := 1.0
	if input < 0 {
		sign = -1
		input *= -1
	}

	// Use the places arg to get the amount of precision wanted
	precision := math.Pow(10, float64(places))

	// Find the decimal place we are looking to round
	digit := input * precision

	// Get the actual decimal number as a fraction to be compared
	_, decimal := math.Modf(digit)

	// If the decimal is less than .5 we round down otherwise up
	if decimal >= 0.5 {
		rounded = math.Ceil(digit)
	} else {
		rounded = math.Floor(digit)
	}

	// Finally we do the math to actually create a rounded number
	return rounded / precision * sign
}

// Percentile finds the relative standing in a slice of floats
func Percentile(input []float64, percent float64) (percentile float64) {

	// Sort the data
	sort.Float64s(input)

	// Multiple percent by length of input
	index := (percent / 100) * float64(len(input))

	// Check if the index is a whole number
	if index == float64(int64(index)) {

		// Convert float to int
		i := Float64ToInt(index)

		// Find the average of the index and following values
		percentile = Mean([]float64{input[i-1], input[i]})

	} else {

		// Convert float to int
		i := Float64ToInt(index)

		// Find the value at the index
		percentile = input[i-1]

	}

	return percentile

}

// Float64ToInt rounds a float64 to an int
func Float64ToInt(input float64) (output int) {

	// Round input to nearest whole number and convert to int
	return int(Round(input, 0))

}
<commit_msg>Add check for slices with a single item<commit_after>package stats

import (
	"math"
	"sort"
)

// Min finds the lowest number in a slice
func Min(input []float64) (min float64) {

	// Get the initial value
	// @todo add error if no length
	if len(input) > 0 {
		min = input[0]
	}

	// Iterate until done checking for a lower value
	for i := 1; i < len(input); i++ {
		if input[i] < min {
			min = input[i]
		}
	}
	return min
}

// Max finds the highest number in a slice
func Max(input []float64) (max float64) {
	if len(input) > 0 {
		max = input[0]
	}
	for i := 1; i < len(input); i++ {
		if input[i] > max {
			max = input[i]
		}
	}
	return max
}

// Sum adds all the numbers of a slice together
func Sum(input []float64) (sum float64) {
	for _, n := range input {
		sum += float64(n)
	}
	return sum
}

// Mean gets the average of a slice of numbers
func Mean(input []float64) (mean float64) {
	if len(input) == 0 {
		return 0.0
	}

	sum := Sum(input)

	return sum / float64(len(input))
}

// Median gets the median number in a slice of numbers
func Median(input []float64) (median float64) {

	// Start by sorting the slice
	sort.Float64s(input)

	l := len(input)

	// No math is needed if there are no numbers
	// For even numbers we add the two middle numbers
	// and divide by two using the mean function above
	// For odd numbers we just use the middle number
	if l == 0 {
		return 0.0
	} else if l%2 == 0 {
		median = Mean(input[l/2-1 : l/2+1])
	} else {
		median = float64(input[l/2])
	}

	return median
}

// Mode gets the mode of a slice of numbers
func Mode(input []float64) (mode []float64) {

	// Return the input if there's only one number
	// @todo add error for empty slice
	l := len(input)
	if l == 1 {
		return input
	}

	// Create a map with the counts for each number
	m := make(map[float64]int)
	for _, v := range input {
		m[v]++
	}

	// Find the highest counts to return as a slice
	// of ints to accomodate duplicate counts
	var current int
	for k, v := range m {

		// Depending if the count is lower, higher
		// or equal to the current numbers count
		// we return nothing, start a new mode or
		// append to the current mode
		switch {
		case v < current:
		case v > current:
			current = v
			mode = append(mode[:0], k)
		default:
			mode = append(mode, k)
		}
	}

	// Finally we check to see if there actually was
	// a mode by checking the length of the input and
	// mode against eachother
	lm := len(mode)
	if l == lm {
		return []float64{}
	}
	return mode
}

// Variance finds the variance for both population and sample data
func Variance(input []float64, sample int) (variance float64) {
	if len(input) == 0 {
		return 0.0
	}

	// Sum the square of the mean subtracted from each number
	m := Mean(input)
	for _, n := range input {
		variance += (float64(n) - m) * (float64(n) - m)
	}

	// When getting the mean of the squared differences
	// "sample" will allow us to know if it's a sample
	// or population and wether to subtract by one or not
	return variance / float64((len(input) - (1 * sample)))
}

// VarP finds the amount of variance within a population
func VarP(input []float64) (sdev float64) {
	return Variance(input, 0)
}

// VarS finds the amount of variance within a sample
func VarS(input []float64) (sdev float64) {
	return Variance(input, 1)
}

// StdDevP finds the amount of variation from the population
func StdDevP(input []float64) (sdev float64) {
	if len(input) == 0 {
		return 0.0
	}

	// Get the population variance
	m := VarP(input)

	// Return the population standard deviation
	return math.Pow(m, 0.5)
}

// StdDevS finds the amount of variation from a sample
func StdDevS(input []float64) (sdev float64) {
	if len(input) == 0 {
		return 0.0
	}

	// Get the sample variance
	m := VarS(input)

	// Return the sample standard deviation
	return math.Pow(m, 0.5)
}

// Round a float to a specific decimal place or precision
func Round(input float64, places int) (rounded float64) {

	// If the float is not a number we just return it
	// @todo add an error
	if math.IsNaN(input) {
		return input
	}

	// Find out the actual sign and correct the input for later
	sign := 1.0
	if input < 0 {
		sign = -1
		input *= -1
	}

	// Use the places arg to get the amount of precision wanted
	precision := math.Pow(10, float64(places))

	// Find the decimal place we are looking to round
	digit := input * precision

	// Get the actual decimal number as a fraction to be compared
	_, decimal := math.Modf(digit)

	// If the decimal is less than .5 we round down otherwise up
	if decimal >= 0.5 {
		rounded = math.Ceil(digit)
	} else {
		rounded = math.Floor(digit)
	}

	// Finally we do the math to actually create a rounded number
	return rounded / precision * sign
}

// Percentile finds the relative standing in a slice of floats
func Percentile(input []float64, percent float64) (percentile float64) {

	// Sort the data
	sort.Float64s(input)

	// Multiple percent by length of input
	index := (percent / 100) * float64(len(input))

	// Check if the index is a whole number
	if index == float64(int64(index)) {

		// Convert float to int
		i := Float64ToInt(index)

		// Find the average of the index and following values
		percentile = Mean([]float64{input[i-1], input[i]})

	} else {

		// Convert float to int
		i := Float64ToInt(index)

		// Find the value at the index
		percentile = input[i-1]

	}

	return percentile

}

// Float64ToInt rounds a float64 to an int
func Float64ToInt(input float64) (output int) {

	// Round input to nearest whole number and convert to int
	return int(Round(input, 0))

}
<|endoftext|>


<commit_before>package stein

import (
	"bufio"
	"encoding/json"
	"fmt"
	"gopkg.in/v1/yaml"
	"io"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	docSep      = regexp.MustCompile(`(?m)^---$|^\.\.\.$`)
	tapTestLine = regexp.MustCompile(`(?m)^(ok|not ok)(?: (\d+))?( [^#]+)?(?:\s*#(.*))?$`)
	tapPlan     = regexp.MustCompile(`(?m)^\d..(\d+|N).*$`)
	tapVersion  = regexp.MustCompile(`(?m)^TAP version \d+$`)
	tapBailOut  = regexp.MustCompile(`(?m)^Bail out!\s*(.*)$`)
)

func Parse(r io.Reader) (*Suite, error) {
	parser := Parser{&DefaultHandler{}}
	return parser.Parse(r)
}

type Parser struct {
	Handler Handler
}

func (p *Parser) Parse(r io.Reader) (*Suite, error) {
	rd := bufio.NewReader(r)
	b, err := rd.Peek(1)
	if err != nil {
		panic(err)
	}

	if p.Handler == nil {
		p.Handler = &DefaultHandler{}
	}

	s := new(Suite)
	switch b[0] {
	case '{':
		return s, s.parse(newDecoder("json", rd), p.Handler)
	case '-':
		return s, s.parse(newDecoder("yaml", rd), p.Handler)
	default:
		return ParseTap(rd, p.Handler)
	}
}

type Time struct {
	time.Time
}

func (t *Time) parse(val string) (err error) {
	var ti time.Time
	if ti, err = time.Parse("2006-01-02 15:04:05", val); err != nil {
		if ti, err = time.Parse(time.RFC3339, val); err != nil {
			return
		}
	}
	(*t).Time = ti
	return
}

func (t *Time) UnmarshalJSON(data []byte) (err error) {
	// parse time, stripping the quotes
	return t.parse(string(data[1 : len(data)-1]))
}
func (t *Time) GetYAML() (tag string, value interface{}) {
	return "", t.String()
}
func (t *Time) SetYAML(tag string, value interface{}) bool {
	if v, ok := value.(string); ok {
		return t.parse(strings.Trim(v, `'"`)) == nil
	}
	return false
}

type Suite struct {
	Type  string
	Start *Time
	Count int
	Seed  int
	Rev   int
	Extra interface{} `yaml:",omitempty" json:",omitempty"`

	Tests []*Test
	Cases []*Case
	Notes []Note
	Final Tally
}

func (s *Suite) parse(dec decoder, handler Handler) (err error) {
	handler.HandleBeforeStream()
	var lastCase *Case
	for dec.Scan() {
		doc := dec.Scanner.Text()
		handler.HandleDoc(doc, dec.format)

		if strings.TrimSpace(doc) == "" {
			continue
		}

		if s.Start == nil {
			if err = dec.Unmarshal([]byte(doc), &s); err != nil {
				return
			}
			handler.HandleSuite(s)
			continue
		}

		var typ string
		if typ, err = dec.Type([]byte(doc)); err != nil {
			return
		}

		switch typ {
		case "case":
			var c Case
			if err = dec.Unmarshal([]byte(doc), &c); err != nil {
				return
			}

			if c.Level == 0 {
				s.Cases = append(s.Cases, &c)
				lastCase = &c
			} else if lastCase == nil {
				// TODO: possibly just add this case to the suite
				return fmt.Errorf("No parent found for test case: %v", c)
			} else {
				// find the parent
				for c.Level <= lastCase.Level {
					lastCase = lastCase.parent
				}
				// TODO: strict check to ensure c.Level == lastCase.Level + 1
				c.parent = lastCase
				lastCase.Subcases = append(lastCase.Subcases, &c)
			}
			handler.HandleCase(&c)
		case "test":
			var t Test
			if err = dec.Unmarshal([]byte(doc), &t); err != nil {
				return
			}
			if lastCase != nil {
				lastCase.Tests = append(lastCase.Tests, &t)
			} else {
				s.Tests = append(s.Tests, &t)
			}
			handler.HandleTest(&t)
		case "note":
			var n Note
			if err = dec.Unmarshal([]byte(doc), &n); err != nil {
				return
			}
			s.Notes = append(s.Notes, n)
			handler.HandleNote(&n)
		case "tally":
			var t Tally
			if err = dec.Unmarshal([]byte(doc), &t); err != nil {
				return
			}
			handler.HandleTally(&t)
		case "final":
			var t Tally
			if err = dec.Unmarshal([]byte(doc), &t); err != nil {
				return
			}
			s.Final = t
			handler.HandleFinal(&t)
		default:
			err = fmt.Errorf("Invalid type: %s", typ)
			return
		}
	}
	handler.HandleAfterStream()
	return
}

func (s Suite) Docs() (ret []interface{}) {
	ret = append(ret, s)
	for _, t := range s.Tests {
		ret = append(ret, t)
	}

	for _, c := range s.Cases {
		ret = append(ret, c.Docs()...)
	}

	for _, n := range s.Notes {
		ret = append(ret, n)
	}

	ret = append(ret, s.Final)
	return
}

func (s Suite) ToTapY() ([]byte, error) {
	var ret []byte
	for _, doc := range s.Docs() {
		if b, err := yaml.Marshal(doc); err != nil {
			return nil, err
		} else {
			ret = append(ret, []byte("---\n")...)
			ret = append(ret, b...)
			ret = append(ret, '\n')
		}
	}
	return append(ret, []byte("...")...), nil
}

type Case struct {
	Type    string
	Subtype string
	Label   string
	Level   int
	Extra   interface{} `yaml:",omitempty" json:",omitempty"`

	Tests    []*Test `yaml:"-" json:"-"`
	Subcases []*Case `yaml:"-" json:"-"`

	parent *Case
}

func (c Case) Docs() (ret []interface{}) {
	ret = append(ret, c)
	for _, t := range c.Tests {
		ret = append(ret, t)
	}
	for _, c := range c.Subcases {
		ret = append(ret, c.Docs()...)
	}
	return
}

// TODO: allow strings via UnmarshalJSON/YAML
type Snippet []map[string]string

type Test struct {
	Type     string
	Subtype  string `yaml:",omitempty" json:",omitempty"`
	Status   string
	Setup    string `yaml:",omitempty" json:",omitempty"`
	Label    string
	Expected interface{} `yaml:",omitempty" json:",omitempty"`
	Returned interface{} `yaml:",omitempty" json:",omitempty"`
	File     string      `yaml:",omitempty" json:",omitempty"`
	Line     int         `yaml:",omitempty" json:",omitempty"`
	Source   string      `yaml:",omitempty" json:",omitempty"`
	Snippet  Snippet     `yaml:",omitempty" json:",omitempty"`
	Coverage struct {
		File string
		Line interface{}
		Code string
	} `yaml:",omitempty" json:",omitempty"`
	Exception struct {
		Message   string
		File      string
		Line      int
		Source    string
		Snippet   Snippet
		Backtrace interface{}
	} `yaml:",omitempty" json:",omitempty"`
	Stdout string `yaml:",omitempty" json:",omitempty"`
	Stderr string `yaml:",omitempty" json:",omitempty"`
	Time   float64
	Extra  interface{} `yaml:",omitempty" json:",omitempty"`
}

type Note struct {
	Type  string
	Text  string
	Extra interface{} `yaml:",omitempty" json:",omitempty"`
}

type Tally struct {
	Type   string
	Time   float64
	Counts struct {
		Total int
		Pass  int
		Fail  int
		Error int
		Omit  int
		Todo  int
	}
	Extra interface{} `yaml:",omitempty" json:",omitempty"`
}

type decoder struct {
	*bufio.Scanner
	format    string
	Unmarshal func([]byte, interface{}) error
}

func scanYAMLDoc(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if loc := docSep.FindIndex(data); loc != nil {
		if loc[0] == 0 {
			// just a start document marker
			return loc[1], nil, nil
		}
		// We found a complete document
		return loc[1], data[0:loc[0]], nil
	}
	if atEOF {
		// if we're at EOF without having matched docSep, there's nothing else to get
		return len(data), nil, nil
	}
	// Request more data.
	return 0, nil, nil
}

func newDecoder(typ string, r io.Reader) decoder {
	dec := decoder{Scanner: bufio.NewScanner(r)}
	// TODO: handle plain TAP with a decoder as well
	switch typ {
	case "json":
		dec.format = "json"
		dec.Unmarshal = json.Unmarshal
	case "yaml":
		dec.format = "yaml"
		dec.Unmarshal = yaml.Unmarshal
		dec.Scanner.Split(scanYAMLDoc)
	default:
		panic("Unsupported decoder")
	}
	return dec
}

func (dec decoder) Type(b []byte) (string, error) {
	m := make(map[string]interface{})
	if err := dec.Unmarshal(b, &m); err != nil {
		return "", fmt.Errorf("Invalid document: %s", err)
	}
	if typ, ok := m["type"].(string); !ok {
		return "", fmt.Errorf("Missing 'type' key in document")
	} else {
		return typ, nil
	}
}

func ParseTap(r io.Reader, handler Handler) (*Suite, error) {
	if handler == nil {
		handler = &DefaultHandler{}
	}
	handler.HandleBeforeStream()
	rd := bufio.NewReader(r)
	var s Suite
	s.Start = &Time{time.Now()}
	first := true
	var totalTests int
	for {
		line, err := rd.ReadString('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		handler.HandleDoc(line, "tap")

		if line[0] == '#' {
			// TODO: do something with diagnostic lines
			continue
		}

		if matches := tapBailOut.FindStringSubmatch(line); matches != nil {
			// TODO: do something with the bail reason
			break
		}

		if first {
			if matches := tapVersion.FindStringSubmatch(line); matches != nil {
				// TAP 13+
				return nil, fmt.Errorf("TAP13 not supported yet")
			}
			first = false
			if matches := tapPlan.FindStringSubmatch(line); matches != nil {
				totalTests, _ = strconv.Atoi(matches[1])
				s.Count = totalTests
				// assuming plan line at beginning of output
				handler.HandleSuite(&s)
				continue
			}
		}

		matches := tapTestLine.FindStringSubmatch(line)
		if matches == nil {
			// TODO: handle extra data
			continue
		}

		t := Test{Type: "test"}
		switch matches[1] {
		case "ok":
			t.Status = "pass"
		case "not ok":
			// TODO: error?
			t.Status = "fail"
		}
		// ignore number
		t.Label = strings.TrimSpace(matches[3])
		directive := strings.TrimSpace(matches[4])
		switch {
		case strings.HasPrefix(strings.ToUpper(directive), "TODO"):
			s.Final.Counts.Todo++
			t.Label = directive
			t.Status = "todo"
		case strings.HasPrefix(strings.ToUpper(directive), "SKIP"):
			s.Final.Counts.Omit++
			t.Label = directive
			t.Status = "omit"
		case t.Status == "pass":
			s.Final.Counts.Pass++
		default:
			s.Final.Counts.Fail++
		}
		handler.HandleTest(&t)
		s.Final.Counts.Total++
		s.Tests = append(s.Tests, &t)
	}

	// fixup missing tests
	if totalTests > 0 && totalTests > s.Final.Counts.Total {
		s.Final.Counts.Fail += (totalTests - s.Final.Counts.Total)
		s.Final.Counts.Total = totalTests
	}

	handler.HandleFinal(&s.Final)
	handler.HandleAfterStream()
	return &s, nil
}
<commit_msg>Store tests and subcases in the database<commit_after>package stein

import (
	"bufio"
	"encoding/json"
	"fmt"
	"gopkg.in/v1/yaml"
	"io"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	docSep      = regexp.MustCompile(`(?m)^---$|^\.\.\.$`)
	tapTestLine = regexp.MustCompile(`(?m)^(ok|not ok)(?: (\d+))?( [^#]+)?(?:\s*#(.*))?$`)
	tapPlan     = regexp.MustCompile(`(?m)^\d..(\d+|N).*$`)
	tapVersion  = regexp.MustCompile(`(?m)^TAP version \d+$`)
	tapBailOut  = regexp.MustCompile(`(?m)^Bail out!\s*(.*)$`)
)

func Parse(r io.Reader) (*Suite, error) {
	parser := Parser{&DefaultHandler{}}
	return parser.Parse(r)
}

type Parser struct {
	Handler Handler
}

func (p *Parser) Parse(r io.Reader) (*Suite, error) {
	rd := bufio.NewReader(r)
	b, err := rd.Peek(1)
	if err != nil {
		panic(err)
	}

	if p.Handler == nil {
		p.Handler = &DefaultHandler{}
	}

	s := new(Suite)
	switch b[0] {
	case '{':
		return s, s.parse(newDecoder("json", rd), p.Handler)
	case '-':
		return s, s.parse(newDecoder("yaml", rd), p.Handler)
	default:
		return ParseTap(rd, p.Handler)
	}
}

type Time struct {
	time.Time
}

func (t *Time) parse(val string) (err error) {
	var ti time.Time
	if ti, err = time.Parse("2006-01-02 15:04:05", val); err != nil {
		if ti, err = time.Parse(time.RFC3339, val); err != nil {
			return
		}
	}
	(*t).Time = ti
	return
}

func (t *Time) UnmarshalJSON(data []byte) (err error) {
	// parse time, stripping the quotes
	return t.parse(string(data[1 : len(data)-1]))
}
func (t *Time) GetYAML() (tag string, value interface{}) {
	return "", t.String()
}
func (t *Time) SetYAML(tag string, value interface{}) bool {
	if v, ok := value.(string); ok {
		return t.parse(strings.Trim(v, `'"`)) == nil
	}
	return false
}

type Suite struct {
	Type  string
	Start *Time
	Count int
	Seed  int
	Rev   int
	Extra interface{} `yaml:",omitempty" json:",omitempty"`

	Tests []*Test
	Cases []*Case
	Notes []Note
	Final Tally
}

func (s *Suite) parse(dec decoder, handler Handler) (err error) {
	handler.HandleBeforeStream()
	var lastCase *Case
	for dec.Scan() {
		doc := dec.Scanner.Text()
		handler.HandleDoc(doc, dec.format)

		if strings.TrimSpace(doc) == "" {
			continue
		}

		if s.Start == nil {
			if err = dec.Unmarshal([]byte(doc), &s); err != nil {
				return
			}
			handler.HandleSuite(s)
			continue
		}

		var typ string
		if typ, err = dec.Type([]byte(doc)); err != nil {
			return
		}

		switch typ {
		case "case":
			var c Case
			if err = dec.Unmarshal([]byte(doc), &c); err != nil {
				return
			}

			if c.Level == 0 {
				s.Cases = append(s.Cases, &c)
				lastCase = &c
			} else if lastCase == nil {
				// TODO: possibly just add this case to the suite
				return fmt.Errorf("No parent found for test case: %v", c)
			} else {
				// find the parent
				for c.Level <= lastCase.Level {
					lastCase = lastCase.parent
				}
				// TODO: strict check to ensure c.Level == lastCase.Level + 1
				c.parent = lastCase
				lastCase.Subcases = append(lastCase.Subcases, &c)
			}
			handler.HandleCase(&c)
		case "test":
			var t Test
			if err = dec.Unmarshal([]byte(doc), &t); err != nil {
				return
			}
			if lastCase != nil {
				lastCase.Tests = append(lastCase.Tests, &t)
			} else {
				s.Tests = append(s.Tests, &t)
			}
			handler.HandleTest(&t)
		case "note":
			var n Note
			if err = dec.Unmarshal([]byte(doc), &n); err != nil {
				return
			}
			s.Notes = append(s.Notes, n)
			handler.HandleNote(&n)
		case "tally":
			var t Tally
			if err = dec.Unmarshal([]byte(doc), &t); err != nil {
				return
			}
			handler.HandleTally(&t)
		case "final":
			var t Tally
			if err = dec.Unmarshal([]byte(doc), &t); err != nil {
				return
			}
			s.Final = t
			handler.HandleFinal(&t)
		default:
			err = fmt.Errorf("Invalid type: %s", typ)
			return
		}
	}
	handler.HandleAfterStream()
	return
}

func (s Suite) Docs() (ret []interface{}) {
	ret = append(ret, s)
	for _, t := range s.Tests {
		ret = append(ret, t)
	}

	for _, c := range s.Cases {
		ret = append(ret, c.Docs()...)
	}

	for _, n := range s.Notes {
		ret = append(ret, n)
	}

	ret = append(ret, s.Final)
	return
}

func (s Suite) ToTapY() ([]byte, error) {
	var ret []byte
	for _, doc := range s.Docs() {
		if b, err := yaml.Marshal(doc); err != nil {
			return nil, err
		} else {
			ret = append(ret, []byte("---\n")...)
			ret = append(ret, b...)
			ret = append(ret, '\n')
		}
	}
	return append(ret, []byte("...")...), nil
}

type Case struct {
	Type    string
	Subtype string
	Label   string
	Level   int
	Extra   interface{} `yaml:",omitempty" json:",omitempty"`

	Tests    []*Test
	Subcases []*Case

	parent *Case
}

func (c Case) Docs() (ret []interface{}) {
	ret = append(ret, c)
	for _, t := range c.Tests {
		ret = append(ret, t)
	}
	for _, c := range c.Subcases {
		ret = append(ret, c.Docs()...)
	}
	return
}

// TODO: allow strings via UnmarshalJSON/YAML
type Snippet []map[string]string

type Test struct {
	Type     string
	Subtype  string `yaml:",omitempty" json:",omitempty"`
	Status   string
	Setup    string `yaml:",omitempty" json:",omitempty"`
	Label    string
	Expected interface{} `yaml:",omitempty" json:",omitempty"`
	Returned interface{} `yaml:",omitempty" json:",omitempty"`
	File     string      `yaml:",omitempty" json:",omitempty"`
	Line     int         `yaml:",omitempty" json:",omitempty"`
	Source   string      `yaml:",omitempty" json:",omitempty"`
	Snippet  Snippet     `yaml:",omitempty" json:",omitempty"`
	Coverage struct {
		File string
		Line interface{}
		Code string
	} `yaml:",omitempty" json:",omitempty"`
	Exception struct {
		Message   string
		File      string
		Line      int
		Source    string
		Snippet   Snippet
		Backtrace interface{}
	} `yaml:",omitempty" json:",omitempty"`
	Stdout string `yaml:",omitempty" json:",omitempty"`
	Stderr string `yaml:",omitempty" json:",omitempty"`
	Time   float64
	Extra  interface{} `yaml:",omitempty" json:",omitempty"`
}

type Note struct {
	Type  string
	Text  string
	Extra interface{} `yaml:",omitempty" json:",omitempty"`
}

type Tally struct {
	Type   string
	Time   float64
	Counts struct {
		Total int
		Pass  int
		Fail  int
		Error int
		Omit  int
		Todo  int
	}
	Extra interface{} `yaml:",omitempty" json:",omitempty"`
}

type decoder struct {
	*bufio.Scanner
	format    string
	Unmarshal func([]byte, interface{}) error
}

func scanYAMLDoc(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if loc := docSep.FindIndex(data); loc != nil {
		if loc[0] == 0 {
			// just a start document marker
			return loc[1], nil, nil
		}
		// We found a complete document
		return loc[1], data[0:loc[0]], nil
	}
	if atEOF {
		// if we're at EOF without having matched docSep, there's nothing else to get
		return len(data), nil, nil
	}
	// Request more data.
	return 0, nil, nil
}

func newDecoder(typ string, r io.Reader) decoder {
	dec := decoder{Scanner: bufio.NewScanner(r)}
	// TODO: handle plain TAP with a decoder as well
	switch typ {
	case "json":
		dec.format = "json"
		dec.Unmarshal = json.Unmarshal
	case "yaml":
		dec.format = "yaml"
		dec.Unmarshal = yaml.Unmarshal
		dec.Scanner.Split(scanYAMLDoc)
	default:
		panic("Unsupported decoder")
	}
	return dec
}

func (dec decoder) Type(b []byte) (string, error) {
	m := make(map[string]interface{})
	if err := dec.Unmarshal(b, &m); err != nil {
		return "", fmt.Errorf("Invalid document: %s", err)
	}
	if typ, ok := m["type"].(string); !ok {
		return "", fmt.Errorf("Missing 'type' key in document")
	} else {
		return typ, nil
	}
}

func ParseTap(r io.Reader, handler Handler) (*Suite, error) {
	if handler == nil {
		handler = &DefaultHandler{}
	}
	handler.HandleBeforeStream()
	rd := bufio.NewReader(r)
	var s Suite
	s.Start = &Time{time.Now()}
	first := true
	var totalTests int
	for {
		line, err := rd.ReadString('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		handler.HandleDoc(line, "tap")

		if line[0] == '#' {
			// TODO: do something with diagnostic lines
			continue
		}

		if matches := tapBailOut.FindStringSubmatch(line); matches != nil {
			// TODO: do something with the bail reason
			break
		}

		if first {
			if matches := tapVersion.FindStringSubmatch(line); matches != nil {
				// TAP 13+
				return nil, fmt.Errorf("TAP13 not supported yet")
			}
			first = false
			if matches := tapPlan.FindStringSubmatch(line); matches != nil {
				totalTests, _ = strconv.Atoi(matches[1])
				s.Count = totalTests
				// assuming plan line at beginning of output
				handler.HandleSuite(&s)
				continue
			}
		}

		matches := tapTestLine.FindStringSubmatch(line)
		if matches == nil {
			// TODO: handle extra data
			continue
		}

		t := Test{Type: "test"}
		switch matches[1] {
		case "ok":
			t.Status = "pass"
		case "not ok":
			// TODO: error?
			t.Status = "fail"
		}
		// ignore number
		t.Label = strings.TrimSpace(matches[3])
		directive := strings.TrimSpace(matches[4])
		switch {
		case strings.HasPrefix(strings.ToUpper(directive), "TODO"):
			s.Final.Counts.Todo++
			t.Label = directive
			t.Status = "todo"
		case strings.HasPrefix(strings.ToUpper(directive), "SKIP"):
			s.Final.Counts.Omit++
			t.Label = directive
			t.Status = "omit"
		case t.Status == "pass":
			s.Final.Counts.Pass++
		default:
			s.Final.Counts.Fail++
		}
		handler.HandleTest(&t)
		s.Final.Counts.Total++
		s.Tests = append(s.Tests, &t)
	}

	// fixup missing tests
	if totalTests > 0 && totalTests > s.Final.Counts.Total {
		s.Final.Counts.Fail += (totalTests - s.Final.Counts.Total)
		s.Final.Counts.Total = totalTests
	}

	handler.HandleFinal(&s.Final)
	handler.HandleAfterStream()
	return &s, nil
}
<|endoftext|>


<commit_before>package struc

import (
	"encoding/binary"
	"fmt"
	"io"
	"reflect"
)

func prep(data interface{}) (reflect.Value, Packable, error) {
	value := reflect.ValueOf(data)
	for value.Kind() == reflect.Ptr {
		next := value.Elem().Kind()
		if next == reflect.Struct || next == reflect.Ptr {
			value = value.Elem()
		} else {
			break
		}
	}
	switch value.Kind() {
	case reflect.Struct:
		fields, err := parseFields(value)
		return value, fields, err
	default:
		if !value.IsValid() {
			return reflect.Value{}, nil, fmt.Errorf("Invalid reflect.Value for %+v", data)
		}
		return value, &binaryFallback{value, binary.BigEndian}, nil
	}
}

func Pack(w io.Writer, data interface{}) error {
	return PackWithOrder(w, data, nil)
}

// TODO: this is destructive with caching
func PackWithOrder(w io.Writer, data interface{}, order binary.ByteOrder) error {
	val, fields, err := prep(data)
	if err != nil {
		return err
	}
	if order != nil {
		fields.SetByteOrder(order)
	}
	size := fields.Sizeof(val)
	buf := make([]byte, size)
	if err := fields.Pack(buf, val); err != nil {
		return err
	}
	_, err = w.Write(buf)
	return err
}

func Unpack(r io.Reader, data interface{}) error {
	return UnpackWithOrder(r, data, nil)
}

func UnpackWithOrder(r io.Reader, data interface{}, order binary.ByteOrder) error {
	val, fields, err := prep(data)
	if err != nil {
		return err
	}
	if order != nil {
		fields.SetByteOrder(order)
	}
	return fields.Unpack(r, val)
}

func Sizeof(data interface{}) (int, error) {
	val, fields, err := prep(data)
	if err != nil {
		return 0, err
	}
	return fields.Sizeof(val), nil
}
<commit_msg>fix raw packing strings<commit_after>package struc

import (
	"encoding/binary"
	"fmt"
	"io"
	"reflect"
)

func prep(data interface{}) (reflect.Value, Packable, error) {
	value := reflect.ValueOf(data)
	for value.Kind() == reflect.Ptr {
		next := value.Elem().Kind()
		if next == reflect.Struct || next == reflect.Ptr {
			value = value.Elem()
		} else {
			break
		}
	}
	switch value.Kind() {
	case reflect.Struct:
		fields, err := parseFields(value)
		return value, fields, err
	default:
		if !value.IsValid() {
			return reflect.Value{}, nil, fmt.Errorf("Invalid reflect.Value for %+v", data)
		}
		return value, &binaryFallback{value, binary.BigEndian}, nil
	}
}

func Pack(w io.Writer, data interface{}) error {
	return PackWithOrder(w, data, nil)
}

// TODO: this is destructive with caching
func PackWithOrder(w io.Writer, data interface{}, order binary.ByteOrder) error {
	val, fields, err := prep(data)
	if err != nil {
		return err
	}
	if order != nil {
		fields.SetByteOrder(order)
	}
	if val.Type().Kind() == reflect.String {
		val = val.Convert(reflect.TypeOf([]byte{}))
	}
	size := fields.Sizeof(val)
	buf := make([]byte, size)
	if err := fields.Pack(buf, val); err != nil {
		return err
	}
	_, err = w.Write(buf)
	return err
}

func Unpack(r io.Reader, data interface{}) error {
	return UnpackWithOrder(r, data, nil)
}

func UnpackWithOrder(r io.Reader, data interface{}, order binary.ByteOrder) error {
	val, fields, err := prep(data)
	if err != nil {
		return err
	}
	if order != nil {
		fields.SetByteOrder(order)
	}
	return fields.Unpack(r, val)
}

func Sizeof(data interface{}) (int, error) {
	val, fields, err := prep(data)
	if err != nil {
		return 0, err
	}
	return fields.Sizeof(val), nil
}
<|endoftext|>