<commit_before>package main

// TIME=1 HOUR (spawn slaves/observer, compute stats (avgs, ops/sec, hist))
// master:
// master distributes CLIENTS over slaves ("gorotines"), hands slaves a waitgroup
// master waits on the observer to know when to proceed to reporting
// master streams <average latency(msec), ops/sec> every 2 seconds to caller
// master builds a histogram of latencies <msec, count>
// master terminates out after the run completes
//

import (
	"launchpad.net/tomb"
	_ "log"
	"sync"
	"time"
)

type SummaryEvent struct {
	Duration           time.Duration
	MeanResponseTimeMs float64
	OpsPerSecond       float64
}

type master struct {
	t     tomb.Tomb
	conf  *BamConfig
	t0    time.Time
	wg    *sync.WaitGroup
	tm    *taskmaster
	hosts []*sandbox

	// histogram
	hist map[int64]int

	// rolling stats
	curr_lag_sum int64
	last_lag_avg float64
	curr_ops     int64
	last_ops     int64

	// summary events
	statsChan chan *SummaryEvent
}

func NewMaster(conf *BamConfig) *master {
	t0 := time.Now().Add(2 * time.Second)
	wg := &sync.WaitGroup{}

	return &master{
		conf:      conf,
		t0:        t0,
		wg:        wg,
		tm:        NewTaskMaster(wg),
		hosts:     make([]*sandbox, conf.Goroutines),
		hist:      make(map[int64]int),
		statsChan: make(chan *SummaryEvent),
	}
}

func (this *master) Start() {
	go this.loop()
}

func (this *master) Stop() (err error) {
	this.t.Kill(nil)
	return this.t.Wait()
}

func (this *master) SummaryEvents() <-chan *SummaryEvent {
	return this.statsChan
}

func (this *master) Histogram() map[int64]int {
	return this.hist
}

func (this *master) loop() {
	const ProgressInterval = 1 * time.Second

	defer this.t.Done()

	this.setup()

	rtChan := this.tm.ResponseTimes()
	prChan := time.After(3 * time.Second)

	for {
		select {
		case <-this.t.Dying():
			this.shutdown()
			return

		case <-this.tm.t.Dead():
			this.t.Kill(nil)

		case <-prChan:
			this.statsChan <- this.summarize()
			prChan = time.After(ProgressInterval)

		default:
			// Attempt to read from the channel a bunch of times
			// between each death check.
			this.capture(rtChan, 64)
		}
	}
}

func (this *master) setup() {
	// Describe sandboxes
	infos := make([]*SandboxInfo, this.conf.Goroutines)
	for i := 0; i < this.conf.Goroutines; i += 1 {
		infos[i] = &SandboxInfo{
			Id:          i,
			Properties:  this.conf.Properties,
			Duration:    this.conf.d,
			StartTime:   this.t0,
			Emitter:     this.tm,
			WaitGroup:   this.wg,
			ClientCount: 0,
		}
	}

	// Distribute clients evenly over sandboxes
	j := 0
	for i := 0; i < this.conf.Clients; i += 1 {
		infos[j].ClientCount += 1
		j += 1
	}

	// Initialize sandboxes
	this.hosts = make([]*sandbox, this.conf.Goroutines)
	for i, info := range infos {
		this.hosts[i] = NewSandbox(info)
		this.wg.Add(1)
	}

	// Spawn the taskmaster
	this.tm.Start()

	// Spawn each sandbox
	for _, host := range this.hosts {
		host.Start()
	}
}

func (this *master) shutdown() {
	close(this.statsChan)
}

func (this *master) capture(ch LatencyEventsChannel, reads int) {
	hist := this.hist
	chunk_lag := int64(0)
	chunk_ops := int64(0)

	for i := 0; i < reads; i += 1 {
		usec, ok := <-ch
		if !ok {
			break
		}

		// Update the histogram
		if count, ok := hist[usec]; ok {
			hist[usec] = count + 1
		} else {
			hist[usec] = 1
		}

		chunk_lag += usec
		chunk_ops += 1
	}

	// Update the intermediate sums
	this.curr_lag_sum += chunk_lag
	this.curr_ops += chunk_ops
}

func (this *master) summarize() *SummaryEvent {
	run_time := time.Since(this.t0)
	next_ops := this.last_ops + this.curr_ops

	// Compute the weighted average response time
	w0 := float64(this.last_ops) / float64(next_ops)
	w1 := float64(this.curr_ops) / float64(next_ops)
	curr_lag_avg := float64(this.curr_lag_sum) / float64(this.curr_ops)
	next_lag_avg := (w0 * this.last_lag_avg) + (w1 * curr_lag_avg)

	// Compute the current throughput ops/sec
	next_ops_sec := float64(next_ops) / run_time.Seconds()

	// Update
	this.last_lag_avg = next_lag_avg
	this.last_ops = next_ops

	// Reset counters
	this.curr_lag_sum = 0
	this.curr_ops = 0

	return &SummaryEvent{run_time, next_lag_avg, next_ops_sec}
}
<commit_msg>fix a bug that was throwing off throughput<commit_after>package main

// TIME=1 HOUR (spawn slaves/observer, compute stats (avgs, ops/sec, hist))
// master:
// master distributes CLIENTS over slaves ("gorotines"), hands slaves a waitgroup
// master waits on the observer to know when to proceed to reporting
// master streams <average latency(msec), ops/sec> every 2 seconds to caller
// master builds a histogram of latencies <msec, count>
// master terminates out after the run completes
//

import (
	"launchpad.net/tomb"
	_ "log"
	"sync"
	"time"
)

type SummaryEvent struct {
	Duration           time.Duration
	MeanResponseTimeMs float64
	OpsPerSecond       float64
}

type master struct {
	t     tomb.Tomb
	conf  *BamConfig
	t0    time.Time
	wg    *sync.WaitGroup
	tm    *taskmaster
	hosts []*sandbox

	// histogram
	hist map[int64]int

	// rolling stats
	curr_lag_sum int64
	last_lag_avg float64
	curr_ops     int64
	last_ops     int64

	// summary events
	statsChan chan *SummaryEvent
}

func NewMaster(conf *BamConfig) *master {
	wg := &sync.WaitGroup{}

	return &master{
		conf:      conf,
		wg:        wg,
		tm:        NewTaskMaster(wg),
		hosts:     make([]*sandbox, conf.Goroutines),
		hist:      make(map[int64]int),
		statsChan: make(chan *SummaryEvent),
	}
}

func (this *master) Start() {
	go this.loop()
}

func (this *master) Stop() (err error) {
	this.t.Kill(nil)
	return this.t.Wait()
}

func (this *master) SummaryEvents() <-chan *SummaryEvent {
	return this.statsChan
}

func (this *master) Histogram() map[int64]int {
	return this.hist
}

func (this *master) loop() {
	const ProgressInterval = 1 * time.Second

	defer this.t.Done()

	this.setup()

	rtChan := this.tm.ResponseTimes()
	prChan := time.After(2 * time.Second)

	for {
		select {
		case <-this.t.Dying():
			this.shutdown()
			return

		case <-this.tm.t.Dead():
			this.t.Kill(nil)

		case <-prChan:
			this.statsChan <- this.summarize()
			prChan = time.After(ProgressInterval)

		default:
			// Attempt to read from the channel a bunch of times
			// between each death check.
			this.capture(rtChan, 64)
		}
	}
}

func (this *master) setup() {
	// Record approximate start time
	this.t0 = time.Now()

	// Describe sandboxes
	infos := make([]*SandboxInfo, this.conf.Goroutines)
	for i := 0; i < this.conf.Goroutines; i += 1 {
		infos[i] = &SandboxInfo{
			Id:          i,
			Properties:  this.conf.Properties,
			Duration:    this.conf.d,
			StartTime:   this.t0,
			Emitter:     this.tm,
			WaitGroup:   this.wg,
			ClientCount: 0,
		}
	}

	// Distribute clients evenly over sandboxes
	j := 0
	for i := 0; i < this.conf.Clients; i += 1 {
		infos[j].ClientCount += 1
		j += 1
	}

	// Initialize sandboxes
	this.hosts = make([]*sandbox, this.conf.Goroutines)
	for i, info := range infos {
		this.hosts[i] = NewSandbox(info)
		this.wg.Add(1)
	}

	// Spawn the taskmaster
	this.tm.Start()

	// Spawn each sandbox
	for _, host := range this.hosts {
		host.Start()
	}
}

func (this *master) shutdown() {
	close(this.statsChan)
}

func (this *master) capture(ch LatencyEventsChannel, reads int) {
	hist := this.hist
	chunk_lag := int64(0)
	chunk_ops := int64(0)

	for i := 0; i < reads; i += 1 {
		usec, ok := <-ch
		if !ok {
			break
		}

		// Update the histogram
		if count, ok := hist[usec]; ok {
			hist[usec] = count + 1
		} else {
			hist[usec] = 1
		}

		chunk_lag += usec
		chunk_ops += 1
	}

	// Update the intermediate sums
	this.curr_lag_sum += chunk_lag
	this.curr_ops += chunk_ops
}

func (this *master) summarize() *SummaryEvent {
	run_time := time.Since(this.t0)
	next_ops := this.last_ops + this.curr_ops

	// Compute the weighted average response time
	w0 := float64(this.last_ops) / float64(next_ops)
	w1 := float64(this.curr_ops) / float64(next_ops)
	curr_lag_avg := float64(this.curr_lag_sum) / float64(this.curr_ops)
	next_lag_avg := (w0 * this.last_lag_avg) + (w1 * curr_lag_avg)

	// Compute the current throughput ops/sec
	next_ops_sec := float64(next_ops) / run_time.Seconds()

	// Update
	this.last_lag_avg = next_lag_avg
	this.last_ops = next_ops

	// Reset counters
	this.curr_lag_sum = 0
	this.curr_ops = 0

	return &SummaryEvent{run_time, next_lag_avg, next_ops_sec}
}
<|endoftext|>


<commit_before>package zerolog

import (
	"encoding/json"
	"fmt"

	"github.com/rs/zerolog"
	"github.com/tychoish/grip/level"
	"github.com/tychoish/grip/message"
	"github.com/tychoish/grip/send"
)

type shim struct {
	zl zerolog.Logger
	*send.Base
}

func NewSender(name string, l send.LevelInfo, zl zerolog.Logger) (send.Sender, error) {
	s := &shim{
		Base: send.NewBase(name),
	}

	if err := s.SetLevel(l); err != nil {
		return nil, fmt.Errorf("problem seeting level on new sender: %w", err)
	}

	return s, nil
}

func MakeSender(zl zerolog.Logger) send.Sender {
	s, _ := NewSender("", send.LevelInfo{Threshold: level.Trace, Default: level.Debug}, zl)
	return s
}

func convertLevel(in level.Priority) zerolog.Level {
	switch in {
	case level.Emergency:
		return zerolog.ErrorLevel
	case level.Alert:
		return zerolog.ErrorLevel
	case level.Critical:
		return zerolog.ErrorLevel
	case level.Error:
		return zerolog.ErrorLevel
	case level.Warning:
		return zerolog.WarnLevel
	case level.Notice:
		return zerolog.InfoLevel
	case level.Info:
		return zerolog.InfoLevel
	case level.Debug:
		return zerolog.DebugLevel
	case level.Trace:
		return zerolog.DebugLevel
	case level.Invalid:
		return zerolog.Disabled
	default:
		return zerolog.NoLevel
	}

}

func (s *shim) Send(m message.Composer) {
	if !s.Level().ShouldLog(m) {
		return
	}
	// unwind group messages
	if grp, ok := m.(*message.GroupComposer); ok {
		for _, msg := range grp.Messages() {
			s.Send(msg)
		}
		return
	}

	event := s.zl.WithLevel(convertLevel(m.Priority()))
	if !m.Structured() {
		out, err := s.Formatter()(m)
		if err != nil {
			s.ErrorHandler()(err, m)
			return
		}
		event.Msg(out)
		return
	}

	// handle payloads to take advantage of the fast paths
	payload := m.Raw()
	switch data := payload.(type) {
	case zerolog.LogObjectMarshaler:
		event.EmbedObject(data)
	case zerolog.LogArrayMarshaler:
		event.Array("payload", data)
	case message.KVs:
		// opted to call event.Fields many times rather than
		// build a new slice. probably.
		pair := make([]any, 2)
		for _, kv := range data {
			pair[0], pair[1] = kv.Key, kv.Value
			event.Fields(pair)
		}
	case message.Fields:
		event.Fields(data)
	case map[string]any:
		event.Fields(data)
	case json.Marshaler:
		// message.KVs are json.Marshalers so make sure this
		// clause stays last.
		r, err := data.MarshalJSON()
		if err != nil {
			s.ErrorHandler()(err, m)
			return
		}
		event.RawJSON("payload", r)
	default:
		// in most cases this uses json.Marshler and
		// reflection, so this will end up being a slower path
		// than any of the above paths.
		event.Interface("payload", payload)
	}
	event.Send()
}
<commit_msg>zerolog: add documentation<commit_after>package zerolog

import (
	"encoding/json"
	"fmt"

	"github.com/rs/zerolog"
	"github.com/tychoish/grip/level"
	"github.com/tychoish/grip/message"
	"github.com/tychoish/grip/send"
)

type shim struct {
	zl zerolog.Logger
	*send.Base
}

// NewSener provides a simple shim around zerolog that's compatible
// with other grip interfaces. The primary differences between zerolog
// and grip are: a different level system and Zerolog contains a
// fast-path for JSON marshalling of structured log paths. The shim
// translates grip message types into appropriate Zerolog message
// building calls to preserve the fast path.
func NewSender(name string, l send.LevelInfo, zl zerolog.Logger) (send.Sender, error) {
	s := &shim{
		Base: send.NewBase(name),
	}

	if err := s.SetLevel(l); err != nil {
		return nil, fmt.Errorf("problem seeting level on new sender: %w", err)
	}

	return s, nil
}

// MakeSender constructs a sender object as NewSender but without the
// error type or level configuration, consistent with other grip
// sender constructors.
func MakeSender(zl zerolog.Logger) send.Sender {
	s, _ := NewSender("", send.LevelInfo{Threshold: level.Trace, Default: level.Debug}, zl)
	return s
}

func convertLevel(in level.Priority) zerolog.Level {
	switch in {
	case level.Emergency:
		return zerolog.ErrorLevel
	case level.Alert:
		return zerolog.ErrorLevel
	case level.Critical:
		return zerolog.ErrorLevel
	case level.Error:
		return zerolog.ErrorLevel
	case level.Warning:
		return zerolog.WarnLevel
	case level.Notice:
		return zerolog.InfoLevel
	case level.Info:
		return zerolog.InfoLevel
	case level.Debug:
		return zerolog.DebugLevel
	case level.Trace:
		return zerolog.DebugLevel
	case level.Invalid:
		return zerolog.Disabled
	default:
		return zerolog.NoLevel
	}

}

func (s *shim) Send(m message.Composer) {
	if !s.Level().ShouldLog(m) {
		return
	}
	// unwind group messages
	if grp, ok := m.(*message.GroupComposer); ok {
		for _, msg := range grp.Messages() {
			s.Send(msg)
		}
		return
	}

	event := s.zl.WithLevel(convertLevel(m.Priority()))
	if !m.Structured() {
		out, err := s.Formatter()(m)
		if err != nil {
			s.ErrorHandler()(err, m)
			return
		}
		event.Msg(out)
		return
	}

	// handle payloads to take advantage of the fast paths
	payload := m.Raw()
	switch data := payload.(type) {
	case zerolog.LogObjectMarshaler:
		event.EmbedObject(data)
	case zerolog.LogArrayMarshaler:
		event.Array("payload", data)
	case message.KVs:
		// opted to call event.Fields many times rather than
		// build a new slice. probably.
		pair := make([]any, 2)
		for _, kv := range data {
			pair[0], pair[1] = kv.Key, kv.Value
			event.Fields(pair)
		}
	case message.Fields:
		event.Fields(data)
	case map[string]any:
		event.Fields(data)
	case json.Marshaler:
		// message.KVs are json.Marshalers so make sure this
		// clause stays last.
		r, err := data.MarshalJSON()
		if err != nil {
			s.ErrorHandler()(err, m)
			return
		}
		event.RawJSON("payload", r)
	default:
		// in most cases this uses json.Marshler and
		// reflection, so this will end up being a slower path
		// than any of the above paths.
		event.Interface("payload", payload)
	}
	event.Send()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/alecthomas/kingpin"
	"github.com/ammario/mcping"
	"github.com/fatih/color"
	"time"
)

func main() {
	//Parse command line flags
	var (
		_host     = kingpin.Flag("host", "Hostname").Short('h').Required().String()
		_port     = kingpin.Flag("port", "Port number").Short('p').Default("25565").Int()
		_timeout  = kingpin.Flag("timeout", "Timeout per ping (ms)").Short('t').Default("300").Int()
		_interval = kingpin.Flag("interval", "Interval between pings (ms)").Short('i').Default("1000").Int()
		_count    = kingpin.Flag("count", "Total amount of pings to send").Short('c').Default("4294967295").Int()
	)

	kingpin.Parse()

	//Standardize input variable access

	host := *_host
	port := uint16(*_port)
	timeout := time.Duration(*_timeout)
	interval := time.Duration(*_interval)
	count := uint32(*_count)

	//Sequentiating ping responses
	var id uint32

	//Output colorer
	failOut := color.New(color.FgRed)
	successOut := color.New(color.FgGreen)

	//Output format strings
	failFmt := "(%x) %s; %s\n"
	successFmt := "(%x) %s; latency=%vms players=(%s)\n"

	fullAddr := fmt.Sprint(host, ":", port)

	for id = 0; id < count; id++ {
		//Have each request asynchronous
		go func(id uint32) {
			err := errors.New("")
			timeoutChan := make(chan bool, 1)

			//Default response
			resp := mcping.PingResponse{}

			//Race timeout against ping
			go func() {
				time.Sleep(timeout * time.Millisecond)
				timeoutChan <- true
			}()
			go func() {
				resp, err = mcping.Ping(host, port)
				timeoutChan <- false
			}()
			if <-timeoutChan {
				err = mcping.TimeoutErr
			}
			if err != nil {
				failOut.Printf(failFmt, id, fullAddr, err)
			} else {
				playerCount := fmt.Sprint(resp.Online, "/", resp.Max)
				latency := resp.Latency
				successOut.Printf(successFmt, id, fullAddr, latency, playerCount)
			}
		}(id)
		time.Sleep(interval * time.Millisecond)
	}
}
<commit_msg>First ping has id 1 instead of 0<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/alecthomas/kingpin"
	"github.com/ammario/mcping"
	"github.com/fatih/color"
	"time"
)

func main() {
	//Parse command line flags
	var (
		_host     = kingpin.Flag("host", "Hostname").Short('h').Required().String()
		_port     = kingpin.Flag("port", "Port number").Short('p').Default("25565").Int()
		_timeout  = kingpin.Flag("timeout", "Timeout per ping (ms)").Short('t').Default("300").Int()
		_interval = kingpin.Flag("interval", "Interval between pings (ms)").Short('i').Default("1000").Int()
		_count    = kingpin.Flag("count", "Total amount of pings to send").Short('c').Default("4294967295").Int()
	)

	kingpin.Parse()

	//Standardize input variable access

	host := *_host
	port := uint16(*_port)
	timeout := time.Duration(*_timeout)
	interval := time.Duration(*_interval)
	count := uint32(*_count)

	//Sequentiating ping responses
	var id uint32

	//Output colorer
	failOut := color.New(color.FgRed)
	successOut := color.New(color.FgGreen)

	//Output format strings
	failFmt := "(%x) %s; %s\n"
	successFmt := "(%x) %s; latency=%vms players=(%s)\n"

	fullAddr := fmt.Sprint(host, ":", port)

	for id = 1; id < count; id++ {
		//Have each request asynchronous
		go func(id uint32) {
			err := errors.New("")
			timeoutChan := make(chan bool, 1)

			//Default response
			resp := mcping.PingResponse{}

			//Race timeout against ping
			go func() {
				time.Sleep(timeout * time.Millisecond)
				timeoutChan <- true
			}()
			go func() {
				resp, err = mcping.Ping(host, port)
				timeoutChan <- false
			}()
			if <-timeoutChan {
				err = mcping.TimeoutErr
			}
			if err != nil {
				failOut.Printf(failFmt, id, fullAddr, err)
			} else {
				playerCount := fmt.Sprint(resp.Online, "/", resp.Max)
				latency := resp.Latency
				successOut.Printf(successFmt, id, fullAddr, latency, playerCount)
			}
		}(id)
		time.Sleep(interval * time.Millisecond)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vreplication

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
	"vitess.io/vitess/go/vt/binlog/binlogplayer"
	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
)

func TestExternalConnectorCopy(t *testing.T) {
	execStatements(t, []string{
		"create table tab1(id int, val varbinary(128), primary key(id))",
		fmt.Sprintf("create table %s.tab1(id int, val varbinary(128), primary key(id))", vrepldb),
		"insert into tab1 values(1, 'a'), (2, 'b')",

		"create table tab2(id int, val varbinary(128), primary key(id))",
		fmt.Sprintf("create table %s.tab2(id int, val varbinary(128), primary key(id))", vrepldb),
		"insert into tab2 values(1, 'a'), (2, 'b')",

		"create table tab3(id int, val varbinary(128), primary key(id))",
		fmt.Sprintf("create table %s.tab3(id int, val varbinary(128), primary key(id))", vrepldb),
		"insert into tab3 values(1, 'a'), (2, 'b')",
	})
	defer execStatements(t, []string{
		"drop table tab1",
		fmt.Sprintf("drop table %s.tab1", vrepldb),
		"drop table tab2",
		fmt.Sprintf("drop table %s.tab2", vrepldb),
		"drop table tab3",
		fmt.Sprintf("drop table %s.tab3", vrepldb),
	})

	filter1 := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "tab1",
			Filter: "",
		}},
	}
	bls1 := &binlogdatapb.BinlogSource{
		ExternalMysql: "exta",
		Filter:        filter1,
	}
	cancel1 := startExternalVReplication(t, bls1, "")

	expectDBClientQueries(t, []string{
		"begin",
		"insert into tab1(id,val) values (1,'a'), (2,'b')",
		"/update _vt.copy_state",
		"commit",
		"/delete from _vt.copy_state",
		"/update _vt.vreplication set state='Running'",
	})
	execStatements(t, []string{"insert into tab1 values(3, 'c')"})
	expectDBClientQueries(t, []string{
		"begin",
		"insert into tab1(id,val) values (3,'c')",
		"/update _vt.vreplication set pos=",
		"commit",
	})
	// Cancel immediately so we don't deal with spurious updates.
	cancel1()

	// Check that only one connector was created.
	assert.Equal(t, 1, len(playerEngine.ec.connectors))

	filter2 := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "tab2",
			Filter: "",
		}},
	}
	bls2 := &binlogdatapb.BinlogSource{
		ExternalMysql: "exta",
		Filter:        filter2,
	}
	cancel2 := startExternalVReplication(t, bls2, "")

	expectDBClientQueries(t, []string{
		"begin",
		"insert into tab2(id,val) values (1,'a'), (2,'b')",
		"/update _vt.copy_state",
		"commit",
		"/delete from _vt.copy_state",
		"/update _vt.vreplication set state='Running'",
	})
	cancel2()

	// Check that only one connector was created.
	assert.Equal(t, 1, len(playerEngine.ec.connectors))

	filter3 := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "tab3",
			Filter: "",
		}},
	}
	bls3 := &binlogdatapb.BinlogSource{
		ExternalMysql: "extb",
		Filter:        filter3,
	}
	cancel3 := startExternalVReplication(t, bls3, "")

	expectDBClientQueries(t, []string{
		"begin",
		"insert into tab3(id,val) values (1,'a'), (2,'b')",
		"/update _vt.copy_state",
		"commit",
		"/delete from _vt.copy_state",
		"/update _vt.vreplication set state='Running'",
	})
	cancel3()

	// Check that there now two connectors.
	assert.Equal(t, 2, len(playerEngine.ec.connectors))
}

func TestExternalConnectorPlay(t *testing.T) {
	execStatements(t, []string{
		"create table tab1(id int, val varbinary(128), primary key(id))",
		fmt.Sprintf("create table %s.tab1(id int, val varbinary(128), primary key(id))", vrepldb),
	})
	defer execStatements(t, []string{
		"drop table tab1",
		fmt.Sprintf("drop table %s.tab1", vrepldb),
	})

	filter1 := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "tab1",
			Filter: "",
		}},
	}
	bls1 := &binlogdatapb.BinlogSource{
		ExternalMysql: "exta",
		Filter:        filter1,
	}
	pos := masterPosition(t)
	cancel1 := startExternalVReplication(t, bls1, pos)
	defer cancel1()

	execStatements(t, []string{
		"insert into tab1 values(1, 'a'), (2, 'b')",
	})

	expectDBClientQueries(t, []string{
		"begin",
		"insert into tab1(id,val) values (1,'a')",
		"insert into tab1(id,val) values (2,'b')",
		"/update _vt.vreplication set pos=",
		"commit",
	})
}

func startExternalVReplication(t *testing.T, bls *binlogdatapb.BinlogSource, pos string) (cancelr func()) {
	query := binlogplayer.CreateVReplication("test", bls, pos, 9223372036854775807, 9223372036854775807, 0, vrepldb)
	qr, err := playerEngine.Exec(query)
	if err != nil {
		t.Fatal(err)
	}
	if pos == "" {
		expectDBClientQueries(t, []string{
			"/insert into _vt.vreplication",
			"begin",
			"/insert into _vt.copy_state",
			"/update _vt.vreplication set state='Copying'",
			"commit",
			"/update _vt.vreplication set pos=",
		})
	} else {
		expectDBClientQueries(t, []string{
			"/insert into _vt.vreplication",
			"/update _vt.vreplication set state='Running'",
		})
	}
	return func() {
		t.Helper()
		query := fmt.Sprintf("delete from _vt.vreplication where id = %d", qr.InsertID)
		if _, err := playerEngine.Exec(query); err != nil {
			t.Fatal(err)
		}
		expectDeleteQueries(t)
	}
}
<commit_msg>Fix race conditions while expecting queries<commit_after>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vreplication

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
	"vitess.io/vitess/go/vt/binlog/binlogplayer"
	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
)

func TestExternalConnectorCopy(t *testing.T) {
	execStatements(t, []string{
		"create table tab1(id int, val varbinary(128), primary key(id))",
		fmt.Sprintf("create table %s.tab1(id int, val varbinary(128), primary key(id))", vrepldb),
		"insert into tab1 values(1, 'a'), (2, 'b')",

		"create table tab2(id int, val varbinary(128), primary key(id))",
		fmt.Sprintf("create table %s.tab2(id int, val varbinary(128), primary key(id))", vrepldb),
		"insert into tab2 values(1, 'a'), (2, 'b')",

		"create table tab3(id int, val varbinary(128), primary key(id))",
		fmt.Sprintf("create table %s.tab3(id int, val varbinary(128), primary key(id))", vrepldb),
		"insert into tab3 values(1, 'a'), (2, 'b')",
	})
	defer execStatements(t, []string{
		"drop table tab1",
		fmt.Sprintf("drop table %s.tab1", vrepldb),
		"drop table tab2",
		fmt.Sprintf("drop table %s.tab2", vrepldb),
		"drop table tab3",
		fmt.Sprintf("drop table %s.tab3", vrepldb),
	})

	filter1 := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "tab1",
			Filter: "",
		}},
	}
	bls1 := &binlogdatapb.BinlogSource{
		ExternalMysql: "exta",
		Filter:        filter1,
	}
	cancel1 := startExternalVReplication(t, bls1, "")

	expectDBClientAndVreplicationQueries(t, []string{
		"begin",
		"insert into tab1(id,val) values (1,'a'), (2,'b')",
		"/update _vt.copy_state",
		"commit",
		"/delete from _vt.copy_state",
		"/update _vt.vreplication set state='Running'",
	}, "")
	execStatements(t, []string{"insert into tab1 values(3, 'c')"})
	expectDBClientQueries(t, []string{
		"begin",
		"insert into tab1(id,val) values (3,'c')",
		"/update _vt.vreplication set pos=",
		"commit",
	})
	// Cancel immediately so we don't deal with spurious updates.
	cancel1()

	// Check that only one connector was created.
	assert.Equal(t, 1, len(playerEngine.ec.connectors))
	filter2 := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "tab2",
			Filter: "",
		}},
	}
	bls2 := &binlogdatapb.BinlogSource{
		ExternalMysql: "exta",
		Filter:        filter2,
	}
	cancel2 := startExternalVReplication(t, bls2, "")

	expectDBClientAndVreplicationQueries(t, []string{
		"begin",
		"insert into tab2(id,val) values (1,'a'), (2,'b')",
		"/update _vt.copy_state",
		"commit",
		"/delete from _vt.copy_state",
		"/update _vt.vreplication set state='Running'",
	}, "")
	cancel2()

	// Check that only one connector was created.
	assert.Equal(t, 1, len(playerEngine.ec.connectors))

	filter3 := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "tab3",
			Filter: "",
		}},
	}
	bls3 := &binlogdatapb.BinlogSource{
		ExternalMysql: "extb",
		Filter:        filter3,
	}
	cancel3 := startExternalVReplication(t, bls3, "")

	expectDBClientAndVreplicationQueries(t, []string{
		"begin",
		"insert into tab3(id,val) values (1,'a'), (2,'b')",
		"/update _vt.copy_state",
		"commit",
		"/delete from _vt.copy_state",
		"/update _vt.vreplication set state='Running'",
	}, "")
	cancel3()

	// Check that there now two connectors.
	assert.Equal(t, 2, len(playerEngine.ec.connectors))
}

func TestExternalConnectorPlay(t *testing.T) {
	execStatements(t, []string{
		"create table tab1(id int, val varbinary(128), primary key(id))",
		fmt.Sprintf("create table %s.tab1(id int, val varbinary(128), primary key(id))", vrepldb),
	})
	defer execStatements(t, []string{
		"drop table tab1",
		fmt.Sprintf("drop table %s.tab1", vrepldb),
	})

	filter1 := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "tab1",
			Filter: "",
		}},
	}
	bls1 := &binlogdatapb.BinlogSource{
		ExternalMysql: "exta",
		Filter:        filter1,
	}
	pos := masterPosition(t)
	cancel1 := startExternalVReplication(t, bls1, pos)
	defer cancel1()

	execStatements(t, []string{
		"insert into tab1 values(1, 'a'), (2, 'b')",
	})

	expectDBClientAndVreplicationQueries(t, []string{
		"begin",
		"insert into tab1(id,val) values (1,'a')",
		"insert into tab1(id,val) values (2,'b')",
		"/update _vt.vreplication set pos=",
		"commit",
	}, pos)
}

func expectDBClientAndVreplicationQueries(t *testing.T, queries []string, pos string) {
	t.Helper()
	vrepQueries := getExpectedVreplicationQueries(t, pos)
	expectedQueries := append(vrepQueries, queries...)
	expectDBClientQueries(t, expectedQueries)
}

func getExpectedVreplicationQueries(t *testing.T, pos string) []string {
	if pos == "" {
		return []string{
			"/insert into _vt.vreplication",
			"begin",
			"/insert into _vt.copy_state",
			"/update _vt.vreplication set state='Copying'",
			"commit",
			"/update _vt.vreplication set pos=",
		}
	}
	return []string{
		"/insert into _vt.vreplication",
		"/update _vt.vreplication set state='Running'",
	}
}

func startExternalVReplication(t *testing.T, bls *binlogdatapb.BinlogSource, pos string) (cancelr func()) {
	query := binlogplayer.CreateVReplication("test", bls, pos, 9223372036854775807, 9223372036854775807, 0, vrepldb)
	qr, err := playerEngine.Exec(query)
	if err != nil {
		t.Fatal(err)
	}
	return func() {
		t.Helper()
		query := fmt.Sprintf("delete from _vt.vreplication where id = %d", qr.InsertID)
		if _, err := playerEngine.Exec(query); err != nil {
			t.Fatal(err)
		}
		expectDeleteQueries(t)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/koding/kite"
	"fmt"
	"os"
	"sync"
	"log"
	"github.com/bitly/go-nsq"
)

func main() {
	go startMessaging()
	go startKite()
}

func startKite() {
	k := kite.New("robber", "1.0.0")
	k.Config.Port = 6001
	k.Config.DisableAuthentication = true
	k.HandleFunc("hello", hello)

	fmt.Println("Robber starting kite")
	k.Run()
}

func startMessaging() {
	fmt.Println("Robber configuring NSQ")
	wg := &sync.WaitGroup{}
	wg.Add(1)

	config := nsq.NewConfig()
	q, _ := nsq.NewConsumer("tick", "ch", config)
	q.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error {
		log.Printf("Got a message: %v", message)
		wg.Done()
		return nil
	}))

	err := q.ConnectToNSQD(os.Getenv("MESSAGING_SERVICE_HOST" + ":4150"))
	if err != nil {
		log.Panic("Could not connect")
	}
	fmt.Println("Robber starting NSQ")
	wg.Wait()
}

func hello(r *kite.Request) (interface{}, error) {

	fmt.Println("Robber got hello")

	// You can return anything as result, as long as it is JSON marshalable.
	return nil, nil
}
<commit_msg>debugging threading<commit_after>package main

import (
	"github.com/koding/kite"
	"fmt"
	"os"
	"sync"
	"log"
	"github.com/bitly/go-nsq"
)

func main() {
	fmt.Println("Starting Robber")
	go startKite()
	go startMessaging()
}

func startKite() {
	k := kite.New("robber", "1.0.0")
	k.Config.Port = 6001
	k.Config.DisableAuthentication = true
	k.HandleFunc("hello", hello)

	fmt.Println("Robber starting kite")
	k.Run()
}

func startMessaging() {
	fmt.Println("Robber configuring NSQ")
	wg := &sync.WaitGroup{}
	wg.Add(1)

	config := nsq.NewConfig()
	q, _ := nsq.NewConsumer("tick", "ch", config)
	q.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error {
		log.Printf("Got a message: %v", message)
		wg.Done()
		return nil
	}))

	err := q.ConnectToNSQD(os.Getenv("MESSAGING_SERVICE_HOST" + ":4150"))
	if err != nil {
		log.Panic("Could not connect")
	}
	fmt.Println("Robber starting NSQ")
	wg.Wait()
}

func hello(r *kite.Request) (interface{}, error) {

	fmt.Println("Robber got hello")

	// You can return anything as result, as long as it is JSON marshalable.
	return nil, nil
}
<|endoftext|>


<commit_before>package watchers

import (
	"context"
	"fmt"
	"github.com/fsnotify/fsnotify"
	"github.com/oklahomer/go-sarah"
	"github.com/oklahomer/go-sarah/log"
	"golang.org/x/xerrors"
	"io/ioutil"
	stdLogger "log"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"
)

func TestMain(m *testing.M) {
	oldLogger := log.GetLogger()
	defer log.SetLogger(oldLogger)

	l := stdLogger.New(ioutil.Discard, "dummyLog", 0)
	logger := log.NewWithStandardLogger(l)
	log.SetLogger(logger)

	code := m.Run()

	os.Exit(code)
}

type dummyFsWatcher struct {
	AddFunc    func(string) error
	RemoveFunc func(string) error
	CloseFunc  func() error
}

func (w *dummyFsWatcher) Add(dir string) error {
	return w.AddFunc(dir)
}

func (w *dummyFsWatcher) Remove(dir string) error {
	return w.RemoveFunc(dir)
}

func (w *dummyFsWatcher) Close() error {
	return w.CloseFunc()
}

func TestNewFileWatcher(t *testing.T) {
	rootCtx := context.Background()
	watcherCtx, cancel := context.WithCancel(rootCtx)
	defer cancel()

	w, err := NewFileWatcher(watcherCtx, "testdata")

	if err != nil {
		t.Fatalf("Unexpected error on Run: %s", err.Error())
	}

	if w == nil {
		t.Error("ConfigWatcher is not returned")
	}
}

func TestFileWatcher_Read(t *testing.T) {
	tests := []struct {
		id     string
		hasErr bool
	}{
		{
			id: "jsonHello",
		},
		{
			id: "yamlHello",
		},
		{
			id:     "invalid",
			hasErr: true,
		},
	}

	dirName, err := filepath.Abs(filepath.Join("..", "testdata", "config"))
	if err != nil {
		t.Fatalf("Unexpected error returned: %s.", err.Error())
	}

	var botType sarah.BotType = "dummy"
	type helloConfig struct {
		Text string `json:"text" yaml:"text"`
	}
	for _, tt := range tests {
		t.Run(tt.id, func(t *testing.T) {
			w := &fileWatcher{
				baseDir: dirName,
			}
			configPtr := &helloConfig{}

			err := w.Read(context.TODO(), botType, tt.id, configPtr)

			if tt.hasErr {
				if err == nil {
					t.Error("Expected error is not returned.")
				}
				return
			}

			if err != nil {
				t.Errorf("Failed to read config file: %s.", err.Error())
			}

			if configPtr.Text != "HELLO" {
				t.Error("Configuration file content is not reflected to the struct.")
			}
		})
	}
}

func TestFileWatcher_Watch(t *testing.T) {
	tests := []struct {
		err error
	}{
		{
			err: xerrors.New("err"),
		},
		{
			err: nil,
		},
	}

	var botType sarah.BotType = "dummy"
	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			subscr := make(chan *subscription, 1)
			w := &fileWatcher{
				baseDir:   filepath.Join("path", "to", "dummy", "dir"),
				subscribe: subscr,
			}

			go func(e error) {
				select {
				case dir := <-subscr:
					dir.initErr <- e

				case <-time.NewTimer(100 * time.Millisecond).C:
					t.Error("Expected error value is not passed")

				}
			}(tt.err)

			callback := func() {}
			err := w.Watch(context.TODO(), botType, "hello", callback)

			if tt.err == nil && err != nil {
				t.Errorf("Unexpected error is returned: %s", err.Error())
				return
			}

			if tt.err != err {
				t.Errorf("Expected error is not returned: %s", err)
			}
		})
	}
}

func TestFileWatcher_Unwatch(t *testing.T) {
	tests := []struct {
		panic bool
	}{
		{
			panic: false,
		},
		{
			panic: true,
		},
	}

	var botType sarah.BotType = "dummy"
	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			unsubscr := make(chan sarah.BotType, 1)
			w := &fileWatcher{
				unsubscribe: unsubscr,
			}

			if tt.panic {
				// Should be recovered and translated to an error
				close(w.unsubscribe)
			}

			err := w.Unwatch(botType)
			if tt.panic {
				if err != sarah.ErrWatcherNotRunning {
					t.Errorf("Expected error is not returned: %s", err)
				}
				return
			}

			if err != nil {
				t.Errorf("Unexpected error is returned: %s", err.Error())
			}
		})
	}
}

func TestFileWatcher_run(t *testing.T) {
	dir, _ := filepath.Abs(filepath.Join("..", "testdata", "config", "dummy"))
	invalidDir, _ := filepath.Abs(filepath.Join("..", "testdata", "config", "invalid"))
	validId := "hello"
	subscriptions := []struct {
		absdir     string
		id         string
		watcherErr error
		err        error
		notify     chan struct{}
	}{
		{
			absdir: dir,
			id:     validId,
			notify: make(chan struct{}, 1),
		},
		{
			absdir: dir,
			id:     validId,
			err:    sarah.ErrAlreadySubscribing,
			notify: nil,
		},
		{
			absdir: dir,
			id:     "invalid",
			notify: make(chan struct{}, 1),
		},
		{
			absdir:     invalidDir,
			id:         "error",
			watcherErr: xerrors.New("error"),
			notify:     nil,
		},
	}
	add := make(chan struct{}, len(subscriptions))
	remove := make(chan string, 2)
	fsWatcher := &dummyFsWatcher{
		AddFunc: func(absDir string) error {
			add <- struct{}{}
			s := subscriptions[len(add)-1]
			return s.watcherErr
		},
		RemoveFunc: func(absDir string) error {
			remove <- absDir
			return nil
		},
		CloseFunc: func() error {
			return nil
		},
	}

	subscr := make(chan *subscription, 1)
	w := &fileWatcher{
		fsWatcher:   fsWatcher,
		subscribe:   subscr,
		unsubscribe: make(chan sarah.BotType, 1),
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	events := make(chan fsnotify.Event, 1)
	errs := make(chan error, 1)
	go w.run(ctx, events, errs)
	go func() {
		e := xerrors.New("watccher error")
		for {
			select {
			case <-ctx.Done():
				return

			default:
				// Error should not affect main operation
				select {
				case errs <- e:
					// O.K.

				case <-time.NewTimer(100 * time.Millisecond).C:
					t.Error("Error sending should not block")

				}
			}
		}

	}()

	// Two subscriptions are added for a directory
	var botType sarah.BotType = "dummyBotType"
	for i, s := range subscriptions {
		copied := s // Not to refer the last element in the loop
		err := make(chan error, 1)
		w.subscribe <- &subscription{
			botType: botType,
			id:      copied.id,
			initErr: err,
			absDir:  copied.absdir,
			callback: func() {
				copied.notify <- struct{}{}
			},
		}

		select {
		case e := <-err:
			if copied.watcherErr != nil {
				if copied.watcherErr != e {
					t.Errorf("Unexpected error state on trial %d: %s", i, e)
				}
			}

			if copied.err != nil {
				if copied.err != e {
					t.Errorf("Unexpected error state on trial %d: %s", i, e)
				}
			}

		case <-time.NewTimer(100 * time.Millisecond).C:
			t.Errorf("Result is not returned.")

		}
	}

	// Valid write event occurs
	events <- fsnotify.Event{
		Op:   fsnotify.Write,
		Name: filepath.Join(dir, fmt.Sprintf("%s.json", validId)),
	}
	for _, s := range subscriptions {
		if s.notify == nil {
			continue
		}

		if s.id == validId {
			select {
			case <-s.notify:
				// O.K.

			case <-time.NewTimer(100 * time.Millisecond).C:
				t.Errorf("Event is not notified for %s.", s.id)

			}
		} else {
			select {
			case id := <-s.notify:
				t.Errorf("Unexpected notification is sent: %s.", id)

			case <-time.NewTimer(100 * time.Millisecond).C:
				// O.K.

			}
		}
	}

	// Invalid valid write events occur
	events <- fsnotify.Event{
		Op:   fsnotify.Write,
		Name: filepath.Join(dir, "noExtension"),
	}
	events <- fsnotify.Event{
		Op:   fsnotify.Write,
		Name: filepath.Join(dir, "noSubscribingDir", "invalid.json"),
	}
	events <- fsnotify.Event{
		Op:   fsnotify.Remove,
		Name: filepath.Join(dir, "noSubscribingDir", "invalid.json"),
	}

	// Unwatch
	w.unsubscribe <- "invalidBotType"
	w.unsubscribe <- botType
	select {
	case d := <-remove:
		if d != dir {
			t.Errorf("Unexpected directory name is given.")
		}

	case <-time.NewTimer(100 * time.Millisecond).C:
		t.Error("Subscription is not removed.")

	}
}

func TestFileWatcher_run_cancel(t *testing.T) {
	tests := []struct {
		err error
	}{
		{
			err: xerrors.New(""),
		},
		{
			err: nil,
		},
	}

	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			dummyWatcher := &dummyFsWatcher{
				CloseFunc: func() error {
					return tt.err
				},
			}
			w := &fileWatcher{
				fsWatcher:   dummyWatcher,
				unsubscribe: make(chan sarah.BotType),
			}

			finished := make(chan struct{})
			go func() {
				w.run(ctx, make(chan fsnotify.Event), make(chan error))
				finished <- struct{}{}
			}()

			// Should not cause panic
			cancel()

			select {
			case <-finished:
				// O.K.

			case <-time.NewTimer(100 * time.Millisecond).C:
				t.Error("ConfigWatcher is not stopped.")

			}
		})
	}
}

func Test_plainPathToFile(t *testing.T) {
	tests := []struct {
		path     string
		hasErr   bool
		fileType fileType
	}{
		{
			path:   "/extension/is/empty",
			hasErr: true,
		},
		{
			path:     "/path/to/json/file.json",
			hasErr:   false,
			fileType: jsonFile,
		},
		{
			path:     "/path/to/yaml/file.yml",
			hasErr:   false,
			fileType: yamlFile,
		},
		{
			path:     "/path/to/yaml/file.yaml",
			hasErr:   false,
			fileType: yamlFile,
		},
		{
			path:   "/path/to/yaml/file.html",
			hasErr: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.path, func(t *testing.T) {
			configFile, err := plainPathToFile(tt.path)
			if tt.hasErr {
				if err == nil {
					t.Fatal("Expected error is not returned.")
				}
				return
			}

			if err != nil {
				t.Errorf("Unexpected error is returned: %s.", err.Error())
			}

			if configFile == nil {
				t.Fatalf("Expected object is not returned.")
			}

			if configFile.fileType != tt.fileType {
				t.Error("Expected fileType is not returned.")
			}
		})
	}

}
<commit_msg>Fix test caused by context cancellation timing<commit_after>package watchers

import (
	"context"
	"fmt"
	"github.com/fsnotify/fsnotify"
	"github.com/oklahomer/go-sarah"
	"github.com/oklahomer/go-sarah/log"
	"golang.org/x/xerrors"
	"io/ioutil"
	stdLogger "log"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"
)

func TestMain(m *testing.M) {
	oldLogger := log.GetLogger()
	defer log.SetLogger(oldLogger)

	l := stdLogger.New(ioutil.Discard, "dummyLog", 0)
	logger := log.NewWithStandardLogger(l)
	log.SetLogger(logger)

	code := m.Run()

	os.Exit(code)
}

type dummyFsWatcher struct {
	AddFunc    func(string) error
	RemoveFunc func(string) error
	CloseFunc  func() error
}

func (w *dummyFsWatcher) Add(dir string) error {
	return w.AddFunc(dir)
}

func (w *dummyFsWatcher) Remove(dir string) error {
	return w.RemoveFunc(dir)
}

func (w *dummyFsWatcher) Close() error {
	return w.CloseFunc()
}

func TestNewFileWatcher(t *testing.T) {
	rootCtx := context.Background()
	watcherCtx, cancel := context.WithCancel(rootCtx)
	defer cancel()

	w, err := NewFileWatcher(watcherCtx, "testdata")

	if err != nil {
		t.Fatalf("Unexpected error on Run: %s", err.Error())
	}

	if w == nil {
		t.Error("ConfigWatcher is not returned")
	}
}

func TestFileWatcher_Read(t *testing.T) {
	tests := []struct {
		id     string
		hasErr bool
	}{
		{
			id: "jsonHello",
		},
		{
			id: "yamlHello",
		},
		{
			id:     "invalid",
			hasErr: true,
		},
	}

	dirName, err := filepath.Abs(filepath.Join("..", "testdata", "config"))
	if err != nil {
		t.Fatalf("Unexpected error returned: %s.", err.Error())
	}

	var botType sarah.BotType = "dummy"
	type helloConfig struct {
		Text string `json:"text" yaml:"text"`
	}
	for _, tt := range tests {
		t.Run(tt.id, func(t *testing.T) {
			w := &fileWatcher{
				baseDir: dirName,
			}
			configPtr := &helloConfig{}

			err := w.Read(context.TODO(), botType, tt.id, configPtr)

			if tt.hasErr {
				if err == nil {
					t.Error("Expected error is not returned.")
				}
				return
			}

			if err != nil {
				t.Errorf("Failed to read config file: %s.", err.Error())
			}

			if configPtr.Text != "HELLO" {
				t.Error("Configuration file content is not reflected to the struct.")
			}
		})
	}
}

func TestFileWatcher_Watch(t *testing.T) {
	tests := []struct {
		err error
	}{
		{
			err: xerrors.New("err"),
		},
		{
			err: nil,
		},
	}

	var botType sarah.BotType = "dummy"
	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			subscr := make(chan *subscription, 1)
			w := &fileWatcher{
				baseDir:   filepath.Join("path", "to", "dummy", "dir"),
				subscribe: subscr,
			}

			go func(e error) {
				select {
				case dir := <-subscr:
					dir.initErr <- e

				case <-time.NewTimer(100 * time.Millisecond).C:
					t.Error("Expected error value is not passed")

				}
			}(tt.err)

			callback := func() {}
			err := w.Watch(context.TODO(), botType, "hello", callback)

			if tt.err == nil && err != nil {
				t.Errorf("Unexpected error is returned: %s", err.Error())
				return
			}

			if tt.err != err {
				t.Errorf("Expected error is not returned: %s", err)
			}
		})
	}
}

func TestFileWatcher_Unwatch(t *testing.T) {
	tests := []struct {
		panic bool
	}{
		{
			panic: false,
		},
		{
			panic: true,
		},
	}

	var botType sarah.BotType = "dummy"
	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			unsubscr := make(chan sarah.BotType, 1)
			w := &fileWatcher{
				unsubscribe: unsubscr,
			}

			if tt.panic {
				// Should be recovered and translated to an error
				close(w.unsubscribe)
			}

			err := w.Unwatch(botType)
			if tt.panic {
				if err != sarah.ErrWatcherNotRunning {
					t.Errorf("Expected error is not returned: %s", err)
				}
				return
			}

			if err != nil {
				t.Errorf("Unexpected error is returned: %s", err.Error())
			}
		})
	}
}

func TestFileWatcher_run(t *testing.T) {
	dir, _ := filepath.Abs(filepath.Join("..", "testdata", "config", "dummy"))
	invalidDir, _ := filepath.Abs(filepath.Join("..", "testdata", "config", "invalid"))
	validId := "hello"
	subscriptions := []struct {
		absdir     string
		id         string
		watcherErr error
		err        error
		notify     chan struct{}
	}{
		{
			absdir: dir,
			id:     validId,
			notify: make(chan struct{}, 1),
		},
		{
			absdir: dir,
			id:     validId,
			err:    sarah.ErrAlreadySubscribing,
			notify: nil,
		},
		{
			absdir: dir,
			id:     "invalid",
			notify: make(chan struct{}, 1),
		},
		{
			absdir:     invalidDir,
			id:         "error",
			watcherErr: xerrors.New("error"),
			notify:     nil,
		},
	}
	add := make(chan struct{}, len(subscriptions))
	remove := make(chan string, 2)
	fsWatcher := &dummyFsWatcher{
		AddFunc: func(absDir string) error {
			add <- struct{}{}
			s := subscriptions[len(add)-1]
			return s.watcherErr
		},
		RemoveFunc: func(absDir string) error {
			remove <- absDir
			return nil
		},
		CloseFunc: func() error {
			return nil
		},
	}

	subscr := make(chan *subscription, 1)
	w := &fileWatcher{
		fsWatcher:   fsWatcher,
		subscribe:   subscr,
		unsubscribe: make(chan sarah.BotType, 1),
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	events := make(chan fsnotify.Event, 1)
	errs := make(chan error, 1)
	go w.run(ctx, events, errs)
	go func() {
		e := xerrors.New("watccher error")
		for {
			select {
			case <-ctx.Done():
				return

			default:
				// Error should not affect main operation
				select {
				case errs <- e:
					// O.K.

				case <-time.NewTimer(100 * time.Millisecond).C:
					if ctx.Err() == context.Canceled {
						// This is acceptable. errs simply stack because watcher stopped.
						return
					}
					t.Error("Error sending should not block")

				}
			}
		}

	}()

	// Two subscriptions are added for a directory
	var botType sarah.BotType = "dummyBotType"
	for i, s := range subscriptions {
		copied := s // Not to refer the last element in the loop
		err := make(chan error, 1)
		w.subscribe <- &subscription{
			botType: botType,
			id:      copied.id,
			initErr: err,
			absDir:  copied.absdir,
			callback: func() {
				copied.notify <- struct{}{}
			},
		}

		select {
		case e := <-err:
			if copied.watcherErr != nil {
				if copied.watcherErr != e {
					t.Errorf("Unexpected error state on trial %d: %s", i, e)
				}
			}

			if copied.err != nil {
				if copied.err != e {
					t.Errorf("Unexpected error state on trial %d: %s", i, e)
				}
			}

		case <-time.NewTimer(100 * time.Millisecond).C:
			t.Errorf("Result is not returned.")

		}
	}

	// Valid write event occurs
	events <- fsnotify.Event{
		Op:   fsnotify.Write,
		Name: filepath.Join(dir, fmt.Sprintf("%s.json", validId)),
	}
	for _, s := range subscriptions {
		if s.notify == nil {
			continue
		}

		if s.id == validId {
			select {
			case <-s.notify:
				// O.K.

			case <-time.NewTimer(100 * time.Millisecond).C:
				t.Errorf("Event is not notified for %s.", s.id)

			}
		} else {
			select {
			case id := <-s.notify:
				t.Errorf("Unexpected notification is sent: %s.", id)

			case <-time.NewTimer(100 * time.Millisecond).C:
				// O.K.

			}
		}
	}

	// Invalid valid write events occur
	events <- fsnotify.Event{
		Op:   fsnotify.Write,
		Name: filepath.Join(dir, "noExtension"),
	}
	events <- fsnotify.Event{
		Op:   fsnotify.Write,
		Name: filepath.Join(dir, "noSubscribingDir", "invalid.json"),
	}
	events <- fsnotify.Event{
		Op:   fsnotify.Remove,
		Name: filepath.Join(dir, "noSubscribingDir", "invalid.json"),
	}

	// Unwatch
	w.unsubscribe <- "invalidBotType"
	w.unsubscribe <- botType
	select {
	case d := <-remove:
		if d != dir {
			t.Errorf("Unexpected directory name is given.")
		}

	case <-time.NewTimer(100 * time.Millisecond).C:
		t.Error("Subscription is not removed.")

	}
}

func TestFileWatcher_run_cancel(t *testing.T) {
	tests := []struct {
		err error
	}{
		{
			err: xerrors.New(""),
		},
		{
			err: nil,
		},
	}

	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			dummyWatcher := &dummyFsWatcher{
				CloseFunc: func() error {
					return tt.err
				},
			}
			w := &fileWatcher{
				fsWatcher:   dummyWatcher,
				unsubscribe: make(chan sarah.BotType),
			}

			finished := make(chan struct{})
			go func() {
				w.run(ctx, make(chan fsnotify.Event), make(chan error))
				finished <- struct{}{}
			}()

			// Should not cause panic
			cancel()

			select {
			case <-finished:
				// O.K.

			case <-time.NewTimer(100 * time.Millisecond).C:
				t.Error("ConfigWatcher is not stopped.")

			}
		})
	}
}

func Test_plainPathToFile(t *testing.T) {
	tests := []struct {
		path     string
		hasErr   bool
		fileType fileType
	}{
		{
			path:   "/extension/is/empty",
			hasErr: true,
		},
		{
			path:     "/path/to/json/file.json",
			hasErr:   false,
			fileType: jsonFile,
		},
		{
			path:     "/path/to/yaml/file.yml",
			hasErr:   false,
			fileType: yamlFile,
		},
		{
			path:     "/path/to/yaml/file.yaml",
			hasErr:   false,
			fileType: yamlFile,
		},
		{
			path:   "/path/to/yaml/file.html",
			hasErr: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.path, func(t *testing.T) {
			configFile, err := plainPathToFile(tt.path)
			if tt.hasErr {
				if err == nil {
					t.Fatal("Expected error is not returned.")
				}
				return
			}

			if err != nil {
				t.Errorf("Unexpected error is returned: %s.", err.Error())
			}

			if configFile == nil {
				t.Fatalf("Expected object is not returned.")
			}

			if configFile.fileType != tt.fileType {
				t.Error("Expected fileType is not returned.")
			}
		})
	}

}
<|endoftext|>


<commit_before>package zog

import "fmt"

type Memory struct {
	buf       []byte
	debug     bool
	watches   Regions
	watchFunc func(addr uint16, old byte, new byte)
	readonly  Regions
}

func NewMemory(size uint16) *Memory {
	intSize := int(size)
	if intSize == 0 {
		intSize = 64 * 1024
	}
	m := &Memory{
		buf:      make([]byte, intSize),
		readonly: make([]Region, 0),
	}
	return m
}

func (m *Memory) SetDebug(debug bool) {
	m.debug = debug
}

func (m *Memory) SetWatchFunc(wf func(uint16, byte, byte)) {
	m.watchFunc = wf
}

func (m *Memory) Len() int {
	return len(m.buf)
}

func (m *Memory) Peek(addr uint16) (byte, error) {
	if int(addr) >= m.Len() {
		return 0, fmt.Errorf("Out of bounds memory read: %d", addr)
	}
	n := m.buf[addr]
	//	if m.debug || m.watches.contains(addr) {
	//		fmt.Printf("MEM: %04X -> %02X\n", addr, n)
	//	}
	return n, nil
}

func (m *Memory) Poke(addr uint16, n byte) error {
	// Poke to ROM is a NOP
	if m.readonly.contains(addr) {
		fmt.Printf("Deny RO POKE [%04X] [%02X]\n", addr, n)
		return nil
	}

	if int(addr) >= m.Len() {
		return fmt.Errorf("Out of bounds memory write: %d (%d)", addr, n)
	}
	if m.debug || m.watches.contains(addr) {
		//		fmt.Printf("MEM: %04X <- %02X\n", addr, n)
		m.watchFunc(addr, m.buf[addr], n)
	}
	m.buf[addr] = n
	return nil
}

func (m *Memory) Peek16(addr uint16) (uint16, error) {
	l, err := m.Peek(addr)
	if err != nil {
		return 0, err
	}
	h, err := m.Peek(addr + 1) // overflow correct
	if err != nil {
		return 0, err
	}
	return uint16(h)<<8 | uint16(l), nil
}

func (m *Memory) Poke16(addr uint16, nn uint16) error {
	if m.readonly.contains(addr) {
		return nil
	}

	l := byte(nn)
	h := byte(nn >> 8)
	err := m.Poke(addr, l)
	if err != nil {
		return err
	}
	err = m.Poke(addr+1, h)
	if err != nil {
		return err
	}
	return nil
}

func (m *Memory) Clear() {
	m.buf = make([]byte, int(m.Len()))
}

func (m *Memory) Copy(addr uint16, buf []byte) error {
	if int(addr)+len(buf) >= int(m.Len()) {
		panic(fmt.Sprintf("Can't load - base addr %04X length %04X memsize %04X", addr, len(buf), m.Len()))
	}
	for i := 0; i < len(buf); i++ {
		m.buf[addr+uint16(i)] = buf[i]
	}
	return nil
}

// Fetch a chunk of memory. Error if overflows end.
// Don't write to this please.
func (m *Memory) PeekBuf(addr uint16, size int) ([]byte, error) {
	if size <= 0 || size > 64*1024*1024 {
		return nil, fmt.Errorf("PeekBuf invalid size: %d", size)
	}
	if size+int(addr) > len(m.buf) {
		return nil, fmt.Errorf("PeekBuf invalid size+addr: %04x - %04x", addr, size)
	}
	return m.buf[addr : int(addr)+size], nil
}
<commit_msg>more info on mem oob<commit_after>package zog

import "fmt"

type Memory struct {
	buf       []byte
	debug     bool
	watches   Regions
	watchFunc func(addr uint16, old byte, new byte)
	readonly  Regions
}

func NewMemory(size uint16) *Memory {
	intSize := int(size)
	if intSize == 0 {
		intSize = 64 * 1024
	}
	m := &Memory{
		buf:      make([]byte, intSize),
		readonly: make([]Region, 0),
	}
	return m
}

func (m *Memory) SetDebug(debug bool) {
	m.debug = debug
}

func (m *Memory) SetWatchFunc(wf func(uint16, byte, byte)) {
	m.watchFunc = wf
}

func (m *Memory) Len() int {
	return len(m.buf)
}

func (m *Memory) Peek(addr uint16) (byte, error) {
	if int(addr) >= m.Len() {
		return 0, fmt.Errorf("Out of bounds memory read: 0x%04X > 0x%04X", addr, m.Len())
	}
	n := m.buf[addr]
	//	if m.debug || m.watches.contains(addr) {
	//		fmt.Printf("MEM: %04X -> %02X\n", addr, n)
	//	}
	return n, nil
}

func (m *Memory) Poke(addr uint16, n byte) error {
	// Poke to ROM is a NOP
	if m.readonly.contains(addr) {
		fmt.Printf("Deny RO POKE [%04X] [%02X]\n", addr, n)
		return nil
	}

	if int(addr) >= m.Len() {
		return fmt.Errorf("Out of bounds memory write: %d (%d)", addr, n)
	}
	if m.debug || m.watches.contains(addr) {
		//		fmt.Printf("MEM: %04X <- %02X\n", addr, n)
		m.watchFunc(addr, m.buf[addr], n)
	}
	m.buf[addr] = n
	return nil
}

func (m *Memory) Peek16(addr uint16) (uint16, error) {
	l, err := m.Peek(addr)
	if err != nil {
		return 0, err
	}
	h, err := m.Peek(addr + 1) // overflow correct
	if err != nil {
		return 0, err
	}
	return uint16(h)<<8 | uint16(l), nil
}

func (m *Memory) Poke16(addr uint16, nn uint16) error {
	if m.readonly.contains(addr) {
		return nil
	}

	l := byte(nn)
	h := byte(nn >> 8)
	err := m.Poke(addr, l)
	if err != nil {
		return err
	}
	err = m.Poke(addr+1, h)
	if err != nil {
		return err
	}
	return nil
}

func (m *Memory) Clear() {
	m.buf = make([]byte, int(m.Len()))
}

func (m *Memory) Copy(addr uint16, buf []byte) error {
	if int(addr)+len(buf) >= int(m.Len()) {
		panic(fmt.Sprintf("Can't load - base addr %04X length %04X memsize %04X", addr, len(buf), m.Len()))
	}
	for i := 0; i < len(buf); i++ {
		m.buf[addr+uint16(i)] = buf[i]
	}
	return nil
}

// Fetch a chunk of memory. Error if overflows end.
// Don't write to this please.
func (m *Memory) PeekBuf(addr uint16, size int) ([]byte, error) {
	if size <= 0 || size > 64*1024*1024 {
		return nil, fmt.Errorf("PeekBuf invalid size: %d", size)
	}
	if size+int(addr) > len(m.buf) {
		return nil, fmt.Errorf("PeekBuf invalid size+addr: %04x - %04x", addr, size)
	}
	return m.buf[addr : int(addr)+size], nil
}
<|endoftext|>


<commit_before>package dockerVolumeRbd

import (
	"github.com/Sirupsen/logrus"
	"github.com/ceph/go-ceph/rbd"
	"golang.org/x/sys/unix"
	"path/filepath"
	"fmt"
)

func (d *rbdDriver) mapImage(imageName string) error {
	logrus.Debugf("volume-rbd Name=%s Message=rbd map", imageName)

	_, err := d.rbdsh("map", imageName)

	return err
}

func (d *rbdDriver) unmapImage(imageName string) error {
	logrus.Debugf("volume-rbd Name=%s Message=rbd unmap", imageName)

	_, err := d.rbdsh("unmap", imageName)

	if err != nil {
		// NOTE: rbd unmap exits 16 if device is still being used - unlike umount.  try to recover differently in that case
		if rbdUnmapBusyRegexp.MatchString(err.Error()) {
			return err
		}

		logrus.Errorf("volume-rbd Name=%s Message=rbd unmap: %s", imageName, err.Error())
		// other error, continue and fail safe
	}

	return nil
}

func (d *rbdDriver) mountImage(imageName string, mountOptions string) error {

    device := d.getTheDevice(imageName)
    mountpoint := d.GetMountPointPath(imageName)

	logrus.Debugf("volume-rbd Name=%s Message=mount %s %s %s", imageName, mountOptions, device, mountpoint)

	// err := unix.Mount(device, mountpoint, "auto", 0, "")
    // note unix.Mount does not work with our aliased device, we user the sh version.
    _, err := shWithDefaultTimeout("mount", mountOptions, device, mountpoint)

    return err
}

func (d *rbdDriver) unmountDevice(imageName string) error {

    mountpoint := d.GetMountPointPath(imageName)
	logrus.Debugf("volume-rbd Message=umount %s", mountpoint)

	err := unix.Unmount(mountpoint, 0)

	return err
}

func (d *rbdDriver) errIfRbdImageHasWatchers(imageName string) error {

	status, err := d.rbdsh("status", imageName)
	if err != nil {
		return err
	}

    if rbdHasNoWatchersRegexp.MatchString(status) {
        return nil
    }

    return fmt.Errorf("image with %s", status)
}

func (d *rbdDriver) removeRbdImage(imageName string) error {
	logrus.Debugf("volume-rbd Name=%s Message=remove rbd image", imageName)

	rbdImage := rbd.GetImage(d.ioctx, imageName)

	return rbdImage.Remove()
}

// rbdsh will call rbd with the given command arguments, also adding config, user and pool flags
func (d *rbdDriver) rbdsh(command string, args ...string) (string, error) {

	args = append([]string{"--pool", d.conf["pool"], "--name", d.conf["keyring_user"], command}, args...)

	return shWithDefaultTimeout("rbd", args...)
}

// returns the aliased device under device_map_root
func (d *rbdDriver) getTheDevice(imageName string) string {
	return filepath.Join(d.conf["device_map_root"], d.conf["pool"], imageName)
}
<commit_msg>Pass the cluster name to rbd invocations.<commit_after>package dockerVolumeRbd

import (
	"github.com/Sirupsen/logrus"
	"github.com/ceph/go-ceph/rbd"
	"golang.org/x/sys/unix"
	"path/filepath"
	"fmt"
)

func (d *rbdDriver) mapImage(imageName string) error {
	logrus.Debugf("volume-rbd Name=%s Message=rbd map", imageName)

	_, err := d.rbdsh("map", imageName)

	return err
}

func (d *rbdDriver) unmapImage(imageName string) error {
	logrus.Debugf("volume-rbd Name=%s Message=rbd unmap", imageName)

	_, err := d.rbdsh("unmap", imageName)

	if err != nil {
		// NOTE: rbd unmap exits 16 if device is still being used - unlike umount.  try to recover differently in that case
		if rbdUnmapBusyRegexp.MatchString(err.Error()) {
			return err
		}

		logrus.Errorf("volume-rbd Name=%s Message=rbd unmap: %s", imageName, err.Error())
		// other error, continue and fail safe
	}

	return nil
}

func (d *rbdDriver) mountImage(imageName string, mountOptions string) error {

    device := d.getTheDevice(imageName)
    mountpoint := d.GetMountPointPath(imageName)

	logrus.Debugf("volume-rbd Name=%s Message=mount %s %s %s", imageName, mountOptions, device, mountpoint)

	// err := unix.Mount(device, mountpoint, "auto", 0, "")
    // note unix.Mount does not work with our aliased device, we user the sh version.
    _, err := shWithDefaultTimeout("mount", mountOptions, device, mountpoint)

    return err
}

func (d *rbdDriver) unmountDevice(imageName string) error {

    mountpoint := d.GetMountPointPath(imageName)
	logrus.Debugf("volume-rbd Message=umount %s", mountpoint)

	err := unix.Unmount(mountpoint, 0)

	return err
}

func (d *rbdDriver) errIfRbdImageHasWatchers(imageName string) error {

	status, err := d.rbdsh("status", imageName)
	if err != nil {
		return err
	}

    if rbdHasNoWatchersRegexp.MatchString(status) {
        return nil
    }

    return fmt.Errorf("image with %s", status)
}

func (d *rbdDriver) removeRbdImage(imageName string) error {
	logrus.Debugf("volume-rbd Name=%s Message=remove rbd image", imageName)

	rbdImage := rbd.GetImage(d.ioctx, imageName)

	return rbdImage.Remove()
}

// rbdsh will call rbd with the given command arguments, also adding config, user and pool flags
func (d *rbdDriver) rbdsh(command string, args ...string) (string, error) {

	args = append([]string{"--cluster", d.conf["cluster"], "--pool", d.conf["pool"], "--name", d.conf["keyring_user"], command}, args...)

	return shWithDefaultTimeout("rbd", args...)
}

// returns the aliased device under device_map_root
func (d *rbdDriver) getTheDevice(imageName string) string {
	return filepath.Join(d.conf["device_map_root"], d.conf["pool"], imageName)
}
<|endoftext|>


<commit_before>// Package romulus contains all the logic for monitoring the kubernetes api
// and registering services with vulcan proxy
//
//    c, e := romulus.NewRegistrar(&romulus.Config{
//      PeerList:            eps,
//      APIVersion:          *kv,
//      KubeConfig:          kcc,
//      Selector:            *sl,
//      VulcanEtcdNAMESPACE: *vk,
//    })
//    if e != nil {
//      // handle configuration error
//    }
//
//    if e := romulus.Start(c); e != nil {
//      // handle runtime error
//    }
//    // NOTE: romulus DOES NOT clean up etcd on shutdown
//
// vulcan proxy documentation: https://docs.vulcand.io/index.html
//
// kubernetes documentation: http://kubernetes.io/v1.0/
package romulus

var (
	version = "v0.1.2"

	// SHA is the build sha
	SHA string
)
<commit_msg>bump to 0.1.3<commit_after>// Package romulus contains all the logic for monitoring the kubernetes api
// and registering services with vulcan proxy
//
//    c, e := romulus.NewRegistrar(&romulus.Config{
//      PeerList:            eps,
//      APIVersion:          *kv,
//      KubeConfig:          kcc,
//      Selector:            *sl,
//      VulcanEtcdNAMESPACE: *vk,
//    })
//    if e != nil {
//      // handle configuration error
//    }
//
//    if e := romulus.Start(c); e != nil {
//      // handle runtime error
//    }
//    // NOTE: romulus DOES NOT clean up etcd on shutdown
//
// vulcan proxy documentation: https://docs.vulcand.io/index.html
//
// kubernetes documentation: http://kubernetes.io/v1.0/
package romulus

var (
	version = "v0.1.3"

	// SHA is the build sha
	SHA string
)
<|endoftext|>


<commit_before>package gobrake

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"
)

type ctxKey string

const traceCtxKey ctxKey = "ab_route_trace"

type routeBreakdownKey struct {
	Method   string    `json:"method"`
	Route    string    `json:"route"`
	RespType string    `json:"responseType"`
	Time     time.Time `json:"time"`
}

type routeBreakdown struct {
	routeBreakdownKey

	routeStat
	Groups map[string]*routeStat `json:"groups"`
}

func (b *routeBreakdown) Add(total time.Duration, groups map[string]time.Duration) {
	b.mu.Lock()
	defer b.mu.Unlock()

	if b.Groups == nil {
		b.Groups = make(map[string]*routeStat)
	}

	_ = b.routeStat.Add(durInMs(total))

	var sum time.Duration
	for _, dur := range groups {
		sum += dur
	}

	other := total - sum
	if other < 0 {
		other = time.Microsecond
	}

	if groups == nil {
		groups = make(map[string]time.Duration)
	}
	groups["other"] = other

	for name, dur := range groups {
		s, ok := b.Groups[name]
		if !ok {
			s = newRouteStat()
			b.Groups[name] = s
		}
		_ = s.Add(durInMs(dur))
	}
}

func (b *routeBreakdown) Pack() error {
	err := b.routeStat.Pack()
	if err != nil {
		return err
	}

	for _, v := range b.Groups {
		err = v.Pack()
		if err != nil {
			return err
		}
	}

	return nil
}

type routeBreakdowns struct {
	opt    *NotifierOptions
	apiURL string

	flushTimer *time.Timer
	addWG      *sync.WaitGroup

	mu sync.Mutex
	m  map[routeBreakdownKey]*routeBreakdown
}

func newRouteBreakdowns(opt *NotifierOptions) *routeBreakdowns {
	return &routeBreakdowns{
		opt: opt,
		apiURL: fmt.Sprintf("%s/api/v5/projects/%d/routes-breakdowns",
			opt.Host, opt.ProjectId),
	}
}

func (s *routeBreakdowns) init() {
	if s.flushTimer == nil {
		s.flushTimer = time.AfterFunc(flushPeriod, s.Flush)
		s.addWG = new(sync.WaitGroup)
		s.m = make(map[routeBreakdownKey]*routeBreakdown)
	}
}

// Flush sends to Airbrake route stats.
func (s *routeBreakdowns) Flush() {
	s.mu.Lock()

	s.flushTimer = nil
	addWG := s.addWG
	s.addWG = nil
	m := s.m
	s.m = nil

	s.mu.Unlock()

	if m == nil {
		return
	}

	addWG.Wait()
	err := s.send(m)
	if err != nil {
		logger.Printf("routeBreakdowns.send failed: %s", err)
	}
}

type breakdownsOut struct {
	Env    string            `json:"environment"`
	Routes []*routeBreakdown `json:"routes"`
}

func (s *routeBreakdowns) send(m map[routeBreakdownKey]*routeBreakdown) error {
	var routes []*routeBreakdown
	for _, v := range m {
		err := v.Pack()
		if err != nil {
			return err
		}
		routes = append(routes, v)
	}

	buf := buffers.Get().(*bytes.Buffer)
	defer buffers.Put(buf)
	buf.Reset()

	out := breakdownsOut{
		Env:    s.opt.Environment,
		Routes: routes,
	}
	err := json.NewEncoder(buf).Encode(out)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", s.apiURL, buf)
	if err != nil {
		return err
	}

	req.Header.Set("Authorization", "Bearer "+s.opt.ProjectKey)
	req.Header.Set("Content-Type", "application/json")
	resp, err := s.opt.HTTPClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	buf.Reset()
	_, err = buf.ReadFrom(resp.Body)
	if err != nil {
		return err
	}

	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return nil
	}

	switch resp.StatusCode {
	case http.StatusUnauthorized:
		return errUnauthorized
	}

	err = fmt.Errorf("got unexpected response status=%q", resp.Status)
	return err
}

func (s *routeBreakdowns) Notify(c context.Context, trace *RouteTrace) error {
	if trace.StatusCode < 200 || (trace.StatusCode >= 300 && trace.StatusCode < 400) {
		// ignore
		return nil
	}

	key := routeBreakdownKey{
		Method:   trace.Method,
		Route:    trace.Route,
		RespType: trace.respType(),
		Time:     trace.Start.UTC().Truncate(time.Minute),
	}

	s.mu.Lock()
	s.init()
	b, ok := s.m[key]
	if !ok {
		b = &routeBreakdown{
			routeBreakdownKey: key,
		}
		s.m[key] = b
	}
	addWG := s.addWG
	addWG.Add(1)
	s.mu.Unlock()

	total := trace.End.Sub(trace.Start)
	groups := trace.flushGroups()

	b.Add(total, groups)
	addWG.Done()

	return nil
}

type RouteTrace struct {
	Method      string
	Route       string
	StatusCode  int
	ContentType string

	Start time.Time
	End   time.Time

	spansMu sync.RWMutex
	spans   map[string]Span

	groupsMu sync.Mutex
	groups   map[string]time.Duration
}

func NewRouteTrace(c context.Context, trace *RouteTrace) (context.Context, *RouteTrace) {
	if trace.Start.IsZero() {
		trace.Start = time.Now()
	}
	c = context.WithValue(c, traceCtxKey, trace)
	return c, trace
}

func RouteTraceFromContext(c context.Context) *RouteTrace {
	if c == nil {
		return nil
	}
	t, _ := c.Value(traceCtxKey).(*RouteTrace)
	return t
}

func (t *RouteTrace) Span(name string) Span {
	if t == nil {
		return noopSpan{}
	}

	s := &span{
		trace: t,
		name:  name,
		start: time.Now(),
	}

	t.spansMu.Lock()
	if t.spans == nil {
		t.spans = make(map[string]Span)
	}
	t.spans[name] = s
	t.spansMu.Unlock()

	return s
}

func (t *RouteTrace) StartSpan(name string) {
	_ = t.Span(name)
}

func (t *RouteTrace) FinishSpan(name string) {
	if t == nil {
		return
	}

	t.spansMu.RLock()
	s := t.spans[name]
	t.spansMu.RUnlock()

	if s == nil {
		log.Printf("no span with name=%q is in progress", name)
		return
	}
	s.Finish()
}

func (t *RouteTrace) IncGroup(name string, dur time.Duration) {
	t.groupsMu.Lock()
	if t.groups == nil {
		t.groups = make(map[string]time.Duration)
	}
	t.groups[name] += dur
	t.groupsMu.Unlock()
}

func (t *RouteTrace) flushGroups() map[string]time.Duration {
	t.groupsMu.Lock()
	groups := t.groups
	t.groups = nil
	t.groupsMu.Unlock()
	return groups
}

func (t *RouteTrace) respType() string {
	if t.StatusCode >= 400 {
		return "error"
	}
	if t.ContentType == "" {
		return ""
	}
	ind := strings.LastIndexByte(t.ContentType, '/')
	if ind != -1 {
		return t.ContentType[ind+1:]
	}
	return t.ContentType
}

type Span interface {
	Finish()
}

type span struct {
	trace *RouteTrace
	name  string
	start time.Time
}

var _ Span = (*span)(nil)

func (s *span) Finish() {
	since := time.Since(s.start)
	s.trace.IncGroup(s.name, since)
}

type noopSpan struct{}

var _ Span = noopSpan{}

func (noopSpan) Finish() {}

func durInMs(dur time.Duration) float64 {
	return float64(dur) / float64(time.Millisecond)
}
<commit_msg>Remove other group<commit_after>package gobrake

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"
)

type ctxKey string

const traceCtxKey ctxKey = "ab_route_trace"

type routeBreakdownKey struct {
	Method   string    `json:"method"`
	Route    string    `json:"route"`
	RespType string    `json:"responseType"`
	Time     time.Time `json:"time"`
}

type routeBreakdown struct {
	routeBreakdownKey

	routeStat
	Groups map[string]*routeStat `json:"groups"`
}

func (b *routeBreakdown) Add(total time.Duration, groups map[string]time.Duration) {
	b.mu.Lock()
	defer b.mu.Unlock()

	if b.Groups == nil {
		b.Groups = make(map[string]*routeStat)
	}

	_ = b.routeStat.Add(durInMs(total))

	if groups == nil {
		groups = make(map[string]time.Duration)
	}

	for name, dur := range groups {
		s, ok := b.Groups[name]
		if !ok {
			s = newRouteStat()
			b.Groups[name] = s
		}
		_ = s.Add(durInMs(dur))
	}
}

func (b *routeBreakdown) Pack() error {
	err := b.routeStat.Pack()
	if err != nil {
		return err
	}

	for _, v := range b.Groups {
		err = v.Pack()
		if err != nil {
			return err
		}
	}

	return nil
}

type routeBreakdowns struct {
	opt    *NotifierOptions
	apiURL string

	flushTimer *time.Timer
	addWG      *sync.WaitGroup

	mu sync.Mutex
	m  map[routeBreakdownKey]*routeBreakdown
}

func newRouteBreakdowns(opt *NotifierOptions) *routeBreakdowns {
	return &routeBreakdowns{
		opt: opt,
		apiURL: fmt.Sprintf("%s/api/v5/projects/%d/routes-breakdowns",
			opt.Host, opt.ProjectId),
	}
}

func (s *routeBreakdowns) init() {
	if s.flushTimer == nil {
		s.flushTimer = time.AfterFunc(flushPeriod, s.Flush)
		s.addWG = new(sync.WaitGroup)
		s.m = make(map[routeBreakdownKey]*routeBreakdown)
	}
}

// Flush sends to Airbrake route stats.
func (s *routeBreakdowns) Flush() {
	s.mu.Lock()

	s.flushTimer = nil
	addWG := s.addWG
	s.addWG = nil
	m := s.m
	s.m = nil

	s.mu.Unlock()

	if m == nil {
		return
	}

	addWG.Wait()
	err := s.send(m)
	if err != nil {
		logger.Printf("routeBreakdowns.send failed: %s", err)
	}
}

type breakdownsOut struct {
	Env    string            `json:"environment"`
	Routes []*routeBreakdown `json:"routes"`
}

func (s *routeBreakdowns) send(m map[routeBreakdownKey]*routeBreakdown) error {
	var routes []*routeBreakdown
	for _, v := range m {
		err := v.Pack()
		if err != nil {
			return err
		}
		routes = append(routes, v)
	}

	buf := buffers.Get().(*bytes.Buffer)
	defer buffers.Put(buf)
	buf.Reset()

	out := breakdownsOut{
		Env:    s.opt.Environment,
		Routes: routes,
	}
	err := json.NewEncoder(buf).Encode(out)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", s.apiURL, buf)
	if err != nil {
		return err
	}

	req.Header.Set("Authorization", "Bearer "+s.opt.ProjectKey)
	req.Header.Set("Content-Type", "application/json")
	resp, err := s.opt.HTTPClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	buf.Reset()
	_, err = buf.ReadFrom(resp.Body)
	if err != nil {
		return err
	}

	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return nil
	}

	switch resp.StatusCode {
	case http.StatusUnauthorized:
		return errUnauthorized
	}

	err = fmt.Errorf("got unexpected response status=%q", resp.Status)
	return err
}

func (s *routeBreakdowns) Notify(c context.Context, trace *RouteTrace) error {
	if trace.StatusCode < 200 || (trace.StatusCode >= 300 && trace.StatusCode < 400) {
		// ignore
		return nil
	}

	key := routeBreakdownKey{
		Method:   trace.Method,
		Route:    trace.Route,
		RespType: trace.respType(),
		Time:     trace.Start.UTC().Truncate(time.Minute),
	}

	s.mu.Lock()
	s.init()
	b, ok := s.m[key]
	if !ok {
		b = &routeBreakdown{
			routeBreakdownKey: key,
		}
		s.m[key] = b
	}
	addWG := s.addWG
	addWG.Add(1)
	s.mu.Unlock()

	total := trace.End.Sub(trace.Start)
	groups := trace.flushGroups()

	b.Add(total, groups)
	addWG.Done()

	return nil
}

type RouteTrace struct {
	Method      string
	Route       string
	StatusCode  int
	ContentType string

	Start time.Time
	End   time.Time

	spansMu sync.RWMutex
	spans   map[string]Span

	groupsMu sync.Mutex
	groups   map[string]time.Duration
}

func NewRouteTrace(c context.Context, trace *RouteTrace) (context.Context, *RouteTrace) {
	if trace.Start.IsZero() {
		trace.Start = time.Now()
	}
	c = context.WithValue(c, traceCtxKey, trace)
	return c, trace
}

func RouteTraceFromContext(c context.Context) *RouteTrace {
	if c == nil {
		return nil
	}
	t, _ := c.Value(traceCtxKey).(*RouteTrace)
	return t
}

func (t *RouteTrace) Span(name string) Span {
	if t == nil {
		return noopSpan{}
	}

	s := &span{
		trace: t,
		name:  name,
		start: time.Now(),
	}

	t.spansMu.Lock()
	if t.spans == nil {
		t.spans = make(map[string]Span)
	}
	t.spans[name] = s
	t.spansMu.Unlock()

	return s
}

func (t *RouteTrace) StartSpan(name string) {
	_ = t.Span(name)
}

func (t *RouteTrace) FinishSpan(name string) {
	if t == nil {
		return
	}

	t.spansMu.RLock()
	s := t.spans[name]
	t.spansMu.RUnlock()

	if s == nil {
		log.Printf("no span with name=%q is in progress", name)
		return
	}
	s.Finish()
}

func (t *RouteTrace) IncGroup(name string, dur time.Duration) {
	t.groupsMu.Lock()
	if t.groups == nil {
		t.groups = make(map[string]time.Duration)
	}
	t.groups[name] += dur
	t.groupsMu.Unlock()
}

func (t *RouteTrace) flushGroups() map[string]time.Duration {
	t.groupsMu.Lock()
	groups := t.groups
	t.groups = nil
	t.groupsMu.Unlock()
	return groups
}

func (t *RouteTrace) respType() string {
	if t.StatusCode >= 400 {
		return "error"
	}
	if t.ContentType == "" {
		return ""
	}
	ind := strings.LastIndexByte(t.ContentType, '/')
	if ind != -1 {
		return t.ContentType[ind+1:]
	}
	return t.ContentType
}

type Span interface {
	Finish()
}

type span struct {
	trace *RouteTrace
	name  string
	start time.Time
}

var _ Span = (*span)(nil)

func (s *span) Finish() {
	since := time.Since(s.start)
	s.trace.IncGroup(s.name, since)
}

type noopSpan struct{}

var _ Span = noopSpan{}

func (noopSpan) Finish() {}

func durInMs(dur time.Duration) float64 {
	return float64(dur) / float64(time.Millisecond)
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"net/http"
	"path"
	"regexp"
	"strings"
)

// Used internally to configure router, a RouterGroup is associated with a prefix
// and an array of handlers (middlewares)
type RouterGroup struct {
	Handlers HandlersChain
	BasePath string
	engine   *Engine
}

// Adds middlewares to the group, see example code in github.
func (group *RouterGroup) Use(middlewares ...HandlerFunc) {
	group.Handlers = append(group.Handlers, middlewares...)
}

// Creates a new router group. You should add all the routes that have common middlwares or the same path prefix.
// For example, all the routes that use a common middlware for authorization could be grouped.
func (group *RouterGroup) Group(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	return &RouterGroup{
		Handlers: group.combineHandlers(handlers),
		BasePath: group.calculateAbsolutePath(relativePath),
		engine:   group.engine,
	}
}

// Handle registers a new request handle and middlewares with the given path and method.
// The last handler should be the real handler, the other ones should be middlewares that can and should be shared among different routes.
// See the example code in github.
//
// For GET, POST, PUT, PATCH and DELETE requests the respective shortcut
// functions can be used.
//
// This function is intended for bulk loading and to allow the usage of less
// frequently used, non-standardized or custom methods (e.g. for internal
// communication with a proxy).
func (group *RouterGroup) handle(httpMethod, relativePath string, handlers HandlersChain) {
	absolutePath := group.calculateAbsolutePath(relativePath)
	handlers = group.combineHandlers(handlers)
	group.engine.addRoute(httpMethod, absolutePath, handlers)
}

func (group *RouterGroup) Handle(httpMethod, relativePath string, handlers ...HandlerFunc) {
	if matches, err := regexp.MatchString("^[A-Z]+$", httpMethod); !matches || err != nil {
		panic("http method " + httpMethod + " is not valid")
	}
	group.handle(httpMethod, relativePath, handlers)
}

// POST is a shortcut for router.Handle("POST", path, handle)
func (group *RouterGroup) POST(relativePath string, handlers ...HandlerFunc) {
	group.handle("POST", relativePath, handlers)
}

// GET is a shortcut for router.Handle("GET", path, handle)
func (group *RouterGroup) GET(relativePath string, handlers ...HandlerFunc) {
	group.handle("GET", relativePath, handlers)
}

// DELETE is a shortcut for router.Handle("DELETE", path, handle)
func (group *RouterGroup) DELETE(relativePath string, handlers ...HandlerFunc) {
	group.handle("DELETE", relativePath, handlers)
}

// PATCH is a shortcut for router.Handle("PATCH", path, handle)
func (group *RouterGroup) PATCH(relativePath string, handlers ...HandlerFunc) {
	group.handle("PATCH", relativePath, handlers)
}

// PUT is a shortcut for router.Handle("PUT", path, handle)
func (group *RouterGroup) PUT(relativePath string, handlers ...HandlerFunc) {
	group.handle("PUT", relativePath, handlers)
}

// OPTIONS is a shortcut for router.Handle("OPTIONS", path, handle)
func (group *RouterGroup) OPTIONS(relativePath string, handlers ...HandlerFunc) {
	group.handle("OPTIONS", relativePath, handlers)
}

// HEAD is a shortcut for router.Handle("HEAD", path, handle)
func (group *RouterGroup) HEAD(relativePath string, handlers ...HandlerFunc) {
	group.handle("HEAD", relativePath, handlers)
}

func (group *RouterGroup) Any(relativePath string, handlers ...HandlerFunc) {
	// GET, POST, PUT, PATCH, HEAD, OPTIONS, DELETE, CONNECT, TRACE
	group.handle("GET", relativePath, handlers)
	group.handle("POST", relativePath, handlers)
	group.handle("PUT", relativePath, handlers)
	group.handle("PATCH", relativePath, handlers)
	group.handle("HEAD", relativePath, handlers)
	group.handle("OPTIONS", relativePath, handlers)
	group.handle("DELETE", relativePath, handlers)
	group.handle("CONNECT", relativePath, handlers)
	group.handle("TRACE", relativePath, handlers)
}

func (group *RouterGroup) StaticFile(relativePath, filepath string) {
	if strings.Contains(relativePath, ":") || strings.Contains(relativePath, "*") {
		panic("URL parameters can not be used when serving a static file")
	}
	handler := func(c *Context) {
		c.File(filepath)
	}
	group.GET(relativePath, handler)
	group.HEAD(relativePath, handler)
}

// Static serves files from the given file system root.
// Internally a http.FileServer is used, therefore http.NotFound is used instead
// of the Router's NotFound handler.
// To use the operating system's file system implementation,
// use :
//     router.Static("/static", "/var/www")
func (group *RouterGroup) Static(relativePath, root string) {
	group.StaticFS(relativePath, Dir(root, false))
}

func (group *RouterGroup) StaticFS(relativePath string, fs http.FileSystem) {
	if strings.Contains(relativePath, ":") || strings.Contains(relativePath, "*") {
		panic("URL parameters can not be used when serving a static folder")
	}
	handler := group.createStaticHandler(relativePath, fs)
	urlPattern := path.Join(relativePath, "/*filepath")

	// Register GET and HEAD handlers
	group.GET(urlPattern, handler)
	group.HEAD(urlPattern, handler)
}

func (group *RouterGroup) createStaticHandler(relativePath string, fs http.FileSystem) HandlerFunc {
	absolutePath := group.calculateAbsolutePath(relativePath)
	fileServer := http.StripPrefix(absolutePath, http.FileServer(fs))
	_, nolisting := fs.(*onlyfilesFS)
	return func(c *Context) {
		if nolisting {
			c.Writer.WriteHeader(404)
		}
		fileServer.ServeHTTP(c.Writer, c.Request)
	}
}

func (group *RouterGroup) combineHandlers(handlers HandlersChain) HandlersChain {
	finalSize := len(group.Handlers) + len(handlers)
	if finalSize >= int(AbortIndex) {
		panic("too many handlers")
	}
	mergedHandlers := make(HandlersChain, finalSize)
	copy(mergedHandlers, group.Handlers)
	copy(mergedHandlers[len(group.Handlers):], handlers)
	return mergedHandlers
}

func (group *RouterGroup) calculateAbsolutePath(relativePath string) string {
	return joinPaths(group.BasePath, relativePath)
}
<commit_msg>Adds collection pipeline pattern<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"net/http"
	"path"
	"regexp"
	"strings"
)

// Used internally to configure router, a RouterGroup is associated with a prefix
// and an array of handlers (middlewares)
type RouterGroup struct {
	Handlers HandlersChain
	BasePath string
	engine   *Engine
}

// Adds middlewares to the group, see example code in github.
func (group *RouterGroup) Use(middlewares ...HandlerFunc) {
	group.Handlers = append(group.Handlers, middlewares...)
}

// Creates a new router group. You should add all the routes that have common middlwares or the same path prefix.
// For example, all the routes that use a common middlware for authorization could be grouped.
func (group *RouterGroup) Group(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	return &RouterGroup{
		Handlers: group.combineHandlers(handlers),
		BasePath: group.calculateAbsolutePath(relativePath),
		engine:   group.engine,
	}
}

// Handle registers a new request handle and middlewares with the given path and method.
// The last handler should be the real handler, the other ones should be middlewares that can and should be shared among different routes.
// See the example code in github.
//
// For GET, POST, PUT, PATCH and DELETE requests the respective shortcut
// functions can be used.
//
// This function is intended for bulk loading and to allow the usage of less
// frequently used, non-standardized or custom methods (e.g. for internal
// communication with a proxy).
func (group *RouterGroup) handle(httpMethod, relativePath string, handlers HandlersChain) {
	absolutePath := group.calculateAbsolutePath(relativePath)
	handlers = group.combineHandlers(handlers)
	group.engine.addRoute(httpMethod, absolutePath, handlers)
}

func (group *RouterGroup) Handle(httpMethod, relativePath string, handlers ...HandlerFunc) *RouterGroup {
	if matches, err := regexp.MatchString("^[A-Z]+$", httpMethod); !matches || err != nil {
		panic("http method " + httpMethod + " is not valid")
	}
	group.handle(httpMethod, relativePath, handlers)
	return group

}

// POST is a shortcut for router.Handle("POST", path, handle)
func (group *RouterGroup) POST(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	group.handle("POST", relativePath, handlers)
	return group
}

// GET is a shortcut for router.Handle("GET", path, handle)
func (group *RouterGroup) GET(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	group.handle("GET", relativePath, handlers)
	return group
}

// DELETE is a shortcut for router.Handle("DELETE", path, handle)
func (group *RouterGroup) DELETE(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	group.handle("DELETE", relativePath, handlers)
	return group
}

// PATCH is a shortcut for router.Handle("PATCH", path, handle)
func (group *RouterGroup) PATCH(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	group.handle("PATCH", relativePath, handlers)
	return group
}

// PUT is a shortcut for router.Handle("PUT", path, handle)
func (group *RouterGroup) PUT(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	group.handle("PUT", relativePath, handlers)
	return group
}

// OPTIONS is a shortcut for router.Handle("OPTIONS", path, handle)
func (group *RouterGroup) OPTIONS(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	group.handle("OPTIONS", relativePath, handlers)
	return group
}

// HEAD is a shortcut for router.Handle("HEAD", path, handle)
func (group *RouterGroup) HEAD(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	group.handle("HEAD", relativePath, handlers)
	return group
}

func (group *RouterGroup) Any(relativePath string, handlers ...HandlerFunc) *RouterGroup {
	// GET, POST, PUT, PATCH, HEAD, OPTIONS, DELETE, CONNECT, TRACE
	group.handle("GET", relativePath, handlers)
	group.handle("POST", relativePath, handlers)
	group.handle("PUT", relativePath, handlers)
	group.handle("PATCH", relativePath, handlers)
	group.handle("HEAD", relativePath, handlers)
	group.handle("OPTIONS", relativePath, handlers)
	group.handle("DELETE", relativePath, handlers)
	group.handle("CONNECT", relativePath, handlers)
	group.handle("TRACE", relativePath, handlers)
	return group
}

func (group *RouterGroup) StaticFile(relativePath, filepath string) *RouterGroup {
	if strings.Contains(relativePath, ":") || strings.Contains(relativePath, "*") {
		panic("URL parameters can not be used when serving a static file")
	}
	handler := func(c *Context) {
		c.File(filepath)
	}
	group.GET(relativePath, handler)
	group.HEAD(relativePath, handler)
	return group
}

// Static serves files from the given file system root.
// Internally a http.FileServer is used, therefore http.NotFound is used instead
// of the Router's NotFound handler.
// To use the operating system's file system implementation,
// use :
//     router.Static("/static", "/var/www")
func (group *RouterGroup) Static(relativePath, root string) *RouterGroup {
	return group.StaticFS(relativePath, Dir(root, false))
}

func (group *RouterGroup) StaticFS(relativePath string, fs http.FileSystem) *RouterGroup {
	if strings.Contains(relativePath, ":") || strings.Contains(relativePath, "*") {
		panic("URL parameters can not be used when serving a static folder")
	}
	handler := group.createStaticHandler(relativePath, fs)
	urlPattern := path.Join(relativePath, "/*filepath")

	// Register GET and HEAD handlers
	group.GET(urlPattern, handler)
	group.HEAD(urlPattern, handler)
	return group
}

func (group *RouterGroup) createStaticHandler(relativePath string, fs http.FileSystem) HandlerFunc {
	absolutePath := group.calculateAbsolutePath(relativePath)
	fileServer := http.StripPrefix(absolutePath, http.FileServer(fs))
	_, nolisting := fs.(*onlyfilesFS)
	return func(c *Context) {
		if nolisting {
			c.Writer.WriteHeader(404)
		}
		fileServer.ServeHTTP(c.Writer, c.Request)
	}
}

func (group *RouterGroup) combineHandlers(handlers HandlersChain) HandlersChain {
	finalSize := len(group.Handlers) + len(handlers)
	if finalSize >= int(AbortIndex) {
		panic("too many handlers")
	}
	mergedHandlers := make(HandlersChain, finalSize)
	copy(mergedHandlers, group.Handlers)
	copy(mergedHandlers[len(group.Handlers):], handlers)
	return mergedHandlers
}

func (group *RouterGroup) calculateAbsolutePath(relativePath string) string {
	return joinPaths(group.BasePath, relativePath)
}
<|endoftext|>