<commit_before>package main

import (
	"flag"
	"log"
	"net"
	"time"

	"github.com/myzhan/boomer"
)

//                            +------------+
//                            |            |
//                            |   locust   |
//                            |            |
//                            +------+-----+
//                                   ^
//                                   |  qps & timeout
//                                   |
//	+------------+        +------+-----+        +------------+
//	|            |request |            |request |            |
//	| udpcopy    +------->+ udp proxy  +------->+ backend    |
//	|            |        |            |        |            |
//	+------------+        +------------+        +------------+

// While requests from udpcopy passing through this udp server, it keeps track of qps and timeout.
// Also, it can multi-copy the original request for more stress.

// Known Issues:
// 1. Once locust start the test, it can't be stopped by locust. You should restart locust and this udp server if you
//    want to restart.

// See also:
// udpcopy: https://github.com/wangbin579/udpcopy

type worker struct {
	conn     *net.UDPConn
	requests chan []byte
	recvBuff []byte
}

func newWorker(remoteAddr string) *worker {
	addr, err := net.ResolveUDPAddr("udp", remoteAddr)
	conn, err := net.DialUDP("udp", nil, addr)
	if err != nil {
		log.Println("Failed to create worker", err)
		return nil
	}
	requests := make(chan []byte, 1000)
	recvBuff := make([]byte, *udpBufferSize)
	go func() {
		for {
			req := <-requests
			for n := 0; n < *number; n++ {
				startTime := boomer.Now()
				wn, err := conn.Write(req)
				if err != nil {
					boomer.Events.Publish("request_failure", "udp-write", name, 0.0, err.Error())
					return
				}

				if *dontRead {
					elapsed := boomer.Now() - startTime
					boomer.Events.Publish("request_success", "udp-write", name, elapsed, int64(wn))
				} else {
					conn.SetReadDeadline(time.Now().Add(backendTimeout))
					respLength, err := conn.Read(recvBuff)
					if err != nil {
						boomer.Events.Publish("request_failure", "udp-read", name, 0.0, err.Error())
						return
					}
					elapsed := boomer.Now() - startTime
					boomer.Events.Publish("request_success", "udp-resp", name, elapsed, int64(respLength))
				}
			}
		}
	}()

	return &worker{
		conn:     conn,
		requests: requests,
		recvBuff: recvBuff,
	}

}

func createWorkers() {
	workersPool = make([]*worker, *workersCount)
	for n := 0; n < *workersCount; n++ {
		workersPool[n] = newWorker(*backendAddr)
	}
}

func proxy() {

	// Pooling workers
	createWorkers()

	listener, err := net.ListenUDP("udp", &net.UDPAddr{
		IP:   net.ParseIP(*proxyHost),
		Port: *proxyPort,
	})
	if err != nil {
		log.Fatal("error binding on port:", *proxyPort, err)
	}

	workerIndex := uint64(0)

	for {
		data := make([]byte, *udpBufferSize)
		n, _, err := listener.ReadFromUDP(data)
		if err != nil {
			log.Println("error during read, current request is dropped.", err)
			continue
		}
		if *udpBufferSize <= n {
			log.Printf("request size is larger than %d，please enlarge udp-buffer-size. current request is dropped.\n", *udpBufferSize)
			continue
		}
		if !testStarted {
			// test is not started, drop current request.
			continue
		}

		// Round-robin selection
		workerIndex = workerIndex + 1
		selectedWorker := workersPool[workerIndex%(uint64(*workersCount))]
		selectedWorker.requests <- data[:n]
	}
}

func deadend() {
	testStarted = true
	for {
		time.Sleep(time.Second * 100)
	}
}

func main() {

	task := &boomer.Task{
		Name:   "udproxy",
		Weight: 10,
		Fn:     deadend,
	}

	go proxy()

	boomer.Run(task)
}

const name = "udproxy"

var testStarted = false
var workersPool []*worker

// command line arguments
var backendAddr *string
var backendTimeout time.Duration
var proxyHost *string
var proxyPort *int
var udpBufferSize *int
var number *int
var workersCount *int
var dontRead *bool

func init() {

	backendAddr = flag.String("backend-addr", "127.0.0.1:44444", "backend address")
	timeout := flag.Int("backend-timeout", 1000, "backend timeout(ms)")
	proxyHost = flag.String("proxy-host", "0.0.0.0", "proxy bind-host")
	proxyPort = flag.Int("proxy-port", 23333, "proxy bind-port")
	workersCount = flag.Int("workers", 200, "UDP workers")
	udpBufferSize = flag.Int("udp-buffer-size", 4096, "udp recv buffer size")
	number = flag.Int("number", 1, "the number of replication for multi-copying")
	dontRead = flag.Bool("dontread", false, "do not wait for backend's response")
	flag.Parse()

	backendTimeout = time.Duration(*timeout) * time.Millisecond
}
<commit_msg>FIX: dont quit when errors happen<commit_after>package main

import (
	"flag"
	"log"
	"net"
	"time"

	"github.com/myzhan/boomer"
)

//                            +------------+
//                            |            |
//                            |   locust   |
//                            |            |
//                            +------+-----+
//                                   ^
//                                   |  qps & timeout
//                                   |
//	+------------+        +------+-----+        +------------+
//	|            |request |            |request |            |
//	| udpcopy    +------->+ udp proxy  +------->+ backend    |
//	|            |        |            |        |            |
//	+------------+        +------------+        +------------+

// While requests from udpcopy passing through this udp server, it keeps track of qps and timeout.
// Also, it can multi-copy the original request for more stress.

// Known Issues:
// 1. Once locust start the test, it can't be stopped by locust. You should restart locust and this udp server if you
//    want to restart.

// See also:
// udpcopy: https://github.com/wangbin579/udpcopy

type worker struct {
	conn     *net.UDPConn
	requests chan []byte
	recvBuff []byte
}

func newWorker(remoteAddr string) *worker {
	addr, err := net.ResolveUDPAddr("udp", remoteAddr)
	conn, err := net.DialUDP("udp", nil, addr)
	if err != nil {
		log.Println("Failed to create worker", err)
		return nil
	}
	requests := make(chan []byte, 1000)
	recvBuff := make([]byte, *udpBufferSize)
	go func() {
		for {
			req := <-requests
			for n := 0; n < *number; n++ {
				startTime := boomer.Now()
				wn, err := conn.Write(req)
				if err != nil {
					boomer.Events.Publish("request_failure", "udp-write", name, 0.0, err.Error())
					continue
				}

				if *dontRead {
					elapsed := boomer.Now() - startTime
					boomer.Events.Publish("request_success", "udp-write", name, elapsed, int64(wn))
				} else {
					conn.SetReadDeadline(time.Now().Add(backendTimeout))
					respLength, err := conn.Read(recvBuff)
					if err != nil {
						boomer.Events.Publish("request_failure", "udp-read", name, 0.0, err.Error())
						continue
					}
					elapsed := boomer.Now() - startTime
					boomer.Events.Publish("request_success", "udp-resp", name, elapsed, int64(respLength))
				}
			}
		}
	}()

	return &worker{
		conn:     conn,
		requests: requests,
		recvBuff: recvBuff,
	}

}

func createWorkers() {
	workersPool = make([]*worker, *workersCount)
	for n := 0; n < *workersCount; n++ {
		workersPool[n] = newWorker(*backendAddr)
	}
}

func proxy() {

	// Pooling workers
	createWorkers()

	listener, err := net.ListenUDP("udp", &net.UDPAddr{
		IP:   net.ParseIP(*proxyHost),
		Port: *proxyPort,
	})
	if err != nil {
		log.Fatal("error binding on port:", *proxyPort, err)
	}

	workerIndex := uint64(0)

	for {
		data := make([]byte, *udpBufferSize)
		n, _, err := listener.ReadFromUDP(data)
		if err != nil {
			log.Println("error during read, current request is dropped.", err)
			continue
		}
		if *udpBufferSize <= n {
			log.Printf("request size is larger than %d，please enlarge udp-buffer-size. current request is dropped.\n", *udpBufferSize)
			continue
		}
		if !testStarted {
			// test is not started, drop current request.
			continue
		}

		// Round-robin selection
		workerIndex = workerIndex + 1
		selectedWorker := workersPool[workerIndex%(uint64(*workersCount))]
		selectedWorker.requests <- data[:n]
	}
}

func deadend() {
	testStarted = true
	for {
		time.Sleep(time.Second * 100)
	}
}

func main() {

	task := &boomer.Task{
		Name:   "udproxy",
		Weight: 10,
		Fn:     deadend,
	}

	go proxy()

	boomer.Run(task)
}

const name = "udproxy"

var testStarted = false
var workersPool []*worker

// command line arguments
var backendAddr *string
var backendTimeout time.Duration
var proxyHost *string
var proxyPort *int
var udpBufferSize *int
var number *int
var workersCount *int
var dontRead *bool

func init() {

	backendAddr = flag.String("backend-addr", "127.0.0.1:44444", "backend address")
	timeout := flag.Int("backend-timeout", 1000, "backend timeout(ms)")
	proxyHost = flag.String("proxy-host", "0.0.0.0", "proxy bind-host")
	proxyPort = flag.Int("proxy-port", 23333, "proxy bind-port")
	workersCount = flag.Int("workers", 200, "UDP workers")
	udpBufferSize = flag.Int("udp-buffer-size", 4096, "udp recv buffer size")
	number = flag.Int("number", 1, "the number of replication for multi-copying")
	dontRead = flag.Bool("dontread", false, "do not wait for backend's response")
	flag.Parse()

	backendTimeout = time.Duration(*timeout) * time.Millisecond
}
<|endoftext|>


<commit_before>package router

import (
	"bufio"
	"errors"
	"io"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
)

func init() {
	pump := &LogsPump{
		pumps:  make(map[string]*containerPump),
		routes: make(map[chan *update]struct{}),
	}
	LogRouters.Register(pump, "pump")
	Jobs.Register(pump, "pump")
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

func debug(v ...interface{}) {
	if os.Getenv("DEBUG") != "" {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatal(context+": ", err)
	}
}

func normalName(name string) string {
	return name[1:]
}

func normalID(id string) string {
	if len(id) > 12 {
		return id[:12]
	}
	return id
}

func logDriverSupported(container *docker.Container) bool {
	switch container.HostConfig.LogConfig.Type {
	case "json-file", "journald":
		return true
	default:
		return false
	}
}

func ignoreContainer(container *docker.Container) bool {
	for _, kv := range container.Config.Env {
		kvp := strings.SplitN(kv, "=", 2)
		if len(kvp) == 2 && kvp[0] == "LOGSPOUT" && strings.ToLower(kvp[1]) == "ignore" {
			return true
		}
	}
	excludeLabel := getopt("EXCLUDE_LABEL", "")
	if value, ok := container.Config.Labels[excludeLabel]; ok {
		return len(excludeLabel) > 0 && strings.ToLower(value) == "true"
	}
	return false
}

func getInactivityTimeoutFromEnv() time.Duration {
	inactivityTimeout, err := time.ParseDuration(getopt("INACTIVITY_TIMEOUT", "0"))
	assert(err, "Couldn't parse env var INACTIVITY_TIMEOUT. See https://golang.org/pkg/time/#ParseDuration for valid format.")
	return inactivityTimeout
}

type update struct {
	*docker.APIEvents
	pump *containerPump
}

// LogsPump is responsible for "pumping" logs to their configured destinations
type LogsPump struct {
	mu     sync.Mutex
	pumps  map[string]*containerPump
	routes map[chan *update]struct{}
	client *docker.Client
}

// Name returns the name of the pump
func (p *LogsPump) Name() string {
	return "pump"
}

// Setup configures the pump
func (p *LogsPump) Setup() error {
	var err error
	p.client, err = docker.NewClientFromEnv()
	return err
}

func (p *LogsPump) rename(event *docker.APIEvents) {
	p.mu.Lock()
	defer p.mu.Unlock()
	container, err := p.client.InspectContainer(event.ID)
	assert(err, "pump")
	pump, ok := p.pumps[normalID(event.ID)]
	if !ok {
		debug("pump.rename(): ignore: pump not found, state:", container.State.StateString())
		return
	}
	pump.container.Name = container.Name
}

// Run executes the pump
func (p *LogsPump) Run() error {
	inactivityTimeout := getInactivityTimeoutFromEnv()
	debug("pump.Run(): using inactivity timeout: ", inactivityTimeout)

	containers, err := p.client.ListContainers(docker.ListContainersOptions{})
	if err != nil {
		return err
	}
	for _, listing := range containers {
		p.pumpLogs(&docker.APIEvents{
			ID:     normalID(listing.ID),
			Status: "start",
		}, false, inactivityTimeout)
	}
	events := make(chan *docker.APIEvents)
	err = p.client.AddEventListener(events)
	if err != nil {
		return err
	}
	for event := range events {
		debug("pump.Run() event:", normalID(event.ID), event.Status)
		switch event.Status {
		case "start", "restart":
			go p.pumpLogs(event, true, inactivityTimeout)
		case "rename":
			go p.rename(event)
		case "die":
			go p.update(event)
		}
	}
	return errors.New("docker event stream closed")
}

func (p *LogsPump) pumpLogs(event *docker.APIEvents, backlog bool, inactivityTimeout time.Duration) {
	id := normalID(event.ID)
	container, err := p.client.InspectContainer(id)
	assert(err, "pump")
	if container.Config.Tty {
		debug("pump.pumpLogs():", id, "ignored: tty enabled")
		return
	}
	if ignoreContainer(container) {
		debug("pump.pumpLogs():", id, "ignored: environ ignore")
		return
	}
	if !logDriverSupported(container) {
		debug("pump.pumpLogs():", id, "ignored: log driver not supported")
		return
	}

	var sinceTime time.Time
	if backlog {
		sinceTime = time.Unix(0, 0)
	} else {
		sinceTime = time.Now()
	}

	p.mu.Lock()
	if _, exists := p.pumps[id]; exists {
		p.mu.Unlock()
		debug("pump.pumpLogs():", id, "pump exists")
		return
	}
	outrd, outwr := io.Pipe()
	errrd, errwr := io.Pipe()
	p.pumps[id] = newContainerPump(container, outrd, errrd)
	p.mu.Unlock()
	p.update(event)
	go func() {
		for {
			debug("pump.pumpLogs():", id, "started")
			err := p.client.Logs(docker.LogsOptions{
				Container:         id,
				OutputStream:      outwr,
				ErrorStream:       errwr,
				Stdout:            true,
				Stderr:            true,
				Follow:            true,
				Tail:              "all",
				Since:             sinceTime.Unix(),
				InactivityTimeout: inactivityTimeout,
			})
			if err != nil {
				debug("pump.pumpLogs():", id, "stopped with error:", err)
			} else {
				debug("pump.pumpLogs():", id, "stopped")
			}

			sinceTime = time.Now()
			if err == docker.ErrInactivityTimeout {
				sinceTime = sinceTime.Add(-inactivityTimeout)
			}

			container, err := p.client.InspectContainer(id)
			if err != nil {
				_, four04 := err.(*docker.NoSuchContainer)
				if !four04 {
					assert(err, "pump")
				}
			} else if container.State.Running {
				continue
			}

			debug("pump.pumpLogs():", id, "dead")
			outwr.Close()
			errwr.Close()
			p.mu.Lock()
			delete(p.pumps, id)
			p.mu.Unlock()
			return
		}
	}()
}

func (p *LogsPump) update(event *docker.APIEvents) {
	p.mu.Lock()
	defer p.mu.Unlock()
	pump, pumping := p.pumps[normalID(event.ID)]
	if pumping {
		for r := range p.routes {
			select {
			case r <- &update{event, pump}:
			case <-time.After(time.Second * 1):
				debug("pump.update(): route timeout, dropping")
				defer delete(p.routes, r)
			}
		}
	}
}

// RoutingFrom returns whether a container id is routing from this pump
func (p *LogsPump) RoutingFrom(id string) bool {
	p.mu.Lock()
	defer p.mu.Unlock()
	_, monitoring := p.pumps[normalID(id)]
	return monitoring
}

// Route takes a logstream and routes it according to the supplied Route
func (p *LogsPump) Route(route *Route, logstream chan *Message) {
	p.mu.Lock()
	for _, pump := range p.pumps {
		if route.MatchContainer(
			normalID(pump.container.ID),
			normalName(pump.container.Name),
			pump.container.Config.Labels) {

			pump.add(logstream, route)
			defer pump.remove(logstream)
		}
	}
	updates := make(chan *update)
	p.routes[updates] = struct{}{}
	p.mu.Unlock()
	defer func() {
		p.mu.Lock()
		delete(p.routes, updates)
		p.mu.Unlock()
	}()
	for {
		select {
		case event := <-updates:
			switch event.Status {
			case "start", "restart":
				if route.MatchContainer(
					normalID(event.pump.container.ID),
					normalName(event.pump.container.Name),
					event.pump.container.Config.Labels) {

					event.pump.add(logstream, route)
					defer event.pump.remove(logstream)
				}
			case "die":
				if strings.HasPrefix(route.FilterID, event.ID) {
					// If the route is just about a single container,
					// we can stop routing when it dies.
					return
				}
			}
		case <-route.Closer():
			return
		}
	}
}

type containerPump struct {
	sync.Mutex
	container  *docker.Container
	logstreams map[chan *Message]*Route
}

func newContainerPump(container *docker.Container, stdout, stderr io.Reader) *containerPump {
	cp := &containerPump{
		container:  container,
		logstreams: make(map[chan *Message]*Route),
	}
	pump := func(source string, input io.Reader) {
		buf := bufio.NewReader(input)
		for {
			line, err := buf.ReadString('\n')
			if err != nil {
				if err != io.EOF {
					debug("pump.newContainerPump():", normalID(container.ID), source+":", err)
				}
				return
			}
			cp.send(&Message{
				Data:      strings.TrimSuffix(line, "\n"),
				Container: container,
				Time:      time.Now(),
				Source:    source,
			})
		}
	}
	go pump("stdout", stdout)
	go pump("stderr", stderr)
	return cp
}

func (cp *containerPump) send(msg *Message) {
	cp.Lock()
	defer cp.Unlock()
	for logstream, route := range cp.logstreams {
		if !route.MatchMessage(msg) {
			continue
		}
		select {
		case logstream <- msg:
		case <-time.After(time.Second * 1):
			debug("pump.send(): send timeout, closing")
			// normal call to remove() triggered by
			// route.Closer() may not be able to grab
			// lock under heavy load, so we delete here
			defer delete(cp.logstreams, logstream)
		}
	}
}

func (cp *containerPump) add(logstream chan *Message, route *Route) {
	cp.Lock()
	defer cp.Unlock()
	cp.logstreams[logstream] = route
}

func (cp *containerPump) remove(logstream chan *Message) {
	cp.Lock()
	defer cp.Unlock()
	delete(cp.logstreams, logstream)
}
<commit_msg>Allow containers with TTY enabled using environment variable ALLOW_TTY=true closes #231<commit_after>package router

import (
	"bufio"
	"errors"
	"io"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
)

func init() {
	pump := &LogsPump{
		pumps:  make(map[string]*containerPump),
		routes: make(map[chan *update]struct{}),
	}
	LogRouters.Register(pump, "pump")
	Jobs.Register(pump, "pump")
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

func debug(v ...interface{}) {
	if os.Getenv("DEBUG") != "" {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatal(context+": ", err)
	}
}

func normalName(name string) string {
	return name[1:]
}

func normalID(id string) string {
	if len(id) > 12 {
		return id[:12]
	}
	return id
}

func logDriverSupported(container *docker.Container) bool {
	switch container.HostConfig.LogConfig.Type {
	case "json-file", "journald":
		return true
	default:
		return false
	}
}

func ignoreContainer(container *docker.Container) bool {
	for _, kv := range container.Config.Env {
		kvp := strings.SplitN(kv, "=", 2)
		if len(kvp) == 2 && kvp[0] == "LOGSPOUT" && strings.ToLower(kvp[1]) == "ignore" {
			return true
		}
	}
	excludeLabel := getopt("EXCLUDE_LABEL", "")
	if value, ok := container.Config.Labels[excludeLabel]; ok {
		return len(excludeLabel) > 0 && strings.ToLower(value) == "true"
	}
	return false
}

func getInactivityTimeoutFromEnv() time.Duration {
	inactivityTimeout, err := time.ParseDuration(getopt("INACTIVITY_TIMEOUT", "0"))
	assert(err, "Couldn't parse env var INACTIVITY_TIMEOUT. See https://golang.org/pkg/time/#ParseDuration for valid format.")
	return inactivityTimeout
}

type update struct {
	*docker.APIEvents
	pump *containerPump
}

// LogsPump is responsible for "pumping" logs to their configured destinations
type LogsPump struct {
	mu     sync.Mutex
	pumps  map[string]*containerPump
	routes map[chan *update]struct{}
	client *docker.Client
}

// Name returns the name of the pump
func (p *LogsPump) Name() string {
	return "pump"
}

// Setup configures the pump
func (p *LogsPump) Setup() error {
	var err error
	p.client, err = docker.NewClientFromEnv()
	return err
}

func (p *LogsPump) rename(event *docker.APIEvents) {
	p.mu.Lock()
	defer p.mu.Unlock()
	container, err := p.client.InspectContainer(event.ID)
	assert(err, "pump")
	pump, ok := p.pumps[normalID(event.ID)]
	if !ok {
		debug("pump.rename(): ignore: pump not found, state:", container.State.StateString())
		return
	}
	pump.container.Name = container.Name
}

// Run executes the pump
func (p *LogsPump) Run() error {
	inactivityTimeout := getInactivityTimeoutFromEnv()
	debug("pump.Run(): using inactivity timeout: ", inactivityTimeout)

	containers, err := p.client.ListContainers(docker.ListContainersOptions{})
	if err != nil {
		return err
	}
	for _, listing := range containers {
		p.pumpLogs(&docker.APIEvents{
			ID:     normalID(listing.ID),
			Status: "start",
		}, false, inactivityTimeout)
	}
	events := make(chan *docker.APIEvents)
	err = p.client.AddEventListener(events)
	if err != nil {
		return err
	}
	for event := range events {
		debug("pump.Run() event:", normalID(event.ID), event.Status)
		switch event.Status {
		case "start", "restart":
			go p.pumpLogs(event, true, inactivityTimeout)
		case "rename":
			go p.rename(event)
		case "die":
			go p.update(event)
		}
	}
	return errors.New("docker event stream closed")
}

func (p *LogsPump) pumpLogs(event *docker.APIEvents, backlog bool, inactivityTimeout time.Duration) {
	id := normalID(event.ID)
	container, err := p.client.InspectContainer(id)
	assert(err, "pump")
	if container.Config.Tty {
		if allowTty := getopt("ALLOW_TTY", ""); allowTty != "true" {
			debug("pump.pumpLogs():", id, "ignored: tty enabled")
			return
		}
	}
	if ignoreContainer(container) {
		debug("pump.pumpLogs():", id, "ignored: environ ignore")
		return
	}
	if !logDriverSupported(container) {
		debug("pump.pumpLogs():", id, "ignored: log driver not supported")
		return
	}

	var sinceTime time.Time
	if backlog {
		sinceTime = time.Unix(0, 0)
	} else {
		sinceTime = time.Now()
	}

	p.mu.Lock()
	if _, exists := p.pumps[id]; exists {
		p.mu.Unlock()
		debug("pump.pumpLogs():", id, "pump exists")
		return
	}
	outrd, outwr := io.Pipe()
	errrd, errwr := io.Pipe()
	p.pumps[id] = newContainerPump(container, outrd, errrd)
	p.mu.Unlock()
	p.update(event)
	go func() {
		for {
			debug("pump.pumpLogs():", id, "started")
			err := p.client.Logs(docker.LogsOptions{
				Container:         id,
				OutputStream:      outwr,
				ErrorStream:       errwr,
				Stdout:            true,
				Stderr:            true,
				Follow:            true,
				Tail:              "all",
				Since:             sinceTime.Unix(),
				InactivityTimeout: inactivityTimeout,
			})
			if err != nil {
				debug("pump.pumpLogs():", id, "stopped with error:", err)
			} else {
				debug("pump.pumpLogs():", id, "stopped")
			}

			sinceTime = time.Now()
			if err == docker.ErrInactivityTimeout {
				sinceTime = sinceTime.Add(-inactivityTimeout)
			}

			container, err := p.client.InspectContainer(id)
			if err != nil {
				_, four04 := err.(*docker.NoSuchContainer)
				if !four04 {
					assert(err, "pump")
				}
			} else if container.State.Running {
				continue
			}

			debug("pump.pumpLogs():", id, "dead")
			outwr.Close()
			errwr.Close()
			p.mu.Lock()
			delete(p.pumps, id)
			p.mu.Unlock()
			return
		}
	}()
}

func (p *LogsPump) update(event *docker.APIEvents) {
	p.mu.Lock()
	defer p.mu.Unlock()
	pump, pumping := p.pumps[normalID(event.ID)]
	if pumping {
		for r := range p.routes {
			select {
			case r <- &update{event, pump}:
			case <-time.After(time.Second * 1):
				debug("pump.update(): route timeout, dropping")
				defer delete(p.routes, r)
			}
		}
	}
}

// RoutingFrom returns whether a container id is routing from this pump
func (p *LogsPump) RoutingFrom(id string) bool {
	p.mu.Lock()
	defer p.mu.Unlock()
	_, monitoring := p.pumps[normalID(id)]
	return monitoring
}

// Route takes a logstream and routes it according to the supplied Route
func (p *LogsPump) Route(route *Route, logstream chan *Message) {
	p.mu.Lock()
	for _, pump := range p.pumps {
		if route.MatchContainer(
			normalID(pump.container.ID),
			normalName(pump.container.Name),
			pump.container.Config.Labels) {

			pump.add(logstream, route)
			defer pump.remove(logstream)
		}
	}
	updates := make(chan *update)
	p.routes[updates] = struct{}{}
	p.mu.Unlock()
	defer func() {
		p.mu.Lock()
		delete(p.routes, updates)
		p.mu.Unlock()
	}()
	for {
		select {
		case event := <-updates:
			switch event.Status {
			case "start", "restart":
				if route.MatchContainer(
					normalID(event.pump.container.ID),
					normalName(event.pump.container.Name),
					event.pump.container.Config.Labels) {

					event.pump.add(logstream, route)
					defer event.pump.remove(logstream)
				}
			case "die":
				if strings.HasPrefix(route.FilterID, event.ID) {
					// If the route is just about a single container,
					// we can stop routing when it dies.
					return
				}
			}
		case <-route.Closer():
			return
		}
	}
}

type containerPump struct {
	sync.Mutex
	container  *docker.Container
	logstreams map[chan *Message]*Route
}

func newContainerPump(container *docker.Container, stdout, stderr io.Reader) *containerPump {
	cp := &containerPump{
		container:  container,
		logstreams: make(map[chan *Message]*Route),
	}
	pump := func(source string, input io.Reader) {
		buf := bufio.NewReader(input)
		for {
			line, err := buf.ReadString('\n')
			if err != nil {
				if err != io.EOF {
					debug("pump.newContainerPump():", normalID(container.ID), source+":", err)
				}
				return
			}
			cp.send(&Message{
				Data:      strings.TrimSuffix(line, "\n"),
				Container: container,
				Time:      time.Now(),
				Source:    source,
			})
		}
	}
	go pump("stdout", stdout)
	go pump("stderr", stderr)
	return cp
}

func (cp *containerPump) send(msg *Message) {
	cp.Lock()
	defer cp.Unlock()
	for logstream, route := range cp.logstreams {
		if !route.MatchMessage(msg) {
			continue
		}
		select {
		case logstream <- msg:
		case <-time.After(time.Second * 1):
			debug("pump.send(): send timeout, closing")
			// normal call to remove() triggered by
			// route.Closer() may not be able to grab
			// lock under heavy load, so we delete here
			defer delete(cp.logstreams, logstream)
		}
	}
}

func (cp *containerPump) add(logstream chan *Message, route *Route) {
	cp.Lock()
	defer cp.Unlock()
	cp.logstreams[logstream] = route
}

func (cp *containerPump) remove(logstream chan *Message) {
	cp.Lock()
	defer cp.Unlock()
	delete(cp.logstreams, logstream)
}
<|endoftext|>


<commit_before>package r2router

import (
	"github.com/stretchr/testify/assert"
	//"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestRouter(t *testing.T) {
	router := NewRouter()
	router.Get("/user/keys/", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys"))
	})
	router.Post("/user/keys/", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("POST:/user/keys"))
	})
	router.Put("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("PUT:/user/keys/:id," + p.Get("id")))
	})
	router.Delete("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("DELETE:/user/keys/:id," + p.Get("id")))
	})
	router.Get("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys/:id," + p.Get("id")))
	})

	ts := httptest.NewServer(router)
	defer ts.Close()

	// get
	res, err := http.Get(ts.URL + "/user/keys/testing")
	assert.Nil(t, err)
	content, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Nil(t, err)
	assert.Equal(t, content, []byte("GET:/user/keys/:id,testing"))

	// Get all
	res, err = http.Get(ts.URL + "/user/keys")
	assert.Nil(t, err)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()

	assert.Nil(t, err)
	assert.Equal(t, content, []byte("GET:/user/keys"))

	// post
	res, err = http.Post(ts.URL+"/user/keys", "", nil)
	assert.Nil(t, err)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()

	assert.Nil(t, err)
	assert.Equal(t, content, []byte("POST:/user/keys"))

	// put
	client := &http.Client{}
	req, err := http.NewRequest("PUT", ts.URL+"/user/keys/testing", nil)
	res, err = client.Do(req)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, content, []byte("PUT:/user/keys/:id,testing"))

	// delete
	req, err = http.NewRequest("DELETE", ts.URL+"/user/keys/testing", nil)
	res, err = client.Do(req)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, content, []byte("DELETE:/user/keys/:id,testing"))

	// options
	req, err = http.NewRequest("OPTIONS", ts.URL+"/user/keys/testing", nil)
	res, err = client.Do(req)
	res.Body.Close()
	assert.Contains(t, res.Header.Get("Allow"), "GET")
	assert.Contains(t, res.Header.Get("Allow"), "PUT")
	assert.Contains(t, res.Header.Get("Allow"), "DELETE")
}

func TestRouterMethodNotAllowed(t *testing.T) {
	router := NewRouter()

	router.Get("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys/:id," + p.Get("id")))
	})

	ts := httptest.NewServer(router)
	defer ts.Close()

	client := &http.Client{}

	req, err := http.NewRequest("DELETE", ts.URL+"/user/keys/testing", nil)
	assert.Nil(t, err)

	res, err := client.Do(req)
	res.Body.Close()
	assert.Equal(t, res.StatusCode, http.StatusMethodNotAllowed)
}

func TestRouterMethodNotAllowedCustomized(t *testing.T) {
	router := NewRouter()
	router.MethodNotAllowed = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusMethodNotAllowed)
		w.Write([]byte("Hello"))
	}
	router.Get("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys/:id," + p.Get("id")))
	})

	ts := httptest.NewServer(router)
	defer ts.Close()

	client := &http.Client{}

	req, err := http.NewRequest("DELETE", ts.URL+"/user/keys/testing", nil)
	assert.Nil(t, err)

	res, err := client.Do(req)
	content, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, content, []byte("Hello"))
	assert.Equal(t, res.StatusCode, http.StatusMethodNotAllowed)

}

func TestRouterCustomizedOptions(t *testing.T) {
	router := NewRouter()

	router.AddHandler("OPTIONS", "/users/", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Header().Add("Allow", "GET, POST")
	})

	ts := httptest.NewServer(router)
	defer ts.Close()
	client := &http.Client{}
	// custom
	req, _ := http.NewRequest("OPTIONS", ts.URL+"/users/", nil)
	res2, _ := client.Do(req)
	res2.Body.Close()
	assert.Equal(t, res2.Header.Get("Allow"), "GET, POST")
}

func TestRouterNotFound(t *testing.T) {
	router := NewRouter()
	ts := httptest.NewServer(router)
	defer ts.Close()
	res, err := http.Get(ts.URL + "/user/keys/testing")
	assert.Nil(t, err)
	res.Body.Close()
	assert.Equal(t, res.StatusCode, http.StatusNotFound)
}

func TestRouterNotFoundCustomized(t *testing.T) {
	router := NewRouter()
	router.NotFound = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Boo"))
	}
	ts := httptest.NewServer(router)
	defer ts.Close()

	res, err := http.Get(ts.URL + "/user/keys/testing")
	assert.Nil(t, err)
	content, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, res.StatusCode, http.StatusNotFound)
	assert.Equal(t, content, []byte("Boo"))
}
<commit_msg>Adding test for param node in the first node<commit_after>package r2router

import (
	"github.com/stretchr/testify/assert"
	//"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestRouter(t *testing.T) {
	router := NewRouter()
	router.Get("/user/keys/", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys"))
	})
	router.Post("/user/keys/", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("POST:/user/keys"))
	})
	router.Put("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("PUT:/user/keys/:id," + p.Get("id")))
	})
	router.Delete("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("DELETE:/user/keys/:id," + p.Get("id")))
	})
	router.Get("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys/:id," + p.Get("id")))
	})

	ts := httptest.NewServer(router)
	defer ts.Close()

	// get
	res, err := http.Get(ts.URL + "/user/keys/testing")
	assert.Nil(t, err)
	content, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Nil(t, err)
	assert.Equal(t, content, []byte("GET:/user/keys/:id,testing"))

	// Get all
	res, err = http.Get(ts.URL + "/user/keys")
	assert.Nil(t, err)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()

	assert.Nil(t, err)
	assert.Equal(t, content, []byte("GET:/user/keys"))

	// post
	res, err = http.Post(ts.URL+"/user/keys", "", nil)
	assert.Nil(t, err)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()

	assert.Nil(t, err)
	assert.Equal(t, content, []byte("POST:/user/keys"))

	// put
	client := &http.Client{}
	req, err := http.NewRequest("PUT", ts.URL+"/user/keys/testing", nil)
	res, err = client.Do(req)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, content, []byte("PUT:/user/keys/:id,testing"))

	// delete
	req, err = http.NewRequest("DELETE", ts.URL+"/user/keys/testing", nil)
	res, err = client.Do(req)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, content, []byte("DELETE:/user/keys/:id,testing"))

	// options
	req, err = http.NewRequest("OPTIONS", ts.URL+"/user/keys/testing", nil)
	res, err = client.Do(req)
	res.Body.Close()
	assert.Contains(t, res.Header.Get("Allow"), "GET")
	assert.Contains(t, res.Header.Get("Allow"), "PUT")
	assert.Contains(t, res.Header.Get("Allow"), "DELETE")
}

func TestRouterMethodNotAllowed(t *testing.T) {
	router := NewRouter()

	router.Get("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys/:id," + p.Get("id")))
	})

	ts := httptest.NewServer(router)
	defer ts.Close()

	client := &http.Client{}

	req, err := http.NewRequest("DELETE", ts.URL+"/user/keys/testing", nil)
	assert.Nil(t, err)

	res, err := client.Do(req)
	res.Body.Close()
	assert.Equal(t, res.StatusCode, http.StatusMethodNotAllowed)
}

func TestRouterMethodNotAllowedCustomized(t *testing.T) {
	router := NewRouter()
	router.MethodNotAllowed = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusMethodNotAllowed)
		w.Write([]byte("Hello"))
	}
	router.Get("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys/:id," + p.Get("id")))
	})

	ts := httptest.NewServer(router)
	defer ts.Close()

	client := &http.Client{}

	req, err := http.NewRequest("DELETE", ts.URL+"/user/keys/testing", nil)
	assert.Nil(t, err)

	res, err := client.Do(req)
	content, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, content, []byte("Hello"))
	assert.Equal(t, res.StatusCode, http.StatusMethodNotAllowed)

}

func TestRouterCustomizedOptions(t *testing.T) {
	router := NewRouter()

	router.AddHandler("OPTIONS", "/users/", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Header().Add("Allow", "GET, POST")
	})

	ts := httptest.NewServer(router)
	defer ts.Close()
	client := &http.Client{}
	// custom
	req, _ := http.NewRequest("OPTIONS", ts.URL+"/users/", nil)
	res2, _ := client.Do(req)
	res2.Body.Close()
	assert.Equal(t, res2.Header.Get("Allow"), "GET, POST")
}

func TestRouterNotFound(t *testing.T) {
	router := NewRouter()
	ts := httptest.NewServer(router)
	defer ts.Close()
	res, err := http.Get(ts.URL + "/user/keys/testing")
	assert.Nil(t, err)
	res.Body.Close()
	assert.Equal(t, res.StatusCode, http.StatusNotFound)
}

func TestRouterNotFoundCustomized(t *testing.T) {
	router := NewRouter()
	router.NotFound = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Boo"))
	}
	ts := httptest.NewServer(router)
	defer ts.Close()

	res, err := http.Get(ts.URL + "/user/keys/testing")
	assert.Nil(t, err)
	content, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, res.StatusCode, http.StatusNotFound)
	assert.Equal(t, content, []byte("Boo"))
}

func TestRouterFirstNodeParamNode(t *testing.T) {
	router := NewRouter()

	router.Get("/:page", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/:page," + p.Get("page")))
	})

	router.Get("/user/keys/:id", func(w http.ResponseWriter, r *http.Request, p Params) {
		w.Write([]byte("GET:/user/keys/:id," + p.Get("id")))
	})

	ts := httptest.NewServer(router)
	defer ts.Close()

	client := &http.Client{}

	req, err := http.NewRequest("GET", ts.URL+"/testing", nil)
	assert.Nil(t, err)
	res, err := client.Do(req)
	content, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, content, []byte("GET:/:page,testing"))

	req, err = http.NewRequest("GET", ts.URL+"/user/keys/testing", nil)
	assert.Nil(t, err)
	res, err = client.Do(req)
	content, err = ioutil.ReadAll(res.Body)
	res.Body.Close()
	assert.Equal(t, content, []byte("GET:/user/keys/:id,testing"))
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	"github.com/restic/restic/backend"
)

func TestRepositoryName(t *testing.T) {
	var name string
	var err error

	name, err = RepositoryName("")
	if err == nil {
		t.Error("empty string should produce an error")
	}

	name, err = RepositoryName("/")
	if err == nil {
		t.Error("empty repository name should produce an error")
	}

	name, err = RepositoryName("//")
	if err == nil {
		t.Error("empty repository name should produce an error")
	}

	name, err = RepositoryName("/$test")
	if err == nil {
		t.Error("special characters should produce an error")
	}

	name, err = RepositoryName("/test")
	if name != "test" {
		t.Errorf("repository name is %s but should be test", name)
	}

	name, err = RepositoryName("/test-1234")
	if name != "test-1234" {
		t.Errorf("repository name is %s but should be test-1234", name)
	}

	name, err = RepositoryName("/test_1234")
	if name != "test_1234" {
		t.Errorf("repository name is %s but should be test_1234", name)
	}
}

func TestBackendType(t *testing.T) {
	var bt backend.Type

	bt = BackendType("/")
	if bt != "" {
		t.Error("backend type should be nil")
	}

	bt = BackendType("/test")
	if bt != "" {
		t.Error("backend type should be nil")
	}

	bt = BackendType("/test/config")
	if bt != backend.Config {
		t.Error("backend type should be config")
	}

	bt = BackendType("/test/config/")
	if bt != backend.Config {
		t.Error("backend type should be config")
	}

	bt = BackendType("/test/config/test")
	if bt != backend.Config {
		t.Error("backend type should be config")
	}
}

func TestBlobID(t *testing.T) {
	var id backend.ID

	id = BlobID("/")
	if !id.IsNull() {
		t.Error("blob id should be nil")
	}

	id = BlobID("/test")
	if !id.IsNull() {
		t.Error("blob id should be nil")
	}

	id = BlobID("/test/data")
	if !id.IsNull() {
		t.Error("blob id should be nil")
	}

	id = BlobID("/test/data/")
	if !id.IsNull() {
		t.Error("blob id should be nil")
	}

	id = BlobID("/test/data/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
	if id.IsNull() {
		t.Error("blob id should not be nil")
	}
}

func TestRestAPI(t *testing.T) {
	type route struct {
		method string
		path   string
	}

	validEndpoints := []route{
		route{"HEAD", "/repo/config"},
		route{"GET", "/repo/config"},
		route{"POST", "/repo/config"},
		route{"GET", "/repo/data/"},
		route{"HEAD", "/repo/data/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/data/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"POST", "/repo/data/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"DELETE", "/repo/data/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/snapshots/"},
		route{"HEAD", "/repo/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"POST", "/repo/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"DELETE", "/repo/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/index/"},
		route{"HEAD", "/repo/index/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/index/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"POST", "/repo/index/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"DELETE", "/repo/index/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/locks/"},
		route{"HEAD", "/repo/locks/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/locks/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"POST", "/repo/locks/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"DELETE", "/repo/locks/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/keys/"},
		route{"HEAD", "/repo/keys/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/keys/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"POST", "/repo/keys/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"DELETE", "/repo/keys/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
	}

	for _, route := range validEndpoints {
		if RestAPI(route.method, route.path) == nil {
			t.Errorf("request %s %s should return a handler", route.method, route.path)
		}
	}

	invalidEndpoints := []route{
		route{"GET", "/"},
		route{"GET", "/repo"},
		route{"GET", "/repo/config/"},
		route{"GET", "/repo/config/aaaa"},
		route{"GET", "/repo/data"},
		route{"GET", "/repo/data/aaaaaaa"},
		route{"GET", "/repo/keys/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
		route{"GET", "/repo/keys/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/"},
		route{"GET", "/repo/keys/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/test"},
	}

	for _, route := range invalidEndpoints {
		if RestAPI(route.method, route.path) != nil {
			t.Errorf("request %s %s should return nil", route.method, route.path)
		}
	}

}
<commit_msg>some basic tests for the router<commit_after>package main

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"
)

func TestRouter(t *testing.T) {
	router := NewRouter()

	getConfig := []byte("GET /config")
	router.Get("/config", func(w http.ResponseWriter, r *http.Request) {
		w.Write(getConfig)
	})

	postConfig := []byte("POST /config")
	router.Post("/config", func(w http.ResponseWriter, r *http.Request) {
		w.Write(postConfig)
	})

	getBlobs := []byte("GET /blobs/")
	router.Get("/blobs/", func(w http.ResponseWriter, r *http.Request) {
		w.Write(getBlobs)
	})

	getBlob := []byte("GET /blobs/:sha")
	router.Get("/blobs/:sha", func(w http.ResponseWriter, r *http.Request) {
		w.Write(getBlob)
	})

	server := httptest.NewServer(router)
	defer server.Close()

	getConfigResp, _ := http.Get(server.URL + "/config")
	getConfigBody, _ := ioutil.ReadAll(getConfigResp.Body)
	require.Equal(t, string(getConfig), string(getConfigBody))

	postConfigResp, _ := http.Post(server.URL+"/config", "binary/octet-stream", strings.NewReader("post test"))
	postConfigBody, _ := ioutil.ReadAll(postConfigResp.Body)
	require.Equal(t, string(postConfig), string(postConfigBody))

	getBlobsResp, _ := http.Get(server.URL + "/blobs/")
	getBlobsBody, _ := ioutil.ReadAll(getBlobsResp.Body)
	require.Equal(t, string(getBlobs), string(getBlobsBody))

	getBlobResp, _ := http.Get(server.URL + "/blobs/test")
	getBlobBody, _ := ioutil.ReadAll(getBlobResp.Body)
	require.Equal(t, string(getBlob), string(getBlobBody))
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httptrace"
	"os"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/oschwald/geoip2-golang"
	"github.com/tcnksm/go-httpstat"
	"github.com/yanc0/beeping/sslcheck"
)

var VERSION = "0.5.0"
var MESSAGE = "BeePing instance - HTTP Ping as a Service (github.com/yanc0/beeping)"
var USERAGENT = "Beeping " + VERSION + " - https://github.com/yanc0/beeping"

var geodatfile *string
var instance *string
var listen *string
var port *string
var tlsmode *bool

type Beeping struct {
	Version string `json:"version"`
	Message string `json:"message"`
}

// Check defines the check to do
type Check struct {
	URL      string        `json:"url" binding:"required"`
	Pattern  string        `json:"pattern"`
	Header   string        `json:"header"`
	Insecure bool          `json:"insecure"`
	Timeout  time.Duration `json:"timeout"`
}

type Timeline struct {
	NameLookup    int64 `json:"name_lookup"`
	Connect       int64 `json:"connect"`
	Pretransfer   int64 `json:"pretransfer"`
	StartTransfer int64 `json:"starttransfer"`
}

type Geo struct {
	Country string `json:"country"`
	City    string `json:"city,omitempty"`
	IP      string `json:"ip"`
}

// Response defines the response to bring back
type Response struct {
	HTTPStatus      string `json:"http_status"`
	HTTPStatusCode  int    `json:"http_status_code"`
	HTTPBodyPattern bool   `json:"http_body_pattern"`
	HTTPHeader      bool   `json:"http_header"`
	HTTPRequestTime int64  `json:"http_request_time"`

	InstanceName string `json:"instance_name"`

	DNSLookup        int64 `json:"dns_lookup"`
	TCPConnection    int64 `json:"tcp_connection"`
	TLSHandshake     int64 `json:"tls_handshake,omitempty"`
	ServerProcessing int64 `json:"server_processing"`
	ContentTransfer  int64 `json:"content_transfer"`

	Timeline *Timeline          `json:"timeline"`
	Geo      *Geo               `json:"geo,omitempty"`
	SSL      *sslcheck.CheckSSL `json:"ssl,omitempty"`
}

func NewResponse() *Response {
	var response = Response{}
	response.Timeline = &Timeline{}
	return &response
}

func NewCheck() *Check {
	return &Check{Timeout: 10}
}

func main() {
	geodatfile = flag.String("geodatfile", "/opt/GeoIP/GeoLite2-City.mmdb", "geoIP database path")
	instance = flag.String("instance", "", "beeping instance name (default hostname)")
	listen = flag.String("listen", "127.0.0.1", "The host to bind the server to")
	port = flag.String("port", "8080", "The port to bind the server to")
	tlsmode = flag.Bool("tlsmode", false, "Activate SSL/TLS versions and Cipher support checks (slow)")
	flag.Parse()

	gin.SetMode("release")

	router := gin.New()
	router.POST("/check", handlerCheck)
	router.GET("/", handlerDefault)

	log.Println("[INFO] Listening on", *listen, *port)
	router.Run(*listen + ":" + *port)
}

func handlerDefault(c *gin.Context) {
	var beeping Beeping
	beeping.Version = VERSION
	beeping.Message = MESSAGE
	log.Println("[INFO] Beeping version", beeping.Version)
	c.JSON(http.StatusOK, beeping)
}

func handlerCheck(c *gin.Context) {
	var check = NewCheck()
	if c.BindJSON(&check) == nil {
		response, err := CheckHTTP(check)
		if err != nil {
			log.Println("[WARN] Check failed:", err.Error())
			c.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
		} else {
			log.Println("[INFO] Successful check:", check.URL, "-", response.HTTPRequestTime, "ms")
			c.JSON(http.StatusOK, response)
		}
	} else {
		log.Println("[WARN] Invalid JSON sent")
		c.JSON(http.StatusBadRequest, gin.H{"message": "invalid json sent"})
	}
}

// CheckHTTP do HTTP check and return a beeping response
func CheckHTTP(check *Check) (*Response, error) {
	var response = NewResponse()
	var conn net.Conn

	req, err := http.NewRequest("GET", check.URL, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", USERAGENT)
	// Create go-httpstat powered context and pass it to http.Request
	var result httpstat.Result
	ctx := httpstat.WithHTTPStat(req.Context(), &result)

	// Add IP:PORT tracing to the context
	ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
		GotConn: func(i httptrace.GotConnInfo) {
			conn = i.Conn
		},
	})

	req = req.WithContext(ctx)

	// DefaultClient is not suitable cause it caches
	// tcp connection https://golang.org/pkg/net/http/#Client
	// Allow us to close Idle connections and reset network
	// metrics each time
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: check.Insecure,
		},
	}

	timeout := time.Duration(check.Timeout * time.Second)

	client := &http.Client{Transport: tr, Timeout: timeout}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	res.Body.Close()
	timeEndBody := time.Now()
	result.End(timeEndBody)
	var total = result.Total(timeEndBody)

	tr.CloseIdleConnections()

	pattern := true
	if !strings.Contains(string(body), check.Pattern) {
		pattern = false
	}

	header := true
	if check.Header != "" {
		key, value := splitCheckHeader(check.Header)
		if key != "" && value != "" && res.Header.Get(key) != value {
			header = false
		}
	}

	response.HTTPStatus = res.Status
	response.HTTPStatusCode = res.StatusCode
	response.HTTPBodyPattern = pattern
	response.HTTPHeader = header
	response.HTTPRequestTime = milliseconds(total)
	response.Timeline.NameLookup = milliseconds(result.NameLookup)
	response.Timeline.Connect = milliseconds(result.Connect)
	response.Timeline.Pretransfer = milliseconds(result.Pretransfer)
	response.Timeline.StartTransfer = milliseconds(result.StartTransfer)
	response.DNSLookup = milliseconds(result.DNSLookup)
	response.TCPConnection = milliseconds(result.TCPConnection)
	response.TLSHandshake = milliseconds(result.TLSHandshake)
	response.ServerProcessing = milliseconds(result.ServerProcessing)
	response.ContentTransfer = milliseconds(result.ContentTransfer(timeEndBody))

	if res.TLS != nil {
		cTLS := &sslcheck.CheckSSL{}
		if *tlsmode {
			cTLS.CheckCiphers(conn)
			cTLS.CheckVersions(conn)
		}
		cTLS.CertExpiryDate = res.TLS.PeerCertificates[0].NotAfter
		cTLS.CertExpiryDaysLeft = int64(cTLS.CertExpiryDate.Sub(time.Now()).Hours() / 24)
		cTLS.CertSignature = res.TLS.PeerCertificates[0].SignatureAlgorithm.String()
		response.SSL = cTLS
	}

	ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
	if err != nil {
		log.Println("[WARN] Cannot parse IP address", err.Error())
	}

	_ = geoIPCountry(*geodatfile, ip, response)

	err = instanceName(*instance, response)
	if err != nil {
		log.Println("[WARN] Cannot set instance name", err.Error())
	}

	return response, nil
}

func milliseconds(d time.Duration) int64 {
	return d.Nanoseconds() / 1000 / 1000
}

func geoIPCountry(geodatabase string, ip string, response *Response) error {
	db, err := geoip2.Open(geodatabase)
	if err != nil {
		return err
	}
	defer db.Close()
	// If you are using strings that may be invalid, check that ip is not nil
	ipParse := net.ParseIP(ip)
	record, err := db.City(ipParse)
	if err != nil {
		return err
	}
	response.Geo = &Geo{}
	response.Geo.Country = record.Country.IsoCode
	response.Geo.IP = ip
	if record.Country.Names != nil {
		response.Geo.City = record.City.Names["en-EN"]
	}
	return nil
}

func instanceName(name string, response *Response) error {
	var err error
	response.InstanceName = name
	if name == "" {
		response.InstanceName, err = os.Hostname()
		if err != nil {
			return err
		}
	}

	return nil
}

func splitCheckHeader(header string) (string, string) {
	h := strings.SplitN(header, ":", 2)
	if len(h) == 2 {
		return strings.TrimSpace(h[0]), strings.TrimSpace(h[1])
	}
	return "", ""
}
<commit_msg>Added option (enabled by default) to validate target. Current checks mitigate SSRF vulnerability by disallowing local IP addresses (both IPv4 and IPv6).<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/oschwald/geoip2-golang"
	"github.com/tcnksm/go-httpstat"
	"github.com/yanc0/beeping/sslcheck"
)

var VERSION = "0.5.0"
var MESSAGE = "BeePing instance - HTTP Ping as a Service (github.com/yanc0/beeping)"
var USERAGENT = "Beeping " + VERSION + " - https://github.com/yanc0/beeping"

var geodatfile *string
var instance *string
var listen *string
var port *string
var tlsmode *bool
var validatetarget *bool

type Beeping struct {
	Version string `json:"version"`
	Message string `json:"message"`
}

// Check defines the check to do
type Check struct {
	URL      string        `json:"url" binding:"required"`
	Pattern  string        `json:"pattern"`
	Header   string        `json:"header"`
	Insecure bool          `json:"insecure"`
	Timeout  time.Duration `json:"timeout"`
}

type Timeline struct {
	NameLookup    int64 `json:"name_lookup"`
	Connect       int64 `json:"connect"`
	Pretransfer   int64 `json:"pretransfer"`
	StartTransfer int64 `json:"starttransfer"`
}

type Geo struct {
	Country string `json:"country"`
	City    string `json:"city,omitempty"`
	IP      string `json:"ip"`
}

// Response defines the response to bring back
type Response struct {
	HTTPStatus      string `json:"http_status"`
	HTTPStatusCode  int    `json:"http_status_code"`
	HTTPBodyPattern bool   `json:"http_body_pattern"`
	HTTPHeader      bool   `json:"http_header"`
	HTTPRequestTime int64  `json:"http_request_time"`

	InstanceName string `json:"instance_name"`

	DNSLookup        int64 `json:"dns_lookup"`
	TCPConnection    int64 `json:"tcp_connection"`
	TLSHandshake     int64 `json:"tls_handshake,omitempty"`
	ServerProcessing int64 `json:"server_processing"`
	ContentTransfer  int64 `json:"content_transfer"`

	Timeline *Timeline          `json:"timeline"`
	Geo      *Geo               `json:"geo,omitempty"`
	SSL      *sslcheck.CheckSSL `json:"ssl,omitempty"`
}

func NewResponse() *Response {
	var response = Response{}
	response.Timeline = &Timeline{}
	return &response
}

func NewCheck() *Check {
	return &Check{Timeout: 10}
}

// Performs some validation checks on the target.
// Returns nil if valid, returns an error otherwise.
func (check *Check) validateTarget() error {
	targetURL, err := url.Parse(check.URL)
	if err != nil {
		return err
	}
	ip := net.ParseIP(targetURL.Hostname())
	if ip == nil {
		// Hostname provided is not an IP. Without whitelisting, it is not possible to tell
		// whether it is an internal hostname.
		return nil // For now, hostnames are not needed for this check.
	}

	// Check for local network IPs
	switch {
	// Loopback address
	case ip.IsLoopback():
		return fmt.Errorf("Disallowed target")
	// Link-local unicast
	case ip.IsLinkLocalUnicast():
		return fmt.Errorf("Disallowed target")
	// Link-local multicast
	case ip.IsLinkLocalMulticast():
		return fmt.Errorf("Disallowed target")
	// Private network (10.0.0.0/8)
	case len(ip) == 4 && ip[0] == 10:
		return fmt.Errorf("Disallowed target")
	// Private network (Carrier-grade NAT; 100.64.0.0/10)
	case len(ip) == 4 && ip[0] == 100 && ip[1] >= 64 && ip[1] <= 127:
		return fmt.Errorf("Disallowed target")
	// Private network (172.16.0.0/12)
	case len(ip) == 4 && ip[0] == 172 && ip[1] >= 16 && ip[1] <= 31:
		return fmt.Errorf("Disallowed target")
	// Private network (192.168.0.0/16)
	case len(ip) == 4 && ip[0] == 192 && ip[1] == 16:
		return fmt.Errorf("Disallowed target")
	// Private network (fc00::/7)
	case len(ip) == 16 && (ip[0] == 0xfc || ip[0] == 0xfd):
		return fmt.Errorf("Disallowed target")
	}

	return nil
}

func main() {
	geodatfile = flag.String("geodatfile", "/opt/GeoIP/GeoLite2-City.mmdb", "geoIP database path")
	instance = flag.String("instance", "", "beeping instance name (default hostname)")
	listen = flag.String("listen", "127.0.0.1", "The host to bind the server to")
	port = flag.String("port", "8080", "The port to bind the server to")
	tlsmode = flag.Bool("tlsmode", false, "Activate SSL/TLS versions and Cipher support checks (slow)")
	validatetarget = flag.Bool("validatetarget", true, "Perform some security checks on the target provided")
	flag.Parse()

	gin.SetMode("release")

	router := gin.New()
	router.POST("/check", handlerCheck)
	router.GET("/", handlerDefault)

	log.Println("[INFO] Listening on", *listen, *port)
	router.Run(*listen + ":" + *port)
}

func handlerDefault(c *gin.Context) {
	var beeping Beeping
	beeping.Version = VERSION
	beeping.Message = MESSAGE
	log.Println("[INFO] Beeping version", beeping.Version)
	c.JSON(http.StatusOK, beeping)
}

func handlerCheck(c *gin.Context) {
	var check = NewCheck()
	if c.BindJSON(&check) == nil {
		if *validatetarget {
			if err := check.validateTarget(); err != nil {
				log.Println("[WARN] Invalid target:", err.Error())
				c.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
			} else {
				response, err := CheckHTTP(check)
				if err != nil {
					log.Println("[WARN] Check failed:", err.Error())
					c.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
				} else {
					log.Println("[INFO] Successful check:", check.URL, "-", response.HTTPRequestTime, "ms")
					c.JSON(http.StatusOK, response)
				}
			}
		} else {
			response, err := CheckHTTP(check)
			if err != nil {
				log.Println("[WARN] Check failed:", err.Error())
				c.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
			} else {
				log.Println("[INFO] Successful check:", check.URL, "-", response.HTTPRequestTime, "ms")
				c.JSON(http.StatusOK, response)
			}
		}
	} else {
		log.Println("[WARN] Invalid JSON sent")
		c.JSON(http.StatusBadRequest, gin.H{"message": "invalid json sent"})
	}
}

// CheckHTTP do HTTP check and return a beeping response
func CheckHTTP(check *Check) (*Response, error) {
	var response = NewResponse()
	var conn net.Conn

	req, err := http.NewRequest("GET", check.URL, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", USERAGENT)
	// Create go-httpstat powered context and pass it to http.Request
	var result httpstat.Result
	ctx := httpstat.WithHTTPStat(req.Context(), &result)

	// Add IP:PORT tracing to the context
	ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
		GotConn: func(i httptrace.GotConnInfo) {
			conn = i.Conn
		},
	})

	req = req.WithContext(ctx)

	// DefaultClient is not suitable cause it caches
	// tcp connection https://golang.org/pkg/net/http/#Client
	// Allow us to close Idle connections and reset network
	// metrics each time
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: check.Insecure,
		},
	}

	timeout := time.Duration(check.Timeout * time.Second)

	client := &http.Client{Transport: tr, Timeout: timeout}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	res.Body.Close()
	timeEndBody := time.Now()
	result.End(timeEndBody)
	var total = result.Total(timeEndBody)

	tr.CloseIdleConnections()

	pattern := true
	if !strings.Contains(string(body), check.Pattern) {
		pattern = false
	}

	header := true
	if check.Header != "" {
		key, value := splitCheckHeader(check.Header)
		if key != "" && value != "" && res.Header.Get(key) != value {
			header = false
		}
	}

	response.HTTPStatus = res.Status
	response.HTTPStatusCode = res.StatusCode
	response.HTTPBodyPattern = pattern
	response.HTTPHeader = header
	response.HTTPRequestTime = milliseconds(total)
	response.Timeline.NameLookup = milliseconds(result.NameLookup)
	response.Timeline.Connect = milliseconds(result.Connect)
	response.Timeline.Pretransfer = milliseconds(result.Pretransfer)
	response.Timeline.StartTransfer = milliseconds(result.StartTransfer)
	response.DNSLookup = milliseconds(result.DNSLookup)
	response.TCPConnection = milliseconds(result.TCPConnection)
	response.TLSHandshake = milliseconds(result.TLSHandshake)
	response.ServerProcessing = milliseconds(result.ServerProcessing)
	response.ContentTransfer = milliseconds(result.ContentTransfer(timeEndBody))

	if res.TLS != nil {
		cTLS := &sslcheck.CheckSSL{}
		if *tlsmode {
			cTLS.CheckCiphers(conn)
			cTLS.CheckVersions(conn)
		}
		cTLS.CertExpiryDate = res.TLS.PeerCertificates[0].NotAfter
		cTLS.CertExpiryDaysLeft = int64(cTLS.CertExpiryDate.Sub(time.Now()).Hours() / 24)
		cTLS.CertSignature = res.TLS.PeerCertificates[0].SignatureAlgorithm.String()
		response.SSL = cTLS
	}

	ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
	if err != nil {
		log.Println("[WARN] Cannot parse IP address", err.Error())
	}

	_ = geoIPCountry(*geodatfile, ip, response)

	err = instanceName(*instance, response)
	if err != nil {
		log.Println("[WARN] Cannot set instance name", err.Error())
	}

	return response, nil
}

func milliseconds(d time.Duration) int64 {
	return d.Nanoseconds() / 1000 / 1000
}

func geoIPCountry(geodatabase string, ip string, response *Response) error {
	db, err := geoip2.Open(geodatabase)
	if err != nil {
		return err
	}
	defer db.Close()
	// If you are using strings that may be invalid, check that ip is not nil
	ipParse := net.ParseIP(ip)
	record, err := db.City(ipParse)
	if err != nil {
		return err
	}
	response.Geo = &Geo{}
	response.Geo.Country = record.Country.IsoCode
	response.Geo.IP = ip
	if record.Country.Names != nil {
		response.Geo.City = record.City.Names["en-EN"]
	}
	return nil
}

func instanceName(name string, response *Response) error {
	var err error
	response.InstanceName = name
	if name == "" {
		response.InstanceName, err = os.Hostname()
		if err != nil {
			return err
		}
	}

	return nil
}

func splitCheckHeader(header string) (string, string) {
	h := strings.SplitN(header, ":", 2)
	if len(h) == 2 {
		return strings.TrimSpace(h[0]), strings.TrimSpace(h[1])
	}
	return "", ""
}
<|endoftext|>


<commit_before>//
// Copyright 2020, Serena Fang
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// InstanceClustersService handles communication with the
// instance clusters related methods of the GitLab API.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html
type InstanceClustersService struct {
	client *Client
}

// InstanceCluster represents a GitLab Instance Cluster.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/instance_clusters.html
type InstanceCluster struct {
	ID                 int                 `json:"id"`
	Name               string              `json:"name"`
	Domain             string              `json:"domain"`
	CreatedAt          *time.Time          `json:"created_at"`
	ProviderType       string              `json:"provider_type"`
	PlatformType       string              `json:"platform_type"`
	EnvironmentScope   string              `json:"environment_scope"`
	ClusterType        string              `json:"cluster_type"`
	User               *User               `json:"user"`
	PlatformKubernetes *PlatformKubernetes `json:"platform_kubernetes"`
	ManagementProject  *ManagementProject  `json:"management_project"`
}

func (v InstanceCluster) String() string {
	return Stringify(v)
}

// PlatformKubernetes represents a GitLab Instance Cluster PlatformKubernetes.
type InstanceClusterPlatformKubernetes struct {
	APIURL            string `json:"api_url"`
	Token             string `json:"token"`
	CaCert            string `json:"ca_cert"`
	Namespace         string `json:"namespace"`
	AuthorizationType string `json:"authorization_type"`
}

// ListClusters gets a list of all instance clusters.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#list-instance-clusters
func (s *InstanceClustersService) ListClusters(options ...RequestOptionFunc) ([]*InstanceCluster, *Response, error) {
	u := fmt.Sprintf("admin/clusters")

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	var ics []*InstanceCluster
	resp, err := s.client.Do(req, &ics)
	if err != nil {
		return nil, resp, err
	}

	return ics, resp, err
}

// GetCluster gets an instance cluster.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#get-a-single-instance-cluster
func (s *InstanceClustersService) GetCluster(cluster int, options ...RequestOptionFunc) (*InstanceCluster, *Response, error) {
	u := fmt.Sprintf("admin/clusters/%d", cluster)

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	ic := new(InstanceCluster)
	resp, err := s.client.Do(req, &ic)
	if err != nil {
		return nil, resp, err
	}

	return ic, resp, err
}

// AddInstanceClusterOptions represents the available AddCluster() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#add-existing-cluster-to-instance
type AddInstanceClusterOptions struct {
	Name                *string                               `url:"name,omitempty" json:"name,omitempty"`
	Domain              *string                               `url:"domain,omitempty" json:"domain,omitempty"`
	Enabled             *bool                                 `url:"enabled,omitempty" json:"enabled,omitempty"`
	Managed             *bool                                 `url:"managed,omitempty" json:"managed,omitempty"`
	EnvironmentScope    *string                               `url:"environment_scope,omitempty" json:"environment_scope,omitempty"`
	PlatformKubernetes  *AddInstancePlatformKubernetesOptions `url:"platform_kubernetes_attributes,omitempty" json:"platform_kubernetes_attributes,omitempty"`
	ManagementProjectID *string                               `url:"management_project_id,omitempty" json:"management_project_id,omitempty"`
}

// AddInstancePlatformKubernetesOptions represents the available PlatformKubernetes options for adding.
type AddInstancePlatformKubernetesOptions struct {
	APIURL            *string `url:"api_url,omitempty" json:"api_url,omitempty"`
	Token             *string `url:"token,omitempty" json:"token,omitempty"`
	CaCert            *string `url:"ca_cert,omitempty" json:"ca_cert,omitempty"`
	Namespace         *string `url:"namespace,omitempty" json:"namespace,omitempty"`
	AuthorizationType *string `url:"authorization_type,omitempty" json:"authorization_type,omitempty"`
}

// AddCluster adds an existing cluster to the instance.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#add-existing-instance-cluster
func (s *InstanceClustersService) AddCluster(opt *AddInstanceClusterOptions, options ...RequestOptionFunc) (*InstanceCluster, *Response, error) {
	u := fmt.Sprintf("admin/clusters/add")

	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	ic := new(InstanceCluster)
	resp, err := s.client.Do(req, ic)
	if err != nil {
		return nil, resp, err
	}

	return ic, resp, err
}

// EditInstanceClusterOptions represents the available EditCluster() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#edit-instance-cluster
type EditInstanceClusterOptions struct {
	Name                *string                                `url:"name,omitempty" json:"name,omitempty"`
	Domain              *string                                `url:"domain,omitempty" json:"domain,omitempty"`
	EnvironmentScope    *string                                `url:"environment_scope,omitempty" json:"environment_scope,omitempty"`
	ManagementProjectID *string                                `url:"management_project_id,omitempty" json:"management_project_id,omitempty"`
	PlatformKubernetes  *EditInstancePlatformKubernetesOptions `url:"platform_kubernetes_attributes,omitempty" json:"platform_kubernetes_attributes,omitempty"`
}

// EditInstancePlatformKubernetesOptions represents the available PlatformKubernetes options for editing.
type EditInstancePlatformKubernetesOptions struct {
	APIURL    *string `url:"api_url,omitempty" json:"api_url,omitempty"`
	Token     *string `url:"token,omitempty" json:"token,omitempty"`
	CaCert    *string `url:"ca_cert,omitempty" json:"ca_cert,omitempty"`
	Namespace *string `url:"namespace,omitempty" json:"namespace,omitempty"`
}

// EditCluster updates an existing instance cluster.
// Note: name, api_url, ca_cert and token can only be updated if the cluster
// was added through the 'Add existing Kubernetes' cluster option or through the
// 'Add existing instance' cluster endpoint.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#edit-instance-cluster
func (s *InstanceClustersService) EditCluster(cluster int, opt *EditInstanceClusterOptions, options ...RequestOptionFunc) (*InstanceCluster, *Response, error) {
	u := fmt.Sprintf("admin/clusters/%d", cluster)

	req, err := s.client.NewRequest("PUT", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	ic := new(InstanceCluster)
	resp, err := s.client.Do(req, ic)
	if err != nil {
		return nil, resp, err
	}

	return ic, resp, err
}

// DeleteCluster deletes an existing instance cluster.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#delete-instance-cluster
func (s *InstanceClustersService) DeleteCluster(cluster int, options ...RequestOptionFunc) (*Response, error) {
	u := fmt.Sprintf("admin/clusters/%d", cluster)

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}
<commit_msg>Remove redundant docs note<commit_after>//
// Copyright 2020, Serena Fang
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// InstanceClustersService handles communication with the
// instance clusters related methods of the GitLab API.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html
type InstanceClustersService struct {
	client *Client
}

// InstanceCluster represents a GitLab Instance Cluster.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/instance_clusters.html
type InstanceCluster struct {
	ID                 int                 `json:"id"`
	Name               string              `json:"name"`
	Domain             string              `json:"domain"`
	CreatedAt          *time.Time          `json:"created_at"`
	ProviderType       string              `json:"provider_type"`
	PlatformType       string              `json:"platform_type"`
	EnvironmentScope   string              `json:"environment_scope"`
	ClusterType        string              `json:"cluster_type"`
	User               *User               `json:"user"`
	PlatformKubernetes *PlatformKubernetes `json:"platform_kubernetes"`
	ManagementProject  *ManagementProject  `json:"management_project"`
}

func (v InstanceCluster) String() string {
	return Stringify(v)
}

// PlatformKubernetes represents a GitLab Instance Cluster PlatformKubernetes.
type InstanceClusterPlatformKubernetes struct {
	APIURL            string `json:"api_url"`
	Token             string `json:"token"`
	CaCert            string `json:"ca_cert"`
	Namespace         string `json:"namespace"`
	AuthorizationType string `json:"authorization_type"`
}

// ListClusters gets a list of all instance clusters.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#list-instance-clusters
func (s *InstanceClustersService) ListClusters(options ...RequestOptionFunc) ([]*InstanceCluster, *Response, error) {
	u := fmt.Sprintf("admin/clusters")

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	var ics []*InstanceCluster
	resp, err := s.client.Do(req, &ics)
	if err != nil {
		return nil, resp, err
	}

	return ics, resp, err
}

// GetCluster gets an instance cluster.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#get-a-single-instance-cluster
func (s *InstanceClustersService) GetCluster(cluster int, options ...RequestOptionFunc) (*InstanceCluster, *Response, error) {
	u := fmt.Sprintf("admin/clusters/%d", cluster)

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	ic := new(InstanceCluster)
	resp, err := s.client.Do(req, &ic)
	if err != nil {
		return nil, resp, err
	}

	return ic, resp, err
}

// AddInstanceClusterOptions represents the available AddCluster() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#add-existing-cluster-to-instance
type AddInstanceClusterOptions struct {
	Name                *string                               `url:"name,omitempty" json:"name,omitempty"`
	Domain              *string                               `url:"domain,omitempty" json:"domain,omitempty"`
	Enabled             *bool                                 `url:"enabled,omitempty" json:"enabled,omitempty"`
	Managed             *bool                                 `url:"managed,omitempty" json:"managed,omitempty"`
	EnvironmentScope    *string                               `url:"environment_scope,omitempty" json:"environment_scope,omitempty"`
	PlatformKubernetes  *AddInstancePlatformKubernetesOptions `url:"platform_kubernetes_attributes,omitempty" json:"platform_kubernetes_attributes,omitempty"`
	ManagementProjectID *string                               `url:"management_project_id,omitempty" json:"management_project_id,omitempty"`
}

// AddInstancePlatformKubernetesOptions represents the available PlatformKubernetes options for adding.
type AddInstancePlatformKubernetesOptions struct {
	APIURL            *string `url:"api_url,omitempty" json:"api_url,omitempty"`
	Token             *string `url:"token,omitempty" json:"token,omitempty"`
	CaCert            *string `url:"ca_cert,omitempty" json:"ca_cert,omitempty"`
	Namespace         *string `url:"namespace,omitempty" json:"namespace,omitempty"`
	AuthorizationType *string `url:"authorization_type,omitempty" json:"authorization_type,omitempty"`
}

// AddCluster adds an existing cluster to the instance.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#add-existing-instance-cluster
func (s *InstanceClustersService) AddCluster(opt *AddInstanceClusterOptions, options ...RequestOptionFunc) (*InstanceCluster, *Response, error) {
	u := fmt.Sprintf("admin/clusters/add")

	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	ic := new(InstanceCluster)
	resp, err := s.client.Do(req, ic)
	if err != nil {
		return nil, resp, err
	}

	return ic, resp, err
}

// EditInstanceClusterOptions represents the available EditCluster() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#edit-instance-cluster
type EditInstanceClusterOptions struct {
	Name                *string                                `url:"name,omitempty" json:"name,omitempty"`
	Domain              *string                                `url:"domain,omitempty" json:"domain,omitempty"`
	EnvironmentScope    *string                                `url:"environment_scope,omitempty" json:"environment_scope,omitempty"`
	ManagementProjectID *string                                `url:"management_project_id,omitempty" json:"management_project_id,omitempty"`
	PlatformKubernetes  *EditInstancePlatformKubernetesOptions `url:"platform_kubernetes_attributes,omitempty" json:"platform_kubernetes_attributes,omitempty"`
}

// EditInstancePlatformKubernetesOptions represents the available PlatformKubernetes options for editing.
type EditInstancePlatformKubernetesOptions struct {
	APIURL    *string `url:"api_url,omitempty" json:"api_url,omitempty"`
	Token     *string `url:"token,omitempty" json:"token,omitempty"`
	CaCert    *string `url:"ca_cert,omitempty" json:"ca_cert,omitempty"`
	Namespace *string `url:"namespace,omitempty" json:"namespace,omitempty"`
}

// EditCluster updates an existing instance cluster.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#edit-instance-cluster
func (s *InstanceClustersService) EditCluster(cluster int, opt *EditInstanceClusterOptions, options ...RequestOptionFunc) (*InstanceCluster, *Response, error) {
	u := fmt.Sprintf("admin/clusters/%d", cluster)

	req, err := s.client.NewRequest("PUT", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	ic := new(InstanceCluster)
	resp, err := s.client.Do(req, ic)
	if err != nil {
		return nil, resp, err
	}

	return ic, resp, err
}

// DeleteCluster deletes an existing instance cluster.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/instance_clusters.html#delete-instance-cluster
func (s *InstanceClustersService) DeleteCluster(cluster int, options ...RequestOptionFunc) (*Response, error) {
	u := fmt.Sprintf("admin/clusters/%d", cluster)

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}
<|endoftext|>


<commit_before>// +build !windows

package term

import (
	"syscall"

	"github.com/pkg/term/termios"
)

// SetRaw put terminal into a raw mode
func SetRaw(fd int) error {
	n, err := getOriginalTermios(fd)
	if err != nil {
		return err
	}

	n.Iflag &^= syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK |
		syscall.ISTRIP | syscall.INLCR | syscall.IGNCR |
		syscall.ICRNL | syscall.IXON
	n.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG | syscall.ECHONL
	n.Cflag &^= syscall.CSIZE | syscall.PARENB
	n.Cc[syscall.VMIN] = 1
	n.Cc[syscall.VTIME] = 0
	return termios.Tcsetattr(uintptr(fd), termios.TCSANOW, (*syscall.Termios)(&n))
}
<commit_msg>termio: Set character width to 8 bits<commit_after>// +build !windows

package term

import (
	"syscall"

	"github.com/pkg/term/termios"
)

// SetRaw put terminal into a raw mode
func SetRaw(fd int) error {
	n, err := getOriginalTermios(fd)
	if err != nil {
		return err
	}

	n.Iflag &^= syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK |
		syscall.ISTRIP | syscall.INLCR | syscall.IGNCR |
		syscall.ICRNL | syscall.IXON
	n.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG | syscall.ECHONL
	n.Cflag &^= syscall.CSIZE | syscall.PARENB
	n.Cflag |= syscall.CS8 // Set to 8-bit wide.  Typical value for displaying characters.
	n.Cc[syscall.VMIN] = 1
	n.Cc[syscall.VTIME] = 0

	return termios.Tcsetattr(uintptr(fd), termios.TCSANOW, (*syscall.Termios)(&n))
}
<|endoftext|>


<commit_before>// Package tl provides TeamLiquid API wrappers
package tl

import (
	"context"
	"encoding/json"
	"sort"
	"strconv"
	"time"

	cache "github.com/patrickmn/go-cache"
	"github.com/robfig/cron"
	"github.com/sirupsen/logrus"
)

const (
	timelineCacheKey = "timelines"
)

var (
	matchCache = cache.New(6*time.Hour, 12*time.Hour)
)

type mCron struct {
	ctx context.Context
	c   *cron.Cron
}

func NewCron() *mCron {
	return &mCron{
		ctx: context.TODO(),
		c:   cron.New(),
	}
}

type Fetcher struct {
	c     *mCron
	cache *cache.Cache
	dsts  []Sender
}

func NewFetcher(s ...Sender) *Fetcher {
	return &Fetcher{
		c:     new(mCron),
		cache: matchCache,
		dsts:  s,
	}
}

func (f *Fetcher) Do() error {
	f.c = NewCron()
	f.c.c.AddFunc("@every 1m", func() {
		if f.cache.ItemCount() > 0 {
			now := time.Now()
			timeLines, ok := f.cache.Get(timelineCacheKey)
			if !ok {
				logrus.Errorf("get timeline cache failed: %s", "no cache key")
				return
			}
			cn, err := time.LoadLocation("Asia/Shanghai")
			if err != nil {
				logrus.Errorf("tl load location failed: %s", err.Error())
				return
			}

			t := make([]Timeline, 0)
			if err = json.Unmarshal(timeLines.([]byte), &t); err != nil {
				logrus.Errorf("unmarshal cache failed: %s", err.Error())
				return
			}

			if len(t) == 0 {
				logrus.Warn("get 0 timelines")
				return
			}

			if now.In(cn).Unix() < getTheLastestTimeline(t) {
				logrus.Infof("now is %d, the lastest match is at %d, no need to refresh cache.",
					now.In(cn).Unix(), getTheLastestTimeline(t))
				return
			}
		}

		logrus.Infof("warming TL cache...")
		if err := f.refreshCache(); err != nil {
			logrus.Errorf("refresh cache failed: %s", err.Error())
		}
	})
	f.c.c.Run()
	return nil
}

func (f *Fetcher) refreshCache() error {
	f.expireAllMatches()
	p, err := NewMatchParser()
	if err != nil {
		return err
	}
	timelines, err := p.GetTimelines()
	if err != nil {
		return err
	}
	tlJSON, err := json.Marshal(timelines)
	if err != nil {
		return err
	}
	f.cache.Set(timelineCacheKey, tlJSON, -1)
	matches, err := p.GetTimeMatches()
	if err != nil {
		return err
	}

	go f.pushMSG(timelines, matches)

	for t, m := range matches {
		f.cache.Set(strconv.FormatInt(t, 10), m, -1)
	}
	return nil
}

func (f *Fetcher) expireAllMatches() {
	f.cache.Flush()
}

func (f *Fetcher) pushMSG(tls []Timeline, matches map[int64][]Match) {
	sort.SliceStable(tls, func(i, j int) bool {
		return tls[i].T < tls[j].T
	})

	var sortedMatches []string
	for _, tl := range tls {
		ms, ok := matches[tl.T]
		if !ok {
			continue
		}
		for _, m := range ms {
			sortedMatches = append(sortedMatches, m.GetMDMatchInfo())
		}
	}
	f.pushWithLimit(sortedMatches, 5)
}

func (f *Fetcher) pushWithLimit(matches []string, limit int) {
	splitMatches := split(matches, limit)
	if len(splitMatches) == 0 {
		return
	}
	// use n goroutines to send message
	for _, dst := range f.dsts {
		go func(dst Sender) {
			var idx int
		SEND:
			msg := dst.ResolveMessage(splitMatches[idx])
			if err := dst.Send(msg); err != nil {
				logrus.Errorf("sender: %s", err.Error())
			}
			idx++
			if idx < len(splitMatches)-1 {
				goto SEND
			}
		}(dst)
	}
}

func split(buf []string, lim int) [][]string {
	var chunk []string
	chunks := make([][]string, 0, len(buf)/lim+1)
	for len(buf) >= lim {
		chunk, buf = buf[:lim], buf[lim:]
		chunks = append(chunks, chunk)
	}
	if len(buf) > 0 {
		chunks = append(chunks, buf[:len(buf)])
	}
	return chunks
}

func getTheLastestTimeline(tls []Timeline) int64 {
	sort.SliceStable(tls, func(i, j int) bool {
		return tls[i].T < tls[j].T
	})

	for _, tl := range tls {
		if !tl.IsOnGoing {
			return tl.T
		}
	}

	return 0
}
<commit_msg>internal/tl: reuse the cache to fix the streaming caster missing issues<commit_after>// Package tl provides TeamLiquid API wrappers
package tl

import (
	"context"
	"encoding/json"
	"sort"
	"strconv"
	"time"

	cache "github.com/patrickmn/go-cache"
	"github.com/robfig/cron"
	"github.com/sirupsen/logrus"
)

const (
	timelineCacheKey = "timelines"
)

var (
	matchCache = cache.New(6*time.Hour, 12*time.Hour)
)

type mCron struct {
	ctx context.Context
	c   *cron.Cron
}

func NewCron() *mCron {
	return &mCron{
		ctx: context.TODO(),
		c:   cron.New(),
	}
}

type Fetcher struct {
	c     *mCron
	cache *cache.Cache
	dsts  []Sender
}

func NewFetcher(s ...Sender) *Fetcher {
	return &Fetcher{
		c:     new(mCron),
		cache: matchCache,
		dsts:  s,
	}
}

func (f *Fetcher) Do() error {
	f.c = NewCron()
	f.c.c.AddFunc("@every 1m", func() {
		// do some condition tricks to save bandwidth
		if f.cache.ItemCount() > 0 {
			now := time.Now()
			timeLines, ok := f.cache.Get(timelineCacheKey)
			if !ok {
				logrus.Errorf("get timeline cache failed: %s", "no cache key")
				return
			}
			cn, err := time.LoadLocation("Asia/Shanghai")
			if err != nil {
				logrus.Errorf("tl load location failed: %s", err.Error())
				return
			}

			t := make([]Timeline, 0)
			if err = json.Unmarshal(timeLines.([]byte), &t); err != nil {
				logrus.Errorf("unmarshal cache failed: %s", err.Error())
				return
			}

			if len(t) == 0 {
				logrus.Warn("get 0 timelines")
				return
			}

			if now.In(cn).Unix() < getTheLastestTimeline(t) {
				logrus.Infof("now is %d, the lastest match is at %d, no need to refresh cache.",
					now.In(cn).Unix(), getTheLastestTimeline(t))
				return
			}
		}

		logrus.Infof("warming TL cache...")
		if err := f.refreshCache(); err != nil {
			logrus.Errorf("refresh cache failed: %s", err.Error())
		}
	})
	f.c.c.Run()
	return nil
}

func (f *Fetcher) refreshCache() error {
	p, err := NewMatchParser()
	if err != nil {
		return err
	}
	timelines, err := p.GetTimelines()
	if err != nil {
		return err
	}
	tlJSON, err := json.Marshal(timelines)
	if err != nil {
		return err
	}
	f.cache.Set(timelineCacheKey, tlJSON, -1)
	matches, err := p.GetTimeMatches()
	if err != nil {
		return err
	}

	go f.pushMSG(timelines, f.expireCache(timelines, matches))

	for t, m := range matches {
		f.cache.Set(strconv.FormatInt(t, 10), m, -1)
	}
	return nil
}

// expireCache reuse the ongoing match info
// and delete the out-of-date match info
// Due to the reuseable cache, from now on , we should manage our cache carefully TAT
func (f *Fetcher) expireCache(tls []Timeline, matches map[int64][]Match) map[int64][]Match {
	// reuse cache:
	// TL will reset Streaming caster URL after the match is going.
	// We should keep the opening match info until it is closed.
	for t := range matches {
		cachedMatches, ok := f.cache.Get(strconv.FormatInt(t, 10))
		if ok {
			matches[t] = cachedMatches.([]Match)
		}
	}

	if len(tls) == 0 {
		return matches
	}

	sort.SliceStable(tls, func(i, j int) bool {
		return tls[i].T < tls[j].T
	})

	// expire cache : T is less than the index 0 (the lowest one)
	for t := range matches {
		if t >= tls[0].T {
			continue
		}
		delete(matches, t)
		f.cache.Delete(strconv.FormatInt(t, 10))
	}
	return matches
}

func (f *Fetcher) pushMSG(tls []Timeline, matches map[int64][]Match) {
	sort.SliceStable(tls, func(i, j int) bool {
		return tls[i].T < tls[j].T
	})

	var sortedMatches []string
	for _, tl := range tls {
		// matches must be the superset of the tls
		ms, ok := matches[tl.T]
		if !ok {
			continue
		}
		for _, m := range ms {
			sortedMatches = append(sortedMatches, m.GetMDMatchInfo())
		}
	}
	f.pushWithLimit(sortedMatches, 5)
}

func (f *Fetcher) pushWithLimit(matches []string, limit int) {
	splitMatches := split(matches, limit)
	if len(splitMatches) == 0 {
		return
	}
	// use n goroutines to send message
	for _, dst := range f.dsts {
		go func(dst Sender) {
			var idx int
		SEND:
			msg := dst.ResolveMessage(splitMatches[idx])
			if err := dst.Send(msg); err != nil {
				logrus.Errorf("sender: %s", err.Error())
			}
			idx++
			if idx < len(splitMatches)-1 {
				goto SEND
			}
		}(dst)
	}
}

func split(buf []string, lim int) [][]string {
	var chunk []string
	chunks := make([][]string, 0, len(buf)/lim+1)
	for len(buf) >= lim {
		chunk, buf = buf[:lim], buf[lim:]
		chunks = append(chunks, chunk)
	}
	if len(buf) > 0 {
		chunks = append(chunks, buf[:len(buf)])
	}
	return chunks
}

func getTheLastestTimeline(tls []Timeline) int64 {
	sort.SliceStable(tls, func(i, j int) bool {
		return tls[i].T < tls[j].T
	})

	for _, tl := range tls {
		if !tl.IsOnGoing {
			return tl.T
		}
	}

	return 0
}
<|endoftext|>