<commit_before>package aiff

import (
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/go-audio/audio"
)

// TODO(mattetti): benchmark allocations

func TestContainerAttributes(t *testing.T) {
	expectations := []struct {
		input           string
		id              [4]byte
		size            uint32
		format          [4]byte
		commSize        uint32
		numChans        uint16
		numSampleFrames uint32
		sampleSize      uint16
		sampleRate      int
		totalFrames     int64
	}{
		{"fixtures/kick.aif", formID, 9642, aiffID,
			18, 1, 4484, 16, 22050, 4484},
	}

	for _, exp := range expectations {
		path, _ := filepath.Abs(exp.input)
		f, err := os.Open(path)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		d := NewDecoder(f)
		buf, err := d.FullPCMBuffer()
		if err != nil {
			t.Fatal(err)
		}

		if int(d.BitDepth) != int(exp.sampleSize) {
			t.Fatalf("%s of %s didn't match %d, got %d", "Clip bit depth", exp.input, exp.sampleSize, d.BitDepth)
		}

		if int(d.SampleRate) != exp.sampleRate {
			t.Fatalf("%s of %s didn't match %d, got %d", "Clip sample rate", exp.input, exp.sampleRate, d.SampleRate)
		}

		if int(d.NumChans) != int(exp.numChans) {
			t.Fatalf("%s of %s didn't match %d, got %d", "Clip sample channels", exp.input, exp.numChans, d.NumChans)
		}

		if buf.NumFrames() != int(exp.totalFrames) {
			t.Fatalf("%s of %s didn't match %d, got %d", "Clip sample data size", exp.input, exp.totalFrames, buf.NumFrames())
		}

		if d.ID != exp.id {
			t.Fatalf("%s of %s didn't match %s, got %s", "ID", exp.input, exp.id, d.ID)
		}
		if d.Size != exp.size {
			t.Fatalf("%s of %s didn't match %d, got %d", "BlockSize", exp.input, exp.size, d.Size)
		}
		if d.Form != exp.format {
			t.Fatalf("%s of %s didn't match %q, got %q", "Format", exp.input, exp.format, d.Form)
		}
		// comm chunk
		if d.commSize != exp.commSize {
			t.Fatalf("%s of %s didn't match %d, got %d", "comm size", exp.input, exp.commSize, d.commSize)
		}
		if d.NumChans != exp.numChans {
			t.Fatalf("%s of %s didn't match %d, got %d", "NumChans", exp.input, exp.numChans, d.NumChans)
		}
		if d.numSampleFrames != exp.numSampleFrames {
			t.Fatalf("%s of %s didn't match %d, got %d", "NumSampleFrames", exp.input, exp.numSampleFrames, d.numSampleFrames)
		}
		if d.BitDepth != exp.sampleSize {
			t.Fatalf("%s of %s didn't match %d, got %d", "SampleSize", exp.input, exp.sampleSize, d.BitDepth)
		}
		if d.SampleRate != exp.sampleRate {
			t.Fatalf("%s of %s didn't match %d, got %d", "SampleRate", exp.input, exp.sampleRate, d.SampleRate)
		}
	}
}

func TestDecoder_Duration(t *testing.T) {
	expectations := []struct {
		input    string
		duration time.Duration
	}{
		{"fixtures/kick.aif", time.Duration(203356009)},
	}

	for _, exp := range expectations {
		path, _ := filepath.Abs(exp.input)
		f, err := os.Open(path)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		c := NewDecoder(f)
		d, err := c.Duration()
		if err != nil {
			t.Fatal(err)
		}
		if d != exp.duration {
			t.Fatalf("duration of %s didn't match %d milliseconds, got %d", exp.input, exp.duration, d)
		}
	}
}

func TestDecoder_FullPCMBuffer(t *testing.T) {
	testCases := []struct {
		input      string
		desc       string
		samples    []int
		numSamples int
	}{
		{"fixtures/kick.aif",
			"1 ch,  22050 Hz, 'lpcm' (0x0000000E) 16-bit big-endian signed integer",
			[]int{76, 76, 75, 75, 72, 71, 72, 69, 70, 68, 65, 73, 529, 1425, 2245, 2941, 3514, 3952, 4258, 4436, 4486, 4413, 4218, 3903, 3474, 2938, 2295, 1553, 711, -214, -1230, -2321, -3489, -4721, -6007, -7352, -8738, -10172, -11631, -13127, -14642, -16029, -17322, -18528, -19710, -20877},
			4484,
		},
		{"fixtures/delivery.aiff",
			"1 ch,  22050 Hz, 'lpcm' (0x0000000E) 16-bit big-endian signed integer",
			[]int{132, 123, 53, 85, 20, -15, 14, -2, 47, 96, 115, 181, 207, 248, 284, 324, 358, 369, 375, 350, 335, 305, 250, 188, 164, 144, 138, 151, 151, 166, 159, 170, 178, 178, 164, 125, 112, 73, 49, 28, -24, -70, -129, -165, -177, -185, -186, -194, -198, -211, -217, -208, -222, -215, -205, -177, -119, -100, -61, -34, -35, 10, 43, 79, 130, 157, 198, 251, 297, 348, 399, 424},
			17199,
		},
	}

	for i, tc := range testCases {
		t.Logf("%d - %s - %s\n", i, tc.input, tc.desc)
		path, _ := filepath.Abs(tc.input)
		f, err := os.Open(path)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		d := NewDecoder(f)

		buf, err := d.FullPCMBuffer()
		if err != nil {
			t.Fatal(err)
		}
		if len(buf.Data) != tc.numSamples {
			t.Fatalf("the length of the buffer (%d) didn't match what we expected (%d)", len(buf.Data), tc.numSamples)
		}
		for i := 0; i < len(tc.samples); i++ {
			if buf.Data[i] != tc.samples[i] {
				t.Fatalf("Expected %d at position %d, but got %d", tc.samples[i], i, buf.Data[i])
			}
		}
	}
}

func TestDecoderPCMBuffer(t *testing.T) {
	testCases := []struct {
		input   string
		desc    string
		samples []int
	}{
		{"fixtures/kick.aif",
			"1 ch,  22050 Hz, 'lpcm' (0x0000000E) 16-bit big-endian signed integer",
			[]int{76, 76, 75, 75, 72, 71, 72, 69, 70, 68, 65, 73, 529, 1425, 2245, 2941, 3514, 3952, 4258, 4436, 4486, 4413, 4218, 3903, 3474, 2938, 2295, 1553, 711, -214, -1230, -2321, -3489, -4721, -6007, -7352, -8738, -10172, -11631, -13127, -14642, -16029, -17322, -18528, -19710, -20877},
		},
	}

	for i, tc := range testCases {
		t.Logf("%d - %s - %s\n", i, tc.input, tc.desc)
		path, _ := filepath.Abs(tc.input)
		f, err := os.Open(path)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		d := NewDecoder(f)

		intBuf := make([]int, len(tc.samples))
		buf := &audio.IntBuffer{Data: intBuf}
		err = d.PCMBuffer(buf)
		if err != nil {
			t.Fatal(err)
		}
		if len(buf.Data) != len(tc.samples) {
			t.Fatalf("the length of the buffer (%d) didn't match what we expected (%d)", len(buf.Data), len(tc.samples))
		}
		for i := 0; i < len(buf.Data); i++ {
			if buf.Data[i] != tc.samples[i] {
				t.Fatalf("Expected %d at position %d, but got %d", tc.samples[i], i, buf.Data[i])
			}
		}
	}
}

func TestDecoder_IsValidFile(t *testing.T) {
	testCases := []struct {
		in      string
		isValid bool
	}{
		{"fixtures/bloop.aif", true},
		{"fixtures/kick8b.aiff", true},
		{"fixtures/zipper24b.aiff", true},
		{"fixtures/sample.avi", false},
		{"fixtures/kick.wav", false},
	}

	for i, tc := range testCases {
		f, err := os.Open(tc.in)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		d := NewDecoder(f)
		if d.IsValidFile() != tc.isValid {
			t.Fatalf("[%d] validation of the aiff files doesn't match expected %t, got %t - %#v", i, tc.isValid, d.IsValidFile(), d)
		}
	}

}
<commit_msg>fix test suite<commit_after>package aiff

import (
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/go-audio/audio"
)

// TODO(mattetti): benchmark allocations

func TestContainerAttributes(t *testing.T) {
	expectations := []struct {
		input           string
		id              [4]byte
		size            uint32
		format          [4]byte
		commSize        uint32
		numChans        uint16
		numSampleFrames uint32
		sampleSize      uint16
		sampleRate      int
		totalFrames     int64
	}{
		{"fixtures/kick.aif", formID, 9642, aiffID,
			18, 1, 4484, 16, 22050, 4484},
	}

	for _, exp := range expectations {
		path, _ := filepath.Abs(exp.input)
		f, err := os.Open(path)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		d := NewDecoder(f)
		buf, err := d.FullPCMBuffer()
		if err != nil {
			t.Fatal(err)
		}

		if int(d.BitDepth) != int(exp.sampleSize) {
			t.Fatalf("%s of %s didn't match %d, got %d", "Clip bit depth", exp.input, exp.sampleSize, d.BitDepth)
		}

		if int(d.SampleRate) != exp.sampleRate {
			t.Fatalf("%s of %s didn't match %d, got %d", "Clip sample rate", exp.input, exp.sampleRate, d.SampleRate)
		}

		if int(d.NumChans) != int(exp.numChans) {
			t.Fatalf("%s of %s didn't match %d, got %d", "Clip sample channels", exp.input, exp.numChans, d.NumChans)
		}

		if buf.NumFrames() != int(exp.totalFrames) {
			t.Fatalf("%s of %s didn't match %d, got %d", "Clip sample data size", exp.input, exp.totalFrames, buf.NumFrames())
		}

		if d.ID != exp.id {
			t.Fatalf("%s of %s didn't match %s, got %s", "ID", exp.input, exp.id, d.ID)
		}
		if d.Size != exp.size {
			t.Fatalf("%s of %s didn't match %d, got %d", "BlockSize", exp.input, exp.size, d.Size)
		}
		if d.Form != exp.format {
			t.Fatalf("%s of %s didn't match %q, got %q", "Format", exp.input, exp.format, d.Form)
		}
		// comm chunk
		if d.commSize != exp.commSize {
			t.Fatalf("%s of %s didn't match %d, got %d", "comm size", exp.input, exp.commSize, d.commSize)
		}
		if d.NumChans != exp.numChans {
			t.Fatalf("%s of %s didn't match %d, got %d", "NumChans", exp.input, exp.numChans, d.NumChans)
		}
		if d.NumSampleFrames != exp.numSampleFrames {
			t.Fatalf("%s of %s didn't match %d, got %d", "NumSampleFrames", exp.input, exp.numSampleFrames, d.NumSampleFrames)
		}
		if d.BitDepth != exp.sampleSize {
			t.Fatalf("%s of %s didn't match %d, got %d", "SampleSize", exp.input, exp.sampleSize, d.BitDepth)
		}
		if d.SampleRate != exp.sampleRate {
			t.Fatalf("%s of %s didn't match %d, got %d", "SampleRate", exp.input, exp.sampleRate, d.SampleRate)
		}
	}
}

func TestDecoder_Duration(t *testing.T) {
	expectations := []struct {
		input    string
		duration time.Duration
	}{
		{"fixtures/kick.aif", time.Duration(203356009)},
	}

	for _, exp := range expectations {
		path, _ := filepath.Abs(exp.input)
		f, err := os.Open(path)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		c := NewDecoder(f)
		d, err := c.Duration()
		if err != nil {
			t.Fatal(err)
		}
		if d != exp.duration {
			t.Fatalf("duration of %s didn't match %d milliseconds, got %d", exp.input, exp.duration, d)
		}
	}
}

func TestDecoder_FullPCMBuffer(t *testing.T) {
	testCases := []struct {
		input      string
		desc       string
		samples    []int
		numSamples int
	}{
		{"fixtures/kick.aif",
			"1 ch,  22050 Hz, 'lpcm' (0x0000000E) 16-bit big-endian signed integer",
			[]int{76, 76, 75, 75, 72, 71, 72, 69, 70, 68, 65, 73, 529, 1425, 2245, 2941, 3514, 3952, 4258, 4436, 4486, 4413, 4218, 3903, 3474, 2938, 2295, 1553, 711, -214, -1230, -2321, -3489, -4721, -6007, -7352, -8738, -10172, -11631, -13127, -14642, -16029, -17322, -18528, -19710, -20877},
			4484,
		},
		{"fixtures/delivery.aiff",
			"1 ch,  22050 Hz, 'lpcm' (0x0000000E) 16-bit big-endian signed integer",
			[]int{132, 123, 53, 85, 20, -15, 14, -2, 47, 96, 115, 181, 207, 248, 284, 324, 358, 369, 375, 350, 335, 305, 250, 188, 164, 144, 138, 151, 151, 166, 159, 170, 178, 178, 164, 125, 112, 73, 49, 28, -24, -70, -129, -165, -177, -185, -186, -194, -198, -211, -217, -208, -222, -215, -205, -177, -119, -100, -61, -34, -35, 10, 43, 79, 130, 157, 198, 251, 297, 348, 399, 424},
			17199,
		},
	}

	for i, tc := range testCases {
		t.Logf("%d - %s - %s\n", i, tc.input, tc.desc)
		path, _ := filepath.Abs(tc.input)
		f, err := os.Open(path)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		d := NewDecoder(f)

		buf, err := d.FullPCMBuffer()
		if err != nil {
			t.Fatal(err)
		}
		if len(buf.Data) != tc.numSamples {
			t.Fatalf("the length of the buffer (%d) didn't match what we expected (%d)", len(buf.Data), tc.numSamples)
		}
		for i := 0; i < len(tc.samples); i++ {
			if buf.Data[i] != tc.samples[i] {
				t.Fatalf("Expected %d at position %d, but got %d", tc.samples[i], i, buf.Data[i])
			}
		}
	}
}

func TestDecoderPCMBuffer(t *testing.T) {
	testCases := []struct {
		input   string
		desc    string
		samples []int
	}{
		{"fixtures/kick.aif",
			"1 ch,  22050 Hz, 'lpcm' (0x0000000E) 16-bit big-endian signed integer",
			[]int{76, 76, 75, 75, 72, 71, 72, 69, 70, 68, 65, 73, 529, 1425, 2245, 2941, 3514, 3952, 4258, 4436, 4486, 4413, 4218, 3903, 3474, 2938, 2295, 1553, 711, -214, -1230, -2321, -3489, -4721, -6007, -7352, -8738, -10172, -11631, -13127, -14642, -16029, -17322, -18528, -19710, -20877},
		},
	}

	for i, tc := range testCases {
		t.Logf("%d - %s - %s\n", i, tc.input, tc.desc)
		path, _ := filepath.Abs(tc.input)
		f, err := os.Open(path)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		d := NewDecoder(f)

		intBuf := make([]int, len(tc.samples))
		buf := &audio.IntBuffer{Data: intBuf}
		_, err = d.PCMBuffer(buf)
		if err != nil {
			t.Fatal(err)
		}
		if len(buf.Data) != len(tc.samples) {
			t.Fatalf("the length of the buffer (%d) didn't match what we expected (%d)", len(buf.Data), len(tc.samples))
		}
		for i := 0; i < len(buf.Data); i++ {
			if buf.Data[i] != tc.samples[i] {
				t.Fatalf("Expected %d at position %d, but got %d", tc.samples[i], i, buf.Data[i])
			}
		}
	}
}

func TestDecoder_IsValidFile(t *testing.T) {
	testCases := []struct {
		in      string
		isValid bool
	}{
		{"fixtures/bloop.aif", true},
		{"fixtures/kick8b.aiff", true},
		{"fixtures/zipper24b.aiff", true},
		{"fixtures/sample.avi", false},
		{"fixtures/kick.wav", false},
	}

	for i, tc := range testCases {
		f, err := os.Open(tc.in)
		if err != nil {
			t.Fatal(err)
		}
		defer f.Close()
		d := NewDecoder(f)
		if d.IsValidFile() != tc.isValid {
			t.Fatalf("[%d] validation of the aiff files doesn't match expected %t, got %t - %#v", i, tc.isValid, d.IsValidFile(), d)
		}
	}

}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package zmqutil

import (
	zmq "github.com/pebbe/zmq4"
	"sync"
	"time"
)

// structure to hold a poller
type Poller struct {
	sync.Mutex
	sockets map[*zmq.Socket]zmq.State
	poller  *zmq.Poller
}

// create a poller
// this is just to encapsulate the zmq poller to allow removal of a socket from a socket
func NewPoller() *Poller {
	return &Poller{
		sockets: make(map[*zmq.Socket]zmq.State),
		poller:  zmq.NewPoller(),
	}
}

// add a socket to a poller
func (poller *Poller) Add(socket *zmq.Socket, events zmq.State) {

	poller.Lock()
	defer poller.Unlock()

	// protect against duplicate add
	if _, ok := poller.sockets[socket]; ok {
		return
	}

	// preserve the event mask
	poller.sockets[socket] = events

	// add to the internal poller
	poller.poller.Add(socket, events)
}

// remove a socket from a poller
func (poller *Poller) Remove(socket *zmq.Socket) {

	poller.Lock()
	defer poller.Unlock()

	// protect against duplicate remove
	if _, ok := poller.sockets[socket]; !ok {
		return
	}

	// remove the socket
	delete(poller.sockets, socket)

	// rebuild the zmq poller
	p := zmq.NewPoller()
	for s, events := range poller.sockets {
		poller.poller.Add(s, events)
	}
	poller.poller = p
}

// perform a poll
func (poller *Poller) Poll(timeout time.Duration) ([]zmq.Polled, error) {
	poller.Lock()
	p := poller.poller
	poller.Unlock()
	polled, err := p.Poll(timeout)
	return polled, err
}
<commit_msg>[zmqutil] fix the incorrect poller swapping when a socket is removed from a poller<commit_after>// Copyright (c) 2014-2017 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package zmqutil

import (
	zmq "github.com/pebbe/zmq4"
	"sync"
	"time"
)

// structure to hold a poller
type Poller struct {
	sync.Mutex
	sockets map[*zmq.Socket]zmq.State
	poller  *zmq.Poller
}

// create a poller
// this is just to encapsulate the zmq poller to allow removal of a socket from a socket
func NewPoller() *Poller {
	return &Poller{
		sockets: make(map[*zmq.Socket]zmq.State),
		poller:  zmq.NewPoller(),
	}
}

// add a socket to a poller
func (poller *Poller) Add(socket *zmq.Socket, events zmq.State) {

	poller.Lock()
	defer poller.Unlock()

	// protect against duplicate add
	if _, ok := poller.sockets[socket]; ok {
		return
	}

	// preserve the event mask
	poller.sockets[socket] = events

	// add to the internal poller
	poller.poller.Add(socket, events)
}

// remove a socket from a poller
func (poller *Poller) Remove(socket *zmq.Socket) {

	poller.Lock()
	defer poller.Unlock()

	// protect against duplicate remove
	if _, ok := poller.sockets[socket]; !ok {
		return
	}

	// remove the socket
	delete(poller.sockets, socket)

	// rebuild the zmq poller
	p := zmq.NewPoller()
	for s, events := range poller.sockets {
		p.Add(s, events)
	}
	poller.poller = p
}

// perform a poll
func (poller *Poller) Poll(timeout time.Duration) ([]zmq.Polled, error) {
	poller.Lock()
	p := poller.poller
	poller.Unlock()
	polled, err := p.Poll(timeout)
	return polled, err
}
<|endoftext|>


<commit_before>package tessen

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"regexp"
	"strings"

	ui "github.com/gizak/termui"
)

type QueryResult struct {
	Id      string
	Summary string
	Data    interface{}
}

type QueryResultsPage struct {
	BaseListPage
	CommandBarFragment
	StatusBarFragment
	ActiveQuery Query
}

func GetFilteredListOfEvents(filter string, eventData *[]map[string]interface{}) []interface{} {
	template := GetTemplate("event_list")
	results := make([]interface{}, 0)
	b, err := json.Marshal(*eventData)
	if err != nil {
		log.Errorf("%s", err)
		return results
	}

	cmd := exec.Command("jq", fmt.Sprintf(".[] | select( %s ) | ._id", filter))
	cmd.Stdin = bytes.NewReader(b)
	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		log.Errorf("%s", err)
		return results
	}

	for _, v := range strings.Split(out.String(), "\n") {
		id := strings.Trim(v, "\"")
		for _, ev := range *eventData {
			if ev["_id"].(string) == id {
				id := strings.Trim(v, "\"")
				buf := new(bytes.Buffer)
				RunTemplate(template, ev, buf)
				results = append(results, QueryResult{id, buf.String(), ev})
				continue
			}
		}
	}
	return results

}

func (p *QueryResultsPage) GetSelectedQueryResultId() string {
	qr := p.cachedResults.([]interface{})[p.selectedLine]
	return qr.(QueryResult).Id
}

func (p *QueryResultsPage) Search() {
	s := p.ActiveSearch
	n := len(p.cachedResults.([]interface{}))
	if s.command == "" {
		return
	}
	increment := 1
	if s.directionUp {
		increment = -1
	}
	// we use modulo here so we can loop through every line.
	// adding 'n' means we never have '-1 % n'.
	startLine := (p.selectedLine + n + increment) % n
	for i := startLine; i != p.selectedLine; i = (i + increment + n) % n {
		cr := p.cachedResults.([]interface{})[i]
		if s.re.MatchString(cr.(QueryResult).Summary) {
			p.SetSelectedLine(i)
			p.Update()
			break
		}
	}
}

func (p *QueryResultsPage) SelectItem() {
	if len(p.cachedResults.([]interface{})) == 0 {
		return
	}
	q := new(ShowDetailPage)
	q.EventId = p.GetSelectedQueryResultId()
	currentPage = q
	q.Create()
	changePage()
}

func (p *QueryResultsPage) GoBack() {
	currentPage = queryPage
	changePage()
}

func (p *QueryResultsPage) Update() {
	ls := p.uiList
	log.Debugf("QueryResultsPage.Update(): self:        %s (%p), ls: (%p)", p.Id(), p, ls)
	p.markActiveLine()
	ls.Items = p.displayLines[p.firstDisplayLine:]
	ui.Render(ls)
	p.statusBar.Update()
	p.commandBar.Update()
}

func (p *QueryResultsPage) Refresh() {
	pDeref := &p
	q := *pDeref
	q.cachedResults = nil
	queryResultsPage = q
	changePage()
	q.Create()
}

func (p *QueryResultsPage) markActiveLine() {
	for i, v := range p.cachedResults.([]interface{}) {
		selected := ""
		s := v.(QueryResult).Summary
		if i == p.selectedLine {
			selected = "fg-white,bg-blue"
			if s == "" {
				s = " "
			} else if ok, _ := regexp.MatchString(`\[.+\]\((fg|bg)-[a-z]{1,6}\)`, s); ok {
				r := regexp.MustCompile(`\[(.*?)\]\((fg|bg)-[a-z]{1,6}\)`)
				s = r.ReplaceAllString(s, `$1`)
			}
			p.displayLines[i] = fmt.Sprintf("[%s](%s)", s, selected)
		} else {
			p.displayLines[i] = s
		}
	}
}

func (p *QueryResultsPage) Create() {
	log.Debugf("QueryResultsPage.Create(): self:        %s (%p)", p.Id(), p)
	log.Debugf("QueryResultsPage.Create(): currentPage: %s (%p)", currentPage.Id(), currentPage)
	ui.Clear()
	ls := ui.NewList()
	p.uiList = ls
	if p.statusBar == nil {
		p.statusBar = new(StatusBar)
	}
	if p.commandBar == nil {
		p.commandBar = commandBar
	}
	query := p.ActiveQuery.Filter
	if p.cachedResults == nil {
		p.cachedResults = GetFilteredListOfEvents(query, &eventData)
	}
	if p.selectedLine >= len(p.cachedResults.([]interface{})) {
		p.selectedLine = len(p.cachedResults.([]interface{})) - 1
	}
	p.displayLines = make([]string, len(p.cachedResults.([]interface{})))
	ls.ItemFgColor = ui.ColorYellow
	ls.BorderLabel = fmt.Sprintf("%s: %s", p.ActiveQuery.Name, p.ActiveQuery.Filter)
	ls.Height = ui.TermHeight() - 2
	ls.Width = ui.TermWidth()
	ls.Y = 0
	p.statusBar.Create()
	p.commandBar.Create()
	p.Update()
}
<commit_msg>Fix bug where a lack of query results caused page to break<commit_after>package tessen

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"regexp"
	"strings"

	ui "github.com/gizak/termui"
)

type QueryResult struct {
	Id      string
	Summary string
	Data    interface{}
}

type QueryResultsPage struct {
	BaseListPage
	CommandBarFragment
	StatusBarFragment
	ActiveQuery Query
}

func GetFilteredListOfEvents(filter string, eventData *[]map[string]interface{}) []interface{} {
	template := GetTemplate("event_list")
	results := make([]interface{}, 0)
	b, err := json.Marshal(*eventData)
	if err != nil {
		log.Errorf("%s", err)
		return results
	}

	cmd := exec.Command("jq", fmt.Sprintf(".[] | select( %s ) | ._id", filter))
	cmd.Stdin = bytes.NewReader(b)
	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		log.Errorf("%s", err)
		return results
	}

	for _, v := range strings.Split(out.String(), "\n") {
		id := strings.Trim(v, "\"")
		for _, ev := range *eventData {
			if ev["_id"].(string) == id {
				id := strings.Trim(v, "\"")
				buf := new(bytes.Buffer)
				RunTemplate(template, ev, buf)
				results = append(results, QueryResult{id, buf.String(), ev})
				continue
			}
		}
	}

	if len(results) == 0 {
		results = append(results, QueryResult{"", "No results found", nil})
	}
	return results

}

func (p *QueryResultsPage) GetSelectedQueryResultId() string {
	qr := p.cachedResults.([]interface{})[p.selectedLine]
	return qr.(QueryResult).Id
}

func (p *QueryResultsPage) Search() {
	s := p.ActiveSearch
	n := len(p.cachedResults.([]interface{}))
	if s.command == "" {
		return
	}
	increment := 1
	if s.directionUp {
		increment = -1
	}
	// we use modulo here so we can loop through every line.
	// adding 'n' means we never have '-1 % n'.
	startLine := (p.selectedLine + n + increment) % n
	for i := startLine; i != p.selectedLine; i = (i + increment + n) % n {
		cr := p.cachedResults.([]interface{})[i]
		if s.re.MatchString(cr.(QueryResult).Summary) {
			p.SetSelectedLine(i)
			p.Update()
			break
		}
	}
}

func (p *QueryResultsPage) SelectItem() {
	if len(p.cachedResults.([]interface{})) == 0 {
		return
	}
	id := p.GetSelectedQueryResultId()
	if id == "" {
		return
	}
	q := new(ShowDetailPage)
	q.EventId = id
	currentPage = q
	q.Create()
	changePage()
}

func (p *QueryResultsPage) GoBack() {
	currentPage = queryPage
	changePage()
}

func (p *QueryResultsPage) Update() {
	ls := p.uiList
	log.Debugf("QueryResultsPage.Update(): self:        %s (%p), ls: (%p)", p.Id(), p, ls)
	p.markActiveLine()
	ls.Items = p.displayLines[p.firstDisplayLine:]
	ui.Render(ls)
	p.statusBar.Update()
	p.commandBar.Update()
}

func (p *QueryResultsPage) Refresh() {
	pDeref := &p
	q := *pDeref
	q.cachedResults = nil
	queryResultsPage = q
	changePage()
	q.Create()
}

func (p *QueryResultsPage) markActiveLine() {
	for i, v := range p.cachedResults.([]interface{}) {
		selected := ""
		s := v.(QueryResult).Summary
		if i == p.selectedLine {
			selected = "fg-white,bg-blue"
			if s == "" {
				s = " "
			} else if ok, _ := regexp.MatchString(`\[.+\]\((fg|bg)-[a-z]{1,6}\)`, s); ok {
				r := regexp.MustCompile(`\[(.*?)\]\((fg|bg)-[a-z]{1,6}\)`)
				s = r.ReplaceAllString(s, `$1`)
			}
			p.displayLines[i] = fmt.Sprintf("[%s](%s)", s, selected)
		} else {
			p.displayLines[i] = s
		}
	}
}

func (p *QueryResultsPage) Create() {
	log.Debugf("QueryResultsPage.Create(): self:        %s (%p)", p.Id(), p)
	log.Debugf("QueryResultsPage.Create(): currentPage: %s (%p)", currentPage.Id(), currentPage)
	ui.Clear()
	ls := ui.NewList()
	p.uiList = ls
	if p.statusBar == nil {
		p.statusBar = new(StatusBar)
	}
	if p.commandBar == nil {
		p.commandBar = commandBar
	}
	query := p.ActiveQuery.Filter
	if p.cachedResults == nil {
		p.cachedResults = GetFilteredListOfEvents(query, &eventData)
	}
	if p.selectedLine >= len(p.cachedResults.([]interface{})) {
		p.selectedLine = len(p.cachedResults.([]interface{})) - 1
	}
	p.displayLines = make([]string, len(p.cachedResults.([]interface{})))
	ls.ItemFgColor = ui.ColorYellow
	ls.BorderLabel = fmt.Sprintf("%s: %s", p.ActiveQuery.Name, p.ActiveQuery.Filter)
	ls.Height = ui.TermHeight() - 2
	ls.Width = ui.TermWidth()
	ls.Y = 0
	p.statusBar.Create()
	p.commandBar.Create()
	p.Update()
}
<|endoftext|>


<commit_before>package amqp

import (
	"fmt"
	"github.com/streadway/amqp"
)

// Channel wraps an AMQP channel
type Channel struct {
	ch *amqp.Channel
}

// Queue creates a named queue on a given chennel
func (c *Channel) Queue(name string) (*Queue, error) {
	deadQueue := fmt.Sprintf("%s-dead", name)

	args := amqp.Table{
		"x-max-priority":            9,                   // Enable all 9 priorities
		"x-message-ttl":             1000 * 60 * 60 * 24, // Expire messages after 24 hours
		"x-dead-letter-exchange":    "",                  // Anything failing or expiring goes here
		"x-dead-letter-routing-key": deadQueue,
	}

	_, err := c.ch.QueueDeclare(
		name,  // name
		true,  // durable
		false, // delete when unused
		false, // exclusive
		false, // no-wait
		args,  // arguments
	)
	if err != nil {
		return nil, err
	}

	_, err = c.ch.QueueDeclare(
		deadQueue, // name
		true,      // durable
		false,     // delete when unused
		false,     // exclusive
		false,     // no-wait
		nil,       // arguments
	)
	if err != nil {
		return nil, err
	}

	return &Queue{
		channel: c,
		name:    name,
	}, nil
}

// Close closes a Channel
func (c *Channel) Close() error {
	return c.ch.Close()
}
<commit_msg>Prevent resource exhaustion of RabbitMQ.<commit_after>package amqp

import (
	"fmt"
	"github.com/streadway/amqp"
)

// Channel wraps an AMQP channel
type Channel struct {
	ch *amqp.Channel
}

// Queue creates a named queue on a given chennel
func (c *Channel) Queue(name string) (*Queue, error) {
	deadQueue := fmt.Sprintf("%s-dead", name)

	_, err := c.ch.QueueDeclare(
		name,  // name
		true,  // durable
		false, // delete when unused
		false, // exclusive
		false, // no-wait
		amqp.Table{
			"x-max-priority":            9,                   // Enable all 9 priorities
			"x-message-ttl":             1000 * 60 * 60 * 24, // Expire messages after 24 hours
			"x-dead-letter-exchange":    "",                  // Anything failing or expiring goes here
			"x-dead-letter-routing-key": deadQueue,
			"x-max-length":              100 * 1000, // Max 100.000 messages; prevent resource exhaustion in queue.
		},
	)
	if err != nil {
		return nil, err
	}

	_, err = c.ch.QueueDeclare(
		deadQueue, // name
		true,      // durable
		false,     // delete when unused
		false,     // exclusive
		false,     // no-wait
		amqp.Table{
			"x-message-ttl": 1000 * 60 * 60 * 24, // Expire messages after 24 hours,
			"x-max-length":  100 * 1000,          // Max 100.000 messages; prevent resource exhaustion in queue.
		},
	)
	if err != nil {
		return nil, err
	}

	return &Queue{
		channel: c,
		name:    name,
	}, nil
}

// Close closes a Channel
func (c *Channel) Close() error {
	return c.ch.Close()
}
<|endoftext|>


<commit_before>package bitbucket

import (
	"context"
	"crypto/sha256"
	"fmt"
	"net/http"
	"path/filepath"
	"strings"
	"sync"

	"github.com/reviewdog/reviewdog"
	"github.com/reviewdog/reviewdog/proto/rdf"
	"github.com/reviewdog/reviewdog/service/bitbucket/openapi"
)

var _ reviewdog.CommentService = &ReportAnnotator{}

const (
	// avatar from https://github.com/apps/reviewdog
	logoURL  = "https://avatars1.githubusercontent.com/in/12131"
	reporter = "reviewdog"
)

// ReportAnnotator is a comment service for Bitbucket Code Insights reports.
//
// API:
//  https://developer.atlassian.com/bitbucket/api/2/reference/resource/repositories/%7Bworkspace%7D/%7Brepo_slug%7D/commit/%7Bcommit%7D/reports/%7BreportId%7D/annotations#post
//  POST /2.0/repositories/{username}/{repo_slug}/commit/{commit}/reports/{reportId}/annotations
type ReportAnnotator struct {
	cli         *openapi.APIClient
	sha         string
	owner, repo string

	muAnnotations sync.Mutex
	annotations   map[string][]openapi.ReportAnnotation
	issuesCount   map[rdf.Severity]int
	severityMap   map[rdf.Severity]string

	// wd is working directory relative to root of repository.
	wd string
}

// NewReportAnnotator creates new Bitbucket Report Annotator
func NewReportAnnotator(cli *openapi.APIClient, reportTitle, owner, repo, sha string) *ReportAnnotator {
	return &ReportAnnotator{
		cli:         cli,
		sha:         sha,
		owner:       owner,
		repo:        repo,
		annotations: make(map[string][]openapi.ReportAnnotation),
		issuesCount: make(map[rdf.Severity]int),
		severityMap: map[rdf.Severity]string{
			rdf.Severity_INFO:    annotationSeverityLow,
			rdf.Severity_WARNING: annotationSeverityMedium,
			rdf.Severity_ERROR:   annotationSeverityHigh,
		},
	}
}

// Post accepts a comment and holds it. Flush method actually posts comments to
// Bitbucket in batch.
func (r *ReportAnnotator) Post(_ context.Context, c *reviewdog.Comment) error {
	c.Result.Diagnostic.GetLocation().Path = filepath.ToSlash(
		filepath.Join(r.wd, c.Result.Diagnostic.GetLocation().GetPath()))
	r.muAnnotations.Lock()
	defer r.muAnnotations.Unlock()

	r.issuesCount[c.Result.Diagnostic.GetSeverity()]++
	r.annotations[c.ToolName] = append(r.annotations[c.ToolName], r.annotationFromReviewDogComment(*c))

	return nil
}

// Flush posts comments which has not been posted yet.
func (r *ReportAnnotator) Flush(ctx context.Context) error {
	r.muAnnotations.Lock()
	defer r.muAnnotations.Unlock()

	// create/update/annotate report per tool
	for tool, annotations := range r.annotations {
		reportID := reportID(reporter, tool)
		title := fmt.Sprintf("[%s] %s report", tool, reporter)
		if len(r.annotations) == 0 {
			// if no annotation, create Passed report
			if err := r.createOrUpdateReport(ctx, reportID, title, reportResultPassed); err != nil {
				return err
			}
			// and move one
			continue
		}

		reportStatus := reportResultPending
		if r.issuesCount[rdf.Severity_ERROR] > 0 {
			reportStatus = reportResultFailed
		}

		// create report or update report first, with the proper status
		if err := r.createOrUpdateReport(ctx, reportID, title, reportStatus); err != nil {
			return err
		}

		// add annotations to the report
		_, resp, err := r.cli.ReportsApi.BulkCreateOrUpdateAnnotations(
			ctx, r.owner, r.repo, r.sha, reportID,
		).Body(annotations).Execute()

		if err := checkAPIError(err, resp, http.StatusOK); err != nil {
			return fmt.Errorf("bitbucket.BulkCreateOrUpdateAnnotations: %s", err)
		}
	}

	return nil
}

func (r *ReportAnnotator) annotationFromReviewDogComment(c reviewdog.Comment) openapi.ReportAnnotation {
	a := openapi.NewReportAnnotation()
	switch c.ToolName {
	// TODO: different type of annotation based on tool?
	default:
		a.SetAnnotationType(annotationTypeCodeSmell)
	}

	// hash the output of linter and use it as external id
	a.SetExternalId(hashString(c.Result.Diagnostic.OriginalOutput))
	a.SetSummary(c.Result.Diagnostic.GetMessage())
	a.SetDetails(fmt.Sprintf(`[%s] %s`, c.ToolName, c.Result.Diagnostic.GetMessage()))
	a.SetLine(c.Result.Diagnostic.GetLocation().GetRange().GetStart().GetLine())
	a.SetPath(c.Result.Diagnostic.GetLocation().GetPath())
	if v, ok := r.severityMap[c.Result.Diagnostic.GetSeverity()]; ok {
		a.SetSeverity(v)
	}
	if link := c.Result.Diagnostic.GetCode().GetUrl(); link != "" {
		a.SetLink(link)
	}

	return *a
}

func (r *ReportAnnotator) createOrUpdateReport(ctx context.Context, redportID, title, reportStatus string) error {
	var report = openapi.NewReport()
	report.SetTitle(title)
	// TODO: different report types?
	report.SetReportType(reportTypeBug)
	report.SetReporter(reporter)
	report.SetLogoUrl(logoURL)
	report.SetResult(reportStatus)
	report.SetDetails("Woof-Woof! This report generated for you by reviewdog")

	_, resp, err := r.cli.ReportsApi.CreateOrUpdateReport(
		ctx, r.owner, r.repo, r.sha, redportID,
	).Body(*report).Execute()

	if err := checkAPIError(err, resp, http.StatusOK); err != nil {
		return fmt.Errorf("bitbucket.CreateOrUpdateReport: %s", err)
	}

	return nil
}

func hashString(str string) string {
	h := sha256.New()
	_, _ = h.Write([]byte(str))
	return fmt.Sprintf("%x", h.Sum(nil))
}

func reportID(ids ...string) string {
	return strings.ReplaceAll(strings.ToLower(strings.Join(ids, "-")), " ", "_")
}
<commit_msg>remove severity counter<commit_after>package bitbucket

import (
	"context"
	"crypto/sha256"
	"fmt"
	"net/http"
	"path/filepath"
	"strings"
	"sync"

	"github.com/reviewdog/reviewdog"
	"github.com/reviewdog/reviewdog/proto/rdf"
	"github.com/reviewdog/reviewdog/service/bitbucket/openapi"
)

var _ reviewdog.CommentService = &ReportAnnotator{}

const (
	// avatar from https://github.com/apps/reviewdog
	logoURL  = "https://avatars1.githubusercontent.com/in/12131"
	reporter = "reviewdog"
)

// ReportAnnotator is a comment service for Bitbucket Code Insights reports.
//
// API:
//  https://developer.atlassian.com/bitbucket/api/2/reference/resource/repositories/%7Bworkspace%7D/%7Brepo_slug%7D/commit/%7Bcommit%7D/reports/%7BreportId%7D/annotations#post
//  POST /2.0/repositories/{username}/{repo_slug}/commit/{commit}/reports/{reportId}/annotations
type ReportAnnotator struct {
	cli         *openapi.APIClient
	sha         string
	owner, repo string

	muAnnotations sync.Mutex
	annotations   map[string][]openapi.ReportAnnotation
	severityMap   map[rdf.Severity]string

	// wd is working directory relative to root of repository.
	wd string
}

// NewReportAnnotator creates new Bitbucket Report Annotator
func NewReportAnnotator(cli *openapi.APIClient, reportTitle, owner, repo, sha string) *ReportAnnotator {
	return &ReportAnnotator{
		cli:         cli,
		sha:         sha,
		owner:       owner,
		repo:        repo,
		annotations: make(map[string][]openapi.ReportAnnotation),
		severityMap: map[rdf.Severity]string{
			rdf.Severity_INFO:    annotationSeverityLow,
			rdf.Severity_WARNING: annotationSeverityMedium,
			rdf.Severity_ERROR:   annotationSeverityHigh,
		},
	}
}

// Post accepts a comment and holds it. Flush method actually posts comments to
// Bitbucket in batch.
func (r *ReportAnnotator) Post(_ context.Context, c *reviewdog.Comment) error {
	c.Result.Diagnostic.GetLocation().Path = filepath.ToSlash(
		filepath.Join(r.wd, c.Result.Diagnostic.GetLocation().GetPath()))
	r.muAnnotations.Lock()
	defer r.muAnnotations.Unlock()

	r.annotations[c.ToolName] = append(r.annotations[c.ToolName], r.annotationFromReviewDogComment(*c))

	return nil
}

// Flush posts comments which has not been posted yet.
func (r *ReportAnnotator) Flush(ctx context.Context) error {
	r.muAnnotations.Lock()
	defer r.muAnnotations.Unlock()

	// create/update/annotate report per tool
	for tool, annotations := range r.annotations {
		reportID := reportID(reporter, tool)
		title := fmt.Sprintf("[%s] %s report", tool, reporter)
		if len(r.annotations) == 0 {
			// if no annotation, create Passed report
			if err := r.createOrUpdateReport(ctx, reportID, title, reportResultPassed); err != nil {
				return err
			}
			// and move one
			continue
		}

		// create report or update report first, with the failed status
		if err := r.createOrUpdateReport(ctx, reportID, title, reportResultFailed); err != nil {
			return err
		}

		// add annotations to the report
		_, resp, err := r.cli.ReportsApi.BulkCreateOrUpdateAnnotations(
			ctx, r.owner, r.repo, r.sha, reportID,
		).Body(annotations).Execute()

		if err := checkAPIError(err, resp, http.StatusOK); err != nil {
			return fmt.Errorf("bitbucket.BulkCreateOrUpdateAnnotations: %s", err)
		}
	}

	return nil
}

func (r *ReportAnnotator) annotationFromReviewDogComment(c reviewdog.Comment) openapi.ReportAnnotation {
	a := openapi.NewReportAnnotation()
	switch c.ToolName {
	// TODO: different type of annotation based on tool?
	default:
		a.SetAnnotationType(annotationTypeCodeSmell)
	}

	// hash the output of linter and use it as external id
	a.SetExternalId(hashString(c.Result.Diagnostic.OriginalOutput))
	a.SetSummary(c.Result.Diagnostic.GetMessage())
	a.SetDetails(fmt.Sprintf(`[%s] %s`, c.ToolName, c.Result.Diagnostic.GetMessage()))
	a.SetLine(c.Result.Diagnostic.GetLocation().GetRange().GetStart().GetLine())
	a.SetPath(c.Result.Diagnostic.GetLocation().GetPath())
	if v, ok := r.severityMap[c.Result.Diagnostic.GetSeverity()]; ok {
		a.SetSeverity(v)
	}
	if link := c.Result.Diagnostic.GetCode().GetUrl(); link != "" {
		a.SetLink(link)
	}

	return *a
}

func (r *ReportAnnotator) createOrUpdateReport(ctx context.Context, redportID, title, reportStatus string) error {
	var report = openapi.NewReport()
	report.SetTitle(title)
	// TODO: different report types?
	report.SetReportType(reportTypeBug)
	report.SetReporter(reporter)
	report.SetLogoUrl(logoURL)
	report.SetResult(reportStatus)
	report.SetDetails("Woof-Woof! This report generated for you by reviewdog")

	_, resp, err := r.cli.ReportsApi.CreateOrUpdateReport(
		ctx, r.owner, r.repo, r.sha, redportID,
	).Body(*report).Execute()

	if err := checkAPIError(err, resp, http.StatusOK); err != nil {
		return fmt.Errorf("bitbucket.CreateOrUpdateReport: %s", err)
	}

	return nil
}

func hashString(str string) string {
	h := sha256.New()
	_, _ = h.Write([]byte(str))
	return fmt.Sprintf("%x", h.Sum(nil))
}

func reportID(ids ...string) string {
	return strings.ReplaceAll(strings.ToLower(strings.Join(ids, "-")), " ", "_")
}
<|endoftext|>


<commit_before>package mongo

import (
	"errors"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/json-iterator/go"
	"github.com/robfig/cron"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/qiniu/log"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/reader"
	. "github.com/qiniu/logkit/reader/config"
	"github.com/qiniu/logkit/utils"
	. "github.com/qiniu/logkit/utils/models"
)

var (
	_ reader.DaemonReader = &Reader{}
	_ reader.StatsReader  = &Reader{}
	_ reader.Reader       = &Reader{}
)

func init() {
	reader.RegisterConstructor(ModeMongo, NewReader)
}

// CollectionFilter is just a typed map of strings of map[string]interface{}
type CollectionFilter map[string]interface{}

const (
	DefaultOffsetKey = "_id"
	EmptyOffsetKey   = ""
)

const (
	Release int32 = iota
	NoRelease
)

type Reader struct {
	meta *reader.Meta
	// Note: 原子操作，用于表示 reader 整体的运行状态
	status int32
	/*
		Note: 原子操作，用于表示获取数据的线程运行状态

		- StatusInit: 当前没有任务在执行
		- StatusRunning: 当前有任务正在执行
		- StatusStopping: 数据管道已经由上层关闭，执行中的任务完成时直接退出无需再处理
	*/
	routineStatus    int32
	isReleaseCluster int32 //Note: 原子操作，用于表示mgo session的连接池是否被释放

	stopChan chan struct{}
	readChan chan []byte //bson
	errChan  chan error

	stats     StatsInfo
	statsLock sync.RWMutex

	host              string
	database          string
	collection        string
	offsetkey         string
	readBatch         int // 每次读取的数据量
	collectionFilters map[string]CollectionFilter

	isLoop       bool
	loopDuration time.Duration
	execOnStart  bool
	Cron         *cron.Cron //定时任务
	session      *mgo.Session
	offset       interface{} //对于默认的offset_key: "_id", 是objectID作为offset，存储的表现形式是string，其他则是int64
	timeout      int
}

func NewReader(meta *reader.Meta, conf conf.MapConf) (reader.Reader, error) {
	readBatch, _ := conf.GetIntOr(KeyMongoReadBatch, 100)
	database, err := conf.GetString(KeyMongoDatabase)
	if err != nil {
		return nil, err
	}
	collection, err := conf.GetString(KeyMongoCollection)
	if err != nil {
		return nil, err
	}
	host, err := conf.GetPasswordEnvString(KeyMongoHost)
	if err != nil {
		return nil, err
	}
	offsetkey, _ := conf.GetStringOr(KeyMongoOffsetKey, EmptyOffsetKey)
	cronSched, _ := conf.GetStringOr(KeyMongoCron, "")
	execOnStart, _ := conf.GetBoolOr(KeyMongoExecOnstart, true)
	filters, _ := conf.GetStringOr(KeyMongoFilters, "")
	certfile, _ := conf.GetStringOr(KeyMongoCert, "")
	timeout, _ := conf.GetIntOr(KeyMongoTimeout, 30)

	keyOrObj, offset, err := meta.ReadOffset()
	if err != nil {
		log.Errorf("Runner[%v] %v -meta data is corrupted err: %v, omit meta data...", meta.RunnerName, meta.MetaFile(), err)
	}
	if keyOrObj != offsetkey {
		offset = 0
	}
	if certfile != "" {
		log.Warnf("Runner[%v] MongoDB reader does not support certfile Now", meta.RunnerName)
		//TODO mongo鉴权暂时不支持
	}
	r := &Reader{
		meta:              meta,
		status:            StatusInit,
		routineStatus:     StatusInit,
		isReleaseCluster:  Release,
		stopChan:          make(chan struct{}),
		readChan:          make(chan []byte),
		errChan:           make(chan error),
		host:              host,
		database:          database,
		collection:        collection,
		offsetkey:         offsetkey,
		readBatch:         readBatch, //这个参数目前没有用
		collectionFilters: map[string]CollectionFilter{},
		execOnStart:       execOnStart,
		Cron:              cron.New(),
		timeout:           timeout,
	}
	if offsetkey == DefaultOffsetKey {
		if bson.IsObjectIdHex(keyOrObj) {
			r.offset = bson.ObjectIdHex(keyOrObj)
		} else {
			r.offset = nil
		}
	} else {
		r.offset = offset
	}

	if filters != "" {
		if jerr := jsoniter.Unmarshal([]byte(filters), &r.collectionFilters); jerr != nil {
			return nil, errors.New("malformed collection_filters")
		}
	}
	if len(cronSched) > 0 {
		cronSched = strings.ToLower(cronSched)
		if strings.HasPrefix(cronSched, Loop) {
			r.isLoop = true
			r.loopDuration, err = reader.ParseLoopDuration(cronSched)
			if err != nil {
				log.Errorf("Runner[%v] %v %v", r.meta.RunnerName, r.Name(), err)
			}
			if r.loopDuration.Nanoseconds() <= 0 {
				r.loopDuration = 1 * time.Second
			}
		} else {
			err = r.Cron.AddFunc(cronSched, r.run)
			if err != nil {
				return nil, err
			}
			log.Infof("Runner[%v] %v Cron added with schedule <%v>", r.meta.RunnerName, r.Name(), cronSched)
		}
	}
	return r, nil
}

func (r *Reader) isStopping() bool {
	return atomic.LoadInt32(&r.status) == StatusStopping
}

func (r *Reader) hasStopped() bool {
	return atomic.LoadInt32(&r.status) == StatusStopped
}

func (r *Reader) Name() string {
	return "MongoReader<" + r.Source() + ">"
}

func (*Reader) SetMode(_ string, _ interface{}) error {
	return errors.New("MongoDB Reader does not support read mode")
}

func (r *Reader) setStatsError(err string) {
	r.statsLock.Lock()
	defer r.statsLock.Unlock()
	r.stats.LastError = err
}

func (r *Reader) sendError(err error) {
	if err == nil {
		return
	}
	defer func() {
		if rec := recover(); rec != nil {
			log.Errorf("Reader %q was panicked and recovered from %v", r.Name(), rec)
		}
	}()
	r.errChan <- err
}

func (r *Reader) Start() error {
	if r.isStopping() || r.hasStopped() {
		return errors.New("reader is stopping or has stopped")
	} else if !atomic.CompareAndSwapInt32(&r.status, StatusInit, StatusRunning) {
		log.Warnf("Runner[%v] %q daemon has already started and is running", r.meta.RunnerName, r.Name())
		return nil
	}

	if r.isLoop {
		go func() {
			ticker := time.NewTicker(r.loopDuration)
			defer ticker.Stop()
			for {
				r.run()

				select {
				case <-r.stopChan:
					atomic.StoreInt32(&r.status, StatusStopped)
					log.Infof("Runner[%v] %q daemon has stopped from running", r.meta.RunnerName, r.Name())
					return
				case <-ticker.C:
				}
			}
		}()

	} else {
		if r.execOnStart {
			go r.run()
		}
		r.Cron.Start()
	}
	log.Infof("Runner[%v] %q daemon has started", r.meta.RunnerName, r.Name())
	return nil
}

func (r *Reader) Source() string {
	return r.host + "_" + r.database + "_" + r.collection
}

func (r *Reader) ReadLine() (string, error) {
	timer := time.NewTimer(time.Second)
	defer timer.Stop()
	select {
	case data := <-r.readChan:
		return string(data), nil
	case err := <-r.errChan:
		return "", err
	case <-timer.C:
	}

	return "", nil
}

func (r *Reader) Status() StatsInfo {
	r.statsLock.RLock()
	defer r.statsLock.RUnlock()
	return r.stats
}

// SyncMeta 从队列取数据时同步队列，作用在于保证数据不重复
func (r *Reader) SyncMeta() {
	var key string
	var offset int64
	if r.offsetkey == DefaultOffsetKey {
		if id, ok := r.offset.(bson.ObjectId); ok {
			key = id.Hex()
		}
	} else {
		key = r.offsetkey
		if ofs, ok := r.offset.(int64); ok {
			offset = ofs
		} else if ofs, ok := r.offset.(int); ok {
			offset = int64(ofs)
		}
	}
	if err := r.meta.WriteOffset(key, offset); err != nil {
		log.Errorf("Runner[%v] %v SyncMeta error %v", r.meta.RunnerName, r.Name(), err)
	}
}

func (r *Reader) Close() (err error) {
	if !atomic.CompareAndSwapInt32(&r.status, StatusRunning, StatusStopping) {
		log.Warnf("Runner[%v] reader %q is not running, close operation ignored", r.meta.RunnerName, r.Name())
		return nil
	}
	log.Debugf("Runner[%v] %q daemon is stopping", r.meta.RunnerName, r.Name())
	close(r.stopChan)

	r.Cron.Stop()
	if r.session != nil {
		atomic.StoreInt32(&r.isReleaseCluster, Release)
		r.session.Close()
	}

	// 如果此时没有 routine 正在运行，则在此处关闭数据管道，否则由 routine 在退出时负责关闭
	if atomic.CompareAndSwapInt32(&r.routineStatus, StatusInit, StatusStopping) {
		close(r.readChan)
		close(r.errChan)
	}
	return
}

func (r *Reader) run() {
	// 未在准备状态（StatusInit）时无法执行此次任务
	if !atomic.CompareAndSwapInt32(&r.routineStatus, StatusInit, StatusRunning) {
		if r.isStopping() || r.hasStopped() {
			log.Warnf("Runner[%v] %q daemon has stopped, this task does not need to be executed and is skipped this time", r.meta.RunnerName, r.Name())
		} else {
			errMsg := fmt.Sprintf("Runner[%v] %q daemon is still working on last task, this task will not be executed and is skipped this time", r.meta.RunnerName, r.Name())
			log.Error(errMsg)
			if !r.isLoop {
				// 通知上层 Cron 执行间隔可能过短或任务执行时间过长
				r.sendError(errors.New(errMsg))
			}
		}
		return
	}
	defer func() {
		// 如果 reader 在 routine 运行时关闭，则需要此 routine 负责关闭数据管道
		if r.isStopping() || r.hasStopped() {
			if atomic.CompareAndSwapInt32(&r.routineStatus, StatusRunning, StatusStopping) {
				close(r.readChan)
				close(r.errChan)
			}
			return
		}
		atomic.StoreInt32(&r.routineStatus, StatusInit)
	}()

	// 如果执行失败，最多重试 10 次
	for i := 1; i <= 10; i++ {
		// 判断上层是否已经关闭，先判断 routineStatus 再判断 status 可以保证同时只有一个 r.run 会运行到此处
		if r.isStopping() || r.hasStopped() {
			log.Warnf("Runner[%v] %q daemon has stopped, task is interrupted", r.meta.RunnerName, r.Name())
			return
		}

		err := r.exec()
		if err == nil {
			log.Infof("Runner[%v] %q task has been successfully executed", r.meta.RunnerName, r.Name())
			return
		}

		log.Errorf("Runner[%v] %q task execution failed: %v ", r.meta.RunnerName, r.Name(), err)
		r.setStatsError(err.Error())
		r.sendError(err)

		if r.isLoop {
			return // 循环执行的任务上层逻辑已经等同重试
		}
		time.Sleep(3 * time.Second)
	}
	log.Warnf("Runner[%v] %q task execution failed and gave up after 10 tries", r.meta.RunnerName, r.Name())
}

func (r *Reader) catQuery(c string, lastID interface{}, mgoSession *mgo.Session) *mgo.Query {
	query := bson.M{}
	if f, ok := r.collectionFilters[c]; ok {
		query = bson.M(f)
	}

	if r.offsetkey == EmptyOffsetKey {
		return mgoSession.DB(r.database).C(c).Find(query)
	}
	if lastID != nil {
		query[r.offsetkey] = bson.M{"$gt": lastID}
	}
	return mgoSession.DB(r.database).C(c).Find(query).Sort(r.offsetkey)
}

func (r *Reader) exec() (err error) {
	if r.session == nil {
		r.session, err = utils.MongoDail(r.host, "", 0)
		if err != nil {
			return
		}
		atomic.StoreInt32(&r.isReleaseCluster, NoRelease)
		r.session.SetSocketTimeout(time.Second * time.Duration(r.timeout))
		r.session.SetSyncTimeout(time.Second * time.Duration(r.timeout))
	} else {
		err := r.session.Ping()
		if err != nil {
			r.session.Refresh()
			r.session.SetSocketTimeout(time.Second * time.Duration(r.timeout))
			r.session.SetSyncTimeout(time.Second * time.Duration(r.timeout))
		}
	}
	if atomic.LoadInt32(&r.isReleaseCluster) == Release {
		log.Warnf("Runner[%s] %s mongo session's cluster has released,this exec will be exited", r.meta.RunnerName, r.Name())
		return
	}
	iter := r.catQuery(r.collection, r.offset, r.session).Iter()

	var result bson.M
	for iter.Next(&result) {
		if r.isStopping() || r.hasStopped() {
			log.Warnf("Runner[%v] %q daemon has stopped, iteration is interrupted", r.meta.RunnerName, r.Name())
			return nil
		}
		if id, ok := result[r.offsetkey]; ok {
			r.offset = id
		}
		bytes, ierr := jsoniter.Marshal(result)
		if ierr != nil {
			log.Errorf("Runner[%v] %v json marshal inner error %v", r.meta.RunnerName, result, ierr)
		}
		r.readChan <- bytes
		result = bson.M{}
	}
	if err := iter.Err(); err != nil {
		return err
	}
	return nil
}
<commit_msg>[PDR-11924][feat(reader)]mongo过滤条件中增加对objectId的转化<commit_after>package mongo

import (
	"errors"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/json-iterator/go"
	"github.com/robfig/cron"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/qiniu/log"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/reader"
	. "github.com/qiniu/logkit/reader/config"
	"github.com/qiniu/logkit/utils"
	. "github.com/qiniu/logkit/utils/models"
)

var (
	_ reader.DaemonReader = &Reader{}
	_ reader.StatsReader  = &Reader{}
	_ reader.Reader       = &Reader{}
)

func init() {
	reader.RegisterConstructor(ModeMongo, NewReader)
}

// CollectionFilter is just a typed map of strings of map[string]interface{}
type CollectionFilter map[string]interface{}

const (
	DefaultOffsetKey = "_id"
	EmptyOffsetKey   = ""
)

const (
	Release int32 = iota
	NoRelease
)

type Reader struct {
	meta *reader.Meta
	// Note: 原子操作，用于表示 reader 整体的运行状态
	status int32
	/*
		Note: 原子操作，用于表示获取数据的线程运行状态

		- StatusInit: 当前没有任务在执行
		- StatusRunning: 当前有任务正在执行
		- StatusStopping: 数据管道已经由上层关闭，执行中的任务完成时直接退出无需再处理
	*/
	routineStatus    int32
	isReleaseCluster int32 //Note: 原子操作，用于表示mgo session的连接池是否被释放

	stopChan chan struct{}
	readChan chan []byte //bson
	errChan  chan error

	stats     StatsInfo
	statsLock sync.RWMutex

	host              string
	database          string
	collection        string
	offsetkey         string
	readBatch         int // 每次读取的数据量
	collectionFilters map[string]CollectionFilter

	isLoop       bool
	loopDuration time.Duration
	execOnStart  bool
	Cron         *cron.Cron //定时任务
	session      *mgo.Session
	offset       interface{} //对于默认的offset_key: "_id", 是objectID作为offset，存储的表现形式是string，其他则是int64
	timeout      int
}

func NewReader(meta *reader.Meta, conf conf.MapConf) (reader.Reader, error) {
	readBatch, _ := conf.GetIntOr(KeyMongoReadBatch, 100)
	database, err := conf.GetString(KeyMongoDatabase)
	if err != nil {
		return nil, err
	}
	collection, err := conf.GetString(KeyMongoCollection)
	if err != nil {
		return nil, err
	}
	host, err := conf.GetPasswordEnvString(KeyMongoHost)
	if err != nil {
		return nil, err
	}
	offsetkey, _ := conf.GetStringOr(KeyMongoOffsetKey, EmptyOffsetKey)
	cronSched, _ := conf.GetStringOr(KeyMongoCron, "")
	execOnStart, _ := conf.GetBoolOr(KeyMongoExecOnstart, true)
	filters, _ := conf.GetStringOr(KeyMongoFilters, "")
	certfile, _ := conf.GetStringOr(KeyMongoCert, "")
	timeout, _ := conf.GetIntOr(KeyMongoTimeout, 30)

	keyOrObj, offset, err := meta.ReadOffset()
	if err != nil {
		log.Errorf("Runner[%v] %v -meta data is corrupted err: %v, omit meta data...", meta.RunnerName, meta.MetaFile(), err)
	}
	if keyOrObj != offsetkey {
		offset = 0
	}
	if certfile != "" {
		log.Warnf("Runner[%v] MongoDB reader does not support certfile Now", meta.RunnerName)
		//TODO mongo鉴权暂时不支持
	}
	r := &Reader{
		meta:              meta,
		status:            StatusInit,
		routineStatus:     StatusInit,
		isReleaseCluster:  Release,
		stopChan:          make(chan struct{}),
		readChan:          make(chan []byte),
		errChan:           make(chan error),
		host:              host,
		database:          database,
		collection:        collection,
		offsetkey:         offsetkey,
		readBatch:         readBatch, //这个参数目前没有用
		collectionFilters: map[string]CollectionFilter{},
		execOnStart:       execOnStart,
		Cron:              cron.New(),
		timeout:           timeout,
	}
	if offsetkey == DefaultOffsetKey {
		if bson.IsObjectIdHex(keyOrObj) {
			r.offset = bson.ObjectIdHex(keyOrObj)
		} else {
			r.offset = nil
		}
	} else {
		r.offset = offset
	}

	if filters != "" {
		if jerr := jsoniter.Unmarshal([]byte(filters), &r.collectionFilters); jerr != nil {
			return nil, errors.New("malformed collection_filters")
		}
	}
	if len(cronSched) > 0 {
		cronSched = strings.ToLower(cronSched)
		if strings.HasPrefix(cronSched, Loop) {
			r.isLoop = true
			r.loopDuration, err = reader.ParseLoopDuration(cronSched)
			if err != nil {
				log.Errorf("Runner[%v] %v %v", r.meta.RunnerName, r.Name(), err)
			}
			if r.loopDuration.Nanoseconds() <= 0 {
				r.loopDuration = 1 * time.Second
			}
		} else {
			err = r.Cron.AddFunc(cronSched, r.run)
			if err != nil {
				return nil, err
			}
			log.Infof("Runner[%v] %v Cron added with schedule <%v>", r.meta.RunnerName, r.Name(), cronSched)
		}
	}
	return r, nil
}

func (r *Reader) isStopping() bool {
	return atomic.LoadInt32(&r.status) == StatusStopping
}

func (r *Reader) hasStopped() bool {
	return atomic.LoadInt32(&r.status) == StatusStopped
}

func (r *Reader) Name() string {
	return "MongoReader<" + r.Source() + ">"
}

func (*Reader) SetMode(_ string, _ interface{}) error {
	return errors.New("MongoDB Reader does not support read mode")
}

func (r *Reader) setStatsError(err string) {
	r.statsLock.Lock()
	defer r.statsLock.Unlock()
	r.stats.LastError = err
}

func (r *Reader) sendError(err error) {
	if err == nil {
		return
	}
	defer func() {
		if rec := recover(); rec != nil {
			log.Errorf("Reader %q was panicked and recovered from %v", r.Name(), rec)
		}
	}()
	r.errChan <- err
}

func (r *Reader) Start() error {
	if r.isStopping() || r.hasStopped() {
		return errors.New("reader is stopping or has stopped")
	} else if !atomic.CompareAndSwapInt32(&r.status, StatusInit, StatusRunning) {
		log.Warnf("Runner[%v] %q daemon has already started and is running", r.meta.RunnerName, r.Name())
		return nil
	}

	if r.isLoop {
		go func() {
			ticker := time.NewTicker(r.loopDuration)
			defer ticker.Stop()
			for {
				r.run()

				select {
				case <-r.stopChan:
					atomic.StoreInt32(&r.status, StatusStopped)
					log.Infof("Runner[%v] %q daemon has stopped from running", r.meta.RunnerName, r.Name())
					return
				case <-ticker.C:
				}
			}
		}()

	} else {
		if r.execOnStart {
			go r.run()
		}
		r.Cron.Start()
	}
	log.Infof("Runner[%v] %q daemon has started", r.meta.RunnerName, r.Name())
	return nil
}

func (r *Reader) Source() string {
	return r.host + "_" + r.database + "_" + r.collection
}

func (r *Reader) ReadLine() (string, error) {
	timer := time.NewTimer(time.Second)
	defer timer.Stop()
	select {
	case data := <-r.readChan:
		return string(data), nil
	case err := <-r.errChan:
		return "", err
	case <-timer.C:
	}

	return "", nil
}

func (r *Reader) Status() StatsInfo {
	r.statsLock.RLock()
	defer r.statsLock.RUnlock()
	return r.stats
}

// SyncMeta 从队列取数据时同步队列，作用在于保证数据不重复
func (r *Reader) SyncMeta() {
	var key string
	var offset int64
	if r.offsetkey == DefaultOffsetKey {
		if id, ok := r.offset.(bson.ObjectId); ok {
			key = id.Hex()
		}
	} else {
		key = r.offsetkey
		if ofs, ok := r.offset.(int64); ok {
			offset = ofs
		} else if ofs, ok := r.offset.(int); ok {
			offset = int64(ofs)
		}
	}
	if err := r.meta.WriteOffset(key, offset); err != nil {
		log.Errorf("Runner[%v] %v SyncMeta error %v", r.meta.RunnerName, r.Name(), err)
	}
}

func (r *Reader) Close() (err error) {
	if !atomic.CompareAndSwapInt32(&r.status, StatusRunning, StatusStopping) {
		log.Warnf("Runner[%v] reader %q is not running, close operation ignored", r.meta.RunnerName, r.Name())
		return nil
	}
	log.Debugf("Runner[%v] %q daemon is stopping", r.meta.RunnerName, r.Name())
	close(r.stopChan)

	r.Cron.Stop()
	if r.session != nil {
		atomic.StoreInt32(&r.isReleaseCluster, Release)
		r.session.Close()
	}

	// 如果此时没有 routine 正在运行，则在此处关闭数据管道，否则由 routine 在退出时负责关闭
	if atomic.CompareAndSwapInt32(&r.routineStatus, StatusInit, StatusStopping) {
		close(r.readChan)
		close(r.errChan)
	}
	return
}

func (r *Reader) run() {
	// 未在准备状态（StatusInit）时无法执行此次任务
	if !atomic.CompareAndSwapInt32(&r.routineStatus, StatusInit, StatusRunning) {
		if r.isStopping() || r.hasStopped() {
			log.Warnf("Runner[%v] %q daemon has stopped, this task does not need to be executed and is skipped this time", r.meta.RunnerName, r.Name())
		} else {
			errMsg := fmt.Sprintf("Runner[%v] %q daemon is still working on last task, this task will not be executed and is skipped this time", r.meta.RunnerName, r.Name())
			log.Error(errMsg)
			if !r.isLoop {
				// 通知上层 Cron 执行间隔可能过短或任务执行时间过长
				r.sendError(errors.New(errMsg))
			}
		}
		return
	}
	defer func() {
		// 如果 reader 在 routine 运行时关闭，则需要此 routine 负责关闭数据管道
		if r.isStopping() || r.hasStopped() {
			if atomic.CompareAndSwapInt32(&r.routineStatus, StatusRunning, StatusStopping) {
				close(r.readChan)
				close(r.errChan)
			}
			return
		}
		atomic.StoreInt32(&r.routineStatus, StatusInit)
	}()

	// 如果执行失败，最多重试 10 次
	for i := 1; i <= 10; i++ {
		// 判断上层是否已经关闭，先判断 routineStatus 再判断 status 可以保证同时只有一个 r.run 会运行到此处
		if r.isStopping() || r.hasStopped() {
			log.Warnf("Runner[%v] %q daemon has stopped, task is interrupted", r.meta.RunnerName, r.Name())
			return
		}

		err := r.exec()
		if err == nil {
			log.Infof("Runner[%v] %q task has been successfully executed", r.meta.RunnerName, r.Name())
			return
		}

		log.Errorf("Runner[%v] %q task execution failed: %v ", r.meta.RunnerName, r.Name(), err)
		r.setStatsError(err.Error())
		r.sendError(err)

		if r.isLoop {
			return // 循环执行的任务上层逻辑已经等同重试
		}
		time.Sleep(3 * time.Second)
	}
	log.Warnf("Runner[%v] %q task execution failed and gave up after 10 tries", r.meta.RunnerName, r.Name())
}

func (r *Reader) catQuery(c string, lastID interface{}, mgoSession *mgo.Session) *mgo.Query {
	query := bson.M{}
	if f, ok := r.collectionFilters[c]; ok {
		query = bson.M(f)
	}

	if r.offsetkey == EmptyOffsetKey {
		return mgoSession.DB(r.database).C(c).Find(query)
	}
	if lastID != nil {
		query[r.offsetkey] = bson.M{"$gt": lastID}
	}
	for key, value := range query {
		if key == DefaultOffsetKey {
			if v, ok := value.(map[string]interface{}); ok {
				for opt, val := range v {
					if bson.IsObjectIdHex(val.(string)) {
						v[opt] = bson.ObjectIdHex(val.(string))
					}
				}
				query[key] = v
			}
			break
		}
	}
	return mgoSession.DB(r.database).C(c).Find(query).Sort(r.offsetkey)
}

func (r *Reader) exec() (err error) {
	if r.session == nil {
		r.session, err = utils.MongoDail(r.host, "", 0)
		if err != nil {
			return
		}
		atomic.StoreInt32(&r.isReleaseCluster, NoRelease)
		r.session.SetSocketTimeout(time.Second * time.Duration(r.timeout))
		r.session.SetSyncTimeout(time.Second * time.Duration(r.timeout))
	} else {
		err := r.session.Ping()
		if err != nil {
			r.session.Refresh()
			r.session.SetSocketTimeout(time.Second * time.Duration(r.timeout))
			r.session.SetSyncTimeout(time.Second * time.Duration(r.timeout))
		}
	}
	if atomic.LoadInt32(&r.isReleaseCluster) == Release {
		log.Warnf("Runner[%s] %s mongo session's cluster has released,this exec will be exited", r.meta.RunnerName, r.Name())
		return
	}
	iter := r.catQuery(r.collection, r.offset, r.session).Iter()

	var result bson.M
	for iter.Next(&result) {
		if r.isStopping() || r.hasStopped() {
			log.Warnf("Runner[%v] %q daemon has stopped, iteration is interrupted", r.meta.RunnerName, r.Name())
			return nil
		}
		if id, ok := result[r.offsetkey]; ok {
			r.offset = id
		}
		bytes, ierr := jsoniter.Marshal(result)
		if ierr != nil {
			log.Errorf("Runner[%v] %v json marshal inner error %v", r.meta.RunnerName, result, ierr)
		}
		r.readChan <- bytes
		result = bson.M{}
	}
	if err := iter.Err(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package sort

import(
  "reflect"
)

//This is a custom Merge Sort with parallel compution in 8 threads.
//You can define the comparion regulations all by yourself.
//
//you should write your own compare function with definition
//" func(i,j interface{})bool "
//then MergeSort() will launch Merge Sort with the regulation you define
//(*you can use i.(YourInputType) to convert i to original type)
//
//the output will be a slice of interface{},you should use o.(YourInputType) to convert o to origibal type
//
//you should use runtime.GOMAXPROS(int) to set the maximum number of CPUs that can be executing simultaneously.
//
//The sort is not guaranteed to be stable.
func MergeSortPara(data interface{}, cmp func(i,j interface{})bool)[]interface{}{
  //data : interface{} to []interface
  value:=reflect.ValueOf(data)
  N:=make([]interface{},value.Len())
  for a:=0;a<value.Len();a++{
    N[a]=value.Index(a).Interface()
  }
  ch:=make(chan bool)
  go msHandlePara8(N,cmp,ch,0)
  <-ch
  return N
}

func msHandlePara8(data []interface{}, cmp func(i,j interface{})bool, chparent chan bool,count int){
  M:=len(data)/2
  if len(data)==1 {
    chparent<-true
    return
  }
  if count<4 {
    chchild1:=make(chan bool)
    chchild2:=make(chan bool)
    go msHandlePara8(data[:M],cmp,chchild1,count+1)
    go msHandlePara8(data[M:],cmp,chchild2,count+1)
    <-chchild1
    <-chchild2
  }else{
    msHandle(data[:M],cmp)
    msHandle(data[M:],cmp)
  }
  merge(data,M,cmp)
  chparent<-true
}
<commit_msg>add MergeSortPara()<commit_after>package sort

import(
  "reflect"
)

//This is a custom Merge Sort with parallel compution in 8 threads.
//You can define the comparion regulations all by yourself.
//
//you should write your own compare function with definition
//" func(i,j interface{})bool "
//then MergeSort() will launch Merge Sort with the regulation you define
//(*you can use i.(YourInputType) to convert i to original type)
//
//the output will be a slice of interface{},you should use o.(YourInputType) to convert o to origibal type
//
//you should use runtime.GOMAXPROS(int) to set the maximum number of CPUs that can be executing simultaneously.
func MergeSortPara(data interface{}, cmp func(i,j interface{})bool)[]interface{}{
  //data : interface{} to []interface
  value:=reflect.ValueOf(data)
  N:=make([]interface{},value.Len())
  for a:=0;a<value.Len();a++{
    N[a]=value.Index(a).Interface()
  }
  ch:=make(chan bool)
  go msHandlePara8(N,cmp,ch,0)
  <-ch
  return N
}

func msHandlePara8(data []interface{}, cmp func(i,j interface{})bool, chparent chan bool,count int){
  M:=len(data)/2
  if len(data)==1 {
    chparent<-true
    return
  }
  if count<4 {
    chchild1:=make(chan bool)
    chchild2:=make(chan bool)
    go msHandlePara8(data[:M],cmp,chchild1,count+1)
    go msHandlePara8(data[M:],cmp,chchild2,count+1)
    <-chchild1
    <-chchild2
  }else{
    msHandle(data[:M],cmp)
    msHandle(data[M:],cmp)
  }
  merge(data,M,cmp)
  chparent<-true
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
)

var (
	service = flag.String("service", "", "Name of the service")
	required = flag.String("required", "", "Comma-separated list of required parameters (name:type)")
	options = flag.String("options", "", "Comma-separated list of options (name:type)")
	call = flag.String("call", "", "Call name")
	method = flag.String("method", "GET", "HTTP method (GET or POST)")
	ret = flag.String("ret", "interface{}", "Return type")
	endpoint = flag.String("endpoint", "", "Twitter RESP API endpoint")
)

func camel(s string) string {
	var news string
	for _, w := range strings.Split(s, "_") {
		news += strings.ToUpper(w[0:1]) + w[1:]
	}
	return news
}

func printComment() {
	fmt.Println("// Automatically generated")
	fmt.Printf("// %s\n", strings.Join(os.Args, " "))
}

func optSnippets() (funcs string, assigns string) {
	for _, opt := range strings.Split(*options, ",") {
		optName := strings.Split(opt, ":")[0]
		optType := strings.Split(opt, ":")[1]
		funcs += `
func (c *`+*service+*call+`Call) `+camel(optName)+`(`+optName+` `+optType+`) *`+*service+*call+`Call {
	c.opt_["`+optName+`"] = `+optName+`
	return c
}
`
		assigns += `
	if v, ok := c.opt_["`+optName+`"]; ok {
		params.Set("`+optName+`", fmt.Sprintf("%v", v))
	}
`

	}
	return
}

func printType() {

	fmt.Println(`
type `+*service+*call+`Call struct {
	s    *Service`)
	if *required != "" {
	for _, r := range strings.Split(*required, ",") {
		rName := strings.Split(r, ":")[0]
		rType := strings.Split(r, ":")[1]
		fmt.Printf("\t%s %s\n", rName, rType)
	}}
	fmt.Println("\topt_ map[string]interface{}\n}\n\n")
}

func printCall() {
	var assigns string
	if *required != "" {
	for _, r := range strings.Split(*required, ",") {
		rName := strings.Split(r, ":")[0]
		assigns += fmt.Sprintf("\tc.%s = %s\n", rName, rName)
	}}
	pars := strings.Replace(*required, ",", " ", -1)
	fmt.Println(`func (r *`+*service+`Service) `+*call+`(`+pars+`) *`+*service+*call+`Call {
	c := &`+*service+*call+`Call{s: r.s, opt_: make(map[string]interface{})}`)
	fmt.Printf("%s\treturn c\n}\n", assigns)
}

func printDo(assigns string) {
	fmt.Print(`
func (c *`+*service+*call+`Call) Do() (*`+*ret+`, os.Error) {
	var body io.Reader = nil
	params := make(url.Values)
`)

	if *required != "" {
		for _, r := range strings.Split(*required, ",") {
			rName := strings.Split(r, ":")[0]
			fmt.Printf("\tparams.Set(\"%s\", fmt.Sprintf(\"%%v\", c.%s))\n", rName, rName)
		}
	}

	fmt.Print(assigns)

	if *method == "POST" {
		fmt.Println(`
	urls := fmt.Sprintf("%s/%s.json", apiURL, "`+*endpoint+`")
	urls += "?" + params.Encode()
	body = bytes.NewBuffer([]byte(params.Encode()))
	ctype := "application/x-www-form-urlencoded"
	req, _ := http.NewRequest("POST", urls, body)
	req.Header.Set("Content-Type", ctype)
	res, err := c.s.client.Do(req)`)
	} else {
		fmt.Println(`
	urls := fmt.Sprintf("%s/%s.json", apiURL, "`+*endpoint+`")
	urls += "?" + params.Encode()
	req, _ := http.NewRequest("GET", urls, body)
	res, err := c.s.client.Do(req)`)

	fmt.Println(`
	if err != nil {
		return nil, err
	}
	if err := checkResponse(res); err != nil {
		return nil, err
	}
	ret := new(`+*ret+`)
	if err := json.NewDecoder(res.Body).Decode(ret); err != nil {
		return ret, err
	}
	return ret, nil
}`)
}

}

func errorReq(s string) {
	fmt.Printf("Parameter '%s' is required.\n", s)
	flag.PrintDefaults()
}

func main() {

	flag.Parse()

	if *service == "" {
		errorReq("service")
		return
	}
	if *call == "" {
		errorReq("call")
		return
	}
	if *method != "GET" && *method != "POST" {
		errorReq("method (GET or POST)")
		return
	}

	funcs, assigns := optSnippets()

	printComment()
	printType()
	printCall()
	fmt.Println(funcs)
	printDo(assigns)
}
<commit_msg>Fixed bug missing error checking on GET calls<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
)

var (
	service = flag.String("service", "", "Name of the service")
	required = flag.String("required", "", "Comma-separated list of required parameters (name:type)")
	options = flag.String("options", "", "Comma-separated list of options (name:type)")
	call = flag.String("call", "", "Call name")
	method = flag.String("method", "GET", "HTTP method (GET or POST)")
	ret = flag.String("ret", "interface{}", "Return type")
	endpoint = flag.String("endpoint", "", "Twitter RESP API endpoint")
)

func camel(s string) string {
	var news string
	for _, w := range strings.Split(s, "_") {
		news += strings.ToUpper(w[0:1]) + w[1:]
	}
	return news
}

func printComment() {
	fmt.Println("// Automatically generated")
	fmt.Printf("// %s\n", strings.Join(os.Args, " "))
}

func optSnippets() (funcs string, assigns string) {
	for _, opt := range strings.Split(*options, ",") {
		optName := strings.Split(opt, ":")[0]
		optType := strings.Split(opt, ":")[1]
		funcs += `
func (c *`+*service+*call+`Call) `+camel(optName)+`(`+optName+` `+optType+`) *`+*service+*call+`Call {
	c.opt_["`+optName+`"] = `+optName+`
	return c
}
`
		assigns += `
	if v, ok := c.opt_["`+optName+`"]; ok {
		params.Set("`+optName+`", fmt.Sprintf("%v", v))
	}
`

	}
	return
}

func printType() {

	fmt.Println(`
type `+*service+*call+`Call struct {
	s    *Service`)
	if *required != "" {
	for _, r := range strings.Split(*required, ",") {
		rName := strings.Split(r, ":")[0]
		rType := strings.Split(r, ":")[1]
		fmt.Printf("\t%s %s\n", rName, rType)
	}}
	fmt.Println("\topt_ map[string]interface{}\n}\n\n")
}

func printCall() {
	var assigns string
	if *required != "" {
	for _, r := range strings.Split(*required, ",") {
		rName := strings.Split(r, ":")[0]
		assigns += fmt.Sprintf("\tc.%s = %s\n", rName, rName)
	}}
	pars := strings.Replace(*required, ",", " ", -1)
	fmt.Println(`func (r *`+*service+`Service) `+*call+`(`+pars+`) *`+*service+*call+`Call {
	c := &`+*service+*call+`Call{s: r.s, opt_: make(map[string]interface{})}`)
	fmt.Printf("%s\treturn c\n}\n", assigns)
}

func printDo(assigns string) {
	fmt.Print(`
func (c *`+*service+*call+`Call) Do() (*`+*ret+`, os.Error) {
	var body io.Reader = nil
	params := make(url.Values)
`)

	if *required != "" {
		for _, r := range strings.Split(*required, ",") {
			rName := strings.Split(r, ":")[0]
			fmt.Printf("\tparams.Set(\"%s\", fmt.Sprintf(\"%%v\", c.%s))\n", rName, rName)
		}
	}

	fmt.Print(assigns)

	if *method == "POST" {
		fmt.Println(`
	urls := fmt.Sprintf("%s/%s.json", apiURL, "`+*endpoint+`")
	urls += "?" + params.Encode()
	body = bytes.NewBuffer([]byte(params.Encode()))
	ctype := "application/x-www-form-urlencoded"
	req, _ := http.NewRequest("POST", urls, body)
	req.Header.Set("Content-Type", ctype)
	res, err := c.s.client.Do(req)`)
	} else {
		fmt.Println(`
	urls := fmt.Sprintf("%s/%s.json", apiURL, "`+*endpoint+`")
	urls += "?" + params.Encode()
	req, _ := http.NewRequest("GET", urls, body)
	res, err := c.s.client.Do(req)`)
	}
	fmt.Println(`
	if err != nil {
		return nil, err
	}
	if err := checkResponse(res); err != nil {
		return nil, err
	}
	ret := new(`+*ret+`)
	if err := json.NewDecoder(res.Body).Decode(ret); err != nil {
		return ret, err
	}
	return ret, nil
}`)

}

func errorReq(s string) {
	fmt.Printf("Parameter '%s' is required.\n", s)
	flag.PrintDefaults()
}

func main() {

	flag.Parse()

	if *service == "" {
		errorReq("service")
		return
	}
	if *call == "" {
		errorReq("call")
		return
	}
	if *method != "GET" && *method != "POST" {
		errorReq("method (GET or POST)")
		return
	}

	funcs, assigns := optSnippets()

	printComment()
	printType()
	printCall()
	fmt.Println(funcs)
	printDo(assigns)
}
<|endoftext|>


<commit_before>package discovery

import (
	"errors"
	"log"
)

type Watcher struct {
	service string
	nodes   []Node

	newNodes chan []Node
	nextNode chan Node

	nodesManager NodesManager
}

type Node struct {
	Uri string
	Err error
}

var emptyNode = Node{
	Err: errors.New("No nodes available"),
}

type NodesManager interface {
	NewNodes(service string) ([]string, error)
}

func NewWatcher(service string, nextNode chan Node, nodesManager NodesManager) *Watcher {
	return &Watcher{
		service:      service,
		newNodes:     make(chan []Node),
		nextNode:     nextNode,
		nodesManager: nodesManager,
	}
}

func (w *Watcher) Start() {
	go w.watchNodes()
	go w.run()
}

func (w *Watcher) run() {
	nextNode := emptyNode

	for {
		select {
		case newNodes := <-w.newNodes:
			log.Println("Received nodes update:", newNodes)
			w.nodes = newNodes
			nextNode = w.getNextNode()
		default:
			select {
			case w.nextNode <- nextNode:
				nextNode = w.getNextNode()
			default:
			}
		}
	}
}

func (w *Watcher) watchNodes() {
	for {
		nodes, err := w.nodesManager.NewNodes(w.service)
		if err != nil {
			log.Println("Nodes manager error:", err.Error())
			continue
		}

		var newNodes []Node
		for _, node := range nodes {
			newNodes = append(newNodes, Node{
				Uri: node,
			})
		}

		w.newNodes <- newNodes
	}
}

func (w *Watcher) getNextNode() Node {
	if len(w.nodes) == 0 {
		return emptyNode
	} else {
		node := w.nodes[0]

		// Cycle the nodes
		if len(w.nodes) > 1 {
			w.nodes = append(w.nodes[1:], node)
		}

		return node
	}
}
<commit_msg>Discovery/Watcher: Do initial nodes update before serving the main loop.<commit_after>package discovery

import (
	"errors"
	"log"
)

type Watcher struct {
	service string
	nodes   []Node

	newNodes chan []Node
	nextNode chan Node

	nodesManager NodesManager
}

type Node struct {
	Uri string
	Err error
}

var emptyNode = Node{
	Err: errors.New("No nodes available"),
}

type NodesManager interface {
	NewNodes(service string) ([]string, error)
}

func NewWatcher(service string, nextNode chan Node, nodesManager NodesManager) *Watcher {
	return &Watcher{
		service:      service,
		newNodes:     make(chan []Node),
		nextNode:     nextNode,
		nodesManager: nodesManager,
	}
}

func (w *Watcher) Start() {
	go w.watchNodes()
	go w.run()
}

func (w *Watcher) run() {
	w.nodes = <-w.newNodes
	nextNode := w.getNextNode()

	for {
		select {
		case newNodes := <-w.newNodes:
			log.Println("Received nodes update:", newNodes)
			w.nodes = newNodes
			nextNode = w.getNextNode()
		default:
			select {
			case w.nextNode <- nextNode:
				nextNode = w.getNextNode()
			default:
			}
		}
	}
}

func (w *Watcher) watchNodes() {
	for {
		nodes, err := w.nodesManager.NewNodes(w.service)
		if err != nil {
			log.Println("Nodes manager error:", err.Error())
			continue
		}

		var newNodes []Node
		for _, node := range nodes {
			newNodes = append(newNodes, Node{
				Uri: node,
			})
		}

		w.newNodes <- newNodes
	}
}

func (w *Watcher) getNextNode() Node {
	if len(w.nodes) == 0 {
		return emptyNode
	} else {
		node := w.nodes[0]

		// Cycle the nodes
		if len(w.nodes) > 1 {
			w.nodes = append(w.nodes[1:], node)
		}

		return node
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"errors"
	"fmt"
	"io"
	"time"

	"golang.org/x/net/context"
)

var errInboundRequestAlreadyActive = errors.New("inbound request is already active; possible duplicate client id")

// handleCallReq handles an incoming call request, registering a message
// exchange to receive further fragments for that call, and dispatching it in
// another goroutine
func (c *Connection) handleCallReq(frame *Frame) bool {
	switch state := c.readState(); state {
	case connectionActive:
		break
	case connectionStartClose, connectionInboundClosed, connectionClosed:
		c.SendSystemError(frame.Header.ID, nil, ErrChannelClosed)
		return true
	case connectionWaitingToRecvInitReq, connectionWaitingToSendInitReq, connectionWaitingToRecvInitRes:
		c.SendSystemError(frame.Header.ID, nil, NewSystemError(ErrCodeDeclined, "connection not ready"))
		return true
	default:
		panic(fmt.Errorf("unknown connection state for call req: %v", state))
	}

	callReq := new(callReq)
	initialFragment, err := parseInboundFragment(c.framePool, frame, callReq)
	if err != nil {
		// TODO(mmihic): Probably want to treat this as a protocol error
		c.log.Errorf("could not decode %s: %v", frame.Header, err)
		return true
	}

	c.log.Debugf("span=%s", callReq.Tracing)
	call := new(InboundCall)
	ctx, cancel := newIncomingContext(call, callReq.TimeToLive, &callReq.Tracing)

	mex, err := c.inbound.newExchange(ctx, c.framePool, callReq.messageType(), frame.Header.ID, 2)
	if err != nil {
		if err == errDuplicateMex {
			err = errInboundRequestAlreadyActive
		}
		c.log.Errorf("could not register exchange for %s", frame.Header)
		c.SendSystemError(frame.Header.ID, nil, err)
		return true
	}

	// Close may have been called between the time we checked the state and us creating the exchange.
	if c.readState() != connectionActive {
		mex.shutdown()
		return true
	}

	response := new(InboundCallResponse)
	response.Annotations = Annotations{
		reporter: c.traceReporter,
		span:     callReq.Tracing,
		endpoint: TargetEndpoint{
			HostPort:    c.localPeerInfo.HostPort,
			ServiceName: callReq.Service,
		},
		binaryAnnotations: []BinaryAnnotation{
			{Key: "cn", Value: callReq.Headers[CallerName]},
			{Key: "as", Value: callReq.Headers[ArgScheme]},
		},
	}
	response.AddAnnotation(AnnotationKeyServerReceive)
	response.mex = mex
	response.conn = c
	response.cancel = cancel
	response.span = callReq.Tracing
	response.log = c.log.WithFields(LogField{"In-Response", callReq.ID()})
	response.contents = newFragmentingWriter(response.log, response, initialFragment.checksumType.New())
	response.headers = transportHeaders{}
	response.messageForFragment = func(initial bool) message {
		if initial {
			callRes := new(callRes)
			callRes.Headers = response.headers
			callRes.ResponseCode = responseOK
			if response.applicationError {
				callRes.ResponseCode = responseApplicationError
			}
			return callRes
		}

		return new(callResContinue)
	}

	call.mex = mex
	call.initialFragment = initialFragment
	call.serviceName = string(callReq.Service)
	call.headers = callReq.Headers
	call.span = callReq.Tracing
	call.response = response
	call.log = c.log.WithFields(LogField{"In-Call", callReq.ID()})
	call.messageForFragment = func(initial bool) message { return new(callReqContinue) }
	call.contents = newFragmentingReader(call.log, call)
	call.statsReporter = c.statsReporter
	call.createStatsTags(c.commonStatsTags)

	response.statsReporter = c.statsReporter
	response.commonStatsTags = call.commonStatsTags

	setResponseHeaders(call.headers, response.headers)
	go c.dispatchInbound(c.connID, callReq.ID(), call)
	return false
}

// handleCallReqContinue handles the continuation of a call request, forwarding
// it to the request channel for that request, where it can be pulled during
// defragmentation
func (c *Connection) handleCallReqContinue(frame *Frame) bool {
	if err := c.inbound.forwardPeerFrame(frame); err != nil {
		c.inbound.removeExchange(frame.Header.ID)
		return true
	}
	return false
}

// createStatsTags creates the common stats tags, if they are not already created.
func (call *InboundCall) createStatsTags(connectionTags map[string]string) {
	call.commonStatsTags = map[string]string{
		"calling-service": call.CallerName(),
	}
	for k, v := range connectionTags {
		call.commonStatsTags[k] = v
	}
}

// dispatchInbound ispatches an inbound call to the appropriate handler
func (c *Connection) dispatchInbound(_ uint32, _ uint32, call *InboundCall) {
	c.log.Debugf("Received incoming call for %s from %s", call.ServiceName(), c.remotePeerInfo)

	if err := call.readOperation(); err != nil {
		c.log.Errorf("Could not read operation from %s: %v", c.remotePeerInfo, err)
		return
	}

	call.commonStatsTags["endpoint"] = string(call.operation)
	call.statsReporter.IncCounter("inbound.calls.recvd", call.commonStatsTags, 1)
	call.response.calledAt = timeNow()
	call.response.SetOperation(string(call.operation))

	// NB(mmihic): Don't cast operation name to string here - this will
	// create a copy of the byte array, where as aliasing to string in the
	// map look up can be optimized by the compiler to avoid the copy.  See
	// https://github.com/golang/go/issues/3512
	h := c.handlers.find(call.ServiceName(), call.Operation())
	if h == nil {
		// CHeck the subchannel map to see if we find one there
		c.log.Debugf("Checking the subchannel's handlers for %s:%s", call.ServiceName(), call.Operation())
		h = c.subchannels.find(call.ServiceName(), call.Operation())
	}
	if h == nil {
		c.log.Errorf("Could not find handler for %s:%s", call.ServiceName(), call.Operation())
		call.Response().SendSystemError(
			NewSystemError(ErrCodeBadRequest, "no handler for service %q and operation %q", call.ServiceName(), call.Operation()))
		return
	}

	// TODO(prashant): This is an expensive way to check for cancellation. Use a heap for timeouts.
	go func() {
		if <-call.mex.ctx.Done(); call.mex.ctx.Err() == context.DeadlineExceeded {
			call.mex.inboundTimeout()
		}
	}()

	c.log.Debugf("Dispatching %s:%s from %s", call.ServiceName(), call.Operation(), c.remotePeerInfo)
	h.Handle(call.mex.ctx, call)
}

// An InboundCall is an incoming call from a peer
type InboundCall struct {
	reqResReader

	response        *InboundCallResponse
	serviceName     string
	operation       []byte
	headers         transportHeaders
	span            Span
	statsReporter   StatsReporter
	commonStatsTags map[string]string
}

// ServiceName returns the name of the service being called
func (call *InboundCall) ServiceName() string {
	return call.serviceName
}

// Operation returns the operation being called
func (call *InboundCall) Operation() []byte {
	return call.operation
}

// Format the format of the request from the ArgScheme transport header.
func (call *InboundCall) Format() Format {
	return Format(call.headers[ArgScheme])
}

// CallerName returns the caller name from the CallerName transport header.
func (call *InboundCall) CallerName() string {
	return call.headers[CallerName]
}

// ShardKey returns the shard key from the ShardKey transport header.
func (call *InboundCall) ShardKey() string {
	return call.headers[ShardKey]
}

// Reads the entire operation name (arg1) from the request stream.
func (call *InboundCall) readOperation() error {
	var arg1 []byte
	if err := NewArgReader(call.arg1Reader()).Read(&arg1); err != nil {
		return call.failed(err)
	}

	call.operation = arg1
	return nil
}

// Arg2Reader returns an io.ReadCloser to read the second argument.
// The ReadCloser must be closed once the argument has been read.
func (call *InboundCall) Arg2Reader() (io.ReadCloser, error) {
	return call.arg2Reader()
}

// Arg3Reader returns an io.ReadCloser to read the last argument.
// The ReadCloser must be closed once the argument has been read.
func (call *InboundCall) Arg3Reader() (io.ReadCloser, error) {
	return call.arg3Reader()
}

// Response provides access to the InboundCallResponse object which can be used
// to write back to the calling peer
func (call *InboundCall) Response() *InboundCallResponse {
	return call.response
}

func (call *InboundCall) doneReading(unexpected error) {}

// An InboundCallResponse is used to send the response back to the calling peer
type InboundCallResponse struct {
	reqResWriter
	Annotations

	cancel context.CancelFunc
	// calledAt is the time the inbound call was routed to the application.
	calledAt         time.Time
	applicationError bool
	systemError      bool
	headers          transportHeaders
	span             Span
	statsReporter    StatsReporter
	commonStatsTags  map[string]string
}

// SendSystemError returns a system error response to the peer.  The call is considered
// complete after this method is called, and no further data can be written.
func (response *InboundCallResponse) SendSystemError(err error) error {
	// Fail all future attempts to read fragments
	response.cancel()
	response.state = reqResWriterComplete
	response.systemError = true
	response.doneSending()
	return response.conn.SendSystemError(response.mex.msgID, CurrentSpan(response.mex.ctx), err)
}

// SetApplicationError marks the response as being an application error.  This method can
// only be called before any arguments have been sent to the calling peer.
func (response *InboundCallResponse) SetApplicationError() error {
	if response.state > reqResWriterPreArg2 {
		return response.failed(errReqResWriterStateMismatch{
			state:         response.state,
			expectedState: reqResWriterPreArg2,
		})
	}
	response.applicationError = true
	return nil
}

// Arg2Writer returns a WriteCloser that can be used to write the second argument.
// The returned writer must be closed once the write is complete.
func (response *InboundCallResponse) Arg2Writer() (ArgWriter, error) {
	if err := NewArgWriter(response.arg1Writer()).Write(nil); err != nil {
		return nil, err
	}
	return response.arg2Writer()
}

// Arg3Writer returns a WriteCloser that can be used to write the last argument.
// The returned writer must be closed once the write is complete.
func (response *InboundCallResponse) Arg3Writer() (ArgWriter, error) {
	return response.arg3Writer()
}

// doneSending shuts down the message exchange for this call.
// For incoming calls, the last message is sending the call response.
func (response *InboundCallResponse) doneSending() {
	// TODO(prashant): Move this to when the message is actually being sent.
	response.AddAnnotation(AnnotationKeyServerSend)
	response.Report()

	latency := timeNow().Sub(response.calledAt)
	response.statsReporter.RecordTimer("inbound.calls.latency", response.commonStatsTags, latency)

	if response.systemError {
		// TODO(prashant): Report the error code type as per metrics doc and enable.
		// response.statsReporter.IncCounter("inbound.calls.system-errors", response.commonStatsTags, 1)
	} else if response.applicationError {
		response.statsReporter.IncCounter("inbound.calls.app-errors", response.commonStatsTags, 1)
	} else {
		response.statsReporter.IncCounter("inbound.calls.success", response.commonStatsTags, 1)
	}

	// The message exchange is still open if there are no errors, call shutdown.
	if response.err == nil {
		response.mex.shutdown()
	}
}
<commit_msg>Cancel context when done sending<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"errors"
	"fmt"
	"io"
	"time"

	"golang.org/x/net/context"
)

var errInboundRequestAlreadyActive = errors.New("inbound request is already active; possible duplicate client id")

// handleCallReq handles an incoming call request, registering a message
// exchange to receive further fragments for that call, and dispatching it in
// another goroutine
func (c *Connection) handleCallReq(frame *Frame) bool {
	switch state := c.readState(); state {
	case connectionActive:
		break
	case connectionStartClose, connectionInboundClosed, connectionClosed:
		c.SendSystemError(frame.Header.ID, nil, ErrChannelClosed)
		return true
	case connectionWaitingToRecvInitReq, connectionWaitingToSendInitReq, connectionWaitingToRecvInitRes:
		c.SendSystemError(frame.Header.ID, nil, NewSystemError(ErrCodeDeclined, "connection not ready"))
		return true
	default:
		panic(fmt.Errorf("unknown connection state for call req: %v", state))
	}

	callReq := new(callReq)
	initialFragment, err := parseInboundFragment(c.framePool, frame, callReq)
	if err != nil {
		// TODO(mmihic): Probably want to treat this as a protocol error
		c.log.Errorf("could not decode %s: %v", frame.Header, err)
		return true
	}

	c.log.Debugf("span=%s", callReq.Tracing)
	call := new(InboundCall)
	ctx, cancel := newIncomingContext(call, callReq.TimeToLive, &callReq.Tracing)

	mex, err := c.inbound.newExchange(ctx, c.framePool, callReq.messageType(), frame.Header.ID, 2)
	if err != nil {
		if err == errDuplicateMex {
			err = errInboundRequestAlreadyActive
		}
		c.log.Errorf("could not register exchange for %s", frame.Header)
		c.SendSystemError(frame.Header.ID, nil, err)
		return true
	}

	// Close may have been called between the time we checked the state and us creating the exchange.
	if c.readState() != connectionActive {
		mex.shutdown()
		return true
	}

	response := new(InboundCallResponse)
	response.Annotations = Annotations{
		reporter: c.traceReporter,
		span:     callReq.Tracing,
		endpoint: TargetEndpoint{
			HostPort:    c.localPeerInfo.HostPort,
			ServiceName: callReq.Service,
		},
		binaryAnnotations: []BinaryAnnotation{
			{Key: "cn", Value: callReq.Headers[CallerName]},
			{Key: "as", Value: callReq.Headers[ArgScheme]},
		},
	}
	response.AddAnnotation(AnnotationKeyServerReceive)
	response.mex = mex
	response.conn = c
	response.cancel = cancel
	response.span = callReq.Tracing
	response.log = c.log.WithFields(LogField{"In-Response", callReq.ID()})
	response.contents = newFragmentingWriter(response.log, response, initialFragment.checksumType.New())
	response.headers = transportHeaders{}
	response.messageForFragment = func(initial bool) message {
		if initial {
			callRes := new(callRes)
			callRes.Headers = response.headers
			callRes.ResponseCode = responseOK
			if response.applicationError {
				callRes.ResponseCode = responseApplicationError
			}
			return callRes
		}

		return new(callResContinue)
	}

	call.mex = mex
	call.initialFragment = initialFragment
	call.serviceName = string(callReq.Service)
	call.headers = callReq.Headers
	call.span = callReq.Tracing
	call.response = response
	call.log = c.log.WithFields(LogField{"In-Call", callReq.ID()})
	call.messageForFragment = func(initial bool) message { return new(callReqContinue) }
	call.contents = newFragmentingReader(call.log, call)
	call.statsReporter = c.statsReporter
	call.createStatsTags(c.commonStatsTags)

	response.statsReporter = c.statsReporter
	response.commonStatsTags = call.commonStatsTags

	setResponseHeaders(call.headers, response.headers)
	go c.dispatchInbound(c.connID, callReq.ID(), call)
	return false
}

// handleCallReqContinue handles the continuation of a call request, forwarding
// it to the request channel for that request, where it can be pulled during
// defragmentation
func (c *Connection) handleCallReqContinue(frame *Frame) bool {
	if err := c.inbound.forwardPeerFrame(frame); err != nil {
		c.inbound.removeExchange(frame.Header.ID)
		return true
	}
	return false
}

// createStatsTags creates the common stats tags, if they are not already created.
func (call *InboundCall) createStatsTags(connectionTags map[string]string) {
	call.commonStatsTags = map[string]string{
		"calling-service": call.CallerName(),
	}
	for k, v := range connectionTags {
		call.commonStatsTags[k] = v
	}
}

// dispatchInbound ispatches an inbound call to the appropriate handler
func (c *Connection) dispatchInbound(_ uint32, _ uint32, call *InboundCall) {
	c.log.Debugf("Received incoming call for %s from %s", call.ServiceName(), c.remotePeerInfo)

	if err := call.readOperation(); err != nil {
		c.log.Errorf("Could not read operation from %s: %v", c.remotePeerInfo, err)
		return
	}

	call.commonStatsTags["endpoint"] = string(call.operation)
	call.statsReporter.IncCounter("inbound.calls.recvd", call.commonStatsTags, 1)
	call.response.calledAt = timeNow()
	call.response.SetOperation(string(call.operation))

	// NB(mmihic): Don't cast operation name to string here - this will
	// create a copy of the byte array, where as aliasing to string in the
	// map look up can be optimized by the compiler to avoid the copy.  See
	// https://github.com/golang/go/issues/3512
	h := c.handlers.find(call.ServiceName(), call.Operation())
	if h == nil {
		// CHeck the subchannel map to see if we find one there
		c.log.Debugf("Checking the subchannel's handlers for %s:%s", call.ServiceName(), call.Operation())
		h = c.subchannels.find(call.ServiceName(), call.Operation())
	}
	if h == nil {
		c.log.Errorf("Could not find handler for %s:%s", call.ServiceName(), call.Operation())
		call.Response().SendSystemError(
			NewSystemError(ErrCodeBadRequest, "no handler for service %q and operation %q", call.ServiceName(), call.Operation()))
		return
	}

	// TODO(prashant): This is an expensive way to check for cancellation. Use a heap for timeouts.
	go func() {
		if <-call.mex.ctx.Done(); call.mex.ctx.Err() == context.DeadlineExceeded {
			call.mex.inboundTimeout()
		}
	}()

	c.log.Debugf("Dispatching %s:%s from %s", call.ServiceName(), call.Operation(), c.remotePeerInfo)
	h.Handle(call.mex.ctx, call)
}

// An InboundCall is an incoming call from a peer
type InboundCall struct {
	reqResReader

	response        *InboundCallResponse
	serviceName     string
	operation       []byte
	headers         transportHeaders
	span            Span
	statsReporter   StatsReporter
	commonStatsTags map[string]string
}

// ServiceName returns the name of the service being called
func (call *InboundCall) ServiceName() string {
	return call.serviceName
}

// Operation returns the operation being called
func (call *InboundCall) Operation() []byte {
	return call.operation
}

// Format the format of the request from the ArgScheme transport header.
func (call *InboundCall) Format() Format {
	return Format(call.headers[ArgScheme])
}

// CallerName returns the caller name from the CallerName transport header.
func (call *InboundCall) CallerName() string {
	return call.headers[CallerName]
}

// ShardKey returns the shard key from the ShardKey transport header.
func (call *InboundCall) ShardKey() string {
	return call.headers[ShardKey]
}

// Reads the entire operation name (arg1) from the request stream.
func (call *InboundCall) readOperation() error {
	var arg1 []byte
	if err := NewArgReader(call.arg1Reader()).Read(&arg1); err != nil {
		return call.failed(err)
	}

	call.operation = arg1
	return nil
}

// Arg2Reader returns an io.ReadCloser to read the second argument.
// The ReadCloser must be closed once the argument has been read.
func (call *InboundCall) Arg2Reader() (io.ReadCloser, error) {
	return call.arg2Reader()
}

// Arg3Reader returns an io.ReadCloser to read the last argument.
// The ReadCloser must be closed once the argument has been read.
func (call *InboundCall) Arg3Reader() (io.ReadCloser, error) {
	return call.arg3Reader()
}

// Response provides access to the InboundCallResponse object which can be used
// to write back to the calling peer
func (call *InboundCall) Response() *InboundCallResponse {
	return call.response
}

func (call *InboundCall) doneReading(unexpected error) {}

// An InboundCallResponse is used to send the response back to the calling peer
type InboundCallResponse struct {
	reqResWriter
	Annotations

	cancel context.CancelFunc
	// calledAt is the time the inbound call was routed to the application.
	calledAt         time.Time
	applicationError bool
	systemError      bool
	headers          transportHeaders
	span             Span
	statsReporter    StatsReporter
	commonStatsTags  map[string]string
}

// SendSystemError returns a system error response to the peer.  The call is considered
// complete after this method is called, and no further data can be written.
func (response *InboundCallResponse) SendSystemError(err error) error {
	// Fail all future attempts to read fragments
	response.state = reqResWriterComplete
	response.systemError = true
	response.doneSending()
	return response.conn.SendSystemError(response.mex.msgID, CurrentSpan(response.mex.ctx), err)
}

// SetApplicationError marks the response as being an application error.  This method can
// only be called before any arguments have been sent to the calling peer.
func (response *InboundCallResponse) SetApplicationError() error {
	if response.state > reqResWriterPreArg2 {
		return response.failed(errReqResWriterStateMismatch{
			state:         response.state,
			expectedState: reqResWriterPreArg2,
		})
	}
	response.applicationError = true
	return nil
}

// Arg2Writer returns a WriteCloser that can be used to write the second argument.
// The returned writer must be closed once the write is complete.
func (response *InboundCallResponse) Arg2Writer() (ArgWriter, error) {
	if err := NewArgWriter(response.arg1Writer()).Write(nil); err != nil {
		return nil, err
	}
	return response.arg2Writer()
}

// Arg3Writer returns a WriteCloser that can be used to write the last argument.
// The returned writer must be closed once the write is complete.
func (response *InboundCallResponse) Arg3Writer() (ArgWriter, error) {
	return response.arg3Writer()
}

// doneSending shuts down the message exchange for this call.
// For incoming calls, the last message is sending the call response.
func (response *InboundCallResponse) doneSending() {
	// TODO(prashant): Move this to when the message is actually being sent.
	response.AddAnnotation(AnnotationKeyServerSend)
	response.Report()

	latency := timeNow().Sub(response.calledAt)
	response.statsReporter.RecordTimer("inbound.calls.latency", response.commonStatsTags, latency)

	if response.systemError {
		// TODO(prashant): Report the error code type as per metrics doc and enable.
		// response.statsReporter.IncCounter("inbound.calls.system-errors", response.commonStatsTags, 1)
	} else if response.applicationError {
		response.statsReporter.IncCounter("inbound.calls.app-errors", response.commonStatsTags, 1)
	} else {
		response.statsReporter.IncCounter("inbound.calls.success", response.commonStatsTags, 1)
	}

	// Cancel the context since the response is complete.
	response.cancel()

	// The message exchange is still open if there are no errors, call shutdown.
	if response.err == nil {
		response.mex.shutdown()
	}
}
<|endoftext|>