<commit_before>package paasio

import (
	"bytes"
	"crypto/rand"
	"io"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"
)

// testVersion identifies the API tested by the test program.
const targetTestVersion = 3

func TestMultiThreaded(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Errorf("Found testVersion = %v, want %v.", testVersion, targetTestVersion)
	}
	mincpu := 2
	minproc := 2
	ncpu := runtime.NumCPU()
	if ncpu < mincpu {
		t.Fatalf("at least %d cpu cores are required", mincpu)
	}
	nproc := runtime.GOMAXPROCS(0)
	if nproc < minproc {
		t.Errorf("at least %d threads are required; rerun the tests", minproc)
		t.Errorf("")
		t.Errorf("\tgo test -cpu %d ...", minproc)
	}
}

// this test could be improved to test that error conditions are preserved.
func testWrite(t *testing.T, writer func(io.Writer) WriteCounter) {
	for i, test := range []struct {
		writes []string
	}{
		{nil},
		{[]string{""}},
		{[]string{"I", " ", "never met ", "", "a gohper"}},
	} {
		var buf bytes.Buffer
		buft := writer(&buf)
		for _, s := range test.writes {
			n, err := buft.Write([]byte(s))
			if err != nil {
				t.Errorf("test %d: Write(%q) unexpected error: %v", i, s, err)
				continue
			}
			if n != len(s) {
				t.Errorf("test %d: Write(%q) unexpected number of bytes written: %v", i, s, n)
				continue
			}
		}
		out := buf.String()
		if out != strings.Join(test.writes, "") {
			t.Errorf("test %d: unexpected content in underlying writer: %q", i, out)
		}
	}
}

func TestWriteWriter(t *testing.T) {
	testWrite(t, NewWriteCounter)
}

func TestWriteReadWriter(t *testing.T) {
	testWrite(t, func(w io.Writer) WriteCounter {
		var r nopReader
		return NewReadWriteCounter(readWriter{r, w})
	})
}

// this test could be improved to test exact number of operations as well as
// ensure that error conditions are preserved.
func testRead(t *testing.T, reader func(io.Reader) ReadCounter) {
	chunkLen := 10 << 20 // 10MB
	orig := make([]byte, 10<<20)
	_, err := rand.Read(orig)
	if err != nil {
		t.Fatalf("error reading random data")
	}
	buf := bytes.NewBuffer(orig)
	rc := reader(buf)
	var obuf bytes.Buffer
	ncopy, err := io.Copy(&obuf, rc)
	if err != nil {
		t.Fatalf("error reading: %v", err)
	}
	if ncopy != int64(chunkLen) {
		t.Fatalf("copied %d bytes instead of %d", ncopy, chunkLen)
	}
	if string(orig) != obuf.String() {
		t.Fatalf("unexpected output from Read()")
	}
	n, nops := rc.ReadCount()
	if n != int64(chunkLen) {
		t.Fatalf("reported %d bytes read instead of %d", n, chunkLen)
	}
	if nops < 2 {
		t.Fatalf("unexpected number of reads: %v", nops)
	}
}

func TestReadReader(t *testing.T) {
	testRead(t, NewReadCounter)
}

func TestReadReadWriter(t *testing.T) {
	testRead(t, func(r io.Reader) ReadCounter {
		var w nopWriter
		return NewReadWriteCounter(readWriter{r, w})
	})
}

func testReadTotal(t *testing.T, rc ReadCounter) {
	numGo := 8000
	numBytes := 50
	totalBytes := int64(numGo) * int64(numBytes)
	p := make([]byte, numBytes)

	t.Logf("Calling Read() for %d*%d=%d bytes", numGo, numBytes, totalBytes)
	wg := new(sync.WaitGroup)
	wg.Add(numGo)
	start := make(chan struct{})
	for i := 0; i < numGo; i++ {
		go func() {
			<-start
			rc.Read(p)
			wg.Done()
		}()
	}
	close(start)

	wg.Wait()
	n, nops := rc.ReadCount()
	if n != totalBytes {
		t.Errorf("expected %d bytes read; %d bytes reported", totalBytes, n)
	}
	if nops != numGo {
		t.Errorf("expected %d read operations; %d operations reported", numGo, nops)
	}
}

func TestReadTotalReader(t *testing.T) {
	var r nopReader
	testReadTotal(t, NewReadCounter(r))
}

func TestReadTotalReadWriter(t *testing.T) {
	var rw nopReadWriter
	testReadTotal(t, NewReadWriteCounter(rw))
}

func testWriteTotal(t *testing.T, wt WriteCounter) {
	numGo := 8000
	numBytes := 50
	totalBytes := int64(numGo) * int64(numBytes)
	p := make([]byte, numBytes)

	t.Logf("Calling Write() with %d*%d=%d bytes", numGo, numBytes, totalBytes)
	wg := new(sync.WaitGroup)
	wg.Add(numGo)
	start := make(chan struct{})
	for i := 0; i < numGo; i++ {
		go func() {
			<-start
			wt.Write(p)
			wg.Done()
		}()
	}
	close(start)

	wg.Wait()
	n, nops := wt.WriteCount()
	if n != totalBytes {
		t.Errorf("expected %d bytes written; %d bytes reported", totalBytes, n)
	}
	if nops != numGo {
		t.Errorf("expected %d write operations; %d operations reported", numGo, nops)
	}
}

func TestWriteTotalWriter(t *testing.T) {
	var w nopWriter
	testWriteTotal(t, NewWriteCounter(w))
}

func TestWriteTotalReadWriter(t *testing.T) {
	var rw nopReadWriter
	testWriteTotal(t, NewReadWriteCounter(rw))
}

type nopWriter struct{ error }

func (w nopWriter) Write(p []byte) (int, error) {
	time.Sleep(1)
	if w.error != nil {
		return 0, w.error
	}
	return len(p), nil
}

type nopReader struct{ error }

func (r nopReader) Read(p []byte) (int, error) {
	time.Sleep(1)
	if r.error != nil {
		return 0, r.error
	}
	return len(p), nil
}

type nopReadWriter struct {
	nopReader
	nopWriter
}

type readWriter struct {
	io.Reader
	io.Writer
}
<commit_msg>paasio: Ensure test versioning consistency with other exercises (#542)<commit_after>package paasio

import (
	"bytes"
	"crypto/rand"
	"io"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"
)

// targetTestVersion identifies the API tested by the test program.
const targetTestVersion = 3

func TestTestVersion(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Fatalf("Found testVersion = %v, want %v", testVersion, targetTestVersion)
	}
}

func TestMultiThreaded(t *testing.T) {
	mincpu := 2
	minproc := 2
	ncpu := runtime.NumCPU()
	if ncpu < mincpu {
		t.Fatalf("at least %d cpu cores are required", mincpu)
	}
	nproc := runtime.GOMAXPROCS(0)
	if nproc < minproc {
		t.Errorf("at least %d threads are required; rerun the tests", minproc)
		t.Errorf("")
		t.Errorf("\tgo test -cpu %d ...", minproc)
	}
}

// this test could be improved to test that error conditions are preserved.
func testWrite(t *testing.T, writer func(io.Writer) WriteCounter) {
	for i, test := range []struct {
		writes []string
	}{
		{nil},
		{[]string{""}},
		{[]string{"I", " ", "never met ", "", "a gohper"}},
	} {
		var buf bytes.Buffer
		buft := writer(&buf)
		for _, s := range test.writes {
			n, err := buft.Write([]byte(s))
			if err != nil {
				t.Errorf("test %d: Write(%q) unexpected error: %v", i, s, err)
				continue
			}
			if n != len(s) {
				t.Errorf("test %d: Write(%q) unexpected number of bytes written: %v", i, s, n)
				continue
			}
		}
		out := buf.String()
		if out != strings.Join(test.writes, "") {
			t.Errorf("test %d: unexpected content in underlying writer: %q", i, out)
		}
	}
}

func TestWriteWriter(t *testing.T) {
	testWrite(t, NewWriteCounter)
}

func TestWriteReadWriter(t *testing.T) {
	testWrite(t, func(w io.Writer) WriteCounter {
		var r nopReader
		return NewReadWriteCounter(readWriter{r, w})
	})
}

// this test could be improved to test exact number of operations as well as
// ensure that error conditions are preserved.
func testRead(t *testing.T, reader func(io.Reader) ReadCounter) {
	chunkLen := 10 << 20 // 10MB
	orig := make([]byte, 10<<20)
	_, err := rand.Read(orig)
	if err != nil {
		t.Fatalf("error reading random data")
	}
	buf := bytes.NewBuffer(orig)
	rc := reader(buf)
	var obuf bytes.Buffer
	ncopy, err := io.Copy(&obuf, rc)
	if err != nil {
		t.Fatalf("error reading: %v", err)
	}
	if ncopy != int64(chunkLen) {
		t.Fatalf("copied %d bytes instead of %d", ncopy, chunkLen)
	}
	if string(orig) != obuf.String() {
		t.Fatalf("unexpected output from Read()")
	}
	n, nops := rc.ReadCount()
	if n != int64(chunkLen) {
		t.Fatalf("reported %d bytes read instead of %d", n, chunkLen)
	}
	if nops < 2 {
		t.Fatalf("unexpected number of reads: %v", nops)
	}
}

func TestReadReader(t *testing.T) {
	testRead(t, NewReadCounter)
}

func TestReadReadWriter(t *testing.T) {
	testRead(t, func(r io.Reader) ReadCounter {
		var w nopWriter
		return NewReadWriteCounter(readWriter{r, w})
	})
}

func testReadTotal(t *testing.T, rc ReadCounter) {
	numGo := 8000
	numBytes := 50
	totalBytes := int64(numGo) * int64(numBytes)
	p := make([]byte, numBytes)

	t.Logf("Calling Read() for %d*%d=%d bytes", numGo, numBytes, totalBytes)
	wg := new(sync.WaitGroup)
	wg.Add(numGo)
	start := make(chan struct{})
	for i := 0; i < numGo; i++ {
		go func() {
			<-start
			rc.Read(p)
			wg.Done()
		}()
	}
	close(start)

	wg.Wait()
	n, nops := rc.ReadCount()
	if n != totalBytes {
		t.Errorf("expected %d bytes read; %d bytes reported", totalBytes, n)
	}
	if nops != numGo {
		t.Errorf("expected %d read operations; %d operations reported", numGo, nops)
	}
}

func TestReadTotalReader(t *testing.T) {
	var r nopReader
	testReadTotal(t, NewReadCounter(r))
}

func TestReadTotalReadWriter(t *testing.T) {
	var rw nopReadWriter
	testReadTotal(t, NewReadWriteCounter(rw))
}

func testWriteTotal(t *testing.T, wt WriteCounter) {
	numGo := 8000
	numBytes := 50
	totalBytes := int64(numGo) * int64(numBytes)
	p := make([]byte, numBytes)

	t.Logf("Calling Write() with %d*%d=%d bytes", numGo, numBytes, totalBytes)
	wg := new(sync.WaitGroup)
	wg.Add(numGo)
	start := make(chan struct{})
	for i := 0; i < numGo; i++ {
		go func() {
			<-start
			wt.Write(p)
			wg.Done()
		}()
	}
	close(start)

	wg.Wait()
	n, nops := wt.WriteCount()
	if n != totalBytes {
		t.Errorf("expected %d bytes written; %d bytes reported", totalBytes, n)
	}
	if nops != numGo {
		t.Errorf("expected %d write operations; %d operations reported", numGo, nops)
	}
}

func TestWriteTotalWriter(t *testing.T) {
	var w nopWriter
	testWriteTotal(t, NewWriteCounter(w))
}

func TestWriteTotalReadWriter(t *testing.T) {
	var rw nopReadWriter
	testWriteTotal(t, NewReadWriteCounter(rw))
}

type nopWriter struct{ error }

func (w nopWriter) Write(p []byte) (int, error) {
	time.Sleep(1)
	if w.error != nil {
		return 0, w.error
	}
	return len(p), nil
}

type nopReader struct{ error }

func (r nopReader) Read(p []byte) (int, error) {
	time.Sleep(1)
	if r.error != nil {
		return 0, r.error
	}
	return len(p), nil
}

type nopReadWriter struct {
	nopReader
	nopWriter
}

type readWriter struct {
	io.Reader
	io.Writer
}
<|endoftext|>


<commit_before>package thrift_nats

import (
	"log"
	"runtime/debug"
	"strconv"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/nats-io/nats"
)

const (
	queue                    = "rpc"
	defaultHeartbeatDeadline = 5 * time.Second
	maxMissedHeartbeats      = 3
)

type natsServer struct {
	conn                   *nats.Conn
	subject                string
	clientTimeout          time.Duration
	heartbeatDeadline      time.Duration
	clients                map[string]thrift.TTransport
	mu                     sync.Mutex
	quit                   chan struct{}
	processorFactory       thrift.TProcessorFactory
	serverTransport        *natsServerTransport
	inputTransportFactory  thrift.TTransportFactory
	outputTransportFactory thrift.TTransportFactory
	inputProtocolFactory   thrift.TProtocolFactory
	outputProtocolFactory  thrift.TProtocolFactory
}

// NewNATSServer returns a Thrift TServer which uses the NATS messaging system
// as the underlying transport. The subject is the NATS subject used for
// connection handshakes. The client timeout controls the read timeout on the
// client connection (negative value for no timeout). The heartbeat deadline
// controls how long clients have to respond with a heartbeat (negative value
// for no heartbeats).
func NewNATSServer(
	conn *nats.Conn,
	subject string,
	clientTimeout time.Duration,
	heartbeatDeadline time.Duration,
	processor thrift.TProcessor,
	transportFactory thrift.TTransportFactory,
	protocolFactory thrift.TProtocolFactory) thrift.TServer {

	return NewNATSServerFactory7(
		conn,
		subject,
		clientTimeout,
		heartbeatDeadline,
		thrift.NewTProcessorFactory(processor),
		transportFactory,
		protocolFactory,
	)
}

func NewNATSServerFactory7(
	conn *nats.Conn,
	subject string,
	clientTimeout time.Duration,
	heartbeatDeadline time.Duration,
	processorFactory thrift.TProcessorFactory,
	transportFactory thrift.TTransportFactory,
	protocolFactory thrift.TProtocolFactory) thrift.TServer {

	return NewNATSServerFactory9(
		conn,
		subject,
		clientTimeout,
		heartbeatDeadline,
		processorFactory,
		transportFactory,
		transportFactory,
		protocolFactory,
		protocolFactory,
	)
}

func NewNATSServerFactory9(
	conn *nats.Conn,
	subject string,
	clientTimeout time.Duration,
	heartbeatDeadline time.Duration,
	processorFactory thrift.TProcessorFactory,
	inputTransportFactory thrift.TTransportFactory,
	outputTransportFactory thrift.TTransportFactory,
	inputProtocolFactory thrift.TProtocolFactory,
	outputProtocolFactory thrift.TProtocolFactory) thrift.TServer {

	return &natsServer{
		conn:                   conn,
		subject:                subject,
		clientTimeout:          clientTimeout,
		heartbeatDeadline:      heartbeatDeadline,
		clients:                make(map[string]thrift.TTransport),
		processorFactory:       processorFactory,
		serverTransport:        newNATSServerTransport(conn),
		inputTransportFactory:  inputTransportFactory,
		outputTransportFactory: outputTransportFactory,
		inputProtocolFactory:   inputProtocolFactory,
		outputProtocolFactory:  outputProtocolFactory,
		quit: make(chan struct{}, 1),
	}
}

func (n *natsServer) ProcessorFactory() thrift.TProcessorFactory {
	return n.processorFactory
}

func (n *natsServer) ServerTransport() thrift.TServerTransport {
	return n.serverTransport
}

func (n *natsServer) InputTransportFactory() thrift.TTransportFactory {
	return n.inputTransportFactory
}

func (n *natsServer) OutputTransportFactory() thrift.TTransportFactory {
	return n.outputTransportFactory
}

func (n *natsServer) InputProtocolFactory() thrift.TProtocolFactory {
	return n.inputProtocolFactory
}

func (n *natsServer) OutputProtocolFactory() thrift.TProtocolFactory {
	return n.outputProtocolFactory
}

func (n *natsServer) Listen() error {
	return n.serverTransport.Listen()
}

func (n *natsServer) AcceptLoop() error {
	sub, err := n.conn.QueueSubscribe(n.subject, queue, func(msg *nats.Msg) {
		if msg.Reply != "" {
			var (
				heartbeat   = nats.NewInbox()
				listenTo    = nats.NewInbox()
				client, err = n.accept(listenTo, msg.Reply, heartbeat)
			)
			if err != nil {
				log.Println("thrift_nats: error accepting client transport:", err)
				return
			}

			if n.isHeartbeating() {
				n.mu.Lock()
				n.clients[heartbeat] = client
				n.mu.Unlock()
			}

			connectMsg := heartbeat + " " + strconv.FormatInt(int64(n.heartbeatDeadline), 10)
			if err := n.conn.PublishRequest(msg.Reply, listenTo, []byte(connectMsg)); err != nil {
				log.Println("thrift_nats: error publishing transport inbox:", err)
				if n.isHeartbeating() {
					n.remove(heartbeat)
				}
			} else if n.isHeartbeating() {
				go n.acceptHeartbeat(heartbeat)
			}
		} else {
			log.Printf("thrift_nats: discarding invalid connect message %+v", msg)
		}
	})
	if err != nil {
		return err
	}

	n.conn.Flush()

	log.Println("thrift_nats: server running...")
	<-n.quit
	return sub.Unsubscribe()
}

func (n *natsServer) remove(heartbeat string) {
	n.mu.Lock()
	defer n.mu.Unlock()
	client, ok := n.clients[heartbeat]
	if !ok {
		return
	}
	client.Close()
	delete(n.clients, heartbeat)
}

func (n *natsServer) acceptHeartbeat(heartbeat string) {
	missed := 0
	recvHeartbeat := make(chan struct{})

	sub, err := n.conn.Subscribe(heartbeat, func(msg *nats.Msg) {
		recvHeartbeat <- struct{}{}
	})
	if err != nil {
		log.Println("thrift_nats: error subscribing to heartbeat", heartbeat)
		return
	}

	for {
		select {
		case <-time.After(time.Duration(n.heartbeatDeadline)):
			missed += 1
			if missed >= maxMissedHeartbeats {
				log.Println("thrift_nats: client heartbeat expired")
				n.remove(heartbeat)
				sub.Unsubscribe()
				return
			}
		case <-recvHeartbeat:
			missed = 0
		}
	}
}

func (n *natsServer) accept(listenTo, replyTo, heartbeat string) (thrift.TTransport, error) {
	client := n.serverTransport.AcceptNATS(listenTo, replyTo, n.clientTimeout)
	if err := client.Open(); err != nil {
		return nil, err
	}
	go func() {
		if err := n.processRequests(client); err != nil {
			log.Println("thrift_nats: error processing request:", err)
		}
		n.remove(heartbeat)
	}()
	return client, nil
}

func (n *natsServer) Serve() error {
	if err := n.Listen(); err != nil {
		return err
	}
	n.AcceptLoop()
	return nil
}

func (n *natsServer) Stop() error {
	n.quit <- struct{}{}
	n.serverTransport.Interrupt()
	return nil
}

func (n *natsServer) processRequests(client thrift.TTransport) error {
	processor := n.processorFactory.GetProcessor(client)
	inputTransport := n.inputTransportFactory.GetTransport(client)
	outputTransport := n.outputTransportFactory.GetTransport(client)
	inputProtocol := n.inputProtocolFactory.GetProtocol(inputTransport)
	outputProtocol := n.outputProtocolFactory.GetProtocol(outputTransport)
	defer func() {
		if e := recover(); e != nil {
			log.Printf("panic in processor: %s: %s", e, debug.Stack())
		}
	}()
	if inputTransport != nil {
		defer inputTransport.Close()
	}
	if outputTransport != nil {
		defer outputTransport.Close()
	}
	for {
		ok, err := processor.Process(inputProtocol, outputProtocol)
		if err, ok := err.(thrift.TTransportException); ok && err.TypeId() == thrift.END_OF_FILE {
			return nil
		} else if err != nil {
			return err
		}
		if !ok {
			break
		}
	}
	return nil
}

func (n *natsServer) isHeartbeating() bool {
	return n.heartbeatDeadline > 0
}
<commit_msg>Fix heartbeat deadline to use milliseconds<commit_after>package thrift_nats

import (
	"log"
	"runtime/debug"
	"strconv"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/nats-io/nats"
)

const (
	queue               = "rpc"
	maxMissedHeartbeats = 3
)

type natsServer struct {
	conn                   *nats.Conn
	subject                string
	clientTimeout          time.Duration
	heartbeatDeadline      time.Duration
	clients                map[string]thrift.TTransport
	mu                     sync.Mutex
	quit                   chan struct{}
	processorFactory       thrift.TProcessorFactory
	serverTransport        *natsServerTransport
	inputTransportFactory  thrift.TTransportFactory
	outputTransportFactory thrift.TTransportFactory
	inputProtocolFactory   thrift.TProtocolFactory
	outputProtocolFactory  thrift.TProtocolFactory
}

// NewNATSServer returns a Thrift TServer which uses the NATS messaging system
// as the underlying transport. The subject is the NATS subject used for
// connection handshakes. The client timeout controls the read timeout on the
// client connection (negative value for no timeout). The heartbeat deadline
// controls how long clients have to respond with a heartbeat (negative value
// for no heartbeats).
func NewNATSServer(
	conn *nats.Conn,
	subject string,
	clientTimeout time.Duration,
	heartbeatDeadline time.Duration,
	processor thrift.TProcessor,
	transportFactory thrift.TTransportFactory,
	protocolFactory thrift.TProtocolFactory) thrift.TServer {

	return NewNATSServerFactory7(
		conn,
		subject,
		clientTimeout,
		heartbeatDeadline,
		thrift.NewTProcessorFactory(processor),
		transportFactory,
		protocolFactory,
	)
}

func NewNATSServerFactory7(
	conn *nats.Conn,
	subject string,
	clientTimeout time.Duration,
	heartbeatDeadline time.Duration,
	processorFactory thrift.TProcessorFactory,
	transportFactory thrift.TTransportFactory,
	protocolFactory thrift.TProtocolFactory) thrift.TServer {

	return NewNATSServerFactory9(
		conn,
		subject,
		clientTimeout,
		heartbeatDeadline,
		processorFactory,
		transportFactory,
		transportFactory,
		protocolFactory,
		protocolFactory,
	)
}

func NewNATSServerFactory9(
	conn *nats.Conn,
	subject string,
	clientTimeout time.Duration,
	heartbeatDeadline time.Duration,
	processorFactory thrift.TProcessorFactory,
	inputTransportFactory thrift.TTransportFactory,
	outputTransportFactory thrift.TTransportFactory,
	inputProtocolFactory thrift.TProtocolFactory,
	outputProtocolFactory thrift.TProtocolFactory) thrift.TServer {

	return &natsServer{
		conn:                   conn,
		subject:                subject,
		clientTimeout:          clientTimeout,
		heartbeatDeadline:      heartbeatDeadline,
		clients:                make(map[string]thrift.TTransport),
		processorFactory:       processorFactory,
		serverTransport:        newNATSServerTransport(conn),
		inputTransportFactory:  inputTransportFactory,
		outputTransportFactory: outputTransportFactory,
		inputProtocolFactory:   inputProtocolFactory,
		outputProtocolFactory:  outputProtocolFactory,
		quit: make(chan struct{}, 1),
	}
}

func (n *natsServer) ProcessorFactory() thrift.TProcessorFactory {
	return n.processorFactory
}

func (n *natsServer) ServerTransport() thrift.TServerTransport {
	return n.serverTransport
}

func (n *natsServer) InputTransportFactory() thrift.TTransportFactory {
	return n.inputTransportFactory
}

func (n *natsServer) OutputTransportFactory() thrift.TTransportFactory {
	return n.outputTransportFactory
}

func (n *natsServer) InputProtocolFactory() thrift.TProtocolFactory {
	return n.inputProtocolFactory
}

func (n *natsServer) OutputProtocolFactory() thrift.TProtocolFactory {
	return n.outputProtocolFactory
}

func (n *natsServer) Listen() error {
	return n.serverTransport.Listen()
}

func (n *natsServer) AcceptLoop() error {
	sub, err := n.conn.QueueSubscribe(n.subject, queue, func(msg *nats.Msg) {
		if msg.Reply != "" {
			var (
				heartbeat   = nats.NewInbox()
				listenTo    = nats.NewInbox()
				client, err = n.accept(listenTo, msg.Reply, heartbeat)
			)
			if err != nil {
				log.Println("thrift_nats: error accepting client transport:", err)
				return
			}

			if n.isHeartbeating() {
				n.mu.Lock()
				n.clients[heartbeat] = client
				n.mu.Unlock()
			}

			connectMsg := heartbeat + " " + strconv.FormatInt(int64(n.heartbeatDeadline*time.Millisecond), 10)
			if err := n.conn.PublishRequest(msg.Reply, listenTo, []byte(connectMsg)); err != nil {
				log.Println("thrift_nats: error publishing transport inbox:", err)
				if n.isHeartbeating() {
					n.remove(heartbeat)
				}
			} else if n.isHeartbeating() {
				go n.acceptHeartbeat(heartbeat)
			}
		} else {
			log.Printf("thrift_nats: discarding invalid connect message %+v", msg)
		}
	})
	if err != nil {
		return err
	}

	n.conn.Flush()

	log.Println("thrift_nats: server running...")
	<-n.quit
	return sub.Unsubscribe()
}

func (n *natsServer) remove(heartbeat string) {
	n.mu.Lock()
	defer n.mu.Unlock()
	client, ok := n.clients[heartbeat]
	if !ok {
		return
	}
	client.Close()
	delete(n.clients, heartbeat)
}

func (n *natsServer) acceptHeartbeat(heartbeat string) {
	missed := 0
	recvHeartbeat := make(chan struct{})

	sub, err := n.conn.Subscribe(heartbeat, func(msg *nats.Msg) {
		recvHeartbeat <- struct{}{}
	})
	if err != nil {
		log.Println("thrift_nats: error subscribing to heartbeat", heartbeat)
		return
	}

	for {
		select {
		case <-time.After(time.Duration(n.heartbeatDeadline)):
			missed += 1
			if missed >= maxMissedHeartbeats {
				log.Println("thrift_nats: client heartbeat expired")
				n.remove(heartbeat)
				sub.Unsubscribe()
				return
			}
		case <-recvHeartbeat:
			missed = 0
		}
	}
}

func (n *natsServer) accept(listenTo, replyTo, heartbeat string) (thrift.TTransport, error) {
	client := n.serverTransport.AcceptNATS(listenTo, replyTo, n.clientTimeout)
	if err := client.Open(); err != nil {
		return nil, err
	}
	go func() {
		if err := n.processRequests(client); err != nil {
			log.Println("thrift_nats: error processing request:", err)
		}
		n.remove(heartbeat)
	}()
	return client, nil
}

func (n *natsServer) Serve() error {
	if err := n.Listen(); err != nil {
		return err
	}
	n.AcceptLoop()
	return nil
}

func (n *natsServer) Stop() error {
	n.quit <- struct{}{}
	n.serverTransport.Interrupt()
	return nil
}

func (n *natsServer) processRequests(client thrift.TTransport) error {
	processor := n.processorFactory.GetProcessor(client)
	inputTransport := n.inputTransportFactory.GetTransport(client)
	outputTransport := n.outputTransportFactory.GetTransport(client)
	inputProtocol := n.inputProtocolFactory.GetProtocol(inputTransport)
	outputProtocol := n.outputProtocolFactory.GetProtocol(outputTransport)
	defer func() {
		if e := recover(); e != nil {
			log.Printf("panic in processor: %s: %s", e, debug.Stack())
		}
	}()
	if inputTransport != nil {
		defer inputTransport.Close()
	}
	if outputTransport != nil {
		defer outputTransport.Close()
	}
	for {
		ok, err := processor.Process(inputProtocol, outputProtocol)
		if err, ok := err.(thrift.TTransportException); ok && err.TypeId() == thrift.END_OF_FILE {
			return nil
		} else if err != nil {
			return err
		}
		if !ok {
			break
		}
	}
	return nil
}

func (n *natsServer) isHeartbeating() bool {
	return n.heartbeatDeadline > 0
}
<|endoftext|>


<commit_before>package protoface

type Transform interface {
	ProtoBuff
	// func GetObjects() []*ScriptObject
	func IGetNthObject(n int) ScriptObject
	func ISetNthObject(n int, s ScriptObject)
	func GetPkg() *Package
}<commit_msg>rename this to match the pb name<commit_after><|endoftext|>


<commit_before>package clickhouse

import (
	"context"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/lomik/graphite-clickhouse/pkg/scope"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

type ErrDataParse struct {
	err  string
	data string
}

func NewErrDataParse(err string, data string) error {
	return &ErrDataParse{err, data}
}

func (e *ErrDataParse) Error() string {
	return fmt.Sprintf("%s: %s", e.err, e.data)
}

func (e *ErrDataParse) PrependDescription(test string) {
	e.data = test + e.data
}

type ErrorWithCode struct {
	err  string
	Code int // error code
}

func NewErrorWithCode(err string, code int) error {
	return &ErrorWithCode{err, code}
}

func (e *ErrorWithCode) Error() string { return e.err }

var ErrUvarintRead = errors.New("ReadUvarint: Malformed array")
var ErrUvarintOverflow = errors.New("ReadUvarint: varint overflows a 64-bit integer")
var ErrClickHouseResponse = errors.New("Malformed response from clickhouse")

func HandleError(w http.ResponseWriter, err error) {
	if errors.Is(err, context.Canceled) {
		http.Error(w, "Storage read context canceled", http.StatusGatewayTimeout)
		return
	}
	netErr, ok := err.(net.Error)
	if ok {
		if netErr.Timeout() {
			http.Error(w, "Storage read timeout", http.StatusGatewayTimeout)
		} else if strings.HasSuffix(err.Error(), "connect: no route to host") ||
			strings.HasSuffix(err.Error(), "connect: connection refused") ||
			strings.HasSuffix(err.Error(), ": connection reset by peer") ||
			strings.HasPrefix(err.Error(), "dial tcp: lookup ") { // DNS lookup
			http.Error(w, "Storage error", http.StatusServiceUnavailable)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	errCode, ok := err.(*ErrorWithCode)
	if ok {
		if (errCode.Code > 500 && errCode.Code < 512) ||
			errCode.Code == http.StatusBadRequest || errCode.Code == http.StatusForbidden {
			http.Error(w, html.EscapeString(errCode.Error()), errCode.Code)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	_, ok = err.(*ErrDataParse)
	if ok || strings.HasPrefix(err.Error(), "clickhouse response status 500: Code:") {
		if strings.Contains(err.Error(), ": Limit for ") {
			//logger.Info("limit", zap.Error(err))
			http.Error(w, "Storage read limit", http.StatusForbidden)
		} else if !ok && strings.HasPrefix(err.Error(), "clickhouse response status 500: Code: 170,") {
			// distributed table configuration error
			// clickhouse response status 500: Code: 170, e.displayText() = DB::Exception: Requested cluster 'cluster' not found
			http.Error(w, "Storage configuration error", http.StatusServiceUnavailable)
		}
	} else {
		//logger.Debug("query", zap.Error(err))
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type Options struct {
	Timeout        time.Duration
	ConnectTimeout time.Duration
}

type loggedReader struct {
	reader   io.ReadCloser
	logger   *zap.Logger
	start    time.Time
	finished bool
	queryID  string
}

func (r *loggedReader) Read(p []byte) (int, error) {
	n, err := r.reader.Read(p)
	if err != nil && !r.finished {
		r.finished = true
		r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start)))
	}
	return n, err
}

func (r *loggedReader) Close() error {
	err := r.reader.Close()
	if !r.finished {
		r.finished = true
		r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start)))
	}
	return err
}

func formatSQL(q string) string {
	s := strings.Split(q, "\n")
	for i := 0; i < len(s); i++ {
		s[i] = strings.TrimSpace(s[i])
	}

	return strings.Join(s, " ")
}

func Query(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) ([]byte, error) {
	return Post(ctx, dsn, query, nil, opts, extData)
}

func Post(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, error) {
	return do(ctx, dsn, query, postBody, false, opts, extData)
}

func PostGzip(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, error) {
	return do(ctx, dsn, query, postBody, true, opts, extData)
}

func Reader(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) (io.ReadCloser, error) {
	return reader(ctx, dsn, query, nil, false, opts, extData)
}

func reader(ctx context.Context, dsn string, query string, postBody io.Reader, gzip bool, opts Options, extData *ExternalData) (bodyReader io.ReadCloser, err error) {
	if postBody != nil && extData != nil {
		err = fmt.Errorf("postBody and extData could not be passed in one request")
		return
	}

	var chQueryID string

	start := time.Now()

	requestID := scope.RequestID(ctx)

	queryForLogger := query
	if len(queryForLogger) > 500 {
		queryForLogger = queryForLogger[:395] + "<...>" + queryForLogger[len(queryForLogger)-100:]
	}
	logger := scope.Logger(ctx).With(zap.String("query", formatSQL(queryForLogger)))

	defer func() {
		// fmt.Println(time.Since(start), formatSQL(queryForLogger))
		if err != nil {
			logger.Error("query", zap.Error(err), zap.Duration("time", time.Since(start)))
		}
	}()

	p, err := url.Parse(dsn)
	if err != nil {
		return
	}

	var b [8]byte
	binary.LittleEndian.PutUint64(b[:], rand.Uint64())
	queryID := fmt.Sprintf("%x", b)

	q := p.Query()
	q.Set("query_id", fmt.Sprintf("%s::%s", requestID, queryID))
	// Get X-Clickhouse-Summary header
	// TODO: remove when https://github.com/ClickHouse/ClickHouse/issues/16207 is done
	q.Set("send_progress_in_http_headers", "1")
	q.Set("http_headers_progress_interval_ms", "10000")
	p.RawQuery = q.Encode()

	var contentHeader string
	if postBody != nil {
		q := p.Query()
		q.Set("query", query)
		p.RawQuery = q.Encode()
	} else if extData != nil {
		q := p.Query()
		q.Set("query", query)
		p.RawQuery = q.Encode()
		postBody, contentHeader, err = extData.buildBody(ctx, p)
		if err != nil {
			return
		}
	} else {
		postBody = strings.NewReader(query)
	}

	url := p.String()

	req, err := http.NewRequest("POST", url, postBody)
	if err != nil {
		return
	}

	req.Header.Add("User-Agent", scope.ClickhouseUserAgent(ctx))
	if contentHeader != "" {
		req.Header.Add("Content-Type", contentHeader)
	}

	if gzip {
		req.Header.Add("Content-Encoding", "gzip")
	}

	client := &http.Client{
		Timeout: opts.Timeout,
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: opts.ConnectTimeout,
			}).Dial,
			DisableKeepAlives: true,
		},
	}
	resp, err := client.Do(req)
	if err != nil {
		return
	}

	// chproxy overwrite our query id. So read it again
	chQueryID = resp.Header.Get("X-ClickHouse-Query-Id")

	summary := make(map[string]string)
	err = json.Unmarshal([]byte(resp.Header.Get("X-Clickhouse-Summary")), &summary)
	if err == nil {
		// TODO: use in carbon metrics sender when it will be implemented
		fields := make([]zapcore.Field, 0, len(summary))
		for k, v := range summary {
			fields = append(fields, zap.String(k, v))
		}
		logger = logger.With(fields...)
	}

	// check for return 5xx error, may be 502 code if clickhouse accesed via reverse proxy
	if resp.StatusCode > 500 && resp.StatusCode < 512 {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = NewErrorWithCode(string(body), resp.StatusCode)
		return
	} else if resp.StatusCode != 200 {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = fmt.Errorf("clickhouse response status %d: %s", resp.StatusCode, string(body))
		return
	}

	bodyReader = &loggedReader{
		reader:  resp.Body,
		logger:  logger,
		start:   start,
		queryID: chQueryID,
	}

	return
}

func do(ctx context.Context, dsn string, query string, postBody io.Reader, gzip bool, opts Options, extData *ExternalData) ([]byte, error) {
	bodyReader, err := reader(ctx, dsn, query, postBody, gzip, opts, extData)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(bodyReader)
	bodyReader.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func ReadUvarint(array []byte) (uint64, int, error) {
	var x uint64
	var s uint
	l := len(array) - 1
	for i := 0; ; i++ {
		if i > l {
			return x, i + 1, ErrUvarintRead
		}
		if array[i] < 0x80 {
			if i > 9 || i == 9 && array[i] > 1 {
				return x, i + 1, ErrUvarintOverflow
			}
			return x | uint64(array[i])<<s, i + 1, nil
		}
		x |= uint64(array[i]&0x7f) << s
		s += 7
	}
}
<commit_msg>fix: logging for clickhouse query summary (no header)<commit_after>package clickhouse

import (
	"context"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/lomik/graphite-clickhouse/pkg/scope"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

type ErrDataParse struct {
	err  string
	data string
}

func NewErrDataParse(err string, data string) error {
	return &ErrDataParse{err, data}
}

func (e *ErrDataParse) Error() string {
	return fmt.Sprintf("%s: %s", e.err, e.data)
}

func (e *ErrDataParse) PrependDescription(test string) {
	e.data = test + e.data
}

type ErrorWithCode struct {
	err  string
	Code int // error code
}

func NewErrorWithCode(err string, code int) error {
	return &ErrorWithCode{err, code}
}

func (e *ErrorWithCode) Error() string { return e.err }

var ErrUvarintRead = errors.New("ReadUvarint: Malformed array")
var ErrUvarintOverflow = errors.New("ReadUvarint: varint overflows a 64-bit integer")
var ErrClickHouseResponse = errors.New("Malformed response from clickhouse")

func HandleError(w http.ResponseWriter, err error) {
	if errors.Is(err, context.Canceled) {
		http.Error(w, "Storage read context canceled", http.StatusGatewayTimeout)
		return
	}
	netErr, ok := err.(net.Error)
	if ok {
		if netErr.Timeout() {
			http.Error(w, "Storage read timeout", http.StatusGatewayTimeout)
		} else if strings.HasSuffix(err.Error(), "connect: no route to host") ||
			strings.HasSuffix(err.Error(), "connect: connection refused") ||
			strings.HasSuffix(err.Error(), ": connection reset by peer") ||
			strings.HasPrefix(err.Error(), "dial tcp: lookup ") { // DNS lookup
			http.Error(w, "Storage error", http.StatusServiceUnavailable)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	errCode, ok := err.(*ErrorWithCode)
	if ok {
		if (errCode.Code > 500 && errCode.Code < 512) ||
			errCode.Code == http.StatusBadRequest || errCode.Code == http.StatusForbidden {
			http.Error(w, html.EscapeString(errCode.Error()), errCode.Code)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	_, ok = err.(*ErrDataParse)
	if ok || strings.HasPrefix(err.Error(), "clickhouse response status 500: Code:") {
		if strings.Contains(err.Error(), ": Limit for ") {
			//logger.Info("limit", zap.Error(err))
			http.Error(w, "Storage read limit", http.StatusForbidden)
		} else if !ok && strings.HasPrefix(err.Error(), "clickhouse response status 500: Code: 170,") {
			// distributed table configuration error
			// clickhouse response status 500: Code: 170, e.displayText() = DB::Exception: Requested cluster 'cluster' not found
			http.Error(w, "Storage configuration error", http.StatusServiceUnavailable)
		}
	} else {
		//logger.Debug("query", zap.Error(err))
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type Options struct {
	Timeout        time.Duration
	ConnectTimeout time.Duration
}

type loggedReader struct {
	reader   io.ReadCloser
	logger   *zap.Logger
	start    time.Time
	finished bool
	queryID  string
}

func (r *loggedReader) Read(p []byte) (int, error) {
	n, err := r.reader.Read(p)
	if err != nil && !r.finished {
		r.finished = true
		r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start)))
	}
	return n, err
}

func (r *loggedReader) Close() error {
	err := r.reader.Close()
	if !r.finished {
		r.finished = true
		r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start)))
	}
	return err
}

func formatSQL(q string) string {
	s := strings.Split(q, "\n")
	for i := 0; i < len(s); i++ {
		s[i] = strings.TrimSpace(s[i])
	}

	return strings.Join(s, " ")
}

func Query(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) ([]byte, error) {
	return Post(ctx, dsn, query, nil, opts, extData)
}

func Post(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, error) {
	return do(ctx, dsn, query, postBody, false, opts, extData)
}

func PostGzip(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, error) {
	return do(ctx, dsn, query, postBody, true, opts, extData)
}

func Reader(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) (io.ReadCloser, error) {
	return reader(ctx, dsn, query, nil, false, opts, extData)
}

func reader(ctx context.Context, dsn string, query string, postBody io.Reader, gzip bool, opts Options, extData *ExternalData) (bodyReader io.ReadCloser, err error) {
	if postBody != nil && extData != nil {
		err = fmt.Errorf("postBody and extData could not be passed in one request")
		return
	}

	var chQueryID string

	start := time.Now()

	requestID := scope.RequestID(ctx)

	queryForLogger := query
	if len(queryForLogger) > 500 {
		queryForLogger = queryForLogger[:395] + "<...>" + queryForLogger[len(queryForLogger)-100:]
	}
	logger := scope.Logger(ctx).With(zap.String("query", formatSQL(queryForLogger)))

	defer func() {
		// fmt.Println(time.Since(start), formatSQL(queryForLogger))
		if err != nil {
			logger.Error("query", zap.Error(err), zap.Duration("time", time.Since(start)))
		}
	}()

	p, err := url.Parse(dsn)
	if err != nil {
		return
	}

	var b [8]byte
	binary.LittleEndian.PutUint64(b[:], rand.Uint64())
	queryID := fmt.Sprintf("%x", b)

	q := p.Query()
	q.Set("query_id", fmt.Sprintf("%s::%s", requestID, queryID))
	// Get X-Clickhouse-Summary header
	// TODO: remove when https://github.com/ClickHouse/ClickHouse/issues/16207 is done
	q.Set("send_progress_in_http_headers", "1")
	q.Set("http_headers_progress_interval_ms", "10000")
	p.RawQuery = q.Encode()

	var contentHeader string
	if postBody != nil {
		q := p.Query()
		q.Set("query", query)
		p.RawQuery = q.Encode()
	} else if extData != nil {
		q := p.Query()
		q.Set("query", query)
		p.RawQuery = q.Encode()
		postBody, contentHeader, err = extData.buildBody(ctx, p)
		if err != nil {
			return
		}
	} else {
		postBody = strings.NewReader(query)
	}

	url := p.String()

	req, err := http.NewRequest("POST", url, postBody)
	if err != nil {
		return
	}

	req.Header.Add("User-Agent", scope.ClickhouseUserAgent(ctx))
	if contentHeader != "" {
		req.Header.Add("Content-Type", contentHeader)
	}

	if gzip {
		req.Header.Add("Content-Encoding", "gzip")
	}

	client := &http.Client{
		Timeout: opts.Timeout,
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: opts.ConnectTimeout,
			}).Dial,
			DisableKeepAlives: true,
		},
	}
	resp, err := client.Do(req)
	if err != nil {
		return
	}

	// chproxy overwrite our query id. So read it again
	chQueryID = resp.Header.Get("X-ClickHouse-Query-Id")

	summaryHeader := resp.Header.Get("X-Clickhouse-Summary")
	if len(summaryHeader) > 0 {
		summary := make(map[string]string)
		err = json.Unmarshal([]byte(summaryHeader), &summary)
		if err == nil {
			// TODO: use in carbon metrics sender when it will be implemented
			fields := make([]zapcore.Field, 0, len(summary))
			for k, v := range summary {
				fields = append(fields, zap.String(k, v))
			}
			logger = logger.With(fields...)
		} else {
			logger.Warn("query", zap.Error(err), zap.String("clickhouse-summary", summaryHeader))
			err = nil
		}
	}

	// check for return 5xx error, may be 502 code if clickhouse accesed via reverse proxy
	if resp.StatusCode > 500 && resp.StatusCode < 512 {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = NewErrorWithCode(string(body), resp.StatusCode)
		return
	} else if resp.StatusCode != 200 {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = fmt.Errorf("clickhouse response status %d: %s", resp.StatusCode, string(body))
		return
	}

	bodyReader = &loggedReader{
		reader:  resp.Body,
		logger:  logger,
		start:   start,
		queryID: chQueryID,
	}

	return
}

func do(ctx context.Context, dsn string, query string, postBody io.Reader, gzip bool, opts Options, extData *ExternalData) ([]byte, error) {
	bodyReader, err := reader(ctx, dsn, query, postBody, gzip, opts, extData)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(bodyReader)
	bodyReader.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func ReadUvarint(array []byte) (uint64, int, error) {
	var x uint64
	var s uint
	l := len(array) - 1
	for i := 0; ; i++ {
		if i > l {
			return x, i + 1, ErrUvarintRead
		}
		if array[i] < 0x80 {
			if i > 9 || i == 9 && array[i] > 1 {
				return x, i + 1, ErrUvarintOverflow
			}
			return x | uint64(array[i])<<s, i + 1, nil
		}
		x |= uint64(array[i]&0x7f) << s
		s += 7
	}
}
<|endoftext|>


<commit_before>// This code does not have any restrictions, use it as you please.
// Instead of copyright stuff, let's have some cool phrases:
//     "The absolute transformation of everything that we ever thought about music will
//     take place within 10 years, and nothing is going to be able to stop it. I see
//     absolutely no point in pretending that it's not going to happen.
//     I'm fully confident that copyright, for instance, will no longer exist in 10 years."
//     -BOWIE, David

// Package parse allows you to parse the raw content of a robots.txt file into a Robots struct.
package parse

import (
	"github.com/lucasfcosta/gobotto/models"
	"strings"
)

func Parse(text string) models.Robots {
	robots := models.NewRobots()

	lines := strings.Split(text, "\n")

	var lastUserAgent string

	// For each line
	for _, line := range lines {
		normalized := strings.ToLower(line)

		// Detect the semantic value of a line
		isComment := strings.HasPrefix(normalized, "#")
		isUserAgent := strings.HasPrefix(normalized, "user-agent")
		isAllow := strings.HasPrefix(normalized, "allow")
		isDisallow := strings.HasPrefix(normalized, "disallow")

		// Handle that line according to its semantic value
		if isComment {
			comment := strings.TrimLeft(strings.Split(line, "#")[1], " ")
			robots.Comments = append(robots.Comments, comment)
		} else if isUserAgent {
			lastUserAgent = strings.Split(line, " ")[1]
			_, exists := robots.Rules[lastUserAgent]
			if !exists {
				robots.Rules[lastUserAgent] = models.NewRules()
			}
		} else if isAllow {
			path := strings.Split(line, " ")[1]
			robots.Rules[lastUserAgent].Allow[path] = precision(path)
		} else if isDisallow {
			path := strings.Split(line, " ")[1]
			robots.Rules[lastUserAgent].Disallow[path] = precision(path)
		}
	}

	return robots
}

func precision(path string) int {
	subpaths := strings.Split(path, "/")
	count := 0
	for _, subpath := range subpaths {
		if subpath != "" {
			count++
		}
	}

	return count
}
<commit_msg>Trim trailing and leading spaces on line normalization for parse<commit_after>// This code does not have any restrictions, use it as you please.
// Instead of copyright stuff, let's have some cool phrases:
//     "The absolute transformation of everything that we ever thought about music will
//     take place within 10 years, and nothing is going to be able to stop it. I see
//     absolutely no point in pretending that it's not going to happen.
//     I'm fully confident that copyright, for instance, will no longer exist in 10 years."
//     -BOWIE, David

// Package parse allows you to parse the raw content of a robots.txt file into a Robots struct.
package parse

import (
	"github.com/lucasfcosta/gobotto/models"
	"strings"
)

func Parse(text string) models.Robots {
	robots := models.NewRobots()

	lines := strings.Split(text, "\n")

	var lastUserAgent string

	// For each line
	for _, line := range lines {
		normalized := strings.Trim(strings.ToLower(line), " ")

		// Detect the semantic value of a line
		isComment := strings.HasPrefix(normalized, "#")
		isUserAgent := strings.HasPrefix(normalized, "user-agent")
		isAllow := strings.HasPrefix(normalized, "allow")
		isDisallow := strings.HasPrefix(normalized, "disallow")

		// Handle that line according to its semantic value
		if isComment {
			comment := strings.TrimLeft(strings.Split(line, "#")[1], " ")
			robots.Comments = append(robots.Comments, comment)
		} else if isUserAgent {
			lastUserAgent = strings.Split(line, " ")[1]
			_, exists := robots.Rules[lastUserAgent]
			if !exists {
				robots.Rules[lastUserAgent] = models.NewRules()
			}
		} else if isAllow {
			path := strings.Split(line, " ")[1]
			robots.Rules[lastUserAgent].Allow[path] = precision(path)
		} else if isDisallow {
			path := strings.Split(line, " ")[1]
			robots.Rules[lastUserAgent].Disallow[path] = precision(path)
		}
	}

	return robots
}

func precision(path string) int {
	subpaths := strings.Split(path, "/")
	count := 0
	for _, subpath := range subpaths {
		if subpath != "" {
			count++
		}
	}

	return count
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bytes"
	"fmt"
	"github.com/aarzilli/sandblast"
	"golang.org/x/net/html"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"unicode"
)

func usage() {
	fmt.Fprintf(os.Stderr, "./qa run <dataset.zip>\n")
	fmt.Fprintf(os.Stderr, "./qa rebuild <dataset.zip> <out.zip>\n")
	fmt.Fprintf(os.Stderr, "./qa one <dataset.zip> <testname>\n")
	os.Exit(1)
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}

type test struct {
	name   string
	input  *zip.File
	target *zip.File
}

type Dataset struct {
	datazip *zip.ReadCloser
	index   *zip.File
	tests   []test
}

func (d *Dataset) Close() error {
	return d.datazip.Close()
}

func collapseWhitespace(in []rune) []rune {
	var b []rune = make([]rune, len(in))
	d := 0
	spaceSeen := true
	for s := range in {
		if spaceSeen {
			if !unicode.IsSpace(in[s]) {
				spaceSeen = false
				b[d] = in[s]
				d++
			}
		} else {
			if unicode.IsSpace(in[s]) {
				b[d] = ' '
				d++
				spaceSeen = true
			} else {
				b[d] = in[s]
				d++
			}
		}
	}
	return b[:d]
}

func openDataset(datapath string) *Dataset {
	dataset, err := zip.OpenReader(datapath)
	must(err)

	ins := map[string]*zip.File{}
	outs := map[string]*zip.File{}
	var index *zip.File

	for _, file := range dataset.File {
		if file.Name == "index.txt" {
			index = file
			continue
		}
		v := strings.Split(file.Name, ".")
		if len(v) != 2 {
			panic(fmt.Errorf("wrong name in dataset: %s\n", file.Name))
		}

		switch v[1] {
		case "html":
			ins[v[0]] = file
		case "target":
			outs[v[0]] = file
		default:
			panic(fmt.Errorf("wrong name in dataset: %s\n", file.Name))
		}
	}

	tests := make([]test, 0, len(ins))
	for k := range ins {
		in, inok := ins[k]
		out, outok := outs[k]
		if !inok || !outok {
			panic(fmt.Errorf("problem with dataset: %s", k))
		}
		tests = append(tests, test{name: k, input: in, target: out})
	}

	return &Dataset{index: index, datazip: dataset, tests: tests}
}

func qarun(datapath string) {
	dataset := openDataset(datapath)
	defer dataset.Close()

	os.Mkdir("work", 0770)

	count := 0
	for _, test := range dataset.tests {
		fmt.Printf("Processing %s\n", test.name)
		if !qaruntest(test, false) {
			count++
		}
		if count > 10 {
			return
		}
	}
}

func qaone(datapath string, name string) {
	dataset := openDataset(datapath)
	defer dataset.Close()

	os.Mkdir("work", 0770)

	for _, test := range dataset.tests {
		if test.name == name {
			qaruntest(test, true)
			return
		}
	}
}

func extractTest(test test, writeextract bool) ([]byte, string) {
	in, err := test.input.Open()
	must(err)
	defer in.Close()

	body, err := ioutil.ReadAll(in)
	must(err)

	e, _, _ := charset.DetermineEncoding(body, "UTF-8")
	r := transform.NewReader(bytes.NewReader(body), e.NewDecoder())
	node, err := html.Parse(r)
	must(err)

	_, output, simplified, flattened, cleaned, err := sandblast.ExtractEx(node)
	must(err)

	if writeextract {
		fmt.Printf("SIMPLIFIED:\n%s\n", simplified.DebugString())
		fmt.Printf("FLATTENED:\n%s\n", flattened.DebugString())
		fmt.Printf("CLEANED:\n%s\n", cleaned.DebugString())
	}

	return body, output
}

func qaruntest(test test, writein bool) bool {
	body, output := extractTest(test, writein)

	tgt, err := test.target.Open()
	must(err)
	defer tgt.Close()

	tgtbody, err := ioutil.ReadAll(tgt)
	must(err)
	target := strings.TrimSpace(string(tgtbody))

	if string(collapseWhitespace([]rune(target))) != string(collapseWhitespace([]rune(output))) {
		fmt.Printf("%s output and target differ\n", test.name)
		tgtout, err := os.Create(fmt.Sprintf("work/%s.target", test.name))
		must(err)
		io.WriteString(tgtout, target)
		io.WriteString(tgtout, "\n")
		tgtout.Close()
		outout, err := os.Create(fmt.Sprintf("work/%s.out", test.name))
		must(err)
		io.WriteString(outout, output)
		io.WriteString(outout, "\n")
		outout.Close()

		if writein {
			inout, err := os.Create(fmt.Sprintf("work/%s.html", test.name))
			must(err)
			inout.Write(body)
			inout.Close()
		}

		return false
	}
	return true
}

func qarebuild(datapath, outpath string) {
	dataset := openDataset(datapath)
	defer dataset.Close()

	outw, err := os.Create(outpath)
	must(err)
	defer outw.Close()

	outzip := zip.NewWriter(outw)
	defer outzip.Close()

	copyFile(outzip, "index.txt", dataset.index)

	for _, test := range dataset.tests {
		fmt.Printf("processing %s\n", test.name)
		copyFile(outzip, fmt.Sprintf("%s.html", test.name), test.input)
		_, output := extractTest(test, false)
		w, err := outzip.Create(fmt.Sprintf("%s.target", test.name))
		must(err)
		_, err = io.WriteString(w, output)
		must(err)
	}
}

func copyFile(outzip *zip.Writer, name string, in *zip.File) {
	w, err := outzip.Create(name)
	must(err)
	r, err := in.Open()
	must(err)
	defer r.Close()
	_, err = io.Copy(w, r)
	must(err)
}

func main() {
	if len(os.Args) < 1 {
		usage()
	}
	switch os.Args[1] {
	case "run":
		if len(os.Args) < 3 {
			usage()
		}
		qarun(os.Args[2])
	case "one":
		qaone(os.Args[2], os.Args[3])
	case "rebuild":
		if len(os.Args) < 4 {
			usage()
		}
		qarebuild(os.Args[2], os.Args[3])
	case "help":
		usage()
	default:
		usage()
	}
}
<commit_msg>qa: two messages<commit_after>package main

import (
	"archive/zip"
	"bytes"
	"fmt"
	"github.com/aarzilli/sandblast"
	"golang.org/x/net/html"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"unicode"
)

func usage() {
	fmt.Fprintf(os.Stderr, "./qa run <dataset.zip>\n")
	fmt.Fprintf(os.Stderr, "./qa rebuild <dataset.zip> <out.zip>\n")
	fmt.Fprintf(os.Stderr, "./qa one <dataset.zip> <testname>\n")
	os.Exit(1)
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}

type test struct {
	name   string
	input  *zip.File
	target *zip.File
}

type Dataset struct {
	datazip *zip.ReadCloser
	index   *zip.File
	tests   []test
}

func (d *Dataset) Close() error {
	return d.datazip.Close()
}

func collapseWhitespace(in []rune) []rune {
	var b []rune = make([]rune, len(in))
	d := 0
	spaceSeen := true
	for s := range in {
		if spaceSeen {
			if !unicode.IsSpace(in[s]) {
				spaceSeen = false
				b[d] = in[s]
				d++
			}
		} else {
			if unicode.IsSpace(in[s]) {
				b[d] = ' '
				d++
				spaceSeen = true
			} else {
				b[d] = in[s]
				d++
			}
		}
	}
	return b[:d]
}

func openDataset(datapath string) *Dataset {
	dataset, err := zip.OpenReader(datapath)
	must(err)

	ins := map[string]*zip.File{}
	outs := map[string]*zip.File{}
	var index *zip.File

	for _, file := range dataset.File {
		if file.Name == "index.txt" {
			index = file
			continue
		}
		v := strings.Split(file.Name, ".")
		if len(v) != 2 {
			panic(fmt.Errorf("wrong name in dataset: %s\n", file.Name))
		}

		switch v[1] {
		case "html":
			ins[v[0]] = file
		case "target":
			outs[v[0]] = file
		default:
			panic(fmt.Errorf("wrong name in dataset: %s\n", file.Name))
		}
	}

	tests := make([]test, 0, len(ins))
	for k := range ins {
		in, inok := ins[k]
		out, outok := outs[k]
		if !inok || !outok {
			panic(fmt.Errorf("problem with dataset: %s", k))
		}
		tests = append(tests, test{name: k, input: in, target: out})
	}

	return &Dataset{index: index, datazip: dataset, tests: tests}
}

func qarun(datapath string) {
	dataset := openDataset(datapath)
	defer dataset.Close()

	os.Mkdir("work", 0770)

	count := 0
	for _, test := range dataset.tests {
		fmt.Printf("Processing %s\n", test.name)
		if !qaruntest(test, false) {
			count++
		}
		if count > 10 {
			fmt.Printf("Too many differences\n")
			return
		}
	}
	fmt.Printf("All ok\n")
}

func qaone(datapath string, name string) {
	dataset := openDataset(datapath)
	defer dataset.Close()

	os.Mkdir("work", 0770)

	for _, test := range dataset.tests {
		if test.name == name {
			qaruntest(test, true)
			return
		}
	}
}

func extractTest(test test, writeextract bool) ([]byte, string) {
	in, err := test.input.Open()
	must(err)
	defer in.Close()

	body, err := ioutil.ReadAll(in)
	must(err)

	e, _, _ := charset.DetermineEncoding(body, "UTF-8")
	r := transform.NewReader(bytes.NewReader(body), e.NewDecoder())
	node, err := html.Parse(r)
	must(err)

	_, output, simplified, flattened, cleaned, err := sandblast.ExtractEx(node)
	must(err)

	if writeextract {
		fmt.Printf("SIMPLIFIED:\n%s\n", simplified.DebugString())
		fmt.Printf("FLATTENED:\n%s\n", flattened.DebugString())
		fmt.Printf("CLEANED:\n%s\n", cleaned.DebugString())
	}

	return body, output
}

func qaruntest(test test, writein bool) bool {
	body, output := extractTest(test, writein)

	tgt, err := test.target.Open()
	must(err)
	defer tgt.Close()

	tgtbody, err := ioutil.ReadAll(tgt)
	must(err)
	target := strings.TrimSpace(string(tgtbody))

	if string(collapseWhitespace([]rune(target))) != string(collapseWhitespace([]rune(output))) {
		fmt.Printf("%s output and target differ\n", test.name)
		tgtout, err := os.Create(fmt.Sprintf("work/%s.target", test.name))
		must(err)
		io.WriteString(tgtout, target)
		io.WriteString(tgtout, "\n")
		tgtout.Close()
		outout, err := os.Create(fmt.Sprintf("work/%s.out", test.name))
		must(err)
		io.WriteString(outout, output)
		io.WriteString(outout, "\n")
		outout.Close()

		if writein {
			inout, err := os.Create(fmt.Sprintf("work/%s.html", test.name))
			must(err)
			inout.Write(body)
			inout.Close()
		}

		return false
	}
	return true
}

func qarebuild(datapath, outpath string) {
	dataset := openDataset(datapath)
	defer dataset.Close()

	outw, err := os.Create(outpath)
	must(err)
	defer outw.Close()

	outzip := zip.NewWriter(outw)
	defer outzip.Close()

	copyFile(outzip, "index.txt", dataset.index)

	for _, test := range dataset.tests {
		fmt.Printf("processing %s\n", test.name)
		copyFile(outzip, fmt.Sprintf("%s.html", test.name), test.input)
		_, output := extractTest(test, false)
		w, err := outzip.Create(fmt.Sprintf("%s.target", test.name))
		must(err)
		_, err = io.WriteString(w, output)
		must(err)
	}
}

func copyFile(outzip *zip.Writer, name string, in *zip.File) {
	w, err := outzip.Create(name)
	must(err)
	r, err := in.Open()
	must(err)
	defer r.Close()
	_, err = io.Copy(w, r)
	must(err)
}

func main() {
	if len(os.Args) < 1 {
		usage()
	}
	switch os.Args[1] {
	case "run":
		if len(os.Args) < 3 {
			usage()
		}
		qarun(os.Args[2])
	case "one":
		qaone(os.Args[2], os.Args[3])
	case "rebuild":
		if len(os.Args) < 4 {
			usage()
		}
		qarebuild(os.Args[2], os.Args[3])
	case "help":
		usage()
	default:
		usage()
	}
}
<|endoftext|>


<commit_before>package gosnow

import (
	"encoding/json"
	"io/ioutil"
	"testing"
)

const simpleSrc string = `
# My API
## GET /message
 + Response 200 (text/plain)

         Hello World
`

const namelessSrc string = `
FORMAT: 1A

# Group Messages

# Message [/messages/{id}]

## Retrieve Message [GET]
+ Response 200 (text/plain)

        Hello World!

`

const warningSrc string = `
FORMAT: 1A

# Group Messages

# Message [/messages/{id}]

+ Model (text/plain)

        Hello World

## Retrieve Message [GET]
+ Response 200 (text/plain)

        Hello World!

## Retrieve Message [GET]
+ Response 200 (text/plain)

        Hello World!

`

var (
	apibFile      = "test/fixtures/sample-api.apib"
	astFile       = "test/fixtures/sample-api-ast.json"
	sourcemapFile = "test/fixtures/sample-api-sourcemap.json"
)

// replace the variables with the contents of the file they point to
func init() {
	if c, err := ioutil.ReadFile(apibFile); err != nil {
		panic("apibFile not found")
	} else {
		apibFile = string(c)
	}

	if c, err := ioutil.ReadFile(astFile); err != nil {
		panic("astFile not found")
	} else {
		astFile = string(c)
	}

	if c, err := ioutil.ReadFile(sourcemapFile); err != nil {
		panic("sourcemapFile not found")
	} else {
		sourcemapFile = string(c)
	}
}

func TestParse(t *testing.T) {
	res, err := Parse(apibFile)
	if err != nil {
		t.Fatalf("Parse failed with error: %v", err)
	}
	if res == nil {
		t.Fatal("Parse returned nil result")
	}
	// v, _ := json.MarshalIndent(res, "", "  ")
	// fmt.Println(string(v))
}

// ensure that the option parse with a 0 does the same thing as the simple parse
func TestParseEquality(t *testing.T) {
	res1, err := Parse(simpleSrc)
	if err != nil {
		t.Fatalf("Parse failed with err: %v", err)
	}
	res2, err := OptionParse(simpleSrc, 0)
	if err != nil {
		t.Fatalf("OptionParse failed with err: %v", err)
	}

	v1, err := json.Marshal(res1)
	if err != nil {
		t.Fatalf("json marshal error: %v", err)
	}
	v2, err := json.Marshal(res2)
	if err != nil {
		t.Fatalf("json marshal error: %v", err)
	}
	if string(v1) != string(v2) {
		t.Error("Results should be equal")
	}
}

func TestParseError(t *testing.T) {
	junk := "*#(*(R$#&)$#)R*(Y@#_RH"
	res, err := OptionParse(junk, -1)
	if err == nil {
		t.Errorf("OptionParse did not fail for junk input")
	}
	if res != nil {
		t.Errorf("OptionParse returned non=empty result for junk input")
	}
}

func TestFilesOptionParse(t *testing.T) {
	res, err := OptionParse(apibFile, ScRenderDescriptionsOptionKey)
	if err != nil {
		t.Errorf("OptionParse failed for key ScRenderDescriptionsOptionKey with error: %v", err)
	} else if res == nil {
		t.Errorf("OptionParse for key ScRenderDescriptionsOptionKey returned empty result")
	}

	_, err = OptionParse(namelessSrc, RequireBlueprintNameOptionKey)
	if err == nil {
		t.Errorf("strict OptionParse did not fail for key RequireBlueprintNameOptionKey")
	}

	res, err = OptionParse(apibFile, ExportSourcemapOptionKey)
	if err != nil {
		t.Errorf("OptionParse failed for ExportSourcemapOptionKey with error: %v", err)
	} else if res == nil {
		t.Errorf("OptionParse for key ExportSourcemapOptionKey returned empty result")
	}
}
<commit_msg>added some tests for new parser methods<commit_after>package gosnow

import (
	"encoding/json"
	"io/ioutil"
	"testing"
)

const simpleSrc string = `
# My API
## GET /message
 + Response 200 (text/plain)

         Hello World
`

const namelessSrc string = `
FORMAT: 1A

# Group Messages

# Message [/messages/{id}]

## Retrieve Message [GET]
+ Response 200 (text/plain)

        Hello World!

`

const warningSrc string = `
FORMAT: 1A

# Group Messages

# Message [/messages/{id}]

+ Model (text/plain)

        Hello World

## Retrieve Message [GET]
+ Response 200 (text/plain)

        Hello World!

## Retrieve Message [GET]
+ Response 200 (text/plain)

        Hello World!

`

var (
	apibFile      = "test/fixtures/sample-api.apib"
	astFile       = "test/fixtures/sample-api-ast.json"
	sourcemapFile = "test/fixtures/sample-api-sourcemap.json"
)

// replace the variables with the contents of the file they point to
func init() {
	if c, err := ioutil.ReadFile(apibFile); err != nil {
		panic("apibFile not found")
	} else {
		apibFile = string(c)
	}

	if c, err := ioutil.ReadFile(astFile); err != nil {
		panic("astFile not found")
	} else {
		astFile = string(c)
	}

	if c, err := ioutil.ReadFile(sourcemapFile); err != nil {
		panic("sourcemapFile not found")
	} else {
		sourcemapFile = string(c)
	}
}

func TestSourceAnnotationOk(t *testing.T) {
	sa := new(SourceAnnotation)
	if !sa.Ok() {
		t.Error("empty source annotation should have zero value indicating ok")
	}
	sa.Code = 2
	if sa.Ok() {
		t.Error("source annotation should have non zero value indicating not ok")
	}
}

func TestNewPR(t *testing.T) {
	_, err := newPR([]byte(`{"unrelated": "json"}`))
	if err != nil {
		t.Errorf("newPR errored for valid json %v", err)
	}
}

func TestNewPRFailure(t *testing.T) {
	junk := []byte(`*#(*(R$#&)$#)R*(Y@#_RH`)
	_, err := newPR(junk)
	if err == nil {
		t.Error("newPR should have errored and did not")
	}
	if e, ok := err.(*json.SyntaxError); !ok {
		t.Errorf("Expected json.SyntaxError, got %T", e)
	}
}

func TestParse(t *testing.T) {
	res, err := Parse(apibFile)
	if err != nil {
		t.Fatalf("Parse failed with error: %v", err)
	}
	if res == nil {
		t.Fatal("Parse returned nil result")
	}
	// v, _ := json.MarshalIndent(res, "", "  ")
	// fmt.Println(string(v))
}

// ensure that the option parse with a 0 does the same thing as the simple parse
func TestParseEquality(t *testing.T) {
	res1, err := Parse(simpleSrc)
	if err != nil {
		t.Fatalf("Parse failed with err: %v", err)
	}
	res2, err := OptionParse(simpleSrc, 0)
	if err != nil {
		t.Fatalf("OptionParse failed with err: %v", err)
	}

	v1, err := json.Marshal(res1)
	if err != nil {
		t.Fatalf("json marshal error: %v", err)
	}
	v2, err := json.Marshal(res2)
	if err != nil {
		t.Fatalf("json marshal error: %v", err)
	}
	if string(v1) != string(v2) {
		t.Error("Results should be equal")
	}
}

func TestParseError(t *testing.T) {
	junk := "*#(*(R$#&)$#)R*(Y@#_RH"
	res, err := OptionParse(junk, -1)
	if err == nil {
		t.Errorf("OptionParse did not fail for junk input")
	}
	if res != nil {
		t.Errorf("OptionParse returned non=empty result for junk input")
	}
}

func TestFilesOptionParse(t *testing.T) {
	res, err := OptionParse(apibFile, ScRenderDescriptionsOptionKey)
	if err != nil {
		t.Errorf("OptionParse failed for key ScRenderDescriptionsOptionKey with error: %v", err)
	} else if res == nil {
		t.Errorf("OptionParse for key ScRenderDescriptionsOptionKey returned empty result")
	}

	_, err = OptionParse(namelessSrc, RequireBlueprintNameOptionKey)
	if err == nil {
		t.Errorf("strict OptionParse did not fail for key RequireBlueprintNameOptionKey")
	}

	res, err = OptionParse(apibFile, ExportSourcemapOptionKey)
	if err != nil {
		t.Errorf("OptionParse failed for ExportSourcemapOptionKey with error: %v", err)
	} else if res == nil {
		t.Errorf("OptionParse for key ExportSourcemapOptionKey returned empty result")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/dustin/go-couchstore"
	"github.com/dustin/go-jsonpointer"
)

var timeoutError = errors.New("query timed out")

type ptrval struct {
	di       *couchstore.DocInfo
	val      *string
	included bool
}

type Reducer func(input chan ptrval) interface{}

type processOut struct {
	cacheKey    string
	key         int64
	value       []interface{}
	err         error
	cacheOpaque uint32
}

func (p processOut) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]interface{}{"v": p.value})
}

type processIn struct {
	cacheKey string
	dbname   string
	key      int64
	infos    []*couchstore.DocInfo
	nextInfo *couchstore.DocInfo
	ptrs     []string
	reds     []string
	before   time.Time
	filters	 []string
	filtervals []string
	out      chan<- *processOut
}

type queryIn struct {
	dbname    string
	from      string
	to        string
	group     int
	ptrs      []string
	reds      []string
	start     time.Time
	before    time.Time
	filters	  []string
	filtervals []string
	started   int32
	totalKeys int32
	out       chan *processOut
	cherr     chan error
}

func processDoc(di *couchstore.DocInfo, chs []chan ptrval,
	doc []byte, ptrs []string, filters []string, filtervals []string, included bool) {

	pv := ptrval{di, nil, included}

	j := map[string]interface{}{}
	err := json.Unmarshal(doc, &j)
	if err != nil {
		for i := range ptrs {
			chs[i] <- pv
		}
		return
	}
	for i, p := range filters {
		val := jsonpointer.Get(j, p)
		checkVal := filtervals[i]
		switch val.(type) {
		case string:
			if (val != checkVal) {
				return
			}
		case int, uint, int64, float64, uint64, bool:
			v := fmt.Sprintf("%v", val)
			if (v != checkVal) {
				return
			}
		default:
			return
		}
	}

	for i, p := range ptrs {
		val := jsonpointer.Get(j, p)
		switch x := val.(type) {
		case string:
			pv.val = &x
			chs[i] <- pv
		case int, uint, int64, float64, uint64, bool:
			v := fmt.Sprintf("%v", val)
			pv.val = &v
			chs[i] <- pv
		default:
			log.Printf("Ignoring %T", val)
			chs[i] <- pv
		}
	}
}

func process_docs(pi *processIn) {

	result := processOut{pi.cacheKey, pi.key, nil, nil, 0}

	db, err := dbopen(pi.dbname)
	if err != nil {
		result.err = err
		pi.out <- &result
		return
	}
	defer db.Close()

	chans := make([]chan ptrval, 0, len(pi.ptrs))
	resultchs := make([]chan interface{}, 0, len(pi.ptrs))
	for i, r := range pi.reds {
		chans = append(chans, make(chan ptrval))
		resultchs = append(resultchs, make(chan interface{}))

		go func() {
			resultchs[i] <- reducers[r](chans[i])
		}()
	}

	go func() {
		defer closeAll(chans)

		dodoc := func(di *couchstore.DocInfo, included bool) {
			doc, err := db.GetFromDocInfo(di)
			if err == nil {
				processDoc(di, chans, doc.Value(), pi.ptrs, pi.filters, pi.filtervals, included)
			} else {
				for i := range pi.ptrs {
					chans[i] <- ptrval{di, nil, included}
				}
			}
		}

		for _, di := range pi.infos {
			dodoc(di, true)
		}
		if pi.nextInfo != nil {
			dodoc(pi.nextInfo, false)
		}
	}()

	results := make([]interface{}, len(pi.ptrs))
	for i := 0; i < len(pi.ptrs); i++ {
		results[i] = <-resultchs[i]
		if f, fok := results[i].(float64); fok && math.IsNaN(f) {
			results[i] = nil
		}
	}
	result.value = results

	if result.cacheOpaque == 0 && result.cacheKey != "" {
		// It's OK if we can't store our newly pulled item in
		// the cache, but it's most definitely not OK to stop
		// here because of this.
		select {
		case cacheInputSet <- &result:
		default:
		}
	}
	pi.out <- &result
}

func docProcessor(ch <-chan *processIn) {
	for pi := range ch {
		if time.Now().Before(pi.before) {
			process_docs(pi)
		} else {
			pi.out <- &processOut{"", pi.key, nil, timeoutError, 0}
		}
	}
}

func fetchDocs(dbname string, key int64, infos []*couchstore.DocInfo,
	nextInfo *couchstore.DocInfo, ptrs []string, reds []string,
	before time.Time, filters []string, filtervals []string, out chan<- *processOut) {

	i := processIn{"", dbname, key, infos, nextInfo,
		ptrs, reds, before, filters, filtervals, out}

	cacheInput <- &i
}

func runQuery(q *queryIn) {
	db, err := dbopen(q.dbname)
	if err != nil {
		log.Printf("Error opening db: %v - %v", q.dbname, err)
		q.cherr <- err
		return
	}
	defer db.Close()

	chunk := int64(time.Duration(q.group) * time.Millisecond)

	infos := []*couchstore.DocInfo{}
	g := int64(0)
	nextg := ""

	err = db.Walk(q.from, func(d *couchstore.Couchstore,
		di *couchstore.DocInfo) error {
		kstr := di.ID()
		var err error
		if q.to != "" && kstr >= q.to {
			err = couchstore.StopIteration
		}

		atomic.AddInt32(&q.totalKeys, 1)

		if kstr >= nextg {
			if len(infos) > 0 {
				atomic.AddInt32(&q.started, 1)
				fetchDocs(q.dbname, g, infos, di,
					q.ptrs, q.reds, q.before, q.filters, q.filtervals, q.out)

				infos = make([]*couchstore.DocInfo, 0, len(infos))
			}

			k := parseKey(kstr)
			g = (k / chunk) * chunk
			nextgi := g + chunk
			nextgt := time.Unix(nextgi/1e9, nextgi%1e9).UTC()
			nextg = nextgt.Format(time.RFC3339Nano)
		}
		infos = append(infos, di)

		return err
	})

	if err == nil && len(infos) > 0 {
		atomic.AddInt32(&q.started, 1)
		fetchDocs(q.dbname, g, infos, nil,
			q.ptrs, q.reds, q.before, q.filters, q.filtervals, q.out)
	}

	q.cherr <- err
}

func queryExecutor(ch <-chan *queryIn) {
	for q := range ch {
		if time.Now().Before(q.before) {
			runQuery(q)
		} else {
			log.Printf("Timed out query that's %v late",
				time.Since(q.before))
			q.cherr <- timeoutError
		}
	}
}

func executeQuery(dbname, from, to string, group int,
	ptrs []string, reds []string, filters []string, filtervals []string) *queryIn {

	now := time.Now()

	rv := &queryIn{
		dbname:  dbname,
		from:    from,
		to:      to,
		group:   group,
		ptrs:    ptrs,
		reds:    reds,
		start:   now,
		before:  now.Add(*queryTimeout),
	    filters: filters,
		filtervals: filtervals,
		out:     make(chan *processOut),
		cherr:   make(chan error),
	}
	queryInput <- rv
	return rv
}

var processorInput chan *processIn
var queryInput chan *queryIn

func convertTofloat64(in chan ptrval) chan float64 {
	ch := make(chan float64)
	go func() {
		defer close(ch)
		for v := range in {
			if v.included && v.val != nil {
				x, err := strconv.ParseFloat(*v.val, 64)
				if err == nil {
					ch <- x
				}
			}
		}
	}()

	return ch
}

func convertTofloat64Rate(in chan ptrval) chan float64 {
	ch := make(chan float64)
	go func() {
		defer close(ch)
		var prevts int64
		var preval float64

		// First, find a part of the stream that has usable data.
		for v := range in {
			if v.di != nil && v.val != nil {
				x, err := strconv.ParseFloat(*v.val, 64)
				if err == nil {
					prevts = parseKey(v.di.ID())
					preval = x
					break
				}
			}
		}
		// Then emit floats based on deltas from previous values.
		for v := range in {
			if v.di != nil && v.val != nil {
				x, err := strconv.ParseFloat(*v.val, 64)
				if err == nil {
					thists := parseKey(v.di.ID())

					val := ((x - preval) /
						(float64(thists-prevts) / 1e9))

					if !math.IsNaN(val) {
						ch <- val
					}

					prevts = thists
					preval = x
				}
			}
		}
	}()

	return ch
}

var reducers = map[string]Reducer{
	"identity": func(input chan ptrval) interface{} {
		rv := []*string{}
		for s := range input {
			if s.included {
				rv = append(rv, s.val)
			}
		}
		return rv
	},
	"any": func(input chan ptrval) interface{} {
		for v := range input {
			if v.included && v.val != nil {
				return *v.val
			}
		}
		return nil
	},
	"count": func(input chan ptrval) interface{} {
		rv := 0
		for v := range input {
			if v.included && v.val != nil {
				rv++
			}
		}
		return rv
	},
	"sum": func(input chan ptrval) interface{} {
		rv := float64(0)
		for v := range convertTofloat64(input) {
			rv += v
		}
		return rv
	},
	"sumsq": func(input chan ptrval) interface{} {
		rv := float64(0)
		for v := range convertTofloat64(input) {
			rv += (v * v)
		}
		return rv
	},
	"max": func(input chan ptrval) interface{} {
		rv := math.NaN()
		for v := range convertTofloat64(input) {
			if v > rv || math.IsNaN(rv) {
				rv = v
			}
		}
		return rv
	},
	"min": func(input chan ptrval) interface{} {
		rv := math.NaN()
		for v := range convertTofloat64(input) {
			if v < rv || math.IsNaN(rv) {
				rv = v
			}
		}
		return rv
	},
	"avg": func(input chan ptrval) interface{} {
		nums := float64(0)
		sum := float64(0)
		for v := range convertTofloat64(input) {
			nums++
			sum += v
		}
		if nums > 0 {
			return sum / nums
		}
		return math.NaN()
	},
	"c_min": func(input chan ptrval) interface{} {
		rv := math.NaN()
		for v := range convertTofloat64Rate(input) {
			if v < rv || math.IsNaN(rv) {
				rv = v
			}
		}
		return rv
	},
	"c_avg": func(input chan ptrval) interface{} {
		nums := float64(0)
		sum := float64(0)
		for v := range convertTofloat64Rate(input) {
			nums++
			sum += v
		}
		if nums > 0 {
			return sum / nums
		}
		return math.NaN()
	},
	"c_max": func(input chan ptrval) interface{} {
		rv := math.NaN()
		for v := range convertTofloat64Rate(input) {
			if v > rv || math.IsNaN(rv) {
				rv = v
			}
		}
		return rv
	},
}
<commit_msg>Minor cleanups.<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/dustin/go-couchstore"
	"github.com/dustin/go-jsonpointer"
)

var timeoutError = errors.New("query timed out")

type ptrval struct {
	di       *couchstore.DocInfo
	val      *string
	included bool
}

type Reducer func(input chan ptrval) interface{}

type processOut struct {
	cacheKey    string
	key         int64
	value       []interface{}
	err         error
	cacheOpaque uint32
}

func (p processOut) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]interface{}{"v": p.value})
}

type processIn struct {
	cacheKey   string
	dbname     string
	key        int64
	infos      []*couchstore.DocInfo
	nextInfo   *couchstore.DocInfo
	ptrs       []string
	reds       []string
	before     time.Time
	filters    []string
	filtervals []string
	out        chan<- *processOut
}

type queryIn struct {
	dbname     string
	from       string
	to         string
	group      int
	ptrs       []string
	reds       []string
	start      time.Time
	before     time.Time
	filters    []string
	filtervals []string
	started    int32
	totalKeys  int32
	out        chan *processOut
	cherr      chan error
}

func processDoc(di *couchstore.DocInfo, chs []chan ptrval,
	doc []byte, ptrs []string,
	filters []string, filtervals []string,
	included bool) {

	pv := ptrval{di, nil, included}

	j := map[string]interface{}{}
	err := json.Unmarshal(doc, &j)
	if err != nil {
		for i := range ptrs {
			chs[i] <- pv
		}
		return
	}
	for i, p := range filters {
		val := jsonpointer.Get(j, p)
		checkVal := filtervals[i]
		switch val.(type) {
		case string:
			if val != checkVal {
				return
			}
		case int, uint, int64, float64, uint64, bool:
			v := fmt.Sprintf("%v", val)
			if v != checkVal {
				return
			}
		default:
			return
		}
	}

	for i, p := range ptrs {
		val := jsonpointer.Get(j, p)
		switch x := val.(type) {
		case string:
			pv.val = &x
			chs[i] <- pv
		case int, uint, int64, float64, uint64, bool:
			v := fmt.Sprintf("%v", val)
			pv.val = &v
			chs[i] <- pv
		default:
			log.Printf("Ignoring %T", val)
			chs[i] <- pv
		}
	}
}

func process_docs(pi *processIn) {

	result := processOut{pi.cacheKey, pi.key, nil, nil, 0}

	db, err := dbopen(pi.dbname)
	if err != nil {
		result.err = err
		pi.out <- &result
		return
	}
	defer db.Close()

	chans := make([]chan ptrval, 0, len(pi.ptrs))
	resultchs := make([]chan interface{}, 0, len(pi.ptrs))
	for i, r := range pi.reds {
		chans = append(chans, make(chan ptrval))
		resultchs = append(resultchs, make(chan interface{}))

		go func() {
			resultchs[i] <- reducers[r](chans[i])
		}()
	}

	go func() {
		defer closeAll(chans)

		dodoc := func(di *couchstore.DocInfo, included bool) {
			doc, err := db.GetFromDocInfo(di)
			if err == nil {
				processDoc(di, chans, doc.Value(), pi.ptrs,
					pi.filters, pi.filtervals, included)
			} else {
				for i := range pi.ptrs {
					chans[i] <- ptrval{di, nil, included}
				}
			}
		}

		for _, di := range pi.infos {
			dodoc(di, true)
		}
		if pi.nextInfo != nil {
			dodoc(pi.nextInfo, false)
		}
	}()

	results := make([]interface{}, len(pi.ptrs))
	for i := 0; i < len(pi.ptrs); i++ {
		results[i] = <-resultchs[i]
		if f, fok := results[i].(float64); fok && math.IsNaN(f) {
			results[i] = nil
		}
	}
	result.value = results

	if result.cacheOpaque == 0 && result.cacheKey != "" {
		// It's OK if we can't store our newly pulled item in
		// the cache, but it's most definitely not OK to stop
		// here because of this.
		select {
		case cacheInputSet <- &result:
		default:
		}
	}
	pi.out <- &result
}

func docProcessor(ch <-chan *processIn) {
	for pi := range ch {
		if time.Now().Before(pi.before) {
			process_docs(pi)
		} else {
			pi.out <- &processOut{"", pi.key, nil, timeoutError, 0}
		}
	}
}

func fetchDocs(dbname string, key int64, infos []*couchstore.DocInfo,
	nextInfo *couchstore.DocInfo, ptrs []string, reds []string,
	filters []string, filtervals []string,
	before time.Time, out chan<- *processOut) {

	i := processIn{"", dbname, key, infos, nextInfo,
		ptrs, reds, before, filters, filtervals, out}

	cacheInput <- &i
}

func runQuery(q *queryIn) {
	db, err := dbopen(q.dbname)
	if err != nil {
		log.Printf("Error opening db: %v - %v", q.dbname, err)
		q.cherr <- err
		return
	}
	defer db.Close()

	chunk := int64(time.Duration(q.group) * time.Millisecond)

	infos := []*couchstore.DocInfo{}
	g := int64(0)
	nextg := ""

	err = db.Walk(q.from, func(d *couchstore.Couchstore,
		di *couchstore.DocInfo) error {
		kstr := di.ID()
		var err error
		if q.to != "" && kstr >= q.to {
			err = couchstore.StopIteration
		}

		atomic.AddInt32(&q.totalKeys, 1)

		if kstr >= nextg {
			if len(infos) > 0 {
				atomic.AddInt32(&q.started, 1)
				fetchDocs(q.dbname, g, infos, di,
					q.ptrs, q.reds, q.filters, q.filtervals,
					q.before, q.out)

				infos = make([]*couchstore.DocInfo, 0, len(infos))
			}

			k := parseKey(kstr)
			g = (k / chunk) * chunk
			nextgi := g + chunk
			nextgt := time.Unix(nextgi/1e9, nextgi%1e9).UTC()
			nextg = nextgt.Format(time.RFC3339Nano)
		}
		infos = append(infos, di)

		return err
	})

	if err == nil && len(infos) > 0 {
		atomic.AddInt32(&q.started, 1)
		fetchDocs(q.dbname, g, infos, nil,
			q.ptrs, q.reds, q.filters, q.filtervals,
			q.before, q.out)
	}

	q.cherr <- err
}

func queryExecutor(ch <-chan *queryIn) {
	for q := range ch {
		if time.Now().Before(q.before) {
			runQuery(q)
		} else {
			log.Printf("Timed out query that's %v late",
				time.Since(q.before))
			q.cherr <- timeoutError
		}
	}
}

func executeQuery(dbname, from, to string, group int,
	ptrs []string, reds []string, filters []string, filtervals []string) *queryIn {

	now := time.Now()

	rv := &queryIn{
		dbname:     dbname,
		from:       from,
		to:         to,
		group:      group,
		ptrs:       ptrs,
		reds:       reds,
		start:      now,
		before:     now.Add(*queryTimeout),
		filters:    filters,
		filtervals: filtervals,
		out:        make(chan *processOut),
		cherr:      make(chan error),
	}
	queryInput <- rv
	return rv
}

var processorInput chan *processIn
var queryInput chan *queryIn

func convertTofloat64(in chan ptrval) chan float64 {
	ch := make(chan float64)
	go func() {
		defer close(ch)
		for v := range in {
			if v.included && v.val != nil {
				x, err := strconv.ParseFloat(*v.val, 64)
				if err == nil {
					ch <- x
				}
			}
		}
	}()

	return ch
}

func convertTofloat64Rate(in chan ptrval) chan float64 {
	ch := make(chan float64)
	go func() {
		defer close(ch)
		var prevts int64
		var preval float64

		// First, find a part of the stream that has usable data.
		for v := range in {
			if v.di != nil && v.val != nil {
				x, err := strconv.ParseFloat(*v.val, 64)
				if err == nil {
					prevts = parseKey(v.di.ID())
					preval = x
					break
				}
			}
		}
		// Then emit floats based on deltas from previous values.
		for v := range in {
			if v.di != nil && v.val != nil {
				x, err := strconv.ParseFloat(*v.val, 64)
				if err == nil {
					thists := parseKey(v.di.ID())

					val := ((x - preval) /
						(float64(thists-prevts) / 1e9))

					if !math.IsNaN(val) {
						ch <- val
					}

					prevts = thists
					preval = x
				}
			}
		}
	}()

	return ch
}

var reducers = map[string]Reducer{
	"identity": func(input chan ptrval) interface{} {
		rv := []*string{}
		for s := range input {
			if s.included {
				rv = append(rv, s.val)
			}
		}
		return rv
	},
	"any": func(input chan ptrval) interface{} {
		for v := range input {
			if v.included && v.val != nil {
				return *v.val
			}
		}
		return nil
	},
	"count": func(input chan ptrval) interface{} {
		rv := 0
		for v := range input {
			if v.included && v.val != nil {
				rv++
			}
		}
		return rv
	},
	"sum": func(input chan ptrval) interface{} {
		rv := float64(0)
		for v := range convertTofloat64(input) {
			rv += v
		}
		return rv
	},
	"sumsq": func(input chan ptrval) interface{} {
		rv := float64(0)
		for v := range convertTofloat64(input) {
			rv += (v * v)
		}
		return rv
	},
	"max": func(input chan ptrval) interface{} {
		rv := math.NaN()
		for v := range convertTofloat64(input) {
			if v > rv || math.IsNaN(rv) {
				rv = v
			}
		}
		return rv
	},
	"min": func(input chan ptrval) interface{} {
		rv := math.NaN()
		for v := range convertTofloat64(input) {
			if v < rv || math.IsNaN(rv) {
				rv = v
			}
		}
		return rv
	},
	"avg": func(input chan ptrval) interface{} {
		nums := float64(0)
		sum := float64(0)
		for v := range convertTofloat64(input) {
			nums++
			sum += v
		}
		if nums > 0 {
			return sum / nums
		}
		return math.NaN()
	},
	"c_min": func(input chan ptrval) interface{} {
		rv := math.NaN()
		for v := range convertTofloat64Rate(input) {
			if v < rv || math.IsNaN(rv) {
				rv = v
			}
		}
		return rv
	},
	"c_avg": func(input chan ptrval) interface{} {
		nums := float64(0)
		sum := float64(0)
		for v := range convertTofloat64Rate(input) {
			nums++
			sum += v
		}
		if nums > 0 {
			return sum / nums
		}
		return math.NaN()
	},
	"c_max": func(input chan ptrval) interface{} {
		rv := math.NaN()
		for v := range convertTofloat64Rate(input) {
			if v > rv || math.IsNaN(rv) {
				rv = v
			}
		}
		return rv
	},
}
<|endoftext|>


<commit_before>package lytics

import (
	"net/url"
	"time"
)

const (
	queryEndpoint         = "query/:id"
	queryListEndpoint     = "query"
	queryTestEndpoint     = "query/_test"
	queryValidateEndpoint = "query/_validate"
)

type (
	// Query represents an LQL Statement structure
	Query struct {
		Id      string                `json:"id"`
		Created time.Time             `json:"created"`
		Updated time.Time             `json:"updated"`
		Alias   string                `json:"alias"`
		Table   string                `json:"table"`
		From    string                `json:"from"`
		Text    string                `json:"text"`
		Fields  map[string]QueryField `json:"fields"`
	}
	// A field in a query
	// - very similar to catalog, query fields create catalog fields
	QueryField struct {
		As         string   `json:"as"`
		IsBy       bool     `json:"is_by"`
		Type       string   `json:"type"`
		ShortDesc  string   `json:"shortdesc"`
		LongDesc   string   `json:"longdesc"`
		Identities []string `json:"identities"`
		Froms      []string `json:"froms"`
	}
)

// GetQueries returns a list of all queries associated with this account
// https://www.getlytics.com/developers/rest-api#query
func (l *Client) GetQueries() ([]Query, error) {
	res := ApiResp{}
	data := []Query{}

	// make the request
	err := l.Get(queryListEndpoint, nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetQueries returns a list of all queries associated with this account
// https://www.getlytics.com/developers/rest-api#query
func (l *Client) GetQuery(alias string) (Query, error) {
	res := ApiResp{}
	data := Query{}

	// make the request
	err := l.Get(parseLyticsURL(queryEndpoint, map[string]string{"id": alias}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetQueryTest returns the evaluated entity from given query
// https://www.getlytics.com/developers/rest-api#query
func (l *Client) GetQueryTest(qs url.Values, query string) (Entity, error) {
	res := ApiResp{}
	data := Entity{}

	// make the request
	err := l.Post(queryTestEndpoint, qs, query, &res, &data)

	if err != nil {
		return data, err
	}

	return data, nil
}

// PostQueryValidate returns the query and how it is interpreted
// https://www.getlytics.com/developers/rest-api#query
func (l *Client) PostQueryValidate(query string) (Query, error) {
	res := ApiResp{}
	data := Query{}

	// make the request
	err := l.PostType("text/plain", queryValidateEndpoint, nil, query, &res, &data)

	if err != nil {
		return data, err
	}

	return data, nil
}
<commit_msg>Queries now return arrays<commit_after>package lytics

import (
	"net/url"
	"time"
)

const (
	queryEndpoint         = "query/:id"
	queryListEndpoint     = "query"
	queryTestEndpoint     = "query/_test"
	queryValidateEndpoint = "query/_validate"
)

type (
	// Query represents an LQL Statement structure
	Query struct {
		Id      string                `json:"id"`
		Created time.Time             `json:"created"`
		Updated time.Time             `json:"updated"`
		Alias   string                `json:"alias"`
		Table   string                `json:"table"`
		From    string                `json:"from"`
		Text    string                `json:"text"`
		Fields  map[string]QueryField `json:"fields"`
	}
	// A field in a query
	// - very similar to catalog, query fields create catalog fields
	QueryField struct {
		As         string   `json:"as"`
		IsBy       bool     `json:"is_by"`
		Type       string   `json:"type"`
		ShortDesc  string   `json:"shortdesc"`
		LongDesc   string   `json:"longdesc"`
		Identities []string `json:"identities"`
		Froms      []string `json:"froms"`
	}
)

// GetQueries returns a list of all queries associated with this account
// https://www.getlytics.com/developers/rest-api#query
func (l *Client) GetQueries() ([]Query, error) {
	res := ApiResp{}
	data := []Query{}

	// make the request
	err := l.Get(queryListEndpoint, nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetQueries returns a list of all queries associated with this account
// https://www.getlytics.com/developers/rest-api#query
func (l *Client) GetQuery(alias string) (Query, error) {
	res := ApiResp{}
	data := Query{}

	// make the request
	err := l.Get(parseLyticsURL(queryEndpoint, map[string]string{"id": alias}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetQueryTest returns the evaluated entity from given query
// https://www.getlytics.com/developers/rest-api#query
func (l *Client) GetQueryTest(qs url.Values, query string) (Entity, error) {
	res := ApiResp{}
	data := Entity{}

	// make the request
	err := l.Post(queryTestEndpoint, qs, query, &res, &data)

	if err != nil {
		return data, err
	}

	return data, nil
}

// PostQueryValidate returns the query and how it is interpreted
// https://www.getlytics.com/developers/rest-api#query
func (l *Client) PostQueryValidate(query string) ([]Query, error) {
	res := ApiResp{}
	data := []Query{}

	// make the request
	err := l.PostType("text/plain", queryValidateEndpoint, nil, query, &res, &data)

	if err != nil {
		return data, err
	}

	return data, nil
}
<|endoftext|>