<commit_before>// A memcached binary protocol client.
package memcached

import (
	"bufio"
	"encoding/binary"
	"github.com/dustin/gomemcached"
	"io"
	"log"
	"net"
	"runtime"
)

const bufsize = 1024

// The Client itself.
type Client struct {
	conn   net.Conn
	writer *bufio.Writer

	hdrBuf []byte
}

// Connect to a memcached server.
func Connect(prot, dest string) (rv *Client, err error) {
	conn, err := net.Dial(prot, dest)
	if err != nil {
		return nil, err
	}
	rv = new(Client)
	rv.conn = conn
	rv.writer = bufio.NewWriterSize(rv.conn, bufsize)
	rv.hdrBuf = make([]byte, gomemcached.HDR_LEN)
	return
}

// Close the connection when you're done.
func (c *Client) Close() {
	c.conn.Close()
}

// Send a custom request and get the response.
func (client *Client) Send(req gomemcached.MCRequest) (rv gomemcached.MCResponse) {
	transmitRequest(client.writer, req)
	rv = client.getResponse()
	return
}

// Send a request, but do not wait for a response.
func (client *Client) Transmit(req gomemcached.MCRequest) {
	transmitRequest(client.writer, req)
}

// Receive a response
func (client *Client) Receive() gomemcached.MCResponse {
	return client.getResponse()
}

// Get the value for a key.
func (client *Client) Get(vb uint16, key string) gomemcached.MCResponse {
	var req gomemcached.MCRequest
	req.Opcode = gomemcached.GET
	req.VBucket = vb
	req.Key = []byte(key)
	req.Cas = 0
	req.Opaque = 0
	req.Extras = []byte{}
	req.Body = []byte{}
	return client.Send(req)
}

// Delete a key.
func (client *Client) Del(vb uint16, key string) gomemcached.MCResponse {
	var req gomemcached.MCRequest
	req.Opcode = gomemcached.DELETE
	req.VBucket = vb
	req.Key = []byte(key)
	req.Cas = 0
	req.Opaque = 0
	req.Extras = []byte{}
	req.Body = []byte{}
	return client.Send(req)
}

func (client *Client) store(opcode gomemcached.CommandCode, vb uint16,
	key string, flags int, exp int, body []byte) gomemcached.MCResponse {

	var req gomemcached.MCRequest
	req.Opcode = opcode
	req.VBucket = vb
	req.Cas = 0
	req.Opaque = 0
	req.Key = []byte(key)
	req.Extras = []byte{0, 0, 0, 0, 0, 0, 0, 0}
	binary.BigEndian.PutUint64(req.Extras, uint64(flags)<<32|uint64(exp))
	req.Body = body
	return client.Send(req)
}

// Add a value for a key (store if not exists).
func (client *Client) Add(vb uint16, key string, flags int, exp int,
	body []byte) gomemcached.MCResponse {
	return client.store(gomemcached.ADD, vb, key, flags, exp, body)
}

// Set the value for a key.
func (client *Client) Set(vb uint16, key string, flags int, exp int,
	body []byte) gomemcached.MCResponse {
	return client.store(gomemcached.SET, vb, key, flags, exp, body)
}

// Stats returns a slice of these.
type StatValue struct {
	// The stat key
	Key string
	// The stat value
	Val string
}

// Get stats from the server
// use "" as the stat key for toplevel stats.
func (client *Client) Stats(key string) []StatValue {
	rv := []StatValue{}

	var req gomemcached.MCRequest
	req.Opcode = gomemcached.STAT
	req.VBucket = 0
	req.Cas = 0
	req.Opaque = 918494
	req.Key = []byte(key)
	req.Extras = []byte{}
	req.Body = []byte{}

	transmitRequest(client.writer, req)

	for {
		res := client.getResponse()
		k := string(res.Key)
		if k == "" {
			break
		}
		rv = append(rv, StatValue{
			Key: k,
			Val: string(res.Body),
		})
	}

	return rv
}

func (client *Client) getResponse() gomemcached.MCResponse {
	bytesRead, err := io.ReadFull(client.conn, client.hdrBuf)
	if err != nil || bytesRead != gomemcached.HDR_LEN {
		log.Printf("Error reading message: %s (%d bytes)", err, bytesRead)
		runtime.Goexit()
	}
	res := grokHeader(client.hdrBuf)
	readContents(client.conn, res)
	return res
}

func readContents(s net.Conn, res gomemcached.MCResponse) {
	readOb(s, res.Extras)
	readOb(s, res.Key)
	readOb(s, res.Body)
}

func grokHeader(hdrBytes []byte) (rv gomemcached.MCResponse) {
	if hdrBytes[0] != gomemcached.RES_MAGIC {
		log.Printf("Bad magic: %x", hdrBytes[0])
		runtime.Goexit()
	}
	// rv.Opcode = hdrBytes[1]
	rv.Key = make([]byte, binary.BigEndian.Uint16(hdrBytes[2:]))
	rv.Extras = make([]byte, hdrBytes[4])
	rv.Status = uint16(hdrBytes[7])
	bodyLen := binary.BigEndian.Uint32(hdrBytes[8:]) - uint32(len(rv.Key)) - uint32(len(rv.Extras))
	rv.Body = make([]byte, bodyLen)
	// rv.Opaque = binary.BigEndian.Uint32(hdrBytes[12:])
	rv.Cas = binary.BigEndian.Uint64(hdrBytes[16:])
	return
}

func transmitRequest(o *bufio.Writer, req gomemcached.MCRequest) {
	// 0
	writeByte(o, gomemcached.REQ_MAGIC)
	writeByte(o, byte(req.Opcode))
	writeUint16(o, uint16(len(req.Key)))
	// 4
	writeByte(o, uint8(len(req.Extras)))
	writeByte(o, 0)
	writeUint16(o, req.VBucket)
	// 8
	writeUint32(o, uint32(len(req.Body))+
		uint32(len(req.Key))+
		uint32(len(req.Extras)))
	// 12
	writeUint32(o, req.Opaque)
	// 16
	writeUint64(o, req.Cas)
	// The rest
	writeBytes(o, req.Extras)
	writeBytes(o, req.Key)
	writeBytes(o, req.Body)
	o.Flush()
}

func writeBytes(s *bufio.Writer, data []byte) {
	if len(data) > 0 {
		written, err := s.Write(data)
		if err != nil || written != len(data) {
			log.Printf("Error writing bytes:  %s", err)
			runtime.Goexit()
		}
	}
	return

}

func writeByte(s *bufio.Writer, b byte) {
	data := make([]byte, 1)
	data[0] = b
	writeBytes(s, data)
}

func writeUint16(s *bufio.Writer, n uint16) {
	data := []byte{0, 0}
	binary.BigEndian.PutUint16(data, n)
	writeBytes(s, data)
}

func writeUint32(s *bufio.Writer, n uint32) {
	data := []byte{0, 0, 0, 0}
	binary.BigEndian.PutUint32(data, n)
	writeBytes(s, data)
}

func writeUint64(s *bufio.Writer, n uint64) {
	data := []byte{0, 0, 0, 0, 0, 0, 0, 0}
	binary.BigEndian.PutUint64(data, n)
	writeBytes(s, data)
}

func readOb(s net.Conn, buf []byte) {
	x, err := io.ReadFull(s, buf)
	if err != nil || x != len(buf) {
		log.Printf("Error reading part: %s", err)
		runtime.Goexit()
	}
}
<commit_msg>Send and receive pointers to (potentially large) requests.<commit_after>// A memcached binary protocol client.
package memcached

import (
	"bufio"
	"encoding/binary"
	"github.com/dustin/gomemcached"
	"io"
	"log"
	"net"
	"runtime"
)

const bufsize = 1024

// The Client itself.
type Client struct {
	conn   net.Conn
	writer *bufio.Writer

	hdrBuf []byte
}

// Connect to a memcached server.
func Connect(prot, dest string) (rv *Client, err error) {
	conn, err := net.Dial(prot, dest)
	if err != nil {
		return nil, err
	}
	rv = new(Client)
	rv.conn = conn
	rv.writer = bufio.NewWriterSize(rv.conn, bufsize)
	rv.hdrBuf = make([]byte, gomemcached.HDR_LEN)
	return
}

// Close the connection when you're done.
func (c *Client) Close() {
	c.conn.Close()
}

// Send a custom request and get the response.
func (client *Client) Send(req *gomemcached.MCRequest) (rv gomemcached.MCResponse) {
	transmitRequest(client.writer, req)
	rv = client.getResponse()
	return
}

// Send a request, but do not wait for a response.
func (client *Client) Transmit(req *gomemcached.MCRequest) {
	transmitRequest(client.writer, req)
}

// Receive a response
func (client *Client) Receive() gomemcached.MCResponse {
	return client.getResponse()
}

// Get the value for a key.
func (client *Client) Get(vb uint16, key string) gomemcached.MCResponse {
	var req gomemcached.MCRequest
	req.Opcode = gomemcached.GET
	req.VBucket = vb
	req.Key = []byte(key)
	req.Cas = 0
	req.Opaque = 0
	req.Extras = []byte{}
	req.Body = []byte{}
	return client.Send(&req)
}

// Delete a key.
func (client *Client) Del(vb uint16, key string) gomemcached.MCResponse {
	var req gomemcached.MCRequest
	req.Opcode = gomemcached.DELETE
	req.VBucket = vb
	req.Key = []byte(key)
	req.Cas = 0
	req.Opaque = 0
	req.Extras = []byte{}
	req.Body = []byte{}
	return client.Send(&req)
}

func (client *Client) store(opcode gomemcached.CommandCode, vb uint16,
	key string, flags int, exp int, body []byte) gomemcached.MCResponse {

	var req gomemcached.MCRequest
	req.Opcode = opcode
	req.VBucket = vb
	req.Cas = 0
	req.Opaque = 0
	req.Key = []byte(key)
	req.Extras = []byte{0, 0, 0, 0, 0, 0, 0, 0}
	binary.BigEndian.PutUint64(req.Extras, uint64(flags)<<32|uint64(exp))
	req.Body = body
	return client.Send(&req)
}

// Add a value for a key (store if not exists).
func (client *Client) Add(vb uint16, key string, flags int, exp int,
	body []byte) gomemcached.MCResponse {
	return client.store(gomemcached.ADD, vb, key, flags, exp, body)
}

// Set the value for a key.
func (client *Client) Set(vb uint16, key string, flags int, exp int,
	body []byte) gomemcached.MCResponse {
	return client.store(gomemcached.SET, vb, key, flags, exp, body)
}

// Stats returns a slice of these.
type StatValue struct {
	// The stat key
	Key string
	// The stat value
	Val string
}

// Get stats from the server
// use "" as the stat key for toplevel stats.
func (client *Client) Stats(key string) []StatValue {
	rv := []StatValue{}

	var req gomemcached.MCRequest
	req.Opcode = gomemcached.STAT
	req.VBucket = 0
	req.Cas = 0
	req.Opaque = 918494
	req.Key = []byte(key)
	req.Extras = []byte{}
	req.Body = []byte{}

	transmitRequest(client.writer, &req)

	for {
		res := client.getResponse()
		k := string(res.Key)
		if k == "" {
			break
		}
		rv = append(rv, StatValue{
			Key: k,
			Val: string(res.Body),
		})
	}

	return rv
}

func (client *Client) getResponse() gomemcached.MCResponse {
	bytesRead, err := io.ReadFull(client.conn, client.hdrBuf)
	if err != nil || bytesRead != gomemcached.HDR_LEN {
		log.Printf("Error reading message: %s (%d bytes)", err, bytesRead)
		runtime.Goexit()
	}
	res := grokHeader(client.hdrBuf)
	readContents(client.conn, res)
	return res
}

func readContents(s net.Conn, res gomemcached.MCResponse) {
	readOb(s, res.Extras)
	readOb(s, res.Key)
	readOb(s, res.Body)
}

func grokHeader(hdrBytes []byte) (rv gomemcached.MCResponse) {
	if hdrBytes[0] != gomemcached.RES_MAGIC {
		log.Printf("Bad magic: %x", hdrBytes[0])
		runtime.Goexit()
	}
	// rv.Opcode = hdrBytes[1]
	rv.Key = make([]byte, binary.BigEndian.Uint16(hdrBytes[2:]))
	rv.Extras = make([]byte, hdrBytes[4])
	rv.Status = uint16(hdrBytes[7])
	bodyLen := binary.BigEndian.Uint32(hdrBytes[8:]) - uint32(len(rv.Key)) - uint32(len(rv.Extras))
	rv.Body = make([]byte, bodyLen)
	// rv.Opaque = binary.BigEndian.Uint32(hdrBytes[12:])
	rv.Cas = binary.BigEndian.Uint64(hdrBytes[16:])
	return
}

func transmitRequest(o *bufio.Writer, req *gomemcached.MCRequest) {
	// 0
	writeByte(o, gomemcached.REQ_MAGIC)
	writeByte(o, byte(req.Opcode))
	writeUint16(o, uint16(len(req.Key)))
	// 4
	writeByte(o, uint8(len(req.Extras)))
	writeByte(o, 0)
	writeUint16(o, req.VBucket)
	// 8
	writeUint32(o, uint32(len(req.Body))+
		uint32(len(req.Key))+
		uint32(len(req.Extras)))
	// 12
	writeUint32(o, req.Opaque)
	// 16
	writeUint64(o, req.Cas)
	// The rest
	writeBytes(o, req.Extras)
	writeBytes(o, req.Key)
	writeBytes(o, req.Body)
	o.Flush()
}

func writeBytes(s *bufio.Writer, data []byte) {
	if len(data) > 0 {
		written, err := s.Write(data)
		if err != nil || written != len(data) {
			log.Printf("Error writing bytes:  %s", err)
			runtime.Goexit()
		}
	}
	return

}

func writeByte(s *bufio.Writer, b byte) {
	data := make([]byte, 1)
	data[0] = b
	writeBytes(s, data)
}

func writeUint16(s *bufio.Writer, n uint16) {
	data := []byte{0, 0}
	binary.BigEndian.PutUint16(data, n)
	writeBytes(s, data)
}

func writeUint32(s *bufio.Writer, n uint32) {
	data := []byte{0, 0, 0, 0}
	binary.BigEndian.PutUint32(data, n)
	writeBytes(s, data)
}

func writeUint64(s *bufio.Writer, n uint64) {
	data := []byte{0, 0, 0, 0, 0, 0, 0, 0}
	binary.BigEndian.PutUint64(data, n)
	writeBytes(s, data)
}

func readOb(s net.Conn, buf []byte) {
	x, err := io.ReadFull(s, buf)
	if err != nil || x != len(buf) {
		log.Printf("Error reading part: %s", err)
		runtime.Goexit()
	}
}
<|endoftext|>


<commit_before>package expr

import (
	"math"
	"strconv"
	"testing"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/schema"
	"github.com/grafana/metrictank/test"
)

var datapoints = []schema.Point{
	{Val: 0, Ts: 10},
	{Val: math.NaN(), Ts: 20},
	{Val: -10, Ts: 30},
	{Val: 5.5, Ts: 40},
	{Val: -math.MaxFloat64, Ts: 50},
	{Val: -1234567890, Ts: 60},
	{Val: math.MaxFloat64, Ts: 70},
}

var datapointsInvert = []schema.Point{
	{Val: math.NaN(), Ts: 10},
	{Val: math.NaN(), Ts: 20},
	{Val: math.Pow(-10, -1), Ts: 30},
	{Val: math.Pow(5.5, -1), Ts: 40},
	{Val: math.Pow(-math.MaxFloat64, -1), Ts: 50},
	{Val: math.Pow(-1234567890, -1), Ts: 60},
	{Val: math.Pow(math.MaxFloat64, -1), Ts: 70},
}

func TestInvert(t *testing.T) {
	in := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "queryPattHere",
			Target:     "targetHere",
			Datapoints: getCopy(datapoints),
		},
	}

	// store copy of the original input
	inCopy := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "queryPattHere",
			Target:     "targetHere",
			Datapoints: getCopy(datapoints),
		},
	}

	f := getNewInvert(in)
	out := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "invert(queryPattHere)",
			Target:     "invert(targetHere)",
			Datapoints: getCopy(datapointsInvert),
		},
	}

	got, err := f.Exec(make(map[Req][]models.Series))
	if err := equalOutput(out, got, nil, err); err != nil {
		t.Fatal(err)
	}

	// make sure input hasn't changed after call to invert
	if err := equalOutput(in, inCopy, nil, err); err != nil {
		t.Fatal(err)
	}
}

func getNewInvert(in []models.Series) *FuncInvert {
	f := NewInvert()
	ps := f.(*FuncInvert)
	ps.in = NewMock(in)
	return ps
}

func BenchmarkInvert10k_1NoNulls(b *testing.B) {
	benchmarkInvert(b, 1, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkInvert10k_10NoNulls(b *testing.B) {
	benchmarkInvert(b, 10, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkInvert10k_100NoNulls(b *testing.B) {
	benchmarkInvert(b, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkInvert10k_1000NoNulls(b *testing.B) {
	benchmarkInvert(b, 1000, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkInvert10k_1SomeSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 1, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_10SomeSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 10, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_100SomeSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_1000SomeSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 1000, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_1AllSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 1, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_10AllSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 10, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_100AllSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 100, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_1000AllSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 1000, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func benchmarkInvert(b *testing.B, numSeries int, fn0, fn1 func() []schema.Point) {
	var input []models.Series
	for i := 0; i < numSeries; i++ {
		series := models.Series{
			QueryPatt: strconv.Itoa(i),
		}
		if i%2 == 0 {
			series.Datapoints = fn0()
		} else {
			series.Datapoints = fn1()
		}
		input = append(input, series)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		f := NewInvert()
		f.(*FuncInvert).in = NewMock(input)
		got, err := f.Exec(make(map[Req][]models.Series))
		if err != nil {
			b.Fatalf("%s", err)
		}
		results = got
	}
}
<commit_msg>Flesh out invert test * split out input unchanged check * 0 series, 2 series<commit_after>package expr

import (
	"math"
	"strconv"
	"testing"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/schema"
	"github.com/grafana/metrictank/test"
)

var datapoints = []schema.Point{
	{Val: 0, Ts: 10},
	{Val: math.NaN(), Ts: 20},
	{Val: -10, Ts: 30},
	{Val: 5.5, Ts: 40},
	{Val: -math.MaxFloat64, Ts: 50},
	{Val: -1234567890, Ts: 60},
	{Val: math.MaxFloat64, Ts: 70},
}

var datapointsInvert = []schema.Point{
	{Val: math.NaN(), Ts: 10},
	{Val: math.NaN(), Ts: 20},
	{Val: math.Pow(-10, -1), Ts: 30},
	{Val: math.Pow(5.5, -1), Ts: 40},
	{Val: math.Pow(-math.MaxFloat64, -1), Ts: 50},
	{Val: math.Pow(-1234567890, -1), Ts: 60},
	{Val: math.Pow(math.MaxFloat64, -1), Ts: 70},
}

func TestInvertBasic(t *testing.T) {
	in := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "queryPattHere",
			Target:     "targetHere",
			Datapoints: getCopy(datapoints),
		},
	}

	f := getNewInvert(in)
	out := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "invert(queryPattHere)",
			Target:     "invert(targetHere)",
			Datapoints: getCopy(datapointsInvert),
		},
	}

	got, err := f.Exec(make(map[Req][]models.Series))
	if err := equalOutput(out, got, nil, err); err != nil {
		t.Fatal(err)
	}
}

func TestInvertInputUnchanged(t *testing.T) {
	in := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "queryPattHere",
			Target:     "targetHere",
			Datapoints: getCopy(datapoints),
		},
	}

	// store copy of the original input
	inCopy := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "queryPattHere",
			Target:     "targetHere",
			Datapoints: getCopy(datapoints),
		},
	}

	f := getNewInvert(in)
	_, err := f.Exec(make(map[Req][]models.Series))

	// make sure input hasn't changed after call to invert
	if err := equalOutput(in, inCopy, nil, err); err != nil {
		t.Fatal(err)
	}
}

func TestInvertEmpty(t *testing.T) {
	in := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "queryPattHere",
			Target:     "targetHere",
			Datapoints: []schema.Point{},
		},
	}

	f := getNewInvert(in)

	out := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "invert(queryPattHere)",
			Target:     "invert(targetHere)",
			Datapoints: []schema.Point{},
		},
	}

	got, err := f.Exec(make(map[Req][]models.Series))
	if err := equalOutput(out, got, nil, err); err != nil {
		t.Fatal(err)
	}
}

func TestInvertMultipleSeries(t *testing.T) {
	in := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "queryPattHere",
			Target:     "targetHere",
			Datapoints: getCopy(datapoints),
		},
		{
			Interval:   20,
			QueryPatt:  "queryPattHere2",
			Target:     "targetHere2",
			Datapoints: getCopy(datapoints),
		},
	}

	f := getNewInvert(in)
	out := []models.Series{
		{
			Interval:   10,
			QueryPatt:  "invert(queryPattHere)",
			Target:     "invert(targetHere)",
			Datapoints: getCopy(datapointsInvert),
		},
		{
			Interval:   20,
			QueryPatt:  "invert(queryPattHere2)",
			Target:     "invert(targetHere2)",
			Datapoints: getCopy(datapointsInvert),
		},
	}

	got, err := f.Exec(make(map[Req][]models.Series))
	if err := equalOutput(out, got, nil, err); err != nil {
		t.Fatal(err)
	}
}

func getNewInvert(in []models.Series) *FuncInvert {
	f := NewInvert()
	ps := f.(*FuncInvert)
	ps.in = NewMock(in)
	return ps
}

func BenchmarkInvert10k_1NoNulls(b *testing.B) {
	benchmarkInvert(b, 1, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkInvert10k_10NoNulls(b *testing.B) {
	benchmarkInvert(b, 10, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkInvert10k_100NoNulls(b *testing.B) {
	benchmarkInvert(b, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkInvert10k_1000NoNulls(b *testing.B) {
	benchmarkInvert(b, 1000, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkInvert10k_1SomeSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 1, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_10SomeSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 10, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_100SomeSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_1000SomeSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 1000, test.RandFloats10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_1AllSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 1, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_10AllSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 10, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_100AllSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 100, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func BenchmarkInvert10k_1000AllSeriesHalfNulls(b *testing.B) {
	benchmarkInvert(b, 1000, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
}
func benchmarkInvert(b *testing.B, numSeries int, fn0, fn1 func() []schema.Point) {
	var input []models.Series
	for i := 0; i < numSeries; i++ {
		series := models.Series{
			QueryPatt: strconv.Itoa(i),
		}
		if i%2 == 0 {
			series.Datapoints = fn0()
		} else {
			series.Datapoints = fn1()
		}
		input = append(input, series)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		f := NewInvert()
		f.(*FuncInvert).in = NewMock(input)
		got, err := f.Exec(make(map[Req][]models.Series))
		if err != nil {
			b.Fatalf("%s", err)
		}
		results = got
	}
}
<|endoftext|>


<commit_before>package elk

import (
	"errors"
	// "fmt"
	"io"
	"io/ioutil"
	// "log/syslog"
	"net"
	// "os"
	"strings"
	// "time"
	"encoding/json"

	"github.com/delectable/logspout/router"
)

var HOSTNAME string

func init() {
	router.AdapterFactories.Register(NewElkAdapter, "elk")

	hostname_bytestring, _ := ioutil.ReadFile("/etc/hostname") // this should stick around in memory, not run for every log line
	HOSTNAME = strings.TrimSpace(string(hostname_bytestring))

}

// func getopt(name, dfault string) string {
// 	value := os.Getenv(name)
// 	if value == "" {
// 		value = dfault
// 	}
// 	return value
// }

func NewElkAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}

	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &ElkAdapter{
		route: route,
		conn:  conn,
	}, nil
}

type ElkAdapter struct {
	conn  net.Conn
	route *router.Route
}

func (adapter *ElkAdapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		elkMessage := NewElkMessage(message)
		io.WriteString(adapter.conn, elkMessage.ToString())
	}
}

type ElkMessage struct {
	routerMessage *router.Message
	Object        struct {
		Time     int64  `json: "time"`
		Message  string `json: "message"`
		Hostname string `json: "hostname"`
		Image    string `json: "image"`
		App      string `json: "app"`
	}
}

func NewElkMessage(routerMessage *router.Message) *ElkMessage {
	elkMessage := &ElkMessage{
		routerMessage: routerMessage,
	}

	elkMessage.Object.Time = routerMessage.Time.Unix()
	elkMessage.Object.Message = routerMessage.Data

	elkMessage.Object.Hostname = HOSTNAME

	elkMessage.Object.Image = routerMessage.Container.Config.Image

	env_map := make(map[string]string)
	for _, blob := range routerMessage.Container.Config.Env {
		split_blob := strings.Split(blob, "=")
		env_map[split_blob[0]] = split_blob[1]
	}

	elkMessage.Object.App = env_map["MARATHON_APP_ID"]

	return elkMessage
}

func (elkMessage *ElkMessage) ToString() string {
	return_string, _ := json.Marshal(elkMessage.Object)
	return string(return_string)
}
<commit_msg>Testing ELK Adapter<commit_after>package elk

import (
	"errors"
	// "fmt"
	"io"
	"io/ioutil"
	// "log/syslog"
	"net"
	// "os"
	"strings"
	// "time"
	"encoding/json"

	"github.com/delectable/logspout/router"
)

var HOSTNAME string

func init() {
	router.AdapterFactories.Register(NewElkAdapter, "elk")

	hostname_bytestring, _ := ioutil.ReadFile("/etc/hostname")
	HOSTNAME = strings.TrimSpace(string(hostname_bytestring))

}

// func getopt(name, dfault string) string {
// 	value := os.Getenv(name)
// 	if value == "" {
// 		value = dfault
// 	}
// 	return value
// }

func NewElkAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}

	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &ElkAdapter{
		route: route,
		conn:  conn,
	}, nil
}

type ElkAdapter struct {
	conn  net.Conn
	route *router.Route
}

func (adapter *ElkAdapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		elkMessage := NewElkMessage(message)
		io.WriteString(adapter.conn, elkMessage.ToString())
	}
}

type ElkMessage struct {
	routerMessage *router.Message
	Object        struct {
		Time     int64  `json: "time"`
		Message  string `json: "message"`
		Hostname string `json: "hostname"`
		Image    string `json: "image"`
		App      string `json: "app"`
	}
}

func NewElkMessage(routerMessage *router.Message) *ElkMessage {
	elkMessage := &ElkMessage{
		routerMessage: routerMessage,
	}

	elkMessage.Object.Time = routerMessage.Time.Unix()
	elkMessage.Object.Message = routerMessage.Data

	elkMessage.Object.Hostname = HOSTNAME

	elkMessage.Object.Image = routerMessage.Container.Config.Image

	env_map := make(map[string]string)
	for _, blob := range routerMessage.Container.Config.Env {
		split_blob := strings.Split(blob, "=")
		env_map[split_blob[0]] = split_blob[1]
	}

	elkMessage.Object.App = env_map["MARATHON_APP_ID"][1:]

	return elkMessage
}

func (elkMessage *ElkMessage) ToString() string {
	return_string, _ := json.Marshal(elkMessage.Object)
	return string(return_string)
}
<|endoftext|>


<commit_before>package elk

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"

	"github.com/delectable/logspout/router"
)

var HOSTNAME string

func init() {
	router.AdapterFactories.Register(NewElkAdapter, "elk")

	hostname_bytestring, _ := ioutil.ReadFile("/etc/hostname")
	HOSTNAME = strings.TrimSpace(string(hostname_bytestring))
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

func NewElkAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}

	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &ElkAdapter{
		route: route,
		conn:  conn,
	}, nil
}

type ElkAdapter struct {
	conn  net.Conn
	route *router.Route
}

func (adapter *ElkAdapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		elkMessage := NewElkMessage(message)
		io.WriteString(adapter.conn, elkMessage.ToString())
	}
}

type ElkMessage struct {
	routerMessage *router.Message
	Object        struct {
		Time     float64 `json:"time"`
		Message  string  `json:"message"`
		Hostname string  `json:"hostname"`
		Image    string  `json:"image"`
		App      string  `json:"app"`
		Env      string  `json:"env"`
	}
}

func NewElkMessage(routerMessage *router.Message) *ElkMessage {
	elkMessage := &ElkMessage{
		routerMessage: routerMessage,
	}

	elkMessage.Object.Time = float64(time.Now().UnixNano()) / 1.0e9
	elkMessage.Object.Message = routerMessage.Data

	elkMessage.Object.Hostname = HOSTNAME
	elkMessage.Object.Env = getopt("ENV", "development")

	elkMessage.Object.Image = routerMessage.Container.Config.Image

	env_map := make(map[string]string)
	for _, blob := range routerMessage.Container.Config.Env {
		split_blob := strings.Split(blob, "=")
		env_map[split_blob[0]] = split_blob[1]
	}

	elkMessage.Object.App = env_map["MARATHON_APP_ID"][1:] // Marathon, for some reason, prepends MARATHON_APP_ID with a '/'

	return elkMessage
}

func (elkMessage *ElkMessage) ToString() string {
	return_string, _ := json.Marshal(elkMessage.Object)
	return string(return_string)
}
<commit_msg>renaming "time" as "timestamp"<commit_after>package elk

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"

	"github.com/delectable/logspout/router"
)

var HOSTNAME string

func init() {
	router.AdapterFactories.Register(NewElkAdapter, "elk")

	hostname_bytestring, _ := ioutil.ReadFile("/etc/hostname")
	HOSTNAME = strings.TrimSpace(string(hostname_bytestring))
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

func NewElkAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}

	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &ElkAdapter{
		route: route,
		conn:  conn,
	}, nil
}

type ElkAdapter struct {
	conn  net.Conn
	route *router.Route
}

func (adapter *ElkAdapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		elkMessage := NewElkMessage(message)
		io.WriteString(adapter.conn, elkMessage.ToString())
	}
}

type ElkMessage struct {
	routerMessage *router.Message
	Object        struct {
		Timestamp float64 `json:"timestamp"`
		Message   string  `json:"message"`
		Hostname  string  `json:"hostname"`
		Image     string  `json:"image"`
		App       string  `json:"app"`
		Env       string  `json:"env"`
	}
}

func NewElkMessage(routerMessage *router.Message) *ElkMessage {
	elkMessage := &ElkMessage{
		routerMessage: routerMessage,
	}

	elkMessage.Object.Timestamp = float64(time.Now().UnixNano()) / 1.0e9
	elkMessage.Object.Message = routerMessage.Data

	elkMessage.Object.Hostname = HOSTNAME
	elkMessage.Object.Env = getopt("ENV", "development")

	elkMessage.Object.Image = routerMessage.Container.Config.Image

	env_map := make(map[string]string)
	for _, blob := range routerMessage.Container.Config.Env {
		split_blob := strings.Split(blob, "=")
		env_map[split_blob[0]] = split_blob[1]
	}

	elkMessage.Object.App = env_map["MARATHON_APP_ID"][1:] // Marathon, for some reason, prepends MARATHON_APP_ID with a '/'

	return elkMessage
}

func (elkMessage *ElkMessage) ToString() string {
	return_string, _ := json.Marshal(elkMessage.Object)
	return string(return_string)
}
<|endoftext|>


<commit_before>package utils

import (
	"crypto/x509"
	"encoding/pem"
	"encoding/xml"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/mitchellh/go-homedir"
	"io/ioutil"
	"net/url"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

const windowsServerConfigFile = "c:\\concerto\\client.xml"
const nixServerConfigFile = "/etc/concerto/client.xml"

// Config stores configuration file contents
type Config struct {
	XMLName      xml.Name `xml:"concerto"`
	APIEndpoint  string   `xml:"server,attr"`
	LogFile      string   `xml:"log_file,attr"`
	LogLevel     string   `xml:"log_level,attr"`
	Certificate  Cert     `xml:"ssl"`
	ConfLocation string
	ConfFile     string
	IsHost       bool
	ConcertoURL  string
}

// Cert stores cert files location
type Cert struct {
	Cert string `xml:"cert,attr"`
	Key  string `xml:"key,attr"`
	Ca   string `xml:"server_ca,attr"`
}

var cachedConfig *Config

// GetConcertoConfig returns concerto configuration
func GetConcertoConfig() (*Config, error) {
	if cachedConfig == nil {
		return nil, fmt.Errorf("Configuration hasn't been initialized")
	}
	return cachedConfig, nil
}

// InitializeConcertoConfig creates the concerto configuration structure
func InitializeConcertoConfig(c *cli.Context) (*Config, error) {
	log.Debug("InitializeConcertoConfig")
	if cachedConfig != nil {
		return cachedConfig, nil
	}

	// where config file must me
	cachedConfig = &Config{}
	err := cachedConfig.evaluateConcertoConfigFile(c)
	if err != nil {
		return nil, err
	}

	// read config contents
	log.Debugf("Reading configuration from %s", cachedConfig.ConfFile)
	err = cachedConfig.readConcertoConfig(c)
	if err != nil {
		return nil, err
	}

	// add login URL. Needed for setup
	err = cachedConfig.readConcertoURL()
	if err != nil {
		return nil, err
	}

	// check if isHost. Needed to show appropiate options
	err = cachedConfig.evaluateCertificate()
	if err != nil {
		return nil, err
	}

	log.Debugf("Concerto configuration used: %+v", cachedConfig)
	return cachedConfig, nil
}

// IsConfigReady returns whether configurations items are filled
func (config *Config) IsConfigReady() bool {
	if config.APIEndpoint == "" ||
		config.Certificate.Cert == "" ||
		config.Certificate.Key == "" ||
		config.Certificate.Ca == "" {
		return false
	}
	return true
}

// IsConfigReadySetup returns whether we can use setup command
func (config *Config) IsConfigReadySetup() bool {
	return config.ConcertoURL != ""
}

// readConcertoConfig reads Concerto config file located at fileLocation
func (config *Config) readConcertoConfig(c *cli.Context) error {
	log.Debug("Reading Concerto Configuration")
	if FileExists(config.ConfFile) {
		// file exists, read it's contents

		xmlFile, err := os.Open(config.ConfFile)
		if err != nil {
			return err
		}
		defer xmlFile.Close()
		b, err := ioutil.ReadAll(xmlFile)
		if err != nil {
			return fmt.Errorf("Configuration File %s couldn't be read.", config.ConfFile)
		}

		if err = xml.Unmarshal(b, &config); err != nil {
			return fmt.Errorf("Configuration File %s does not have valid XML format.", config.ConfFile)
		}

	} else {
		log.Debugf("Configuration File %s does not exist. Reading environment variables", config.ConfFile)
	}

	// overwrite with environment/arguments vars
	if overwEP := c.String("concerto-endpoint"); overwEP != "" {
		log.Debug("Concerto APIEndpoint taken from env/args")
		config.APIEndpoint = overwEP
	}

	if overwCert := c.String("client-cert"); overwCert != "" {
		log.Debug("Certificate path taken from env/args")
		config.Certificate.Cert = overwCert
	}

	if overwKey := c.String("client-key"); overwKey != "" {
		log.Debug("Certificate key path taken from env/args")
		config.Certificate.Key = overwKey
	}

	if overwCa := c.String("ca-cert"); overwCa != "" {
		log.Debug("CA certificate path taken from env/args")
		config.Certificate.Ca = overwCa
	}

	return nil
}

// evaluateConcertoConfigFile returns path to concerto config file
func (config *Config) evaluateConcertoConfigFile(c *cli.Context) error {

	if configFile := c.String("concerto-config"); configFile != "" {

		log.Debug("Concerto configuration file location taken from env/args")
		config.ConfFile = configFile

	} else {

		currUser, err := user.Current()
		if err != nil {
			log.Debugf("Couldn't use os.user to get user details: %s", err.Error())
			dir, err := homedir.Dir()
			if err != nil {
				return fmt.Errorf("Couldn't get home dir for current user: %s", err.Error())
			}
			currUser = &user.User{
				Username: getUsername(),
				HomeDir:  dir,
			}
		}

		if runtime.GOOS == "windows" {
			// Server mode Windows
			if (currUser.Gid == "S-1-5-32-544" || currUser.Username == "Administrator") && FileExists(windowsServerConfigFile) {
				config.ConfFile = configFile
			} else {
				// User mode Windows
				config.ConfFile = filepath.Join(currUser.HomeDir, ".concerto/client.xml")
			}
		} else {
			// Server mode *nix
			if currUser.Uid == "0" || currUser.Username == "root" && FileExists(nixServerConfigFile) {
				config.ConfFile = nixServerConfigFile
			} else {
				// User mode *nix
				config.ConfFile = filepath.Join(currUser.HomeDir, ".concerto/client.xml")
			}
		}
	}
	config.ConfLocation = path.Dir(config.ConfFile)
	return nil
}

// getUsername gets username by env variable.
// os.user is dependant on cgo, so cross compiling won't work
func getUsername() string {
	u := "unknown"
	osUser := ""

	switch runtime.GOOS {
	case "darwin", "linux":
		osUser = os.Getenv("USER")
	case "windows":
		osUser = os.Getenv("USERNAME")

		// HACK ugly ... if localized administrator, translate to administrator
		if osUser == "Järjestelmänvalvoja" ||
			osUser == "Administrateur" ||
			osUser == "Rendszergazda" ||
			osUser == "Administrador" ||
			osUser == "Администратор" ||
			osUser == "Administratör" {
			osUser = "Administrator"
		}
	}

	if osUser != "" {
		u = osUser
	}
	return u
}

// readConcertoURL reads URL from CONCERTO_URL envrionment or calculates using API URL
func (config *Config) readConcertoURL() error {

	if config.ConcertoURL != "" {
		return nil
	}

	if overwURL := os.Getenv("CONCERTO_URL"); overwURL != "" {
		config.ConcertoURL = overwURL
		log.Debug("Concerto URL taken from CONCERTO_URL")
		return nil
	}

	cURL, err := url.Parse(config.APIEndpoint)
	if err != nil {
		return err
	}

	tokenHost := strings.Split(cURL.Host, ":")
	tokenFqdn := strings.Split(tokenHost[0], ".")

	if !strings.Contains(cURL.Host, "staging") {
		tokenFqdn[0] = "start"
	}

	config.ConcertoURL = fmt.Sprintf("%s://%s/", cURL.Scheme, strings.Join(tokenFqdn, "."))
	return nil
}

// evaluateCertificate determines if a certificate has been issued for a host
func (config *Config) evaluateCertificate() error {
	if FileExists(config.Certificate.Cert) {

		data, err := ioutil.ReadFile(config.Certificate.Cert)
		if err != nil {
			return err
		}

		block, _ := pem.Decode(data)

		cert, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			return err
		}

		if len(cert.Subject.OrganizationalUnit) > 0 {
			if cert.Subject.OrganizationalUnit[0] == "Hosts" {
				config.IsHost = true
				return nil
			}
		} else if len(cert.Issuer.Organization) > 0 {
			if cert.Issuer.Organization[0] == "Tapp" {
				config.IsHost = true
				return nil
			}
		}
	}
	config.IsHost = false
	return nil
}
<commit_msg>add default endpoint url in case a user is executing setup, but no conf file is present, assume production<commit_after>package utils

import (
	"crypto/x509"
	"encoding/pem"
	"encoding/xml"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/mitchellh/go-homedir"
	"io/ioutil"
	"net/url"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

const windowsServerConfigFile = "c:\\concerto\\client.xml"
const nixServerConfigFile = "/etc/concerto/client.xml"
const defaultConcertoEndpoint = "https://clients.concerto.io:886/"

// Config stores configuration file contents
type Config struct {
	XMLName      xml.Name `xml:"concerto"`
	APIEndpoint  string   `xml:"server,attr"`
	LogFile      string   `xml:"log_file,attr"`
	LogLevel     string   `xml:"log_level,attr"`
	Certificate  Cert     `xml:"ssl"`
	ConfLocation string
	ConfFile     string
	IsHost       bool
	ConcertoURL  string
}

// Cert stores cert files location
type Cert struct {
	Cert string `xml:"cert,attr"`
	Key  string `xml:"key,attr"`
	Ca   string `xml:"server_ca,attr"`
}

var cachedConfig *Config

// GetConcertoConfig returns concerto configuration
func GetConcertoConfig() (*Config, error) {
	if cachedConfig == nil {
		return nil, fmt.Errorf("Configuration hasn't been initialized")
	}
	return cachedConfig, nil
}

// InitializeConcertoConfig creates the concerto configuration structure
func InitializeConcertoConfig(c *cli.Context) (*Config, error) {
	log.Debug("InitializeConcertoConfig")
	if cachedConfig != nil {
		return cachedConfig, nil
	}

	// where config file must me
	cachedConfig = &Config{}
	err := cachedConfig.evaluateConcertoConfigFile(c)
	if err != nil {
		return nil, err
	}

	// read config contents
	log.Debugf("Reading configuration from %s", cachedConfig.ConfFile)
	err = cachedConfig.readConcertoConfig(c)
	if err != nil {
		return nil, err
	}

	// add login URL. Needed for setup
	err = cachedConfig.readConcertoURL()
	if err != nil {
		return nil, err
	}

	// check if isHost. Needed to show appropiate options
	err = cachedConfig.evaluateCertificate()
	if err != nil {
		return nil, err
	}

	log.Debugf("Concerto configuration used: %+v", cachedConfig)
	return cachedConfig, nil
}

// IsConfigReady returns whether configurations items are filled
func (config *Config) IsConfigReady() bool {
	if config.APIEndpoint == "" ||
		config.Certificate.Cert == "" ||
		config.Certificate.Key == "" ||
		config.Certificate.Ca == "" {
		return false
	}
	return true
}

// IsConfigReadySetup returns whether we can use setup command
func (config *Config) IsConfigReadySetup() bool {
	return config.ConcertoURL != ""
}

// readConcertoConfig reads Concerto config file located at fileLocation
func (config *Config) readConcertoConfig(c *cli.Context) error {
	log.Debug("Reading Concerto Configuration")
	if FileExists(config.ConfFile) {
		// file exists, read it's contents

		xmlFile, err := os.Open(config.ConfFile)
		if err != nil {
			return err
		}
		defer xmlFile.Close()
		b, err := ioutil.ReadAll(xmlFile)
		if err != nil {
			return fmt.Errorf("Configuration File %s couldn't be read.", config.ConfFile)
		}

		if err = xml.Unmarshal(b, &config); err != nil {
			return fmt.Errorf("Configuration File %s does not have valid XML format.", config.ConfFile)
		}

	} else {
		log.Debugf("Configuration File %s does not exist. Reading environment variables", config.ConfFile)
	}

	// overwrite with environment/arguments vars
	if overwEP := c.String("concerto-endpoint"); overwEP != "" {
		log.Debug("Concerto APIEndpoint taken from env/args")
		config.APIEndpoint = overwEP
	}

	if overwCert := c.String("client-cert"); overwCert != "" {
		log.Debug("Certificate path taken from env/args")
		config.Certificate.Cert = overwCert
	}

	if overwKey := c.String("client-key"); overwKey != "" {
		log.Debug("Certificate key path taken from env/args")
		config.Certificate.Key = overwKey
	}

	if overwCa := c.String("ca-cert"); overwCa != "" {
		log.Debug("CA certificate path taken from env/args")
		config.Certificate.Ca = overwCa
	}

	// if endpoint empty set default
	// we can't set the default from flags, because it would overwrite config file
	if config.APIEndpoint == "" {
		config.APIEndpoint = defaultConcertoEndpoint
	}

	return nil
}

// evaluateConcertoConfigFile returns path to concerto config file
func (config *Config) evaluateConcertoConfigFile(c *cli.Context) error {

	if configFile := c.String("concerto-config"); configFile != "" {

		log.Debug("Concerto configuration file location taken from env/args")
		config.ConfFile = configFile

	} else {

		currUser, err := user.Current()
		if err != nil {
			log.Debugf("Couldn't use os.user to get user details: %s", err.Error())
			dir, err := homedir.Dir()
			if err != nil {
				return fmt.Errorf("Couldn't get home dir for current user: %s", err.Error())
			}
			currUser = &user.User{
				Username: getUsername(),
				HomeDir:  dir,
			}
		}

		if runtime.GOOS == "windows" {
			// Server mode Windows
			if (currUser.Gid == "S-1-5-32-544" || currUser.Username == "Administrator") && FileExists(windowsServerConfigFile) {
				config.ConfFile = configFile
			} else {
				// User mode Windows
				config.ConfFile = filepath.Join(currUser.HomeDir, ".concerto/client.xml")
			}
		} else {
			// Server mode *nix
			if currUser.Uid == "0" || currUser.Username == "root" && FileExists(nixServerConfigFile) {
				config.ConfFile = nixServerConfigFile
			} else {
				// User mode *nix
				config.ConfFile = filepath.Join(currUser.HomeDir, ".concerto/client.xml")
			}
		}
	}
	config.ConfLocation = path.Dir(config.ConfFile)
	return nil
}

// getUsername gets username by env variable.
// os.user is dependant on cgo, so cross compiling won't work
func getUsername() string {
	u := "unknown"
	osUser := ""

	switch runtime.GOOS {
	case "darwin", "linux":
		osUser = os.Getenv("USER")
	case "windows":
		osUser = os.Getenv("USERNAME")

		// HACK ugly ... if localized administrator, translate to administrator
		if osUser == "Järjestelmänvalvoja" ||
			osUser == "Administrateur" ||
			osUser == "Rendszergazda" ||
			osUser == "Administrador" ||
			osUser == "Администратор" ||
			osUser == "Administratör" {
			osUser = "Administrator"
		}
	}

	if osUser != "" {
		u = osUser
	}
	return u
}

// readConcertoURL reads URL from CONCERTO_URL envrionment or calculates using API URL
func (config *Config) readConcertoURL() error {

	if config.ConcertoURL != "" {
		return nil
	}

	if overwURL := os.Getenv("CONCERTO_URL"); overwURL != "" {
		config.ConcertoURL = overwURL
		log.Debug("Concerto URL taken from CONCERTO_URL")
		return nil
	}

	cURL, err := url.Parse(config.APIEndpoint)
	if err != nil {
		return err
	}

	tokenHost := strings.Split(cURL.Host, ":")
	tokenFqdn := strings.Split(tokenHost[0], ".")

	if !strings.Contains(cURL.Host, "staging") {
		tokenFqdn[0] = "start"
	}

	config.ConcertoURL = fmt.Sprintf("%s://%s/", cURL.Scheme, strings.Join(tokenFqdn, "."))
	return nil
}

// evaluateCertificate determines if a certificate has been issued for a host
func (config *Config) evaluateCertificate() error {
	if FileExists(config.Certificate.Cert) {

		data, err := ioutil.ReadFile(config.Certificate.Cert)
		if err != nil {
			return err
		}

		block, _ := pem.Decode(data)

		cert, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			return err
		}

		if len(cert.Subject.OrganizationalUnit) > 0 {
			if cert.Subject.OrganizationalUnit[0] == "Hosts" {
				config.IsHost = true
				return nil
			}
		} else if len(cert.Issuer.Organization) > 0 {
			if cert.Issuer.Organization[0] == "Tapp" {
				config.IsHost = true
				return nil
			}
		}
	}
	config.IsHost = false
	return nil
}
<|endoftext|>


<commit_before>package fakes

import (
	bosherr "bosh/errors"
	"bytes"
	"errors"
	gouuid "github.com/nu7hatch/gouuid"
	"os"
	"strings"
)

type FakeFileType string

const (
	FakeFileTypeFile    FakeFileType = "file"
	FakeFileTypeSymlink FakeFileType = "symlink"
	FakeFileTypeDir     FakeFileType = "dir"
)

type FakeFileSystem struct {
	Files map[string]*FakeFileStats

	HomeDirUsername string
	HomeDirHomePath string

	FilesToOpen map[string]*os.File

	WriteToFileError error
	MkdirAllError    error
	SymlinkError     error

	CopyDirEntriesError   error
	CopyDirEntriesSrcPath string
	CopyDirEntriesDstPath string

	CopyFileError error

	RenameError    error
	RenameOldPaths []string
	RenameNewPaths []string

	RemoveAllError error

	TempFileError  error
	ReturnTempFile *os.File

	TempDirDir   string
	TempDirError error

	globsMap map[string][][]string
}

type FakeFileStats struct {
	FileMode      os.FileMode
	Username      string
	Content       []byte
	SymlinkTarget string
	FileType      FakeFileType
}

func (stats FakeFileStats) StringContents() string {
	return string(stats.Content)
}

func NewFakeFileSystem() *FakeFileSystem {
	return &FakeFileSystem{
		globsMap: make(map[string][][]string),
	}
}

func (fs *FakeFileSystem) GetFileTestStat(path string) (stats *FakeFileStats) {
	stats = fs.Files[path]
	return
}

func (fs *FakeFileSystem) HomeDir(username string) (path string, err error) {
	fs.HomeDirUsername = username
	path = fs.HomeDirHomePath
	return
}

func (fs *FakeFileSystem) MkdirAll(path string, perm os.FileMode) (err error) {
	if fs.MkdirAllError == nil {
		stats := fs.getOrCreateFile(path)
		stats.FileMode = perm
		stats.FileType = FakeFileTypeDir
	}

	err = fs.MkdirAllError
	return
}

func (fs *FakeFileSystem) Chown(path, username string) (err error) {
	stats := fs.GetFileTestStat(path)
	stats.Username = username
	return
}

func (fs *FakeFileSystem) Chmod(path string, perm os.FileMode) (err error) {
	stats := fs.GetFileTestStat(path)
	stats.FileMode = perm
	return
}

func (fs *FakeFileSystem) WriteFileString(path, content string) (err error) {
	return fs.WriteFile(path, []byte(content))
}

func (fs *FakeFileSystem) WriteFile(path string, content []byte) (err error) {
	if fs.WriteToFileError != nil {
		err = fs.WriteToFileError
		return
	}

	stats := fs.getOrCreateFile(path)
	stats.FileType = FakeFileTypeFile
	stats.Content = content
	return
}

func (fs *FakeFileSystem) ConvergeFileContents(path string, content []byte) (written bool, err error) {
	if fs.WriteToFileError != nil {
		err = fs.WriteToFileError
		return
	}

	stats := fs.getOrCreateFile(path)
	stats.FileType = FakeFileTypeFile

	if bytes.Compare(stats.Content, content) != 0 {
		stats.Content = content
		written = true
	}
	return
}

func (fs *FakeFileSystem) ReadFileString(path string) (content string, err error) {
	bytes, err := fs.ReadFile(path)
	if err != nil {
		return
	}

	content = string(bytes)
	return
}

func (fs *FakeFileSystem) ReadFile(path string) (content []byte, err error) {
	stats := fs.GetFileTestStat(path)
	if stats != nil {
		content = stats.Content
	} else {
		err = errors.New("File not found")
	}
	return
}

func (fs *FakeFileSystem) FileExists(path string) bool {
	return fs.GetFileTestStat(path) != nil
}

func (fs *FakeFileSystem) Rename(oldPath, newPath string) (err error) {
	if fs.RenameError != nil {
		err = fs.RenameError
		return
	}

	stats := fs.GetFileTestStat(oldPath)
	if stats == nil {
		err = errors.New("Old path did not exist")
		return
	}

	fs.RenameOldPaths = append(fs.RenameOldPaths, oldPath)
	fs.RenameNewPaths = append(fs.RenameNewPaths, newPath)

	newStats := fs.getOrCreateFile(newPath)
	newStats.Content = stats.Content
	newStats.FileMode = stats.FileMode
	newStats.FileType = stats.FileType

	fs.RemoveAll(oldPath)

	return
}

func (fs *FakeFileSystem) Symlink(oldPath, newPath string) (err error) {
	if fs.SymlinkError == nil {
		stats := fs.getOrCreateFile(newPath)
		stats.FileType = FakeFileTypeSymlink
		stats.SymlinkTarget = oldPath
		return
	}

	err = fs.SymlinkError
	return
}

func (fs *FakeFileSystem) ReadLink(symlinkPath string) (targetPath string, err error) {
	stat := fs.GetFileTestStat(symlinkPath)
	if stat != nil {
		targetPath = stat.SymlinkTarget
	} else {
		err = os.ErrNotExist
	}

	return
}

func (fs *FakeFileSystem) CopyDirEntries(srcPath, dstPath string) (err error) {
	if fs.CopyDirEntriesError != nil {
		return fs.CopyDirEntriesError
	}

	filesToCopy := []string{}

	for filePath, _ := range fs.Files {
		if strings.HasPrefix(filePath, srcPath) {
			filesToCopy = append(filesToCopy, filePath)
		}
	}

	for _, filePath := range filesToCopy {
		newPath := strings.Replace(filePath, srcPath, dstPath, 1)
		fs.Files[newPath] = fs.Files[filePath]
	}

	return
}

func (fs *FakeFileSystem) CopyFile(srcPath, dstPath string) (err error) {
	if fs.CopyFileError != nil {
		err = fs.CopyFileError
		return
	}

	fs.Files[dstPath] = fs.Files[srcPath]
	return
}

func (fs *FakeFileSystem) TempFile(prefix string) (file *os.File, err error) {
	if fs.TempFileError != nil {
		return nil, fs.TempFileError
	}
	if fs.ReturnTempFile != nil {
		return fs.ReturnTempFile, nil
	} else {
		file, err = os.Open("/dev/null")
		if err != nil {
			err = bosherr.WrapError(err, "Opening /dev/null")
			return
		}

		// Make sure to record a reference for FileExist, etc. to work
		stats := fs.getOrCreateFile(file.Name())
		stats.FileType = FakeFileTypeFile

		return
	}
}

func (fs *FakeFileSystem) TempDir(prefix string) (string, error) {
	if fs.TempDirError != nil {
		return "", fs.TempDirError
	}

	var path string
	if len(fs.TempDirDir) > 0 {
		path = fs.TempDirDir
	} else {
		uuid, err := gouuid.NewV4()
		if err != nil {
			return "", err
		}

		path = uuid.String()
	}

	// Make sure to record a reference for FileExist, etc. to work
	stats := fs.getOrCreateFile(path)
	stats.FileType = FakeFileTypeDir

	return path, nil
}

func (fs *FakeFileSystem) RemoveAll(path string) (err error) {
	if fs.RemoveAllError != nil {
		return fs.RemoveAllError
	}

	filesToRemove := []string{}

	for name, _ := range fs.Files {
		if strings.HasPrefix(name, path) {
			filesToRemove = append(filesToRemove, name)
		}
	}

	for _, name := range filesToRemove {
		delete(fs.Files, name)
	}
	return
}

func (fs *FakeFileSystem) Open(path string) (file *os.File, err error) {
	file = fs.FilesToOpen[path]
	return
}

func (fs *FakeFileSystem) Glob(pattern string) (matches []string, err error) {
	remainingMatches, found := fs.globsMap[pattern]
	if found {
		matches = remainingMatches[0]
		if len(remainingMatches) > 1 {
			fs.globsMap[pattern] = remainingMatches[1:]
		}
	} else {
		matches = []string{}
	}
	return
}

func (fs *FakeFileSystem) SetGlob(pattern string, matches ...[]string) {
	fs.globsMap[pattern] = matches
	return
}

func (fs *FakeFileSystem) getOrCreateFile(path string) (stats *FakeFileStats) {
	stats = fs.GetFileTestStat(path)
	if stats == nil {
		if fs.Files == nil {
			fs.Files = make(map[string]*FakeFileStats)
		}

		stats = new(FakeFileStats)
		fs.Files[path] = stats
	}
	return
}
<commit_msg>added ReadFileError to fake file system<commit_after>package fakes

import (
	bosherr "bosh/errors"
	"bytes"
	"errors"
	gouuid "github.com/nu7hatch/gouuid"
	"os"
	"strings"
)

type FakeFileType string

const (
	FakeFileTypeFile    FakeFileType = "file"
	FakeFileTypeSymlink FakeFileType = "symlink"
	FakeFileTypeDir     FakeFileType = "dir"
)

type FakeFileSystem struct {
	Files map[string]*FakeFileStats

	HomeDirUsername string
	HomeDirHomePath string

	FilesToOpen map[string]*os.File

	ReadFileError    error
	WriteToFileError error
	MkdirAllError    error
	SymlinkError     error

	CopyDirEntriesError   error
	CopyDirEntriesSrcPath string
	CopyDirEntriesDstPath string

	CopyFileError error

	RenameError    error
	RenameOldPaths []string
	RenameNewPaths []string

	RemoveAllError error

	TempFileError  error
	ReturnTempFile *os.File

	TempDirDir   string
	TempDirError error

	globsMap map[string][][]string
}

type FakeFileStats struct {
	FileMode      os.FileMode
	Username      string
	Content       []byte
	SymlinkTarget string
	FileType      FakeFileType
}

func (stats FakeFileStats) StringContents() string {
	return string(stats.Content)
}

func NewFakeFileSystem() *FakeFileSystem {
	return &FakeFileSystem{
		globsMap: make(map[string][][]string),
	}
}

func (fs *FakeFileSystem) GetFileTestStat(path string) (stats *FakeFileStats) {
	stats = fs.Files[path]
	return
}

func (fs *FakeFileSystem) HomeDir(username string) (path string, err error) {
	fs.HomeDirUsername = username
	path = fs.HomeDirHomePath
	return
}

func (fs *FakeFileSystem) MkdirAll(path string, perm os.FileMode) (err error) {
	if fs.MkdirAllError == nil {
		stats := fs.getOrCreateFile(path)
		stats.FileMode = perm
		stats.FileType = FakeFileTypeDir
	}

	err = fs.MkdirAllError
	return
}

func (fs *FakeFileSystem) Chown(path, username string) (err error) {
	stats := fs.GetFileTestStat(path)
	stats.Username = username
	return
}

func (fs *FakeFileSystem) Chmod(path string, perm os.FileMode) (err error) {
	stats := fs.GetFileTestStat(path)
	stats.FileMode = perm
	return
}

func (fs *FakeFileSystem) WriteFileString(path, content string) (err error) {
	return fs.WriteFile(path, []byte(content))
}

func (fs *FakeFileSystem) WriteFile(path string, content []byte) (err error) {
	if fs.WriteToFileError != nil {
		err = fs.WriteToFileError
		return
	}

	stats := fs.getOrCreateFile(path)
	stats.FileType = FakeFileTypeFile
	stats.Content = content
	return
}

func (fs *FakeFileSystem) ConvergeFileContents(path string, content []byte) (written bool, err error) {
	if fs.WriteToFileError != nil {
		err = fs.WriteToFileError
		return
	}

	stats := fs.getOrCreateFile(path)
	stats.FileType = FakeFileTypeFile

	if bytes.Compare(stats.Content, content) != 0 {
		stats.Content = content
		written = true
	}
	return
}

func (fs *FakeFileSystem) ReadFileString(path string) (content string, err error) {
	bytes, err := fs.ReadFile(path)
	if err != nil {
		return
	}

	content = string(bytes)
	return
}

func (fs *FakeFileSystem) ReadFile(path string) ([]byte, error) {
	stats := fs.GetFileTestStat(path)
	if stats != nil {
		if fs.ReadFileError != nil {
			return nil, fs.ReadFileError
		} else {
			return stats.Content, nil
		}
	} else {
		return nil, errors.New("File not found")
	}
}

func (fs *FakeFileSystem) FileExists(path string) bool {
	return fs.GetFileTestStat(path) != nil
}

func (fs *FakeFileSystem) Rename(oldPath, newPath string) (err error) {
	if fs.RenameError != nil {
		err = fs.RenameError
		return
	}

	stats := fs.GetFileTestStat(oldPath)
	if stats == nil {
		err = errors.New("Old path did not exist")
		return
	}

	fs.RenameOldPaths = append(fs.RenameOldPaths, oldPath)
	fs.RenameNewPaths = append(fs.RenameNewPaths, newPath)

	newStats := fs.getOrCreateFile(newPath)
	newStats.Content = stats.Content
	newStats.FileMode = stats.FileMode
	newStats.FileType = stats.FileType

	fs.RemoveAll(oldPath)

	return
}

func (fs *FakeFileSystem) Symlink(oldPath, newPath string) (err error) {
	if fs.SymlinkError == nil {
		stats := fs.getOrCreateFile(newPath)
		stats.FileType = FakeFileTypeSymlink
		stats.SymlinkTarget = oldPath
		return
	}

	err = fs.SymlinkError
	return
}

func (fs *FakeFileSystem) ReadLink(symlinkPath string) (targetPath string, err error) {
	stat := fs.GetFileTestStat(symlinkPath)
	if stat != nil {
		targetPath = stat.SymlinkTarget
	} else {
		err = os.ErrNotExist
	}

	return
}

func (fs *FakeFileSystem) CopyDirEntries(srcPath, dstPath string) (err error) {
	if fs.CopyDirEntriesError != nil {
		return fs.CopyDirEntriesError
	}

	filesToCopy := []string{}

	for filePath, _ := range fs.Files {
		if strings.HasPrefix(filePath, srcPath) {
			filesToCopy = append(filesToCopy, filePath)
		}
	}

	for _, filePath := range filesToCopy {
		newPath := strings.Replace(filePath, srcPath, dstPath, 1)
		fs.Files[newPath] = fs.Files[filePath]
	}

	return
}

func (fs *FakeFileSystem) CopyFile(srcPath, dstPath string) (err error) {
	if fs.CopyFileError != nil {
		err = fs.CopyFileError
		return
	}

	fs.Files[dstPath] = fs.Files[srcPath]
	return
}

func (fs *FakeFileSystem) TempFile(prefix string) (file *os.File, err error) {
	if fs.TempFileError != nil {
		return nil, fs.TempFileError
	}
	if fs.ReturnTempFile != nil {
		return fs.ReturnTempFile, nil
	} else {
		file, err = os.Open("/dev/null")
		if err != nil {
			err = bosherr.WrapError(err, "Opening /dev/null")
			return
		}

		// Make sure to record a reference for FileExist, etc. to work
		stats := fs.getOrCreateFile(file.Name())
		stats.FileType = FakeFileTypeFile

		return
	}
}

func (fs *FakeFileSystem) TempDir(prefix string) (string, error) {
	if fs.TempDirError != nil {
		return "", fs.TempDirError
	}

	var path string
	if len(fs.TempDirDir) > 0 {
		path = fs.TempDirDir
	} else {
		uuid, err := gouuid.NewV4()
		if err != nil {
			return "", err
		}

		path = uuid.String()
	}

	// Make sure to record a reference for FileExist, etc. to work
	stats := fs.getOrCreateFile(path)
	stats.FileType = FakeFileTypeDir

	return path, nil
}

func (fs *FakeFileSystem) RemoveAll(path string) (err error) {
	if fs.RemoveAllError != nil {
		return fs.RemoveAllError
	}

	filesToRemove := []string{}

	for name, _ := range fs.Files {
		if strings.HasPrefix(name, path) {
			filesToRemove = append(filesToRemove, name)
		}
	}

	for _, name := range filesToRemove {
		delete(fs.Files, name)
	}
	return
}

func (fs *FakeFileSystem) Open(path string) (file *os.File, err error) {
	file = fs.FilesToOpen[path]
	return
}

func (fs *FakeFileSystem) Glob(pattern string) (matches []string, err error) {
	remainingMatches, found := fs.globsMap[pattern]
	if found {
		matches = remainingMatches[0]
		if len(remainingMatches) > 1 {
			fs.globsMap[pattern] = remainingMatches[1:]
		}
	} else {
		matches = []string{}
	}
	return
}

func (fs *FakeFileSystem) SetGlob(pattern string, matches ...[]string) {
	fs.globsMap[pattern] = matches
	return
}

func (fs *FakeFileSystem) getOrCreateFile(path string) (stats *FakeFileStats) {
	stats = fs.GetFileTestStat(path)
	if stats == nil {
		if fs.Files == nil {
			fs.Files = make(map[string]*FakeFileStats)
		}

		stats = new(FakeFileStats)
		fs.Files[path] = stats
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"compress/zlib"
	"io"
	"net/http"
	"strings"
)

type compressor struct {
	http.ResponseWriter
	w interface {
		io.Writer
		io.Closer
	}
}

// compress enables gzip and deflate compression for outgoing requests.
func compress(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Vary", strings.Trim(w.Header().Get("Vary")+",Accept-Encoding", ","))

		for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") {
			enc = strings.TrimSpace(enc)
			if enc != "gzip" && enc != "deflate" {
				continue
			}
			w.Header().Set("Content-Encoding", enc)
			c := compressor{ResponseWriter: w}
			switch enc {
			case "gzip":
				c.w = gzip.NewWriter(w)
			case "deflate":
				c.w = zlib.NewWriter(w)
			}
			defer c.w.Close()
			w = c
			break
		}
		next(w, r)
	}
}

// Write calls io.Writer.Write().
func (c compressor) Write(b []byte) (int, error) {
	return c.w.Write(b)
}

func decodeBody(r *http.Response) error {
	// Decode the response:
	switch r.Header.Get("Content-Encoding") {
	case "gzip":
		body, err := gzip.NewReader(r.Body)
		if err != nil {
			return err
		}
		r.Body = body
	case "deflate":
		body, err := zlib.NewReader(r.Body)
		if err != nil {
			return err
		}
		r.Body = body
	}

	return nil
}
<commit_msg>use a sync.pool for encoding responses<commit_after>package main

import (
	"compress/gzip"
	"compress/zlib"
	"io"
	"net/http"
	"strings"
	"sync"
)

var pools = struct {
	gzip, deflate sync.Pool
}{
	gzip: sync.Pool{
		New: func() interface{} {
			return gzip.NewWriter(nil)
		},
	},
	deflate: sync.Pool{
		New: func() interface{} {
			return zlib.NewWriter(nil)
		},
	},
}

type compressor struct {
	http.ResponseWriter
	w encoder
}

type encoder interface {
	io.Writer
	Reset(io.Writer)
	Flush() error
}

// compress enables gzip and deflate compression for outgoing requests.
func compress(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Vary", strings.Trim(w.Header().Get("Vary")+",Accept-Encoding", ","))

		for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") {
			enc = strings.TrimSpace(enc)
			if enc != "gzip" && enc != "deflate" {
				continue
			}
			w.Header().Set("Content-Encoding", enc)
			c := compressor{ResponseWriter: w}
			switch enc {
			case "gzip":
				c.w = pools.gzip.Get().(encoder)
				defer pools.gzip.Put(c.w)
			case "deflate":
				c.w = pools.deflate.Get().(encoder)
				defer pools.deflate.Put(c.w)
			}
			c.w.Reset(w)
			defer c.w.Flush()
			w = c
			break
		}
		next(w, r)
	}
}

// Write calls io.Writer.Write().
func (c compressor) Write(b []byte) (int, error) {
	return c.w.Write(b)
}

func decodeBody(r *http.Response) error {
	// Decode the response:
	switch r.Header.Get("Content-Encoding") {
	case "gzip":
		body, err := gzip.NewReader(r.Body)
		if err != nil {
			return err
		}
		r.Body = body
	case "deflate":
		body, err := zlib.NewReader(r.Body)
		if err != nil {
			return err
		}
		r.Body = body
	}

	return nil
}
<|endoftext|>


<commit_before>package dockerfiles

import (
	"bytes"
	"fmt"
	"text/template"

	"go/format"

	"github.com/cihangir/gene/generators/common"
	"github.com/cihangir/schema"
)

type Generator struct{}

// Generate generates Dockerfile for given schema
func (c *Generator) Generate(context *common.Context, s *schema.Schema) ([]common.Output, error) {
	tmpl := template.New("dockerfile.tmpl").Funcs(context.TemplateFuncs)
	if _, err := tmpl.Parse(DockerfileTemplate); err != nil {
		return nil, err
	}

	moduleName := context.ModuleNameFunc(s.Title)
	outputs := make([]common.Output, 0)

	for _, def := range common.SortedObjectSchemas(s.Definitions) {

		var buf bytes.Buffer

		data := struct {
			Target     string
			ModuleName string
			Schema     *schema.Schema
		}{
			Target:     context.Config.Target,
			ModuleName: moduleName,
			Schema:     def,
		}

		if err := tmpl.Execute(&buf, data); err != nil {
			return nil, err
		}

		f, err := format.Source(buf.Bytes())
		if err != nil {
			return nil, err
		}

		path := fmt.Sprintf(
			"%sdockerfiles/%s/%s/Dockerfile",
			context.Config.Target,
			moduleName,
			context.FileNameFunc(def.Title),
		)

		outputs = append(outputs, common.Output{Content: f, Path: path, DoNotFormat: true})
	}

	return outputs, nil
}

// DockerfileTemplate holds the template for Dockerfile
var DockerfileTemplate = `
{{$schema := .Schema}}
{{$title := $schema.Title}}

# Start from a Debian image with the latest version of Go installed
# and a workspace (GOPATH) configured at /go.
FROM golang

# Copy the local package files to the container's workspace.
ADD . /go/src

# Build the outyet command inside the container.
# (You may fetch or manage dependencies here,
# either manually or with a tool like "godep".)

RUN go install {{.Target}}workers/{{ToLower $title}}

# Run the outyet command by default when the container starts.
ENTRYPOINT /go/bin/{{ToLower $title}}

# Document that the service listens on port 8080.
EXPOSE 8080
`
<commit_msg>Dockerfiles: pass proper command paths<commit_after>package dockerfiles

import (
	"bytes"
	"fmt"
	"text/template"

	"github.com/cihangir/gene/generators/common"
	"github.com/cihangir/schema"
)

type Generator struct {
	// CMDPath holds the path to executable files
	CMDPath string
}

// Generate generates Dockerfile for given schema
func (c *Generator) Generate(context *common.Context, s *schema.Schema) ([]common.Output, error) {
	tmpl := template.New("dockerfile.tmpl").Funcs(context.TemplateFuncs)
	if _, err := tmpl.Parse(DockerfileTemplate); err != nil {
		return nil, err
	}

	moduleName := context.ModuleNameFunc(s.Title)
	outputs := make([]common.Output, 0)

	for _, def := range common.SortedObjectSchemas(s.Definitions) {

		var buf bytes.Buffer

		data := struct {
			CMDPath    string
			ModuleName string
			Schema     *schema.Schema
		}{
			CMDPath:    c.CMDPath,
			ModuleName: moduleName,
			Schema:     def,
		}

		if err := tmpl.Execute(&buf, data); err != nil {
			return nil, err
		}

		path := fmt.Sprintf(
			"%s/%s/Dockerfile",
			context.Config.Target,
			context.FileNameFunc(def.Title),
		)

		outputs = append(outputs, common.Output{Content: buf.Bytes(), Path: path, DoNotFormat: true})
	}

	return outputs, nil
}

// DockerfileTemplate holds the template for Dockerfile
var DockerfileTemplate = `
{{$schema := .Schema}}
{{$title := $schema.Title}}

# Start from a Debian image with the latest version of Go installed
# and a workspace (GOPATH) configured at /go.
FROM golang

# Copy the local package files to the container's workspace.
ADD . /go/src

# Build the outyet command inside the container.
# (You may fetch or manage dependencies here,
# either manually or with a tool like "godep".)

RUN go install {{.CMDPath}}{{ToLower $title}}

# Run the outyet command by default when the container starts.
ENTRYPOINT /go/bin/{{ToLower $title}}

# Document that the service listens on port 8080.
# EXPOSE 8080
# TODO make this configurable
`
<|endoftext|>


<commit_before>/*
   Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at

    http://aws.amazon.com/apache2.0/

   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/
// snippet-start:[sqs.go.create_lp_queue]
package main

// snippet-start:[sqs.go.create_lp_queue.imports]
import (
    "flag"
    "fmt"
    "strconv"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/sqs"
)
// snippet-end:[sqs.go.create_lp_queue.imports]

// CreateLPQueue creates an Amazon SQS queue with long-polling enabled
// Inputs:
//     sess is the current session, which provides configuration for the SDK's service clients
//     queueName is the name of the queue
//     waitTime is the wait time, in seconds, for long polling to wait for messages
// Output:
//     If success, the URL of the queue and nil
//     Otherwise, an empty string and an error from the call to CreateQueue
func CreateLPQueue(sess *session.Session, queueName *string, waitTime *int) (string, error) {
    // Create a SQS service client
    svc := sqs.New(sess)

    // snippet-start:[sqs.go.create_lp_queue.call]
    result, err := svc.CreateQueue(&sqs.CreateQueueInput{
        QueueName: queueName,
        Attributes: aws.StringMap(map[string]string{
            "ReceiveMessageWaitTimeSeconds": strconv.Itoa(*waitTime),
        }),
    })
    // snippet-end:[sqs.go.create_lp_queue]
    if err != nil {
        return "", err
    }

    return *result.QueueUrl, nil
}

func main() {
    queueName := flag.String("n", "", "The name of the queue")
    waitTime := flag.Int("w", 10, "How long, in seconds, to wait for long polling")
    flag.Parse()

    if *queueName == "" {
        fmt.Println("You must supply a queue name (-n QUEUE-NAME")
        return
    }

    if *waitTime < 1 {
        *waitTime = 1
    }

    if *waitTime > 20 {
        *waitTime = 20
    }

    // Create a session that get credential values from ~/.aws/credentials
    // and the default region from ~/.aws/config
    // snippet-start:[sqs.go.create_lp_queue.sess]
    sess := session.Must(session.NewSessionWithOptions(session.Options{
        SharedConfigState: session.SharedConfigEnable,
    }))
    // snippet-end:[sqs.go.create_lp_queue.sess]

    url, err := CreateLPQueue(sess, queueName, waitTime)
    if err != nil {
        fmt.Println("Got an error creating the long-polling queue:")
        fmt.Println(err)
        return
    }

    fmt.Println("URL for long-polling queue " + *queueName + ": " + url)
}
// snippet-end:[sqs.go.create_lp_queue]
<commit_msg>Fixed snippet tag<commit_after>/*
   Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at

    http://aws.amazon.com/apache2.0/

   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/
// snippet-start:[sqs.go.create_lp_queue]
package main

// snippet-start:[sqs.go.create_lp_queue.imports]
import (
    "flag"
    "fmt"
    "strconv"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/sqs"
)
// snippet-end:[sqs.go.create_lp_queue.imports]

// CreateLPQueue creates an Amazon SQS queue with long-polling enabled
// Inputs:
//     sess is the current session, which provides configuration for the SDK's service clients
//     queueName is the name of the queue
//     waitTime is the wait time, in seconds, for long polling to wait for messages
// Output:
//     If success, the URL of the queue and nil
//     Otherwise, an empty string and an error from the call to CreateQueue
func CreateLPQueue(sess *session.Session, queueName *string, waitTime *int) (string, error) {
    // Create a SQS service client
    svc := sqs.New(sess)

    // snippet-start:[sqs.go.create_lp_queue.call]
    result, err := svc.CreateQueue(&sqs.CreateQueueInput{
        QueueName: queueName,
        Attributes: aws.StringMap(map[string]string{
            "ReceiveMessageWaitTimeSeconds": strconv.Itoa(*waitTime),
        }),
    })
    // snippet-end:[sqs.go.create_lp_queue.call]
    if err != nil {
        return "", err
    }

    return *result.QueueUrl, nil
}

func main() {
    queueName := flag.String("n", "", "The name of the queue")
    waitTime := flag.Int("w", 10, "How long, in seconds, to wait for long polling")
    flag.Parse()

    if *queueName == "" {
        fmt.Println("You must supply a queue name (-n QUEUE-NAME")
        return
    }

    if *waitTime < 1 {
        *waitTime = 1
    }

    if *waitTime > 20 {
        *waitTime = 20
    }

    // Create a session that get credential values from ~/.aws/credentials
    // and the default region from ~/.aws/config
    // snippet-start:[sqs.go.create_lp_queue.sess]
    sess := session.Must(session.NewSessionWithOptions(session.Options{
        SharedConfigState: session.SharedConfigEnable,
    }))
    // snippet-end:[sqs.go.create_lp_queue.sess]

    url, err := CreateLPQueue(sess, queueName, waitTime)
    if err != nil {
        fmt.Println("Got an error creating the long-polling queue:")
        fmt.Println(err)
        return
    }

    fmt.Println("URL for long-polling queue " + *queueName + ": " + url)
}
// snippet-end:[sqs.go.create_lp_queue]
<|endoftext|>


<commit_before>package rest

import (
	"encoding/json"
	"fmt"
	kodingmodels "koding/db/models"
	"socialapi/models"
)

func GetInteractions(interactionType string, postId int64) ([]string, error) {
	url := fmt.Sprintf("/message/%d/interaction/%s", postId, interactionType)
	res, err := sendRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	var interactions []string
	err = json.Unmarshal(res, &interactions)
	if err != nil {
		return nil, err
	}

	return interactions, nil
}

func AddInteraction(iType string, postId int64, ses *kodingmodels.Session) (*models.Interaction, error) {
	acc := models.NewAccount()
	if err := acc.ByNick(ses.Username); err != nil {

	}

	cm := models.NewInteraction()
	cm.AccountId = acc.Id
	cm.MessageId = postId

	url := fmt.Sprintf("/message/%d/interaction/%s/add", postId, iType)
	_, err := sendModelWithAuth("POST", url, cm, ses.ClientId)
	if err != nil {
		return cm, err
	}

	return cm, nil
}

func DeleteInteraction(interactionType string, postId int64, ses *kodingmodels.Session) error {
	acc := models.NewAccount()
	if err := acc.ByNick(ses.Username); err != nil {

	}

	cm := models.NewInteraction()
	cm.AccountId = acc.Id
	cm.MessageId = postId

	url := fmt.Sprintf("/message/%d/interaction/%s/delete", postId, interactionType)
	_, err := marshallAndSendRequestWithAuth("POST", url, cm, ses.ClientId)
	if err != nil {
		return err
	}
	return nil
}

func ListMessageInteractionsByType(iType string, accountId int64, token string) ([]*models.ChannelMessageContainer, error) {
	url := fmt.Sprintf("/account/%d/interaction/%s", accountId, iType)

	res, err := sendRequestWithAuth("GET", url, nil, token)
	if err != nil {
		return nil, err
	}

	// var cm []*models.ChannelMessageContainer
	cm := make([]*models.ChannelMessageContainer, 0)
	err = json.Unmarshal(res, &cm)
	if err != nil {
		return nil, err
	}

	return cm, nil
}
<commit_msg>socialapi/rest: add accountId for interactions<commit_after>package rest

import (
	"encoding/json"
	"fmt"
	"socialapi/models"
)

func GetInteractions(interactionType string, postId int64) ([]string, error) {
	url := fmt.Sprintf("/message/%d/interaction/%s", postId, interactionType)
	res, err := sendRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	var interactions []string
	err = json.Unmarshal(res, &interactions)
	if err != nil {
		return nil, err
	}

	return interactions, nil
}

func AddInteraction(iType string, postId, accountId int64, token string) (*models.Interaction, error) {
	cm := models.NewInteraction()
	cm.AccountId = accountId
	cm.MessageId = postId

	url := fmt.Sprintf("/message/%d/interaction/%s/add", postId, iType)
	_, err := sendModelWithAuth("POST", url, cm, token)
	if err != nil {
		return cm, err
	}

	return cm, nil
}

func DeleteInteraction(interactionType string, postId, accountId int64, token string) error {
	cm := models.NewInteraction()
	cm.AccountId = accountId
	cm.MessageId = postId

	url := fmt.Sprintf("/message/%d/interaction/%s/delete", postId, interactionType)
	_, err := marshallAndSendRequestWithAuth("POST", url, cm, token)
	if err != nil {
		return err
	}
	return nil
}

func ListMessageInteractionsByType(iType string, accountId int64, token string) ([]*models.ChannelMessageContainer, error) {
	url := fmt.Sprintf("/account/%d/interaction/%s", accountId, iType)

	res, err := sendRequestWithAuth("GET", url, nil, token)
	if err != nil {
		return nil, err
	}

	// var cm []*models.ChannelMessageContainer
	cm := make([]*models.ChannelMessageContainer, 0)
	err = json.Unmarshal(res, &cm)
	if err != nil {
		return nil, err
	}

	return cm, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package discovery

import (
	"sync"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
)

// TabletStatsCache is a HealthCheckStatsListener that keeps both the
// current list of available TabletStats, and a serving list:
// - for master tablets, only the current master is kept.
// - for non-master tablets, we filter the list using FilterByReplicationLag.
// It keeps entries for all tablets in the cell it's configured to serve for,
// and for the master independently of which cell it's in.
// Note the healthy tablet computation is done when we receive a tablet
// update only, not at serving time.
// Also note the cache may not have the last entry received by the tablet.
// For instance, if a tablet was healthy, and is still healthy, we do not
// keep its new update.
type TabletStatsCache struct {
	// cell is the cell we are keeping all tablets for.
	// Note we keep track of all master tablets in all cells.
	cell string

	// mu protects the entries map. It does not protect individual
	// entries in the map.
	mu sync.RWMutex
	// entries maps from keyspace/shard/tabletType to our cache.
	entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry
}

// tabletStatsCacheEntry is the per keyspace/shard/tabaletType
// entry of the in-memory map for TabletStatsCache.
type tabletStatsCacheEntry struct {
	// mu protects the rest of this structure.
	mu sync.RWMutex
	// all has the valid tablets, indexed by TabletToMapKey(ts.Tablet),
	// as it is the index used by HealthCheck.
	all map[string]*TabletStats
	// healthy only has the healthy ones.
	healthy []*TabletStats
}

// NewTabletStatsCache creates a TabletStatsCache, and registers
// it as HealthCheckStatsListener of the provided healthcheck.
// Note we do the registration in this code to guarantee we call
// SetListener with sendDownEvents=true, as we need these events
// to maintain the integrity of our cache.
func NewTabletStatsCache(hc HealthCheck, cell string) *TabletStatsCache {
	return newTabletStatsCache(hc, cell, true /* setListener */)
}

// NewTabletStatsCacheDoNotSetListener is identical to NewTabletStatsCache
// but does not automatically set the returned object as listener for "hc".
// Instead, it's up to the caller to ensure that TabletStatsCache.StatsUpdate()
// gets called properly. This is useful for chaining multiple listeners.
// When the caller sets its own listener on "hc", they must make sure that they
// set the parameter  "sendDownEvents" to "true" or this cache won't properly
// remove tablets whose tablet type changes.
func NewTabletStatsCacheDoNotSetListener(cell string) *TabletStatsCache {
	return newTabletStatsCache(nil, cell, false /* setListener */)
}

func newTabletStatsCache(hc HealthCheck, cell string, setListener bool) *TabletStatsCache {
	tc := &TabletStatsCache{
		cell:    cell,
		entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry),
	}

	if setListener {
		// We need to set sendDownEvents=true to get the deletes from the map
		// upon type change.
		hc.SetListener(tc, true /*sendDownEvents*/)
	}
	return tc
}

// getEntry returns an existing tabletStatsCacheEntry in the cache, or nil
// if the entry does not exist. It only takes a Read lock on mu.
func (tc *TabletStatsCache) getEntry(keyspace, shard string, tabletType topodatapb.TabletType) *tabletStatsCacheEntry {
	tc.mu.RLock()
	defer tc.mu.RUnlock()

	if s, ok := tc.entries[keyspace]; ok {
		if t, ok := s[shard]; ok {
			if e, ok := t[tabletType]; ok {
				return e
			}
		}
	}
	return nil
}

// getOrCreateEntry returns an existing tabletStatsCacheEntry from the cache,
// or creates it if it doesn't exist.
func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStatsCacheEntry {
	// Fast path (most common path too): Read-lock, return the entry.
	if e := tc.getEntry(target.Keyspace, target.Shard, target.TabletType); e != nil {
		return e
	}

	// Slow path: Lock, will probably have to add the entry at some level.
	tc.mu.Lock()
	defer tc.mu.Unlock()

	s, ok := tc.entries[target.Keyspace]
	if !ok {
		s = make(map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry)
		tc.entries[target.Keyspace] = s
	}
	t, ok := s[target.Shard]
	if !ok {
		t = make(map[topodatapb.TabletType]*tabletStatsCacheEntry)
		s[target.Shard] = t
	}
	e, ok := t[target.TabletType]
	if !ok {
		e = &tabletStatsCacheEntry{
			all: make(map[string]*TabletStats),
		}
		t[target.TabletType] = e
	}
	return e
}

// StatsUpdate is part of the HealthCheckStatsListener interface.
func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) {
	if ts.Target.TabletType != topodatapb.TabletType_MASTER && ts.Tablet.Alias.Cell != tc.cell {
		// this is for a non-master tablet in a different cell, drop it
		return
	}

	e := tc.getOrCreateEntry(ts.Target)
	e.mu.Lock()
	defer e.mu.Unlock()

	// Update our full map.
	trivialNonMasterUpdate := false
	if existing, ok := e.all[ts.Key]; ok {
		if ts.Up {
			// We have an existing entry, and a new entry.
			// Remember if they are both good (most common case).
			trivialNonMasterUpdate = existing.LastError == nil && existing.Serving && ts.LastError == nil && ts.Serving && ts.Target.TabletType != topodatapb.TabletType_MASTER && TrivialStatsUpdate(existing, ts)

			// We already have the entry, update the
			// values if necessary.  (will update both
			// 'all' and 'healthy' as they use pointers).
			if !trivialNonMasterUpdate {
				*existing = *ts
			}
		} else {
			// We have an entry which we shouldn't. Remove it.
			delete(e.all, ts.Key)
		}
	} else {
		if ts.Up {
			// Add the entry.
			e.all[ts.Key] = ts
		} else {
			// We were told to remove an entry which we
			// didn't have anyway, nothing should happen.
			return
		}
	}

	// The healthy list is different for TabletType_MASTER: we
	// only keep the most recent one.
	if ts.Target.TabletType == topodatapb.TabletType_MASTER {
		if ts.Up {
			// We have an Up master
			if len(e.healthy) == 0 {
				// We have a new Up server, just remember it.
				e.healthy = append(e.healthy, ts)
				return
			}

			// We already have one up server, see if we
			// need to replace it.
			if e.healthy[0].TabletExternallyReparentedTimestamp > ts.TabletExternallyReparentedTimestamp {
				// The notification we just got is older than
				// the one we had before, discard it.
				return
			}

			// Just replace it
			e.healthy[0] = ts
		} else {
			// We have a Down master, remove it only if
			// it's exactly the same
			if len(e.healthy) != 0 {
				if ts.Key == e.healthy[0].Key {
					// same guy, remove it
					e.healthy = nil
				}
			}
		}
		return
	}

	// For non-master, we just recompute the healthy list
	// using FilterByReplicationLag, if we need to.
	if trivialNonMasterUpdate {
		return
	}
	allArray := make([]*TabletStats, 0, len(e.all))
	for _, s := range e.all {
		allArray = append(allArray, s)
	}
	e.healthy = FilterByReplicationLag(allArray)
}

// GetTabletStats returns the full list of available targets.
// The returned array is owned by the caller.
func (tc *TabletStatsCache) GetTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats {
	e := tc.getEntry(keyspace, shard, tabletType)
	if e == nil {
		return nil
	}

	e.mu.RLock()
	defer e.mu.RUnlock()
	result := make([]TabletStats, 0, len(e.all))
	for _, s := range e.all {
		result = append(result, *s)
	}
	return result
}

// GetHealthyTabletStats returns only the healthy targets.
// The returned array is owned by the caller.
// For TabletType_MASTER, this will only return at most one entry,
// the most recent tablet of type master.
func (tc *TabletStatsCache) GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats {
	e := tc.getEntry(keyspace, shard, tabletType)
	if e == nil {
		return nil
	}

	e.mu.RLock()
	defer e.mu.RUnlock()
	result := make([]TabletStats, len(e.healthy))
	for i, ts := range e.healthy {
		result[i] = *ts
	}
	return result
}

// ResetForTesting is for use in tests only.
func (tc *TabletStatsCache) ResetForTesting() {
	tc.mu.Lock()
	defer tc.mu.Unlock()

	tc.entries = make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry)
}

// Compile-time interface check.
var _ HealthCheckStatsListener = (*TabletStatsCache)(nil)
<commit_msg>add log message if new tablet doesn't have a sufficiently new timestamp to be marked up<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package discovery

import (
	"sync"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
)

// TabletStatsCache is a HealthCheckStatsListener that keeps both the
// current list of available TabletStats, and a serving list:
// - for master tablets, only the current master is kept.
// - for non-master tablets, we filter the list using FilterByReplicationLag.
// It keeps entries for all tablets in the cell it's configured to serve for,
// and for the master independently of which cell it's in.
// Note the healthy tablet computation is done when we receive a tablet
// update only, not at serving time.
// Also note the cache may not have the last entry received by the tablet.
// For instance, if a tablet was healthy, and is still healthy, we do not
// keep its new update.
type TabletStatsCache struct {
	// cell is the cell we are keeping all tablets for.
	// Note we keep track of all master tablets in all cells.
	cell string

	// mu protects the entries map. It does not protect individual
	// entries in the map.
	mu sync.RWMutex
	// entries maps from keyspace/shard/tabletType to our cache.
	entries map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry
}

// tabletStatsCacheEntry is the per keyspace/shard/tabaletType
// entry of the in-memory map for TabletStatsCache.
type tabletStatsCacheEntry struct {
	// mu protects the rest of this structure.
	mu sync.RWMutex
	// all has the valid tablets, indexed by TabletToMapKey(ts.Tablet),
	// as it is the index used by HealthCheck.
	all map[string]*TabletStats
	// healthy only has the healthy ones.
	healthy []*TabletStats
}

// NewTabletStatsCache creates a TabletStatsCache, and registers
// it as HealthCheckStatsListener of the provided healthcheck.
// Note we do the registration in this code to guarantee we call
// SetListener with sendDownEvents=true, as we need these events
// to maintain the integrity of our cache.
func NewTabletStatsCache(hc HealthCheck, cell string) *TabletStatsCache {
	return newTabletStatsCache(hc, cell, true /* setListener */)
}

// NewTabletStatsCacheDoNotSetListener is identical to NewTabletStatsCache
// but does not automatically set the returned object as listener for "hc".
// Instead, it's up to the caller to ensure that TabletStatsCache.StatsUpdate()
// gets called properly. This is useful for chaining multiple listeners.
// When the caller sets its own listener on "hc", they must make sure that they
// set the parameter  "sendDownEvents" to "true" or this cache won't properly
// remove tablets whose tablet type changes.
func NewTabletStatsCacheDoNotSetListener(cell string) *TabletStatsCache {
	return newTabletStatsCache(nil, cell, false /* setListener */)
}

func newTabletStatsCache(hc HealthCheck, cell string, setListener bool) *TabletStatsCache {
	tc := &TabletStatsCache{
		cell:    cell,
		entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry),
	}

	if setListener {
		// We need to set sendDownEvents=true to get the deletes from the map
		// upon type change.
		hc.SetListener(tc, true /*sendDownEvents*/)
	}
	return tc
}

// getEntry returns an existing tabletStatsCacheEntry in the cache, or nil
// if the entry does not exist. It only takes a Read lock on mu.
func (tc *TabletStatsCache) getEntry(keyspace, shard string, tabletType topodatapb.TabletType) *tabletStatsCacheEntry {
	tc.mu.RLock()
	defer tc.mu.RUnlock()

	if s, ok := tc.entries[keyspace]; ok {
		if t, ok := s[shard]; ok {
			if e, ok := t[tabletType]; ok {
				return e
			}
		}
	}
	return nil
}

// getOrCreateEntry returns an existing tabletStatsCacheEntry from the cache,
// or creates it if it doesn't exist.
func (tc *TabletStatsCache) getOrCreateEntry(target *querypb.Target) *tabletStatsCacheEntry {
	// Fast path (most common path too): Read-lock, return the entry.
	if e := tc.getEntry(target.Keyspace, target.Shard, target.TabletType); e != nil {
		return e
	}

	// Slow path: Lock, will probably have to add the entry at some level.
	tc.mu.Lock()
	defer tc.mu.Unlock()

	s, ok := tc.entries[target.Keyspace]
	if !ok {
		s = make(map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry)
		tc.entries[target.Keyspace] = s
	}
	t, ok := s[target.Shard]
	if !ok {
		t = make(map[topodatapb.TabletType]*tabletStatsCacheEntry)
		s[target.Shard] = t
	}
	e, ok := t[target.TabletType]
	if !ok {
		e = &tabletStatsCacheEntry{
			all: make(map[string]*TabletStats),
		}
		t[target.TabletType] = e
	}
	return e
}

// StatsUpdate is part of the HealthCheckStatsListener interface.
func (tc *TabletStatsCache) StatsUpdate(ts *TabletStats) {
	if ts.Target.TabletType != topodatapb.TabletType_MASTER && ts.Tablet.Alias.Cell != tc.cell {
		// this is for a non-master tablet in a different cell, drop it
		return
	}

	e := tc.getOrCreateEntry(ts.Target)
	e.mu.Lock()
	defer e.mu.Unlock()

	// Update our full map.
	trivialNonMasterUpdate := false
	if existing, ok := e.all[ts.Key]; ok {
		if ts.Up {
			// We have an existing entry, and a new entry.
			// Remember if they are both good (most common case).
			trivialNonMasterUpdate = existing.LastError == nil && existing.Serving && ts.LastError == nil && ts.Serving && ts.Target.TabletType != topodatapb.TabletType_MASTER && TrivialStatsUpdate(existing, ts)

			// We already have the entry, update the
			// values if necessary.  (will update both
			// 'all' and 'healthy' as they use pointers).
			if !trivialNonMasterUpdate {
				*existing = *ts
			}
		} else {
			// We have an entry which we shouldn't. Remove it.
			delete(e.all, ts.Key)
		}
	} else {
		if ts.Up {
			// Add the entry.
			e.all[ts.Key] = ts
		} else {
			// We were told to remove an entry which we
			// didn't have anyway, nothing should happen.
			return
		}
	}

	// The healthy list is different for TabletType_MASTER: we
	// only keep the most recent one.
	if ts.Target.TabletType == topodatapb.TabletType_MASTER {
		if ts.Up {
			// We have an Up master
			if len(e.healthy) == 0 {
				// We have a new Up server, just remember it.
				e.healthy = append(e.healthy, ts)
				return
			}

			// We already have one up server, see if we
			// need to replace it.
			if ts.TabletExternallyReparentedTimestamp < e.healthy[0].TabletExternallyReparentedTimestamp {
				log.Warningf("not marking healthy master as Up because its externally reparented timestamt is smaller than the highest known timestamp from previous MASTERs: %d < %d ",
					ts.TabletExternallyReparentedTimestamp,
					e.healthy[0].TabletExternallyReparentedTimestamp,
					topoproto.KeyspaceShardString(eps.Target.Keyspace, eps.Target.Shard),
					topoproto.TabletAliasString(eps.Tablet.Alias),
					topoproto.TabletAliasString(e.healthy[0].Tablet.Alias))
				return
			}

			// Just replace it
			e.healthy[0] = ts
		} else {
			// We have a Down master, remove it only if
			// it's exactly the same
			if len(e.healthy) != 0 {
				if ts.Key == e.healthy[0].Key {
					// same guy, remove it
					e.healthy = nil
				}
			}
		}
		return
	}

	// For non-master, we just recompute the healthy list
	// using FilterByReplicationLag, if we need to.
	if trivialNonMasterUpdate {
		return
	}
	allArray := make([]*TabletStats, 0, len(e.all))
	for _, s := range e.all {
		allArray = append(allArray, s)
	}
	e.healthy = FilterByReplicationLag(allArray)
}

// GetTabletStats returns the full list of available targets.
// The returned array is owned by the caller.
func (tc *TabletStatsCache) GetTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats {
	e := tc.getEntry(keyspace, shard, tabletType)
	if e == nil {
		return nil
	}

	e.mu.RLock()
	defer e.mu.RUnlock()
	result := make([]TabletStats, 0, len(e.all))
	for _, s := range e.all {
		result = append(result, *s)
	}
	return result
}

// GetHealthyTabletStats returns only the healthy targets.
// The returned array is owned by the caller.
// For TabletType_MASTER, this will only return at most one entry,
// the most recent tablet of type master.
func (tc *TabletStatsCache) GetHealthyTabletStats(keyspace, shard string, tabletType topodatapb.TabletType) []TabletStats {
	e := tc.getEntry(keyspace, shard, tabletType)
	if e == nil {
		return nil
	}

	e.mu.RLock()
	defer e.mu.RUnlock()
	result := make([]TabletStats, len(e.healthy))
	for i, ts := range e.healthy {
		result[i] = *ts
	}
	return result
}

// ResetForTesting is for use in tests only.
func (tc *TabletStatsCache) ResetForTesting() {
	tc.mu.Lock()
	defer tc.mu.Unlock()

	tc.entries = make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry)
}

// Compile-time interface check.
var _ HealthCheckStatsListener = (*TabletStatsCache)(nil)
<|endoftext|>