<commit_before>package tnt

import (
	"bytes"
	"encoding/binary"
	"fmt"
)

var packedInt0 = PackInt(0)
var packedInt1 = PackInt(1)

func packLittle(value uint, bytes int) []byte {
	b := value
	result := make([]byte, bytes)
	for i := 0; i < bytes; i++ {
		result[i] = uint8(b & 0xFF)
		b >>= 8
	}
	return result
}

func PackLittle(value uint, bytes int) []byte {
	return packLittle(value, bytes)
}

func packBig(value int, bytes int) []byte {
	b := value
	result := make([]byte, bytes)
	for i := bytes - 1; i >= 0; i-- {
		result[i] = uint8(b & 0xFF)
		b >>= 8
	}
	return result
}

func PackBig(value int, bytes int) []byte {
	return packBig(value, bytes)
}

func PackB(value uint8) []byte {
	return []byte{value}
}

// PackInt is alias for PackL
func PackInt(value uint32) []byte {
	return packLittle(uint(value), 4)
}

// PackLong is alias for PackQ
func PackLong(value uint64) []byte {
	return packLittle(uint(value), 8)
}

func PackDouble(value float64) []byte {
	buffer := new(bytes.Buffer)
	binary.Write(buffer, binary.LittleEndian, value)
	return buffer.Bytes()
}

func base128len(length int) int {
	switch {
	case length < (1 << 7):
		return 1 + length
	case length < (1 << 14):
		return 2 + length
	case length < (1 << 21):
		return 3 + length
	case length < (1 << 28):
		return 4 + length
	default:
		return 5 + length
	}
}

// PackIntBase128 is port from python pack_int_base128
func PackIntBase128(value uint32) []byte {
	if value < (1 << 7) {
		return []byte{
			uint8(value),
		}
	}

	if value < (1 << 14) {
		return []byte{
			uint8((value>>7)&0xff | 0x80),
			uint8(value & 0x7F),
		}
	}

	if value < (1 << 21) {
		return []byte{
			uint8((value>>14)&0xff | 0x80),
			uint8((value>>7)&0xff | 0x80),
			uint8(value & 0x7F),
		}
	}

	if value < (1 << 28) {
		return []byte{
			uint8((value>>21)&0xff | 0x80),
			uint8((value>>14)&0xff | 0x80),
			uint8((value>>7)&0xff | 0x80),
			uint8(value & 0x7F),
		}
	}

	return []byte{
		uint8((value>>28)&0xff | 0x80),
		uint8((value>>21)&0xff | 0x80),
		uint8((value>>14)&0xff | 0x80),
		uint8((value>>7)&0xff | 0x80),
		uint8(value & 0x7F),
	}
}

func packFieldStr(value []byte) []byte {
	valueLenPacked := PackIntBase128(uint32(len(value)))

	var buffer bytes.Buffer
	buffer.Write(valueLenPacked)
	buffer.Write(value)

	return buffer.Bytes()
}

func packFieldInt(value uint32) []byte {
	var buffer bytes.Buffer
	buffer.Write(PackB(4))
	buffer.Write(PackInt(value))

	return buffer.Bytes()
}

func packTuple(value Tuple) []byte {
	var buffer bytes.Buffer

	fields := len(value)

	buffer.Write(PackInt(uint32(fields)))

	for i := 0; i < fields; i++ {
		buffer.Write(packFieldStr(value[i]))
	}

	return buffer.Bytes()
}

func interfaceToUint32(t interface{}) (uint32, error) {
	switch t := t.(type) {
	default:
		return 0, fmt.Errorf("unexpected type %T\n", t) // %T prints whatever type t has
	case int:
		return uint32(t), nil
	case int64:
		return uint32(t), nil
	case uint:
		return uint32(t), nil
	case uint64:
		return uint32(t), nil
	case int32:
		return uint32(t), nil
	case uint32:
		return t, nil
	}
}

func (q *Select) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	length := q.ByteLength()

	buf := NewFixedBuffer(length + 12)

	buf.WriteUint32(requestTypeSelect)
	buf.WriteUint32(uint32(length))
	buf.WriteUint32(requestID)

	limit := q.Limit
	if limit == 0 {
		limit = 0xffffffff
	}

	if q.Space != nil {
		i, err := interfaceToUint32(q.Space)
		if err != nil {
			return nil, err
		}
		buf.WriteUint32(i)
	} else {
		buf.WriteUint32(defaultSpace)
	}

	buf.WriteUint32(q.Index)
	buf.WriteUint32(q.Offset)
	buf.WriteUint32(limit)

	switch {
	case q.Value != nil:
		buf.WriteUint32(1) // count
		buf.WriteUint32(1) // fields
		vlp := PackIntBase128(uint32(len(q.Value)))
		buf.Write(vlp)
		buf.Write(q.Value)
	case q.Values != nil:
		cnt := len(q.Values)
		buf.WriteUint32(uint32(cnt))
		for i := 0; i < cnt; i++ {
			buf.WriteUint32(1) // fields
			vlp := PackIntBase128(uint32(len(q.Values[i])))
			buf.Write(vlp)
			buf.Write(q.Values[i])
		}
	case q.Tuples != nil:
		cnt := len(q.Tuples)
		buf.WriteUint32(uint32(cnt))
		for i := 0; i < cnt; i++ {
			tuple := q.Tuples[i]
			fields := len(tuple)
			buf.WriteUint32(uint32(fields))
			for j := 0; j < fields; j++ {
				vlp := PackIntBase128(uint32(len(tuple[j])))
				buf.Write(vlp)
				buf.Write(tuple[j])
			}
		}
	default:
		buf.WriteUint32(0) // count
	}

	return buf.Bytes(), nil
}

func (q *Insert) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	var bodyBuffer bytes.Buffer
	var buffer bytes.Buffer

	if q.Space != nil {
		i, err := interfaceToUint32(q.Space)
		if err != nil {
			return nil, err
		}
		bodyBuffer.Write(PackInt(i))
	} else {
		bodyBuffer.Write(PackInt(defaultSpace))
	}

	if q.ReturnTuple {
		bodyBuffer.Write(packedInt1)
	} else {
		bodyBuffer.Write(packedInt0)
	}
	bodyBuffer.Write(packTuple(q.Tuple))

	buffer.Write(PackInt(requestTypeInsert))
	buffer.Write(PackInt(uint32(bodyBuffer.Len())))
	buffer.Write(PackInt(requestID))
	buffer.Write(bodyBuffer.Bytes())

	return buffer.Bytes(), nil

}

func (q *Update) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	var bodyBuffer bytes.Buffer
	var buffer bytes.Buffer

	if q.Space != nil {
		space, err := interfaceToUint32(q.Space)
		if err != nil {
			return nil, err
		}
		bodyBuffer.Write(PackInt(space))
	} else {
		bodyBuffer.Write(PackInt(defaultSpace))
	}

	if q.ReturnTuple {
		bodyBuffer.Write(packedInt1)
	} else {
		bodyBuffer.Write(packedInt0)
	}

	bodyBuffer.Write(packTuple(q.Tuple))

	if len(q.Ops) != 0 {
		cnt := len(q.Ops)
		bodyBuffer.Write(PackInt(uint32(cnt)))
		for i := 0; i < cnt; i++ {
			op := q.Ops[i]
			bodyBuffer.Write(PackInt(op.Field))
			bodyBuffer.Write(PackB(byte(op.OpCode)))
			bodyBuffer.Write(packFieldStr(op.Value))
		}
	}

	var buf bytes.Buffer

	buffer.Write(PackInt(requestTypeUpdate))
	buffer.Write(PackInt(uint32(bodyBuffer.Len())))
	buffer.Write(PackInt(requestID))
	buffer.Write(bodyBuffer.Bytes())

	return buf.Bytes(), nil
}

func (q *Delete) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	var bodyBuffer bytes.Buffer
	var buffer bytes.Buffer

	if q.Space != nil {
		i, err := interfaceToUint32(q.Space)
		if err != nil {
			return nil, err
		}
		bodyBuffer.Write(PackInt(i))
	} else {
		bodyBuffer.Write(PackInt(defaultSpace))
	}

	if q.ReturnTuple {
		bodyBuffer.Write(packedInt1)
	} else {
		bodyBuffer.Write(packedInt0)
	}

	if q.Value != nil {
		bodyBuffer.Write(packTuple(Tuple{q.Value}))
	} else if q.Values != nil {
		cnt := len(q.Values)
		for i := 0; i < cnt; i++ {
			bodyBuffer.Write(packTuple(Tuple{q.Values[i]}))
		}
	}

	buffer.Write(PackInt(requestTypeDelete))
	buffer.Write(PackInt(uint32(bodyBuffer.Len())))
	buffer.Write(PackInt(requestID))
	buffer.Write(bodyBuffer.Bytes())

	return buffer.Bytes(), nil
}

func (q *Call) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	var bodyBuffer bytes.Buffer
	var buffer bytes.Buffer

	if q.ReturnTuple {
		bodyBuffer.Write(packedInt1)
	} else {
		bodyBuffer.Write(packedInt0)
	}
	bodyBuffer.Write(packFieldStr(q.Name))
	bodyBuffer.Write(packTuple(q.Tuple))

	buffer.Write(PackInt(requestTypeCall))
	buffer.Write(PackInt(uint32(bodyBuffer.Len())))
	buffer.Write(PackInt(requestID))
	buffer.Write(bodyBuffer.Bytes())

	return buffer.Bytes(), nil

}
<commit_msg>fix Update query packing<commit_after>package tnt

import (
	"bytes"
	"encoding/binary"
	"fmt"
)

var packedInt0 = PackInt(0)
var packedInt1 = PackInt(1)

func packLittle(value uint, bytes int) []byte {
	b := value
	result := make([]byte, bytes)
	for i := 0; i < bytes; i++ {
		result[i] = uint8(b & 0xFF)
		b >>= 8
	}
	return result
}

func PackLittle(value uint, bytes int) []byte {
	return packLittle(value, bytes)
}

func packBig(value int, bytes int) []byte {
	b := value
	result := make([]byte, bytes)
	for i := bytes - 1; i >= 0; i-- {
		result[i] = uint8(b & 0xFF)
		b >>= 8
	}
	return result
}

func PackBig(value int, bytes int) []byte {
	return packBig(value, bytes)
}

func PackB(value uint8) []byte {
	return []byte{value}
}

// PackInt is alias for PackL
func PackInt(value uint32) []byte {
	return packLittle(uint(value), 4)
}

// PackLong is alias for PackQ
func PackLong(value uint64) []byte {
	return packLittle(uint(value), 8)
}

func PackDouble(value float64) []byte {
	buffer := new(bytes.Buffer)
	binary.Write(buffer, binary.LittleEndian, value)
	return buffer.Bytes()
}

func base128len(length int) int {
	switch {
	case length < (1 << 7):
		return 1 + length
	case length < (1 << 14):
		return 2 + length
	case length < (1 << 21):
		return 3 + length
	case length < (1 << 28):
		return 4 + length
	default:
		return 5 + length
	}
}

// PackIntBase128 is port from python pack_int_base128
func PackIntBase128(value uint32) []byte {
	if value < (1 << 7) {
		return []byte{
			uint8(value),
		}
	}

	if value < (1 << 14) {
		return []byte{
			uint8((value>>7)&0xff | 0x80),
			uint8(value & 0x7F),
		}
	}

	if value < (1 << 21) {
		return []byte{
			uint8((value>>14)&0xff | 0x80),
			uint8((value>>7)&0xff | 0x80),
			uint8(value & 0x7F),
		}
	}

	if value < (1 << 28) {
		return []byte{
			uint8((value>>21)&0xff | 0x80),
			uint8((value>>14)&0xff | 0x80),
			uint8((value>>7)&0xff | 0x80),
			uint8(value & 0x7F),
		}
	}

	return []byte{
		uint8((value>>28)&0xff | 0x80),
		uint8((value>>21)&0xff | 0x80),
		uint8((value>>14)&0xff | 0x80),
		uint8((value>>7)&0xff | 0x80),
		uint8(value & 0x7F),
	}
}

func packFieldStr(value []byte) []byte {
	valueLenPacked := PackIntBase128(uint32(len(value)))

	var buffer bytes.Buffer
	buffer.Write(valueLenPacked)
	buffer.Write(value)

	return buffer.Bytes()
}

func packFieldInt(value uint32) []byte {
	var buffer bytes.Buffer
	buffer.Write(PackB(4))
	buffer.Write(PackInt(value))

	return buffer.Bytes()
}

func packTuple(value Tuple) []byte {
	var buffer bytes.Buffer

	fields := len(value)

	buffer.Write(PackInt(uint32(fields)))

	for i := 0; i < fields; i++ {
		buffer.Write(packFieldStr(value[i]))
	}

	return buffer.Bytes()
}

func interfaceToUint32(t interface{}) (uint32, error) {
	switch t := t.(type) {
	default:
		return 0, fmt.Errorf("unexpected type %T\n", t) // %T prints whatever type t has
	case int:
		return uint32(t), nil
	case int64:
		return uint32(t), nil
	case uint:
		return uint32(t), nil
	case uint64:
		return uint32(t), nil
	case int32:
		return uint32(t), nil
	case uint32:
		return t, nil
	}
}

func (q *Select) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	length := q.ByteLength()

	buf := NewFixedBuffer(length + 12)

	buf.WriteUint32(requestTypeSelect)
	buf.WriteUint32(uint32(length))
	buf.WriteUint32(requestID)

	limit := q.Limit
	if limit == 0 {
		limit = 0xffffffff
	}

	if q.Space != nil {
		i, err := interfaceToUint32(q.Space)
		if err != nil {
			return nil, err
		}
		buf.WriteUint32(i)
	} else {
		buf.WriteUint32(defaultSpace)
	}

	buf.WriteUint32(q.Index)
	buf.WriteUint32(q.Offset)
	buf.WriteUint32(limit)

	switch {
	case q.Value != nil:
		buf.WriteUint32(1) // count
		buf.WriteUint32(1) // fields
		vlp := PackIntBase128(uint32(len(q.Value)))
		buf.Write(vlp)
		buf.Write(q.Value)
	case q.Values != nil:
		cnt := len(q.Values)
		buf.WriteUint32(uint32(cnt))
		for i := 0; i < cnt; i++ {
			buf.WriteUint32(1) // fields
			vlp := PackIntBase128(uint32(len(q.Values[i])))
			buf.Write(vlp)
			buf.Write(q.Values[i])
		}
	case q.Tuples != nil:
		cnt := len(q.Tuples)
		buf.WriteUint32(uint32(cnt))
		for i := 0; i < cnt; i++ {
			tuple := q.Tuples[i]
			fields := len(tuple)
			buf.WriteUint32(uint32(fields))
			for j := 0; j < fields; j++ {
				vlp := PackIntBase128(uint32(len(tuple[j])))
				buf.Write(vlp)
				buf.Write(tuple[j])
			}
		}
	default:
		buf.WriteUint32(0) // count
	}

	return buf.Bytes(), nil
}

func (q *Insert) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	var bodyBuffer bytes.Buffer
	var buffer bytes.Buffer

	if q.Space != nil {
		i, err := interfaceToUint32(q.Space)
		if err != nil {
			return nil, err
		}
		bodyBuffer.Write(PackInt(i))
	} else {
		bodyBuffer.Write(PackInt(defaultSpace))
	}

	if q.ReturnTuple {
		bodyBuffer.Write(packedInt1)
	} else {
		bodyBuffer.Write(packedInt0)
	}
	bodyBuffer.Write(packTuple(q.Tuple))

	buffer.Write(PackInt(requestTypeInsert))
	buffer.Write(PackInt(uint32(bodyBuffer.Len())))
	buffer.Write(PackInt(requestID))
	buffer.Write(bodyBuffer.Bytes())

	return buffer.Bytes(), nil

}

func (q *Update) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	var bodyBuffer bytes.Buffer
	var buffer bytes.Buffer

	if q.Space != nil {
		space, err := interfaceToUint32(q.Space)
		if err != nil {
			return nil, err
		}
		bodyBuffer.Write(PackInt(space))
	} else {
		bodyBuffer.Write(PackInt(defaultSpace))
	}

	if q.ReturnTuple {
		bodyBuffer.Write(packedInt1)
	} else {
		bodyBuffer.Write(packedInt0)
	}

	bodyBuffer.Write(packTuple(q.Tuple))

	if len(q.Ops) != 0 {
		cnt := len(q.Ops)
		bodyBuffer.Write(PackInt(uint32(cnt)))
		for i := 0; i < cnt; i++ {
			op := q.Ops[i]
			bodyBuffer.Write(PackInt(op.Field))
			bodyBuffer.Write(PackB(byte(op.OpCode)))
			vlp := PackIntBase128(uint32(len(op.Value)))
			bodyBuffer.Write(vlp)
			bodyBuffer.Write(op.Value)
		}
	}

	buffer.Write(PackInt(requestTypeUpdate))
	buffer.Write(PackInt(uint32(bodyBuffer.Len())))
	buffer.Write(PackInt(requestID))
	buffer.Write(bodyBuffer.Bytes())

	return buffer.Bytes(), nil
}

func (q *Delete) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	var bodyBuffer bytes.Buffer
	var buffer bytes.Buffer

	if q.Space != nil {
		i, err := interfaceToUint32(q.Space)
		if err != nil {
			return nil, err
		}
		bodyBuffer.Write(PackInt(i))
	} else {
		bodyBuffer.Write(PackInt(defaultSpace))
	}

	if q.ReturnTuple {
		bodyBuffer.Write(packedInt1)
	} else {
		bodyBuffer.Write(packedInt0)
	}

	if q.Value != nil {
		bodyBuffer.Write(packTuple(Tuple{q.Value}))
	} else if q.Values != nil {
		cnt := len(q.Values)
		for i := 0; i < cnt; i++ {
			bodyBuffer.Write(packTuple(Tuple{q.Values[i]}))
		}
	}

	buffer.Write(PackInt(requestTypeDelete))
	buffer.Write(PackInt(uint32(bodyBuffer.Len())))
	buffer.Write(PackInt(requestID))
	buffer.Write(bodyBuffer.Bytes())

	return buffer.Bytes(), nil

}

func (q *Call) Pack(requestID uint32, defaultSpace uint32) ([]byte, error) {
	var bodyBuffer bytes.Buffer
	var buffer bytes.Buffer

	if q.ReturnTuple {
		bodyBuffer.Write(packedInt1)
	} else {
		bodyBuffer.Write(packedInt0)
	}
	bodyBuffer.Write(packFieldStr(q.Name))
	bodyBuffer.Write(packTuple(q.Tuple))

	buffer.Write(PackInt(requestTypeCall))
	buffer.Write(PackInt(uint32(bodyBuffer.Len())))
	buffer.Write(PackInt(requestID))
	buffer.Write(bodyBuffer.Bytes())

	return buffer.Bytes(), nil

}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"sort"
	"sync"

	"github.com/raintank/metrictank/mdata/cache/accnt"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/worldping-api/pkg/log"
)

type CCacheMetric struct {
	sync.RWMutex

	// points to the timestamp of the newest cache chunk currently held
	// the head of the linked list that containes the cache chunks
	newest uint32

	// points to the timestamp of the oldest cache chunk currently held
	// the tail of the linked list that containes the cache chunks
	oldest uint32

	// points at cached data chunks, indexed by their according time stamp
	chunks map[uint32]*CCacheChunk
}

func NewCCacheMetric() *CCacheMetric {
	return &CCacheMetric{
		chunks: make(map[uint32]*CCacheChunk),
	}
}

func (mc *CCacheMetric) Init(prev uint32, itergen chunk.IterGen) {
	mc.Add(prev, itergen)
	mc.oldest = itergen.Ts()
	mc.newest = itergen.Ts()
}

func (mc *CCacheMetric) Del(ts uint32) int {
	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; !ok {
		return len(mc.chunks)
	}

	prev := mc.chunks[ts].Prev
	next := mc.chunks[ts].Next

	if _, ok := mc.chunks[prev]; prev != 0 && ok {
		mc.chunks[prev].Next = 0
	}
	if _, ok := mc.chunks[next]; next != 0 && ok {
		mc.chunks[next].Prev = 0
	}

	delete(mc.chunks, ts)

	return len(mc.chunks)
}

func (mc *CCacheMetric) Add(prev uint32, itergen chunk.IterGen) {
	ts := itergen.Ts()

	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; ok {
		// chunk is already present. no need to error on that, just ignore it
		return
	}

	mc.chunks[ts] = &CCacheChunk{
		Ts:    ts,
		Prev:  0,
		Next:  0,
		Itgen: itergen,
	}

	nextTs := mc.nextTs(ts)

	log.Debug("CCacheMetric Add: caching chunk ts %d, nextTs %d", ts, nextTs)

	// if the previous chunk is cached, link in both directions
	if _, ok := mc.chunks[prev]; ok {
		mc.chunks[prev].Next = ts
		mc.chunks[ts].Prev = prev
	}

	// if nextTs() can't figure out the end date it returns ts
	if nextTs > ts {
		// if the next chunk is cached, link in both directions
		if _, ok := mc.chunks[nextTs]; ok {
			mc.chunks[nextTs].Prev = ts
			mc.chunks[ts].Next = nextTs
		}
	}

	// update list head/tail if necessary
	if ts > mc.newest {
		mc.newest = ts
	} else if ts < mc.oldest {
		mc.oldest = ts
	}

	return
}

// get sorted slice of all chunk timestamps
// assumes we have at least read lock
func (mc *CCacheMetric) sortedTs() []uint32 {
	keys := make([]uint32, 0, len(mc.chunks))
	for k := range mc.chunks {
		keys = append(keys, k)
	}
	sort.Sort(accnt.Uint32Asc(keys))
	return keys
}

// nextTs takes a chunk's ts and returns the ts of the next chunk. (guessing if necessary)
// assumes we already have at least a read lock
func (mc *CCacheMetric) nextTs(ts uint32) uint32 {
	chunk := mc.chunks[ts]
	span := chunk.Itgen.Span()
	if span > 0 {
		// if the chunk is span-aware we don't need anything else
		return chunk.Ts + span
	}

	if chunk.Next == 0 {
		if chunk.Prev == 0 {
			// if a chunk has no next and no previous chunk we have to assume it's length is 0
			return chunk.Ts
		} else {
			// if chunk has no next chunk, but has a previous one, we assume the length of this one is same as the previous one
			return chunk.Ts + (chunk.Ts - chunk.Prev)
		}
	} else {
		// if chunk has a next chunk, then that's the ts we need
		return chunk.Next
	}
}

// seekAsc finds the t0 of the chunk that contains ts, by searching from old to recent
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekAsc(ts uint32, keys []uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekAsc: seeking for %d in the keys %+d", ts, keys)

	for i := 0; i < len(keys) && keys[i] <= ts; i++ {
		if mc.nextTs(keys[i]) > ts {
			log.Debug("CCacheMetric seekAsc: seek found ts %d is between %d and %d", ts, keys[i], mc.nextTs(keys[i]))
			return keys[i], true
		}
	}

	log.Debug("CCacheMetric seekAsc: seekAsc unsuccessful")
	return 0, false
}

// seekDesc finds the t0 of the chunk that contains ts, by searching from recent to old
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekDesc(ts uint32, keys []uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekDesc: seeking for %d in the keys %+d", ts, keys)

	for i := len(keys) - 1; i >= 0 && mc.nextTs(keys[i]) > ts; i-- {
		if keys[i] <= ts {
			log.Debug("CCacheMetric seekDesc: seek found ts %d is between %d and %d", ts, keys[i], mc.nextTs(keys[i]))
			return keys[i], true
		}
	}

	log.Debug("CCacheMetric seekDesc: seekDesc unsuccessful")
	return 0, false
}

func (mc *CCacheMetric) searchForward(from, until uint32, keys []uint32, res *CCSearchResult) {
	ts, ok := mc.seekAsc(from, keys)
	if !ok {
		return
	}

	// add all consecutive chunks to search results, starting at the one containing "from"
	for ; ts != 0; ts = mc.chunks[ts].Next {
		log.Debug("CCacheMetric searchForward: forward search adds chunk ts %d to start", ts)
		res.Start = append(res.Start, mc.chunks[ts].Itgen)
		nextTs := mc.nextTs(ts)
		res.From = nextTs

		if nextTs >= until {
			res.Complete = true
			break
		}
	}
}

func (mc *CCacheMetric) searchBackward(from, until uint32, keys []uint32, res *CCSearchResult) {
	ts, ok := mc.seekDesc(until-1, keys)
	if !ok {
		return
	}

	for ; ts != 0; ts = mc.chunks[ts].Prev {
		log.Debug("CCacheMetric searchBackward: backward search adds chunk ts %d to end", ts)
		res.End = append(res.End, mc.chunks[ts].Itgen)
		startTs := mc.chunks[ts].Ts
		res.Until = startTs

		if startTs <= from {
			break
		}
	}
}

// the idea of this method is that we first look for the chunks where the
// "from" and "until" ts are in. then we seek from the "from" towards "until"
// and add as many cunks as possible to the result, if this did not result
// in all chunks necessary to serve the request we do the same in the reverse
// order from "until" to "from"
// if the first seek in chronological direction already ends up with all the
// chunks we need to serve the request, the second one can be skipped.

// EXAMPLE:
// from ts:                    |
// until ts:                                                   |
// cache:            |---|---|---|   |   |   |   |   |---|---|---|---|---|---|
// chunks returned:          |---|                   |---|---|---|
//
func (mc *CCacheMetric) Search(res *CCSearchResult, from, until uint32) {
	mc.RLock()
	defer mc.RUnlock()

	if len(mc.chunks) < 1 || from == until {
		return
	}

	keys := mc.sortedTs()

	mc.searchForward(from, until, keys, res)
	if !res.Complete {
		mc.searchBackward(from, until, keys, res)
	}

	if !res.Complete && res.From > res.Until {
		fmt.Printf("found from > until (%d/%d), printing chunks\n", res.From, res.Until)
		mc.debugMetric()
	}
}

func (mc *CCacheMetric) debugMetric() {
	keys := mc.sortedTs()
	fmt.Printf("--- debugging metric ---\n")
	fmt.Printf("chunk debug: oldest %d; newest %d\n", mc.oldest, mc.newest)
	for _, key := range keys {
		fmt.Printf("chunk debug: ts %d; prev %d; next %d\n", key, mc.chunks[key].Prev, mc.chunks[key].Next)
	}
	fmt.Printf("------------------------\n")
}
<commit_msg>changing prints to log statements<commit_after>package cache

import (
	"sort"
	"sync"

	"github.com/raintank/metrictank/mdata/cache/accnt"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/worldping-api/pkg/log"
)

type CCacheMetric struct {
	sync.RWMutex

	// points to the timestamp of the newest cache chunk currently held
	// the head of the linked list that containes the cache chunks
	newest uint32

	// points to the timestamp of the oldest cache chunk currently held
	// the tail of the linked list that containes the cache chunks
	oldest uint32

	// points at cached data chunks, indexed by their according time stamp
	chunks map[uint32]*CCacheChunk
}

func NewCCacheMetric() *CCacheMetric {
	return &CCacheMetric{
		chunks: make(map[uint32]*CCacheChunk),
	}
}

func (mc *CCacheMetric) Init(prev uint32, itergen chunk.IterGen) {
	mc.Add(prev, itergen)
	mc.oldest = itergen.Ts()
	mc.newest = itergen.Ts()
}

func (mc *CCacheMetric) Del(ts uint32) int {
	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; !ok {
		return len(mc.chunks)
	}

	prev := mc.chunks[ts].Prev
	next := mc.chunks[ts].Next

	if _, ok := mc.chunks[prev]; prev != 0 && ok {
		mc.chunks[prev].Next = 0
	}
	if _, ok := mc.chunks[next]; next != 0 && ok {
		mc.chunks[next].Prev = 0
	}

	delete(mc.chunks, ts)

	return len(mc.chunks)
}

func (mc *CCacheMetric) Add(prev uint32, itergen chunk.IterGen) {
	ts := itergen.Ts()

	mc.Lock()
	defer mc.Unlock()

	if _, ok := mc.chunks[ts]; ok {
		// chunk is already present. no need to error on that, just ignore it
		return
	}

	mc.chunks[ts] = &CCacheChunk{
		Ts:    ts,
		Prev:  0,
		Next:  0,
		Itgen: itergen,
	}

	nextTs := mc.nextTs(ts)

	log.Debug("CCacheMetric Add: caching chunk ts %d, nextTs %d", ts, nextTs)

	// if the previous chunk is cached, link in both directions
	if _, ok := mc.chunks[prev]; ok {
		mc.chunks[prev].Next = ts
		mc.chunks[ts].Prev = prev
	}

	// if nextTs() can't figure out the end date it returns ts
	if nextTs > ts {
		// if the next chunk is cached, link in both directions
		if _, ok := mc.chunks[nextTs]; ok {
			mc.chunks[nextTs].Prev = ts
			mc.chunks[ts].Next = nextTs
		}
	}

	// update list head/tail if necessary
	if ts > mc.newest {
		mc.newest = ts
	} else if ts < mc.oldest {
		mc.oldest = ts
	}

	return
}

// get sorted slice of all chunk timestamps
// assumes we have at least read lock
func (mc *CCacheMetric) sortedTs() []uint32 {
	keys := make([]uint32, 0, len(mc.chunks))
	for k := range mc.chunks {
		keys = append(keys, k)
	}
	sort.Sort(accnt.Uint32Asc(keys))
	return keys
}

// nextTs takes a chunk's ts and returns the ts of the next chunk. (guessing if necessary)
// assumes we already have at least a read lock
func (mc *CCacheMetric) nextTs(ts uint32) uint32 {
	chunk := mc.chunks[ts]
	span := chunk.Itgen.Span()
	if span > 0 {
		// if the chunk is span-aware we don't need anything else
		return chunk.Ts + span
	}

	if chunk.Next == 0 {
		if chunk.Prev == 0 {
			// if a chunk has no next and no previous chunk we have to assume it's length is 0
			return chunk.Ts
		} else {
			// if chunk has no next chunk, but has a previous one, we assume the length of this one is same as the previous one
			return chunk.Ts + (chunk.Ts - chunk.Prev)
		}
	} else {
		// if chunk has a next chunk, then that's the ts we need
		return chunk.Next
	}
}

// seekAsc finds the t0 of the chunk that contains ts, by searching from old to recent
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekAsc(ts uint32, keys []uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekAsc: seeking for %d in the keys %+d", ts, keys)

	for i := 0; i < len(keys) && keys[i] <= ts; i++ {
		if mc.nextTs(keys[i]) > ts {
			log.Debug("CCacheMetric seekAsc: seek found ts %d is between %d and %d", ts, keys[i], mc.nextTs(keys[i]))
			return keys[i], true
		}
	}

	log.Debug("CCacheMetric seekAsc: seekAsc unsuccessful")
	return 0, false
}

// seekDesc finds the t0 of the chunk that contains ts, by searching from recent to old
// if not found or can't be sure returns 0, false
// assumes we already have at least a read lock
func (mc *CCacheMetric) seekDesc(ts uint32, keys []uint32) (uint32, bool) {
	log.Debug("CCacheMetric seekDesc: seeking for %d in the keys %+d", ts, keys)

	for i := len(keys) - 1; i >= 0 && mc.nextTs(keys[i]) > ts; i-- {
		if keys[i] <= ts {
			log.Debug("CCacheMetric seekDesc: seek found ts %d is between %d and %d", ts, keys[i], mc.nextTs(keys[i]))
			return keys[i], true
		}
	}

	log.Debug("CCacheMetric seekDesc: seekDesc unsuccessful")
	return 0, false
}

func (mc *CCacheMetric) searchForward(from, until uint32, keys []uint32, res *CCSearchResult) {
	ts, ok := mc.seekAsc(from, keys)
	if !ok {
		return
	}

	// add all consecutive chunks to search results, starting at the one containing "from"
	for ; ts != 0; ts = mc.chunks[ts].Next {
		log.Debug("CCacheMetric searchForward: forward search adds chunk ts %d to start", ts)
		res.Start = append(res.Start, mc.chunks[ts].Itgen)
		nextTs := mc.nextTs(ts)
		res.From = nextTs

		if nextTs >= until {
			res.Complete = true
			break
		}
	}
}

func (mc *CCacheMetric) searchBackward(from, until uint32, keys []uint32, res *CCSearchResult) {
	ts, ok := mc.seekDesc(until-1, keys)
	if !ok {
		return
	}

	for ; ts != 0; ts = mc.chunks[ts].Prev {
		log.Debug("CCacheMetric searchBackward: backward search adds chunk ts %d to end", ts)
		res.End = append(res.End, mc.chunks[ts].Itgen)
		startTs := mc.chunks[ts].Ts
		res.Until = startTs

		if startTs <= from {
			break
		}
	}
}

// the idea of this method is that we first look for the chunks where the
// "from" and "until" ts are in. then we seek from the "from" towards "until"
// and add as many cunks as possible to the result, if this did not result
// in all chunks necessary to serve the request we do the same in the reverse
// order from "until" to "from"
// if the first seek in chronological direction already ends up with all the
// chunks we need to serve the request, the second one can be skipped.

// EXAMPLE:
// from ts:                    |
// until ts:                                                   |
// cache:            |---|---|---|   |   |   |   |   |---|---|---|---|---|---|
// chunks returned:          |---|                   |---|---|---|
//
func (mc *CCacheMetric) Search(res *CCSearchResult, from, until uint32) {
	mc.RLock()
	defer mc.RUnlock()

	if len(mc.chunks) < 1 || from == until {
		return
	}

	keys := mc.sortedTs()

	mc.searchForward(from, until, keys, res)
	if !res.Complete {
		mc.searchBackward(from, until, keys, res)
	}

	if !res.Complete && res.From > res.Until {
		log.Debug("CCacheMetric Search: Found from > until (%d/%d), printing chunks\n", res.From, res.Until)
		mc.debugMetric()
	}
}

func (mc *CCacheMetric) debugMetric() {
	keys := mc.sortedTs()
	log.Debug("CCacheMetric debugMetric: --- debugging metric ---\n")
	log.Debug("CCacheMetric debugMetric: oldest %d; newest %d\n", mc.oldest, mc.newest)
	for _, key := range keys {
		log.Debug("CCacheMetric debugMetric: ts %d; prev %d; next %d\n", key, mc.chunks[key].Prev, mc.chunks[key].Next)
	}
	log.Debug("CCacheMetric debugMetric: ------------------------\n")
}
<|endoftext|>


<commit_before>package markov

type Phrase []Morpheme

func (p Phrase) OriginalForm() (ans string) {
	for _, v := range p {
		ans += v.OriginalForm
	}
	return
}

func (p Phrase) Pronounciation() (ans string) {
	for _, v := range p {
		ans += v.Pronounciation
	}
	return
}
<commit_msg>[Pronounciation()] return false when Pronounciation is empty<commit_after>package markov

type Phrase []Morpheme

func (p Phrase) OriginalForm() (ans string) {
	for _, v := range p {
		ans += v.OriginalForm
	}
	return
}

func (p Phrase) Pronounciation() (string, bool) {
	var ans string
	for _, v := range p {
		if v.Pronounciation == "" {
			return "", false
		}
		ans += v.Pronounciation
	}
	return ans, true
}
<|endoftext|>


<commit_before>package environment

import (
	"os"
	"os/user"
	"strings"
)

func GetenvEitherCase(k string) string {
	if k == "" {
		return ""
	}
	if v := os.Getenv(k); v != "" {
		return v
	}
	if v := os.Getenv(strings.ToUpper(k)); v != "" {
		return v
	}
	return os.Getenv(strings.ToLower(k))
}

func UserHome() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}
	home := usr.HomeDir
	return home, nil
}
<commit_msg>use mitchellh/go-homedir instead of os/user<commit_after>package environment

import (
	"os"
	"github.com/mitchellh/go-homedir"
	"strings"
)

func GetenvEitherCase(k string) string {
	if k == "" {
		return ""
	}
	if v := os.Getenv(k); v != "" {
		return v
	}
	if v := os.Getenv(strings.ToUpper(k)); v != "" {
		return v
	}
	return os.Getenv(strings.ToLower(k))
}

func UserHome() (string, error) {
	home, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	return home, nil
}
<|endoftext|>


<commit_before>package metrics

import (
	"fmt"
	"strconv"

	"github.com/fweikert/continuous-integration/metrics/clients"
	"github.com/fweikert/continuous-integration/metrics/data"
	"github.com/fweikert/go-buildkite/buildkite"
)

type BuildsPerChange struct {
	client    *clients.BuildkiteClient
	columns   []Column
	pipelines []string
	builds    int
}

func (bps *BuildsPerChange) Name() string {
	return "builds_per_change"
}

func (bps *BuildsPerChange) Columns() []Column {
	return bps.columns
}

const changelistMetaDataKey = "PiperOrigin-RevId"

func (bps *BuildsPerChange) Collect() (*data.DataSet, error) {
	result := data.CreateDataSet(GetColumnNames(bps.columns))
	buildsPerPipeline := make(map[string]map[int]int)
	for _, pipeline := range bps.pipelines {
		builds, err := bps.client.GetMostRecentBuilds(pipeline, bps.builds)
		if err != nil {
			return nil, fmt.Errorf("Cannot collect builds_per_change statistics for pipeline %s: %v", pipeline, err)
		}
		buildsPerChange := make(map[int]int)
		for _, build := range builds {
			change, err := getChangeNumber(build)
			if err != nil {
				return nil, err
			} else if change < 0 {
				continue
			}
			if _, ok := buildsPerPipeline[pipeline]; !ok {
				buildsPerPipeline[pipeline] = make(map[int]int)
			}
			if _, ok := buildsPerChange[change]; !ok {
				buildsPerChange[change] = 0
			}
			buildsPerChange[change] += 1
		}
		buildsPerPipeline[pipeline] = buildsPerChange
	}
	for pipeline, buildsPerChange := range buildsPerPipeline {
		for change, builds := range buildsPerChange {
			err := result.AddRow(pipeline, change, builds)
			if err != nil {
				return nil, fmt.Errorf("Failed to add result for change %d and pipeline %s: %v", change, pipeline, err)
			}
		}
	}
	return result, nil
}

func getChangeNumber(build buildkite.Build) (int, error) {
	pipeline := *build.Pipeline.Slug
	metaData, ok := build.MetaData.(map[string]interface{})
	if !ok {
		return -1, fmt.Errorf("Invalid meta data on build %d in pipeline %s.", *build.Number, pipeline)
	}
	change, ok := metaData[changelistMetaDataKey]
	if !ok {
		return -1, nil
	}
	changeNumber, err := strconv.Atoi(fmt.Sprintf("%v", change))
	if err != nil {
		return -1, fmt.Errorf("Meta data of build %d in pipeline %s: '%v' is not a valid changelist number.", *build.Number, pipeline, change)
	}
	return changeNumber, nil
}

// CREATE TABLE builds_per_change (pipeline VARCHAR(255), change INT, builds INT, PRIMARY KEY(pipeline, change));
func CreateBuildsPerChange(client *clients.BuildkiteClient, builds int, pipelines ...string) *BuildsPerChange {
	columns := []Column{Column{"pipeline", true}, Column{"change", true}, Column{"builds", false}}
	return &BuildsPerChange{client: client, pipelines: pipelines, columns: columns, builds: builds}
}
<commit_msg>Remove obsolete code.<commit_after>package metrics

import (
	"fmt"
	"strconv"

	"github.com/fweikert/continuous-integration/metrics/clients"
	"github.com/fweikert/continuous-integration/metrics/data"
	"github.com/fweikert/go-buildkite/buildkite"
)

type BuildsPerChange struct {
	client    *clients.BuildkiteClient
	columns   []Column
	pipelines []string
	builds    int
}

func (bps *BuildsPerChange) Name() string {
	return "builds_per_change"
}

func (bps *BuildsPerChange) Columns() []Column {
	return bps.columns
}

const changelistMetaDataKey = "PiperOrigin-RevId"

func (bps *BuildsPerChange) Collect() (*data.DataSet, error) {
	result := data.CreateDataSet(GetColumnNames(bps.columns))
	buildsPerPipeline := make(map[string]map[int]int)
	for _, pipeline := range bps.pipelines {
		builds, err := bps.client.GetMostRecentBuilds(pipeline, bps.builds)
		if err != nil {
			return nil, fmt.Errorf("Cannot collect builds_per_change statistics for pipeline %s: %v", pipeline, err)
		}
		buildsPerChange := make(map[int]int)
		for _, build := range builds {
			change, err := getChangeNumber(build)
			if err != nil {
				return nil, err
			} else if change < 0 {
				continue
			}
			if _, ok := buildsPerChange[change]; !ok {
				buildsPerChange[change] = 0
			}
			buildsPerChange[change] += 1
		}
		buildsPerPipeline[pipeline] = buildsPerChange
	}
	for pipeline, buildsPerChange := range buildsPerPipeline {
		for change, builds := range buildsPerChange {
			err := result.AddRow(pipeline, change, builds)
			if err != nil {
				return nil, fmt.Errorf("Failed to add result for change %d and pipeline %s: %v", change, pipeline, err)
			}
		}
	}
	return result, nil
}

func getChangeNumber(build buildkite.Build) (int, error) {
	pipeline := *build.Pipeline.Slug
	metaData, ok := build.MetaData.(map[string]interface{})
	if !ok {
		return -1, fmt.Errorf("Invalid meta data on build %d in pipeline %s.", *build.Number, pipeline)
	}
	change, ok := metaData[changelistMetaDataKey]
	if !ok {
		return -1, nil
	}
	changeNumber, err := strconv.Atoi(fmt.Sprintf("%v", change))
	if err != nil {
		return -1, fmt.Errorf("Meta data of build %d in pipeline %s: '%v' is not a valid changelist number.", *build.Number, pipeline, change)
	}
	return changeNumber, nil
}

// CREATE TABLE builds_per_change (pipeline VARCHAR(255), change INT, builds INT, PRIMARY KEY(pipeline, change));
func CreateBuildsPerChange(client *clients.BuildkiteClient, builds int, pipelines ...string) *BuildsPerChange {
	columns := []Column{Column{"pipeline", true}, Column{"change", true}, Column{"builds", false}}
	return &BuildsPerChange{client: client, pipelines: pipelines, columns: columns, builds: builds}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"io/ioutil"
	"bytes"
	"encoding/json"
	"regexp"
)

const CONFIG_FILE string = "./config/client.json"

// A global configuration instance. Must be instantiated properly in main().
var Configuration Config

// Verifies a URL as valid (enough)
const URL_REGEX = "(https?://)?(www\\.)?\\w+\\.\\w+"

// In the case that wait.html cannot be served, we will respond with a
// plain text message to the user.
const PLEASE_WAIT_PLAINTEXT = /* Multi-line strings, yeah! */ `
The page you have requested is being prepared.
Plesae refresh this page in a few seconds to check if it is ready.
`

// Result of a bundle lookup from cache server.
type Result struct {
  ErrCode  ErrorCode
  ErrMsg   string
	Complete bool
	Found    bool
	Bundle   string
	// Should add a Created field for the date created
}

// Set a header on responses that indicates that the response
// was served by the CeNo client. The header can be referenced
// by pages, browser plugins, and so on to check if CeNo client
// is running.
func WriteProxyHeader(w http.ResponseWriter) http.ResponseWriter {
  w.Header().Add("X-Ceno-Proxy", "yxorP-oneC-X")
  return w
}

// Serve a page to inform the user that a bundle for the site they requested is
// being prepared. It will automatically initiate new requests to retrieve the same
// URL in an interval.
// The second bool return value specifies whether the response is HTML or not
func pleaseWait(url string) ([]byte, bool) {
	content, err := ioutil.ReadFile(Configuration.PleaseWaitPage)
	if err != nil {
		return []byte(PLEASE_WAIT_PLAINTEXT), false
	} else {
		return bytes.Replace(content, []byte("{{REDIRECT}}"), []byte(url), 1), true
	}
}

// Report that an error occured trying to decode the response from the LCS
// The LCS is expected to respond to this request with just the string "okay",
// so we will ignore it for now.
func reportDecodeError(reportURL, errMsg string) (bool, error) {
	mapping := map[string]interface{} {
		"error": errMsg,
	}
	marshalled, _ := json.Marshal(mapping)
	reader := bytes.NewReader(marshalled)
	req, err := http.NewRequest("POST", reportURL, reader)
	if err != nil {
		return false, err
	}
	req.Header.Set("Content-Type", "application/json")
	client := &http.Client{}
	response, err := client.Do(req)
	return response.StatusCode == 200, err
}

// Check with the local cache server to find a bundle for a given URL.
func lookup(lookupURL string) Result {
	response, err := http.Get(BundleLookupURL(Configuration, lookupURL))
	if err != nil || response.StatusCode != 200 {
	  fmt.Print("error: ")
		fmt.Println(err)
    return Result{ ERR_NO_CONNECT_LCS, err.Error(), false, false, "" }
	}
	decoder := json.NewDecoder(response.Body)
	var result Result
	if err := decoder.Decode(&result); err != nil {
		fmt.Println("Error decoding response from LCS")
		fmt.Println(err)
		reachedLCS, err2 := reportDecodeError(DecodeErrReportURL(Configuration), err.Error())
		if reachedLCS {
			return Result{ ERR_MALFORMED_LCS_RESPONSE, err2.Error(), false, false, "" }
		} else {
      errMsg := "Could not reach the local cache server to report decode eror"
			return Result{ ERR_NO_CONNECT_LCS, errMsg, false, false, "" }
		}
	}
	return result
}

// POST to the request server to have it start making a new bundle.
func requestNewBundle(lookupURL string) error {
	// We can ignore the content of the response since it is not used.
	reader := bytes.NewReader([]byte(lookupURL))
	URL := CreateBundleURL(Configuration, lookupURL)
	req, err := http.NewRequest("POST", URL, reader)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "text/plain")
	client := &http.Client{}
	_, err2 := client.Do(req)
	return err2
}

func execPleaseWait(URL string, w http.ResponseWriter, r *http.Request) {
  body, isHTML := pleaseWait(URL)
  if isHTML {
    w.Header().Set("Content-Type", "text/html")
  } else {
    w.Header().Set("Content-Type", "text/plain")
  }
  w.Write(body)
}

// Handle incoming requests for bundles.
// 1. Initiate bundle lookup process
// 2. Initiate bundle creation process when no bundle exists anywhere
func proxyHandler(w http.ResponseWriter, r *http.Request) {
  w = WriteProxyHeader(w)
	URL := r.URL.String()
  fmt.Printf("Got a request for %s\n", URL)
	matched, err := regexp.MatchString(URL_REGEX, URL)
	if !matched || err != nil {
		fmt.Println("Invalid URL " + URL)
    state := ErrorState{
      "responseWriter": w, "request": r, "errMsg": "Malformed URL \"" + URL + "\"",
    }
    ErrorHandlers[ERR_MALFORMED_URL](state)
		return
	}
	result := lookup(URL)
	if result.ErrCode > 0 {
    fmt.Printf("Got error from LCS: Error %v: %s\n", result.ErrCode, result.ErrMsg)
    // Assuming the reason the response is malformed is because of the formation of the bundle,
    // so we will request that a new bundle be created.
    if result.ErrCode == ERR_MALFORMED_LCS_RESPONSE {
      err = requestNewBundle(URL)
      fmt.Printf("Requested new bundle; Error: ")
      fmt.Println(err)
      if err != nil {
        state := ErrorState{ "responseWriter": w, "request": r, "errMsg": err.Error() }
        ErrorHandlers[ERR_FROM_LCS](state)
      } else {
        execPleaseWait(URL, w, r)
      }
    } else {
      state := ErrorState{ "responseWriter": w, "request": r, "errMsg": result.ErrMsg }
      ErrorHandlers[ERR_FROM_LCS](state)
    }
	} else if result.Complete {
		if result.Found {
			w.Write([]byte(result.Bundle))
		} else {
			err = requestNewBundle(URL)
			fmt.Println("Error information from requestNewbundle")
			fmt.Println(err)
			if err != nil {
        state := ErrorState{ "responseWriter": w, "request": r, "errMsg": err.Error() }
        ErrorHandlers[ERR_FROM_LCS](state)
			} else {
        execPleaseWait(URL, w, r)
			}
		}
	} else {
    execPleaseWait(URL, w, r)
	}
}

func main() {
	// Read an existing configuration file or have the user supply settings
	conf, err := ReadConfigFile(CONFIG_FILE)
	if err != nil {
		fmt.Print("Could not read configuration file at " + CONFIG_FILE)
		Configuration = GetConfigFromUser()
	} else {
		Configuration = conf
	}
	// Create an HTTP proxy server
	http.HandleFunc("/", proxyHandler)
	fmt.Println("CeNo proxy server listening for HTTP requests at http://localhost" + Configuration.PortNumber)
	if err = http.ListenAndServe(Configuration.PortNumber, nil); err != nil {
    panic(err)
  }
}
<commit_msg>Have the client pass on information about the rewriting of requests ti the RR<commit_after>package main

import (
	"fmt"
	"net/http"
	"io/ioutil"
	"bytes"
	"encoding/json"
	"regexp"
)

const CONFIG_FILE string = "./config/client.json"

// A global configuration instance. Must be instantiated properly in main().
var Configuration Config

// Verifies a URL as valid (enough)
const URL_REGEX = "(https?://)?(www\\.)?\\w+\\.\\w+"

// In the case that wait.html cannot be served, we will respond with a
// plain text message to the user.
const PLEASE_WAIT_PLAINTEXT = /* Multi-line strings, yeah! */ `
The page you have requested is being prepared.
Plesae refresh this page in a few seconds to check if it is ready.
`

// The header used to communicate from the browser extension to the bundle server
// that a request for http://site.com was rewritten from one for https://site.com.
const REWRITTEN_HEADER = "X-Ceno-Rewritten"

// Result of a bundle lookup from cache server.
type Result struct {
  ErrCode  ErrorCode
  ErrMsg   string
	Complete bool
	Found    bool
	Bundle   string
	// Should add a Created field for the date created
}

// Set a header on responses that indicates that the response
// was served by the CeNo client. The header can be referenced
// by pages, browser plugins, and so on to check if CeNo client
// is running.
func WriteProxyHeader(w http.ResponseWriter) http.ResponseWriter {
  w.Header().Add("X-Ceno-Proxy", "yxorP-oneC-X")
  return w
}

// Serve a page to inform the user that a bundle for the site they requested is
// being prepared. It will automatically initiate new requests to retrieve the same
// URL in an interval.
// The second bool return value specifies whether the response is HTML or not
func pleaseWait(url string) ([]byte, bool) {
	content, err := ioutil.ReadFile(Configuration.PleaseWaitPage)
	if err != nil {
		return []byte(PLEASE_WAIT_PLAINTEXT), false
	} else {
		return bytes.Replace(content, []byte("{{REDIRECT}}"), []byte(url), 1), true
	}
}

// Report that an error occured trying to decode the response from the LCS
// The LCS is expected to respond to this request with just the string "okay",
// so we will ignore it for now.
func reportDecodeError(reportURL, errMsg string) (bool, error) {
	mapping := map[string]interface{} {
		"error": errMsg,
	}
	marshalled, _ := json.Marshal(mapping)
	reader := bytes.NewReader(marshalled)
	req, err := http.NewRequest("POST", reportURL, reader)
	if err != nil {
		return false, err
	}
	req.Header.Set("Content-Type", "application/json")
	client := &http.Client{}
	response, err := client.Do(req)
	return response.StatusCode == 200, err
}

// Check with the local cache server to find a bundle for a given URL.
func lookup(lookupURL string) Result {
	response, err := http.Get(BundleLookupURL(Configuration, lookupURL))
	if err != nil || response.StatusCode != 200 {
	  fmt.Print("error: ")
		fmt.Println(err)
    return Result{ ERR_NO_CONNECT_LCS, err.Error(), false, false, "" }
	}
	decoder := json.NewDecoder(response.Body)
	var result Result
	if err := decoder.Decode(&result); err != nil {
		fmt.Println("Error decoding response from LCS")
		fmt.Println(err)
		reachedLCS, err2 := reportDecodeError(DecodeErrReportURL(Configuration), err.Error())
		if reachedLCS {
			return Result{ ERR_MALFORMED_LCS_RESPONSE, err2.Error(), false, false, "" }
		} else {
      errMsg := "Could not reach the local cache server to report decode eror"
			return Result{ ERR_NO_CONNECT_LCS, errMsg, false, false, "" }
		}
	}
	return result
}

// POST to the request server to have it start making a new bundle.
func requestNewBundle(lookupURL string, wasRewritten bool) error {
	// We can ignore the content of the response since it is not used.
	reader := bytes.NewReader([]byte(lookupURL))
	URL := CreateBundleURL(Configuration, lookupURL)
	req, err := http.NewRequest("POST", URL, reader)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "text/plain")
  if wasRewritten {
    req.Header.Set(REWRITTEN_HEADER, "true")
  } else {
    req.Header.Set(REWRITTEN_HEADER, "false")
  }
	client := &http.Client{}
	_, err2 := client.Do(req)
	return err2
}

func execPleaseWait(URL string, w http.ResponseWriter, r *http.Request) {
  body, isHTML := pleaseWait(URL)
  if isHTML {
    w.Header().Set("Content-Type", "text/html")
  } else {
    w.Header().Set("Content-Type", "text/plain")
  }
  w.Write(body)
}

// Handle incoming requests for bundles.
// 1. Initiate bundle lookup process
// 2. Initiate bundle creation process when no bundle exists anywhere
func proxyHandler(w http.ResponseWriter, r *http.Request) {
  w = WriteProxyHeader(w)
	URL := r.URL.String()
  fmt.Printf("Got a request for %s\n", URL)
	matched, err := regexp.MatchString(URL_REGEX, URL)
	if !matched || err != nil {
		fmt.Println("Invalid URL " + URL)
    state := ErrorState{
      "responseWriter": w, "request": r, "errMsg": "Malformed URL \"" + URL + "\"",
    }
    ErrorHandlers[ERR_MALFORMED_URL](state)
		return
	}
	result := lookup(URL)
	if result.ErrCode > 0 {
    fmt.Printf("Got error from LCS: Error %v: %s\n", result.ErrCode, result.ErrMsg)
    // Assuming the reason the response is malformed is because of the formation of the bundle,
    // so we will request that a new bundle be created.
    if result.ErrCode == ERR_MALFORMED_LCS_RESPONSE {
      wasRewritten := r.Header().Get(REWRITTEN_HEADER) == "true"
      err = requestNewBundle(URL, wasRewritten)
      fmt.Printf("Requested new bundle; Error: ")
      fmt.Println(err)
      if err != nil {
        state := ErrorState{ "responseWriter": w, "request": r, "errMsg": err.Error() }
        ErrorHandlers[ERR_FROM_LCS](state)
      } else {
        execPleaseWait(URL, w, r)
      }
    } else {
      state := ErrorState{ "responseWriter": w, "request": r, "errMsg": result.ErrMsg }
      ErrorHandlers[ERR_FROM_LCS](state)
    }
	} else if result.Complete {
		if result.Found {
			w.Write([]byte(result.Bundle))
		} else {
			err = requestNewBundle(URL)
			fmt.Println("Error information from requestNewbundle")
			fmt.Println(err)
			if err != nil {
        state := ErrorState{ "responseWriter": w, "request": r, "errMsg": err.Error() }
        ErrorHandlers[ERR_FROM_LCS](state)
			} else {
        execPleaseWait(URL, w, r)
			}
		}
	} else {
    execPleaseWait(URL, w, r)
	}
}

func main() {
	// Read an existing configuration file or have the user supply settings
	conf, err := ReadConfigFile(CONFIG_FILE)
	if err != nil {
		fmt.Print("Could not read configuration file at " + CONFIG_FILE)
		Configuration = GetConfigFromUser()
	} else {
		Configuration = conf
	}
	// Create an HTTP proxy server
	http.HandleFunc("/", proxyHandler)
	fmt.Println("CeNo proxy server listening for HTTP requests at http://localhost" + Configuration.PortNumber)
	if err = http.ListenAndServe(Configuration.PortNumber, nil); err != nil {
    panic(err)
  }
}
<|endoftext|>


<commit_before>package bird

import "sync"

// Operation type of manipulation
type Operation int

// Possible operation types
const (
	Land Operation = iota
	Raise
	Include
	Exclude
)

func (op Operation) String() string {
	switch op {
	case Land:
		return "Land"
	case Raise:
		return "Raise"
	case Include:
		return "Include"
	case Exclude:
		return "Exclude"
	default:
		return "Unknown"
	}
}

// Action details of single manipulation with birds group
type Action struct {
	Birds     []*SmartBird
	Operation Operation
}

// Flock is a collection of smart birds
type Flock struct {
	birds          map[*SmartBird]bool
	access         sync.RWMutex
	journalEnabled bool
	journal        chan Action
}

// NewFlock initializes new flock of smart birds
func NewFlock() *Flock {
	return &Flock{birds: make(map[*SmartBird]bool), journal: make(chan Action)}
}

// Journal if all manipulations with birds in the flock. If journal was invoked,
// reader must read it all otherwise all block
func (f *Flock) Journal() <-chan Action {
	f.journalEnabled = true
	return f.journal
}

// Include new smart bird to a flock or do nothing if it is already in
func (f *Flock) Include(smartBird *SmartBird) {
	if f.birds[smartBird] {
		return
	}
	f.access.Lock()
	defer f.access.Unlock()
	if !f.birds[smartBird] {
		f.birds[smartBird] = true
		f.log(Include, smartBird)
	}
}

// Take out smart bird from a flock, land it (if required) and return it.
// If bird wasn't be in a flock then returns nil
func (f *Flock) Take(birdToExclude *SmartBird, landBird bool) *SmartBird {
	if !f.birds[birdToExclude] {
		return nil
	}
	f.access.Lock()
	defer f.access.Unlock()
	if f.birds[birdToExclude] {
		if landBird {
			birdToExclude.Stop()
			f.log(Land, birdToExclude)
		}
		delete(f.birds, birdToExclude)
		f.log(Exclude, birdToExclude)
		return birdToExclude
	}
	return nil
}

// Exclude some smart birds from a flock by their names and then returns them as list
func (f *Flock) Exclude(landBird bool, birdNames ...string) []*SmartBird {
	var ans []*SmartBird
	f.access.Lock()
	defer f.access.Unlock()
	selected := f.selectUnsafe(birdNames...)
	for _, bird := range selected {
		if landBird {
			bird.Stop()
		}
		delete(f.birds, bird)
	}
	if landBird {
		f.log(Land, selected...)
	}
	f.log(Exclude, selected...)
	return ans
}

// Land some smart birds in a flock by their names.
// If names not specified - all birds are used
func (f *Flock) Land(names ...string) {
	f.access.RLock()
	defer f.access.RUnlock()
	wg := sync.WaitGroup{}
	selected := f.selectUnsafe(names...)
	wg.Add(len(selected))
	for _, bird := range selected {
		go func(bird *SmartBird) {
			defer wg.Done()
			bird.Stop()
		}(bird) // run stop in separate routing because of Stop() may be slow operation
	}
	wg.Wait()
	f.log(Land, selected...)
}

// Raise all smart birds in a flock.
// If names not specified - all birds are used
func (f *Flock) Raise(names ...string) {
	f.access.RLock()
	defer f.access.RUnlock()
	selected := f.selectUnsafe(names...)
	for _, bird := range selected {
		bird.Start()
	}
	f.log(Raise, selected...)
}

// Select some smart birds from a flock by their names
// If names not specified - all birds are used
func (f *Flock) Select(names ...string) []*SmartBird {
	f.access.RLock()
	defer f.access.RUnlock()
	return f.selectUnsafe(names...)
}

// Dissolve all smart birds from a flock and optionally land them
func (f *Flock) Dissolve(land bool) {
	f.Exclude(land)
}

func (f *Flock) log(operation Operation, birds ...*SmartBird) {
	if f.journalEnabled && len(birds) > 0 {
		f.journal <- Action{birds, operation}
	}
}

func (f *Flock) selectUnsafe(names ...string) []*SmartBird {
	ans := make([]*SmartBird, 0, len(f.birds))
	switch { // Some nano-optimization
	case len(names) > 1:
		set := make(map[string]bool)
		for _, name := range names {
			set[name] = true
		}
		for bird := range f.birds {
			if set[bird.name] {
				ans = append(ans, bird)
			}
		}
	case len(names) == 1:
		for bird := range f.birds {
			if names[0] == bird.name {
				ans = append(ans, bird)
			}
		}
	default:
		for bird := range f.birds {
			ans = append(ans, bird)
		}

	}

	return ans
}
<commit_msg>typo fix in comments<commit_after>package bird

import "sync"

// Operation type of manipulation
type Operation int

// Possible operation types
const (
	Land Operation = iota
	Raise
	Include
	Exclude
)

func (op Operation) String() string {
	switch op {
	case Land:
		return "Land"
	case Raise:
		return "Raise"
	case Include:
		return "Include"
	case Exclude:
		return "Exclude"
	default:
		return "Unknown"
	}
}

// Action details of single manipulation with birds group
type Action struct {
	Birds     []*SmartBird
	Operation Operation
}

// Flock is a collection of smart birds
type Flock struct {
	birds          map[*SmartBird]bool
	access         sync.RWMutex
	journalEnabled bool
	journal        chan Action
}

// NewFlock initializes new flock of smart birds
func NewFlock() *Flock {
	return &Flock{birds: make(map[*SmartBird]bool), journal: make(chan Action)}
}

// Journal of all manipulations with birds in the flock. If journal was invoked,
// reader must read it all otherwise all block
func (f *Flock) Journal() <-chan Action {
	f.journalEnabled = true
	return f.journal
}

// Include new smart bird to a flock or do nothing if it is already in
func (f *Flock) Include(smartBird *SmartBird) {
	if f.birds[smartBird] {
		return
	}
	f.access.Lock()
	defer f.access.Unlock()
	if !f.birds[smartBird] {
		f.birds[smartBird] = true
		f.log(Include, smartBird)
	}
}

// Take out smart bird from a flock, land it (if required) and return it.
// If bird wasn't be in a flock then returns nil
func (f *Flock) Take(birdToExclude *SmartBird, landBird bool) *SmartBird {
	if !f.birds[birdToExclude] {
		return nil
	}
	f.access.Lock()
	defer f.access.Unlock()
	if f.birds[birdToExclude] {
		if landBird {
			birdToExclude.Stop()
			f.log(Land, birdToExclude)
		}
		delete(f.birds, birdToExclude)
		f.log(Exclude, birdToExclude)
		return birdToExclude
	}
	return nil
}

// Exclude some smart birds from a flock by their names and then returns them as list
func (f *Flock) Exclude(landBird bool, birdNames ...string) []*SmartBird {
	var ans []*SmartBird
	f.access.Lock()
	defer f.access.Unlock()
	selected := f.selectUnsafe(birdNames...)
	for _, bird := range selected {
		if landBird {
			bird.Stop()
		}
		delete(f.birds, bird)
	}
	if landBird {
		f.log(Land, selected...)
	}
	f.log(Exclude, selected...)
	return ans
}

// Land some smart birds in a flock by their names.
// If names not specified - all birds are used
func (f *Flock) Land(names ...string) {
	f.access.RLock()
	defer f.access.RUnlock()
	wg := sync.WaitGroup{}
	selected := f.selectUnsafe(names...)
	wg.Add(len(selected))
	for _, bird := range selected {
		go func(bird *SmartBird) {
			defer wg.Done()
			bird.Stop()
		}(bird) // run stop in separate routing because of Stop() may be slow operation
	}
	wg.Wait()
	f.log(Land, selected...)
}

// Raise all smart birds in a flock.
// If names not specified - all birds are used
func (f *Flock) Raise(names ...string) {
	f.access.RLock()
	defer f.access.RUnlock()
	selected := f.selectUnsafe(names...)
	for _, bird := range selected {
		bird.Start()
	}
	f.log(Raise, selected...)
}

// Select some smart birds from a flock by their names
// If names not specified - all birds are used
func (f *Flock) Select(names ...string) []*SmartBird {
	f.access.RLock()
	defer f.access.RUnlock()
	return f.selectUnsafe(names...)
}

// Dissolve all smart birds from a flock and optionally land them
func (f *Flock) Dissolve(land bool) {
	f.Exclude(land)
}

func (f *Flock) log(operation Operation, birds ...*SmartBird) {
	if f.journalEnabled && len(birds) > 0 {
		f.journal <- Action{birds, operation}
	}
}

func (f *Flock) selectUnsafe(names ...string) []*SmartBird {
	ans := make([]*SmartBird, 0, len(f.birds))
	switch { // Some nano-optimization
	case len(names) > 1:
		set := make(map[string]bool)
		for _, name := range names {
			set[name] = true
		}
		for bird := range f.birds {
			if set[bird.name] {
				ans = append(ans, bird)
			}
		}
	case len(names) == 1:
		for bird := range f.birds {
			if names[0] == bird.name {
				ans = append(ans, bird)
			}
		}
	default:
		for bird := range f.birds {
			ans = append(ans, bird)
		}

	}

	return ans
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package common

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/uber/cherami-thrift/.generated/go/controller"
	"github.com/uber/cherami-server/common/configure"
	dconfig "github.com/uber/cherami-server/common/dconfigclient"
	"github.com/uber/cherami-server/common/metrics"
	log "github.com/Sirupsen/logrus"
	"github.com/uber-common/bark"
	"github.com/uber/tchannel-go"
	"github.com/uber/tchannel-go/thrift"
)

// roleKey label is set by every single service as soon as it bootstraps its
// ringpop instance. The data for this key is the service name
const roleKey = "serviceName"

// NewService instantiates a ServiceInstance
// TODO: have a better name for Service.
// this is the object which holds all the common stuff
// shared by all the services.
func NewService(serviceName string, uuid string, cfg configure.CommonServiceConfig, resolver UUIDResolver, hostHWInfoReader HostHardwareInfoReader, reporter metrics.Reporter, dClient dconfig.Client) *Service {
	sVice := &Service{
		sName:                  serviceName,
		cfg:                    cfg,
		resolver:               resolver,
		hostHWInfoReader:       hostHWInfoReader,
		hostUUID:               uuid,
		bLimitsEnabled:         true, // limits are enabled by default
		mReporter:              reporter,
		runtimeMetricsReporter: metrics.NewRuntimeMetricsReporter(reporter, time.Minute, cfg.GetLogger()),
		dClient:                dClient,
	}

	// Get the host name and set it on the service.  This is used for emitting metric with a tag for hostname
	if hostName, e := os.Hostname(); e != nil {
		log.Fatal("Error getting hostname")
	} else {
		sVice.hostName = hostName
	}

	sVice.startWg.Add(1)
	return sVice
}

// UpdateAdvertisedName is used to update the advertised name to be
// deployment specific
func (h *Service) UpdateAdvertisedName(deploymentName string) {
	// We'll have multiple deployments(prod,staging,etc) in same datacenter, so we need to advertise
	// frontend with different name for different deployments
	if h.sName == FrontendServiceName {
		if IsDevelopmentEnvironment(deploymentName) {
			h.sName = FrontendServiceName
		} else {
			parts := strings.Split(deploymentName, "_")
			if len(parts) != 2 {
				log.Fatal("Unexpected deployment name")
			}
			tenancy := strings.ToLower(parts[0])
			if tenancy == TenancyProd {
				h.sName = FrontendServiceName
			} else {
				h.sName = fmt.Sprintf("%v_%v", FrontendServiceName, tenancy)
			}
		}
	}
}

// GetTChannel returns the tchannel for this service
func (h *Service) GetTChannel() *tchannel.Channel {
	return h.ch
}

// GetConfig returns the AppConfig for this service
func (h *Service) GetConfig() configure.CommonServiceConfig {
	return h.cfg
}

// GetRingpopMonitor returns the RingpopMonitor for this service
func (h *Service) GetRingpopMonitor() RingpopMonitor {
	// This is needed to prevent overzealous TChannel clients from crashing us,
	// since Start() registers before booting ringpop
	if h.rpm == nil {
		h.startWg.Wait()
	}

	return h.rpm
}

// GetClientFactory returns the ClientFactory interface for this service
func (h *Service) GetClientFactory() ClientFactory {
	return h.cFactory
}

// SetClientFactory allowes change of client factory for getting thrift clients
func (h *Service) SetClientFactory(cf ClientFactory) {
	h.cFactory = cf
}

// GetWSConnector returns websocket connector for establishing websocket connections
func (h *Service) GetWSConnector() WSConnector {
	return h.wsConnector
}

// GetLoadReporterDaemonFactory is the factory interface for creating load reporters
func (h *Service) GetLoadReporterDaemonFactory() LoadReporterDaemonFactory {
	return h.rFactory
}

// GetHostPort returns the host port for this service
func (h *Service) GetHostPort() string {
	return h.hostPort
}

// GetHostName returns the name of host running the service
func (h *Service) GetHostName() string {
	return h.hostName
}

// GetHostUUID returns the uuid for this service
func (h *Service) GetHostUUID() string {
	return h.hostUUID
}

// GetMetricsReporter returns the metrics metrics for this service
func (h *Service) GetMetricsReporter() metrics.Reporter {
	return h.mReporter
}

// IsLimitsEnabled returns whether limits are enabled
func (h *Service) IsLimitsEnabled() bool {
	return h.bLimitsEnabled
}

// GetDConfigClient returns the dconfig client
func (h *Service) GetDConfigClient() dconfig.Client {
	return h.dClient
}

// Start starts a TChannel-Thrift service
func (h *Service) Start(thriftServices []thrift.TChanServer) {
	// bump restart counter (should already have service tagged)
	h.GetMetricsReporter().InitMetrics(metrics.ServiceMetrics)
	h.GetMetricsReporter().IncCounter(metrics.RestartCount, nil, 1)
	h.runtimeMetricsReporter.Start()

	h.ch, h.server = h.cfg.SetupTChannelServer(h.sName, thriftServices)
	h.logger = h.cfg.GetLogger()
	h.bLimitsEnabled = h.cfg.GetLimitsEnabled()

	// use actual listen port (in case service is bound to :0 or 0.0.0.0:0)
	h.hostPort = h.ch.PeerInfo().HostPort
	host, port, _ := SplitHostPort(h.hostPort)
	h.rp = CreateRingpop(h.sName, h.ch, host, port)
	if h.rp == nil {
		h.logger.Fatal("Ringpop creation failed")
	}

	err := BootstrapRingpop(h.rp, host, port, h.cfg)
	if err != nil {
		h.logger.WithFields(bark.Fields{TagErr: err}).Fatal("Ringpop bootstrap failed")
	}

	h.rpm = NewRingpopMonitor(h.rp, services, h.resolver, h.hostHWInfoReader, h.logger)
	// set the role for this service even before we start the rpm
	if err = h.rpm.SetMetadata(roleKey, h.sName); err != nil {
		// there is no point in proceeding further if we cannot set the role label
		h.logger.WithFields(bark.Fields{TagErr: err}).Fatal("Ringpop setting role label failed")
	}
	h.rpm.Start()

	// create the client factory interface
	h.cFactory = NewClientFactory(h.rpm, h.ch, h.logger)

	// create websocket connector
	h.wsConnector = NewWSConnector()

	lg := h.logger.WithField(TagService, FmtService(h.sName)).WithField(TagHostPort, FmtHostPort(h.hostPort))

	// create the LoadReporterDaemonFactory used by downstream components for reporting load
	h.rFactory = NewLoadReporterDaemonFactory(h.hostUUID, controller.SKU_Machine1, controller.Role_IN, h.cFactory,
		NewRealTimeTickerFactory(), NewTokenBucketFactory(), lg)

	// The service is now started up and registered with hyperbahn
	lg.Info("service started")

	// seed the random generator once for this service
	rand.Seed(time.Now().UTC().UnixNano())

	h.startWg.Done()
}

// Stop destroys ringpop for that service, and closes the associated listening tchannel
func (h *Service) Stop() {

	if h.rpm != nil {
		h.rpm.Stop()
	}

	if h.rp != nil {
		h.rp.Destroy()
	}

	if h.ch != nil {
		h.ch.Close()
	}

	h.runtimeMetricsReporter.Stop()
}

// Report is used for reporting Host specific load to controller
func (h *Service) Report(reporter LoadReporter) {
	// TODO: Report Host specific load here like CPU, Memory and Diskspace
}

// IsDevelopmentEnvironment detects if we are running in a development environment
func IsDevelopmentEnvironment(deploymentName string) (devEnv bool) {
	// If deploymentName is empty or "development", meaning the service is running locally, use the default service name
	if len(deploymentName) == 0 || strings.ToLower(deploymentName) == "development" {
		devEnv = true
	}

	return
}
<commit_msg>wait for service completely started in getters<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package common

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/uber/cherami-thrift/.generated/go/controller"
	"github.com/uber/cherami-server/common/configure"
	dconfig "github.com/uber/cherami-server/common/dconfigclient"
	"github.com/uber/cherami-server/common/metrics"
	log "github.com/Sirupsen/logrus"
	"github.com/uber-common/bark"
	"github.com/uber/tchannel-go"
	"github.com/uber/tchannel-go/thrift"
)

// roleKey label is set by every single service as soon as it bootstraps its
// ringpop instance. The data for this key is the service name
const roleKey = "serviceName"

// NewService instantiates a ServiceInstance
// TODO: have a better name for Service.
// this is the object which holds all the common stuff
// shared by all the services.
func NewService(serviceName string, uuid string, cfg configure.CommonServiceConfig, resolver UUIDResolver, hostHWInfoReader HostHardwareInfoReader, reporter metrics.Reporter, dClient dconfig.Client) *Service {
	sVice := &Service{
		sName:                  serviceName,
		cfg:                    cfg,
		resolver:               resolver,
		hostHWInfoReader:       hostHWInfoReader,
		hostUUID:               uuid,
		bLimitsEnabled:         true, // limits are enabled by default
		mReporter:              reporter,
		runtimeMetricsReporter: metrics.NewRuntimeMetricsReporter(reporter, time.Minute, cfg.GetLogger()),
		dClient:                dClient,
	}

	// Get the host name and set it on the service.  This is used for emitting metric with a tag for hostname
	if hostName, e := os.Hostname(); e != nil {
		log.Fatal("Error getting hostname")
	} else {
		sVice.hostName = hostName
	}

	sVice.startWg.Add(1)
	return sVice
}

// UpdateAdvertisedName is used to update the advertised name to be
// deployment specific
func (h *Service) UpdateAdvertisedName(deploymentName string) {
	// We'll have multiple deployments(prod,staging,etc) in same datacenter, so we need to advertise
	// frontend with different name for different deployments
	if h.sName == FrontendServiceName {
		if IsDevelopmentEnvironment(deploymentName) {
			h.sName = FrontendServiceName
		} else {
			parts := strings.Split(deploymentName, "_")
			if len(parts) != 2 {
				log.Fatal("Unexpected deployment name")
			}
			tenancy := strings.ToLower(parts[0])
			if tenancy == TenancyProd {
				h.sName = FrontendServiceName
			} else {
				h.sName = fmt.Sprintf("%v_%v", FrontendServiceName, tenancy)
			}
		}
	}
}

// GetTChannel returns the tchannel for this service
func (h *Service) GetTChannel() *tchannel.Channel {
	h.startWg.Wait()
	return h.ch
}

// GetConfig returns the AppConfig for this service
func (h *Service) GetConfig() configure.CommonServiceConfig {
	return h.cfg
}

// GetRingpopMonitor returns the RingpopMonitor for this service
func (h *Service) GetRingpopMonitor() RingpopMonitor {
	h.startWg.Wait()
	return h.rpm
}

// GetClientFactory returns the ClientFactory interface for this service
func (h *Service) GetClientFactory() ClientFactory {
	h.startWg.Wait()
	return h.cFactory
}

// SetClientFactory allowes change of client factory for getting thrift clients
func (h *Service) SetClientFactory(cf ClientFactory) {
	h.cFactory = cf
}

// GetWSConnector returns websocket connector for establishing websocket connections
func (h *Service) GetWSConnector() WSConnector {
	h.startWg.Wait()
	return h.wsConnector
}

// GetLoadReporterDaemonFactory is the factory interface for creating load reporters
func (h *Service) GetLoadReporterDaemonFactory() LoadReporterDaemonFactory {
	h.startWg.Wait()
	return h.rFactory
}

// GetHostPort returns the host port for this service
func (h *Service) GetHostPort() string {
	h.startWg.Wait()
	return h.hostPort
}

// GetHostName returns the name of host running the service
func (h *Service) GetHostName() string {
	return h.hostName
}

// GetHostUUID returns the uuid for this service
func (h *Service) GetHostUUID() string {
	return h.hostUUID
}

// GetMetricsReporter returns the metrics metrics for this service
func (h *Service) GetMetricsReporter() metrics.Reporter {
	return h.mReporter
}

// IsLimitsEnabled returns whether limits are enabled
func (h *Service) IsLimitsEnabled() bool {
	return h.bLimitsEnabled
}

// GetDConfigClient returns the dconfig client
func (h *Service) GetDConfigClient() dconfig.Client {
	return h.dClient
}

// Start starts a TChannel-Thrift service
func (h *Service) Start(thriftServices []thrift.TChanServer) {
	// bump restart counter (should already have service tagged)
	h.GetMetricsReporter().InitMetrics(metrics.ServiceMetrics)
	h.GetMetricsReporter().IncCounter(metrics.RestartCount, nil, 1)
	h.runtimeMetricsReporter.Start()

	h.ch, h.server = h.cfg.SetupTChannelServer(h.sName, thriftServices)
	h.logger = h.cfg.GetLogger()
	h.bLimitsEnabled = h.cfg.GetLimitsEnabled()

	// use actual listen port (in case service is bound to :0 or 0.0.0.0:0)
	h.hostPort = h.ch.PeerInfo().HostPort
	host, port, _ := SplitHostPort(h.hostPort)
	h.rp = CreateRingpop(h.sName, h.ch, host, port)
	if h.rp == nil {
		h.logger.Fatal("Ringpop creation failed")
	}

	err := BootstrapRingpop(h.rp, host, port, h.cfg)
	if err != nil {
		h.logger.WithFields(bark.Fields{TagErr: err}).Fatal("Ringpop bootstrap failed")
	}

	h.rpm = NewRingpopMonitor(h.rp, services, h.resolver, h.hostHWInfoReader, h.logger)
	// set the role for this service even before we start the rpm
	if err = h.rpm.SetMetadata(roleKey, h.sName); err != nil {
		// there is no point in proceeding further if we cannot set the role label
		h.logger.WithFields(bark.Fields{TagErr: err}).Fatal("Ringpop setting role label failed")
	}
	h.rpm.Start()

	// create the client factory interface
	h.cFactory = NewClientFactory(h.rpm, h.ch, h.logger)

	// create websocket connector
	h.wsConnector = NewWSConnector()

	lg := h.logger.WithField(TagService, FmtService(h.sName)).WithField(TagHostPort, FmtHostPort(h.hostPort))

	// create the LoadReporterDaemonFactory used by downstream components for reporting load
	h.rFactory = NewLoadReporterDaemonFactory(h.hostUUID, controller.SKU_Machine1, controller.Role_IN, h.cFactory,
		NewRealTimeTickerFactory(), NewTokenBucketFactory(), lg)

	// The service is now started up and registered with hyperbahn
	lg.Info("service started")

	// seed the random generator once for this service
	rand.Seed(time.Now().UTC().UnixNano())

	// Now decrements the counter. This should be the last step of this function
	// All getters are blocked until this step
        // This is needed to prevent overzealous TChannel clients from crashing us,
        // since hyperbahn advertisement is done in early stage of this function
	h.startWg.Done()
}

// Stop destroys ringpop for that service, and closes the associated listening tchannel
func (h *Service) Stop() {

	if h.rpm != nil {
		h.rpm.Stop()
	}

	if h.rp != nil {
		h.rp.Destroy()
	}

	if h.ch != nil {
		h.ch.Close()
	}

	h.runtimeMetricsReporter.Stop()
}

// Report is used for reporting Host specific load to controller
func (h *Service) Report(reporter LoadReporter) {
	// TODO: Report Host specific load here like CPU, Memory and Diskspace
}

// IsDevelopmentEnvironment detects if we are running in a development environment
func IsDevelopmentEnvironment(deploymentName string) (devEnv bool) {
	// If deploymentName is empty or "development", meaning the service is running locally, use the default service name
	if len(deploymentName) == 0 || strings.ToLower(deploymentName) == "development" {
		devEnv = true
	}

	return
}
<|endoftext|>


<commit_before>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package common

import (
	"testing"
	"time"
)

func TestHumanizeRate(t *testing.T) {
	// TOOD: more test cases
	s := HumanizeRate(100000*99, time.Second*5))
	if s != "2.0MB/s" {
		t.Fatal("HumanizeRate: expected 2.0MB/s, got "+ s)
	}
}
<commit_msg>fix HumanizeRate test<commit_after>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package common

import (
	"testing"
	"time"
)

func TestHumanizeRate(t *testing.T) {
	// TOOD: more test cases
	s := HumanizeRate(100000*99, time.Second*5)
	if s != "2.0MB/s" {
		t.Fatal("HumanizeRate: expected 2.0MB/s, got " + s)
	}
}
<|endoftext|>


<commit_before>/*
fperf allows you to build your performace tools easily

Three steps to create your own testcase

1. Create the "NewClient" function

	package demo

	import (
		"fmt"
		"github.com/fperf/fperf"
		"time"
	)

	type DemoClient struct{}

	func NewDemoClient(flag *fperf.FlagSet) fperf.Client {
		return &DemoClient{}
	}

2. Implement the UnaryClient or StreamClient

	func (c *DemoClient) Dial(addr string) error {
		fmt.Println("Dial to", addr)
		return nil
	}

	func (c *DemoClient) Request() error {
		time.Sleep(100 * time.Millisecond)
		return nil
	}

3. Register to fperf

	func init() {
		fperf.Register("demo", NewDemoClient, "This is a demo client discription")
	}

Run the buildin testcase

http is a simple builtin testcase to benchmark http servers

	fperf -cpu 8 -connection 10 http http://example.com
*/
package fperf

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"

	hist "github.com/fperf/fperf/stats"
	db "github.com/influxdata/influxdb/client/v2"
	"golang.org/x/net/context"
)

type setting struct {
	Connection int
	Stream     int
	Goroutine  int
	CPU        int
	Burst      int
	N          int //number of requests
	Tick       time.Duration
	Address    string
	Send       bool
	Recv       bool
	Delay      time.Duration
	Async      bool
	Target     string
	CallType   string
	InfluxDB   string
	Seed       int64
}

type statistics struct {
	latencies []time.Duration
	histogram *hist.Histogram
}

//roundtrip will be used in async mode
//the sender and receiver will be in seperate goroutines
type roundtrip struct {
	start time.Time
	end   time.Time
}

//create the testcase clients, n is the number of clients, set by
//flag -connection
func createClients(n int, addr string) []Client {
	addrs := strings.Split(addr, ";")
	getAddr := func(addrs []string) func() string {
		i := -1
		size := len(addrs)
		return func() string {
			i++
			return addrs[i%size]
		}
	}(addrs)
	clients := make([]Client, n)
	for i := 0; i < n; i++ {
		cli := NewClient(s.Target)
		if cli == nil {
			log.Fatalf("Can not find client %q for benchmark\n", s.Target)
		}

		addr := getAddr()
		if err := cli.Dial(addr); err != nil {
			log.Fatalln(addr, err)
		}
		clients[i] = cli
	}
	return clients
}

//create streams for every client. n is the number of streams per client
func createStreams(n int, clients []Client) []Stream {
	streams := make([]Stream, n*len(clients))
	for cur, cli := range clients {
		for i := 0; i < n; i++ {
			if cli, ok := cli.(StreamClient); ok {
				stream, err := cli.CreateStream(context.Background())
				if err != nil {
					log.Fatalf("StreamCall faile to create new stream, %v", err)
				}
				streams[cur*n+i] = stream
			} else {
				log.Fatalln(s.Target, " do not implement the fperf.StreamClient")
			}
		}
	}
	return streams
}

//run benchmark for stream clients, can be in async or sync mode
func benchmarkStream(n int, streams []Stream) {
	var wg sync.WaitGroup
	for _, stream := range streams {
		for i := 0; i < n; i++ {
			//Notice here. we must pass stream as a parameter because the varibale stream
			//would be changed after the goroutine created
			if s.Async {
				wg.Add(2)
				go func(stream Stream) { send(nil, stream); wg.Done() }(stream)
				go func(stream Stream) { recv(nil, stream); wg.Done() }(stream)
			} else {
				wg.Add(1)
				go func(stream Stream) { run(nil, stream); wg.Done() }(stream)
			}
		}
	}
	go statPrint()
	wg.Wait()
}

//run benchmark for unary clients
func benchmarkUnary(n int, clients []Client) {
	var wg sync.WaitGroup
	for _, cli := range clients {
		for i := 0; i < n; i++ {
			wg.Add(1)
			if cli, ok := cli.(UnaryClient); ok {
				go func(cli UnaryClient) { runUnary(nil, cli); wg.Done() }(cli)
			} else {
				log.Fatalln(s.Target, " does not implement the fperf.UnaryClient")
			}
		}
	}
	go statPrint()
	wg.Wait()
}

func runUnary(done <-chan int, cli UnaryClient) {
	for i := 0; s.N == 0 || i < s.N; i++ {
		//select {
		//case <-done:
		//	log.Println("run goroutine exit done")
		//	return
		//default:
		start := time.Now()
		if err := cli.Request(); err != nil {
			log.Println(err)
		}
		eplase := time.Since(start)
		stats.latencies = append(stats.latencies, eplase)
		if s.Delay > 0 {
			time.Sleep(s.Delay)
		}
		//	}
	}
}
func run(done <-chan int, stream Stream) {
	for i := 0; s.N == 0 || i < s.N; i++ {
		select {
		case <-done:
			log.Println("run goroutine exit done")
			return
		default:
			start := time.Now()
			if s.Send {
				stream.DoSend()
			}
			if s.Recv {
				stream.DoRecv()
			}
			eplase := time.Since(start)
			stats.latencies = append(stats.latencies, eplase)
			if s.Delay > 0 {
				time.Sleep(s.Delay)
			}
		}
	}
}

func send(done <-chan int, stream Stream) {
	timer := time.NewTimer(time.Second)
	for i := 0; s.N == 0 || i < s.N; i++ {
		select {
		case <-done:
			log.Println("send goroutine exit, done")
			return
		default:
			select {
			case rtts <- &roundtrip{start: time.Now()}:
				timer.Reset(time.Second)
			case <-timer.C:
				log.Println("blocked on send rtts")
			}

			if burst != nil {
				select {
				case burst <- 0:
					timer.Reset(time.Second)
				case <-timer.C:
					log.Println("blocked on send burst chan")
				}
			}

			stream.DoSend()
			if s.Delay > 0 {
				time.Sleep(s.Delay)
			}
		}
	}
}
func recv(done <-chan int, stream Stream) {
	timer := time.NewTimer(time.Second)
	for i := 0; s.N == 0 || i < s.N; i++ {
		select {
		case <-done:
			log.Println("recv goroutine exit, done")
			return
		default:
			if burst != nil {
				select {
				case <-burst:
					timer.Reset(time.Second)
				case <-timer.C:
					log.Println("blocked on recv burst chan")
				}
			}
			err := stream.DoRecv()
			if err != nil {
				log.Println("recv goroutine exit", err)
				return
			}
			select {
			case rtt := <-rtts:
				timer.Reset(time.Second)
				eplase := time.Since(rtt.start)
				stats.latencies = append(stats.latencies, eplase)
			case <-timer.C:
				log.Println("blocked on recv rtts")
			}
			if s.Delay > 0 {
				time.Sleep(s.Delay)
			}
		}
	}
}

func statPrint() {
	tickc := time.Tick(s.Tick)
	var latencies []time.Duration
	total := int64(0)
	for {
		select {
		case <-tickc:
			latencies = stats.latencies
			stats.latencies = stats.latencies[:0]

			sum := time.Duration(0)
			for _, eplase := range latencies {
				total++
				sum += eplase
				stats.histogram.Add(int64(eplase))
			}
			count := len(latencies)
			if count != 0 {
				log.Printf("latency %v qps %d total %v\n", sum/time.Duration(count), int64(float64(count)/float64(s.Tick)*float64(time.Second)), total)
				if influxdb != nil {
					bp, _ := db.NewBatchPoints(db.BatchPointsConfig{
						Database:  "fperf",
						Precision: "s",
					})
					tags := map[string]string{"latency": "latency", "qps": "qps"}
					fields := map[string]interface{}{
						"latency": float64(sum) / float64(count) / 1000.0,
						"qps":     int64(float64(count) / float64(s.Tick) * float64(time.Second)),
					}
					pt, err := db.NewPoint("benchmark", tags, fields, time.Now())
					if err != nil {
						log.Println("Error: ", err.Error())
					}
					bp.AddPoint(pt)

					// Write the batch
					err = influxdb.Write(bp)
					if err != nil {
						log.Println("Error: ", err.Error())
					}
				}
			} else {
				log.Printf("blocking...")
			}
		}
	}
}

var s setting
var stats statistics
var rtts = make(chan *roundtrip, 10*1024*1024)
var mutex sync.RWMutex
var burst chan int
var influxdb db.Client

func usage() {
	fmt.Printf("Usage: %v [options] <client>\noptions:\n", os.Args[0])
	flag.PrintDefaults()
	fmt.Println("clients:")
	for name, desc := range AllClients() {
		fmt.Printf(" %s", name)
		if len(desc) > 0 {
			fmt.Printf("\t: %s", desc)
		}
		fmt.Println()
	}
}

func Main() {
	flag.IntVar(&s.Connection, "connection", 1, "number of connection")
	flag.IntVar(&s.Stream, "stream", 1, "number of streams per connection")
	flag.IntVar(&s.Goroutine, "goroutine", 1, "number of goroutines per stream")
	flag.IntVar(&s.CPU, "cpu", 0, "set the GOMAXPROCS, use go default if 0")
	flag.IntVar(&s.Burst, "burst", 0, "burst a number of request, use with -async=true")
	flag.IntVar(&s.N, "N", 0, "number of request per goroutine")
	flag.BoolVar(&s.Send, "send", true, "perform send action")
	flag.BoolVar(&s.Recv, "recv", true, "perform recv action")
	flag.DurationVar(&s.Delay, "delay", 0, "wait delay time before send the next request")
	flag.DurationVar(&s.Tick, "tick", 2*time.Second, "interval between statistics")
	flag.StringVar(&s.Address, "server", "127.0.0.1:8804", "address of the target server")
	flag.BoolVar(&s.Async, "async", false, "send and recv in seperate goroutines")
	flag.StringVar(&s.CallType, "type", "auto", "set the call type:unary, stream or auto. default is auto")
	flag.StringVar(&s.InfluxDB, "influxdb", "", "writing stats to influxdb, specify the address in this option")
	flag.Int64Var(&s.Seed, "seed", 0, "seed of the global math/rand")
	flag.Usage = usage
	flag.Parse()

	s.Target = flag.Arg(0)
	if len(s.Target) == 0 {
		flag.Usage()
		return
	}
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		_ = <-c
		stats.histogram.Print(os.Stdout)
		os.Exit(0)
	}()

	runtime.GOMAXPROCS(s.CPU)
	go func() {
		runtime.SetBlockProfileRate(1)
		log.Println(http.ListenAndServe(":6060", nil))
	}()

	rand.Seed(s.Seed)

	if s.Burst > 0 {
		burst = make(chan int, s.Burst)
	}

	if len(s.InfluxDB) > 0 {
		c, err := db.NewHTTPClient(db.HTTPConfig{
			Addr: s.InfluxDB,
		})
		if err != nil {
			log.Fatalf("Error creating InfluxDB Client: %v", err.Error())
		}
		defer c.Close()
		q := db.NewQuery("CREATE DATABASE fperf", "", "")
		if response, err := c.Query(q); err == nil && response.Error() == nil {
			log.Println(response.Results)
		}
		influxdb = c
	}

	stats.latencies = make([]time.Duration, 0, 500000)
	histopt := hist.HistogramOptions{
		NumBuckets:     16,
		GrowthFactor:   1.8,
		BaseBucketSize: 1000,
		MinValue:       10000,
	}
	stats.histogram = hist.NewHistogram(histopt)
	clients := createClients(s.Connection, s.Address)
	cli := clients[0]
	switch s.CallType {
	case "auto":
		switch cli.(type) {
		case StreamClient:
			streams := createStreams(s.Stream, clients)
			benchmarkStream(s.Goroutine, streams)
		case UnaryClient:
			benchmarkUnary(s.Goroutine, clients)
		}
	case "stream":
		streams := createStreams(s.Stream, clients)
		benchmarkStream(s.Goroutine, streams)
	case "unary":
		benchmarkUnary(s.Goroutine, clients)
	}
}
<commit_msg>Remove influxdb<commit_after>/*
fperf allows you to build your performace tools easily

Three steps to create your own testcase

1. Create the "NewClient" function

	package demo

	import (
		"fmt"
		"github.com/fperf/fperf"
		"time"
	)

	type DemoClient struct{}

	func NewDemoClient(flag *fperf.FlagSet) fperf.Client {
		return &DemoClient{}
	}

2. Implement the UnaryClient or StreamClient

	func (c *DemoClient) Dial(addr string) error {
		fmt.Println("Dial to", addr)
		return nil
	}

	func (c *DemoClient) Request() error {
		time.Sleep(100 * time.Millisecond)
		return nil
	}

3. Register to fperf

	func init() {
		fperf.Register("demo", NewDemoClient, "This is a demo client discription")
	}

Run the buildin testcase

http is a simple builtin testcase to benchmark http servers

	fperf -cpu 8 -connection 10 http http://example.com
*/
package fperf

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"

	hist "github.com/fperf/fperf/stats"
	"golang.org/x/net/context"
)

type setting struct {
	Connection int
	Stream     int
	Goroutine  int
	CPU        int
	Burst      int
	N          int //number of requests
	Tick       time.Duration
	Address    string
	Send       bool
	Recv       bool
	Delay      time.Duration
	Async      bool
	Target     string
	CallType   string
	InfluxDB   string
	Seed       int64
}

type statistics struct {
	latencies []time.Duration
	histogram *hist.Histogram
}

//roundtrip will be used in async mode
//the sender and receiver will be in seperate goroutines
type roundtrip struct {
	start time.Time
	end   time.Time
}

//create the testcase clients, n is the number of clients, set by
//flag -connection
func createClients(n int, addr string) []Client {
	addrs := strings.Split(addr, ";")
	getAddr := func(addrs []string) func() string {
		i := -1
		size := len(addrs)
		return func() string {
			i++
			return addrs[i%size]
		}
	}(addrs)
	clients := make([]Client, n)
	for i := 0; i < n; i++ {
		cli := NewClient(s.Target)
		if cli == nil {
			log.Fatalf("Can not find client %q for benchmark\n", s.Target)
		}

		addr := getAddr()
		if err := cli.Dial(addr); err != nil {
			log.Fatalln(addr, err)
		}
		clients[i] = cli
	}
	return clients
}

//create streams for every client. n is the number of streams per client
func createStreams(n int, clients []Client) []Stream {
	streams := make([]Stream, n*len(clients))
	for cur, cli := range clients {
		for i := 0; i < n; i++ {
			if cli, ok := cli.(StreamClient); ok {
				stream, err := cli.CreateStream(context.Background())
				if err != nil {
					log.Fatalf("StreamCall faile to create new stream, %v", err)
				}
				streams[cur*n+i] = stream
			} else {
				log.Fatalln(s.Target, " do not implement the fperf.StreamClient")
			}
		}
	}
	return streams
}

//run benchmark for stream clients, can be in async or sync mode
func benchmarkStream(n int, streams []Stream) {
	var wg sync.WaitGroup
	for _, stream := range streams {
		for i := 0; i < n; i++ {
			//Notice here. we must pass stream as a parameter because the varibale stream
			//would be changed after the goroutine created
			if s.Async {
				wg.Add(2)
				go func(stream Stream) { send(nil, stream); wg.Done() }(stream)
				go func(stream Stream) { recv(nil, stream); wg.Done() }(stream)
			} else {
				wg.Add(1)
				go func(stream Stream) { run(nil, stream); wg.Done() }(stream)
			}
		}
	}
	go statPrint()
	wg.Wait()
}

//run benchmark for unary clients
func benchmarkUnary(n int, clients []Client) {
	var wg sync.WaitGroup
	for _, cli := range clients {
		for i := 0; i < n; i++ {
			wg.Add(1)
			if cli, ok := cli.(UnaryClient); ok {
				go func(cli UnaryClient) { runUnary(nil, cli); wg.Done() }(cli)
			} else {
				log.Fatalln(s.Target, " does not implement the fperf.UnaryClient")
			}
		}
	}
	go statPrint()
	wg.Wait()
}

func runUnary(done <-chan int, cli UnaryClient) {
	for i := 0; s.N == 0 || i < s.N; i++ {
		//select {
		//case <-done:
		//	log.Println("run goroutine exit done")
		//	return
		//default:
		start := time.Now()
		if err := cli.Request(); err != nil {
			log.Println(err)
		}
		eplase := time.Since(start)
		stats.latencies = append(stats.latencies, eplase)
		if s.Delay > 0 {
			time.Sleep(s.Delay)
		}
		//	}
	}
}
func run(done <-chan int, stream Stream) {
	for i := 0; s.N == 0 || i < s.N; i++ {
		select {
		case <-done:
			log.Println("run goroutine exit done")
			return
		default:
			start := time.Now()
			if s.Send {
				stream.DoSend()
			}
			if s.Recv {
				stream.DoRecv()
			}
			eplase := time.Since(start)
			stats.latencies = append(stats.latencies, eplase)
			if s.Delay > 0 {
				time.Sleep(s.Delay)
			}
		}
	}
}

func send(done <-chan int, stream Stream) {
	timer := time.NewTimer(time.Second)
	for i := 0; s.N == 0 || i < s.N; i++ {
		select {
		case <-done:
			log.Println("send goroutine exit, done")
			return
		default:
			select {
			case rtts <- &roundtrip{start: time.Now()}:
				timer.Reset(time.Second)
			case <-timer.C:
				log.Println("blocked on send rtts")
			}

			if burst != nil {
				select {
				case burst <- 0:
					timer.Reset(time.Second)
				case <-timer.C:
					log.Println("blocked on send burst chan")
				}
			}

			stream.DoSend()
			if s.Delay > 0 {
				time.Sleep(s.Delay)
			}
		}
	}
}
func recv(done <-chan int, stream Stream) {
	timer := time.NewTimer(time.Second)
	for i := 0; s.N == 0 || i < s.N; i++ {
		select {
		case <-done:
			log.Println("recv goroutine exit, done")
			return
		default:
			if burst != nil {
				select {
				case <-burst:
					timer.Reset(time.Second)
				case <-timer.C:
					log.Println("blocked on recv burst chan")
				}
			}
			err := stream.DoRecv()
			if err != nil {
				log.Println("recv goroutine exit", err)
				return
			}
			select {
			case rtt := <-rtts:
				timer.Reset(time.Second)
				eplase := time.Since(rtt.start)
				stats.latencies = append(stats.latencies, eplase)
			case <-timer.C:
				log.Println("blocked on recv rtts")
			}
			if s.Delay > 0 {
				time.Sleep(s.Delay)
			}
		}
	}
}

func statPrint() {
	tickc := time.Tick(s.Tick)
	var latencies []time.Duration
	total := int64(0)
	for {
		select {
		case <-tickc:
			latencies = stats.latencies
			stats.latencies = stats.latencies[:0]

			sum := time.Duration(0)
			for _, eplase := range latencies {
				total++
				sum += eplase
				stats.histogram.Add(int64(eplase))
			}
			count := len(latencies)
			if count != 0 {
				log.Printf("latency %v qps %d total %v\n", sum/time.Duration(count), int64(float64(count)/float64(s.Tick)*float64(time.Second)), total)
			} else {
				log.Printf("blocking...")
			}
		}
	}
}

var s setting
var stats statistics
var rtts = make(chan *roundtrip, 10*1024*1024)
var mutex sync.RWMutex
var burst chan int

func usage() {
	fmt.Printf("Usage: %v [options] <client>\noptions:\n", os.Args[0])
	flag.PrintDefaults()
	fmt.Println("clients:")
	for name, desc := range AllClients() {
		fmt.Printf(" %s", name)
		if len(desc) > 0 {
			fmt.Printf("\t: %s", desc)
		}
		fmt.Println()
	}
}

func Main() {
	flag.IntVar(&s.Connection, "connection", 1, "number of connection")
	flag.IntVar(&s.Stream, "stream", 1, "number of streams per connection")
	flag.IntVar(&s.Goroutine, "goroutine", 1, "number of goroutines per stream")
	flag.IntVar(&s.CPU, "cpu", 0, "set the GOMAXPROCS, use go default if 0")
	flag.IntVar(&s.Burst, "burst", 0, "burst a number of request, use with -async=true")
	flag.IntVar(&s.N, "N", 0, "number of request per goroutine")
	flag.BoolVar(&s.Send, "send", true, "perform send action")
	flag.BoolVar(&s.Recv, "recv", true, "perform recv action")
	flag.DurationVar(&s.Delay, "delay", 0, "wait delay time before send the next request")
	flag.DurationVar(&s.Tick, "tick", 2*time.Second, "interval between statistics")
	flag.StringVar(&s.Address, "server", "127.0.0.1:8804", "address of the target server")
	flag.BoolVar(&s.Async, "async", false, "send and recv in seperate goroutines")
	flag.StringVar(&s.CallType, "type", "auto", "set the call type:unary, stream or auto. default is auto")
	flag.Int64Var(&s.Seed, "seed", 0, "seed of the global math/rand")
	flag.Usage = usage
	flag.Parse()

	s.Target = flag.Arg(0)
	if len(s.Target) == 0 {
		flag.Usage()
		return
	}
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		_ = <-c
		stats.histogram.Print(os.Stdout)
		os.Exit(0)
	}()

	runtime.GOMAXPROCS(s.CPU)
	go func() {
		runtime.SetBlockProfileRate(1)
		log.Println(http.ListenAndServe(":6060", nil))
	}()

	rand.Seed(s.Seed)

	if s.Burst > 0 {
		burst = make(chan int, s.Burst)
	}

	stats.latencies = make([]time.Duration, 0, 500000)
	histopt := hist.HistogramOptions{
		NumBuckets:     16,
		GrowthFactor:   1.8,
		BaseBucketSize: 1000,
		MinValue:       10000,
	}
	stats.histogram = hist.NewHistogram(histopt)
	clients := createClients(s.Connection, s.Address)
	cli := clients[0]
	switch s.CallType {
	case "auto":
		switch cli.(type) {
		case StreamClient:
			streams := createStreams(s.Stream, clients)
			benchmarkStream(s.Goroutine, streams)
		case UnaryClient:
			benchmarkUnary(s.Goroutine, clients)
		}
	case "stream":
		streams := createStreams(s.Stream, clients)
		benchmarkStream(s.Goroutine, streams)
	case "unary":
		benchmarkUnary(s.Goroutine, clients)
	}
}
<|endoftext|>