<commit_before>package smux

import (
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
)

// Stream implements net.Conn
type Stream struct {
	id   uint32
	sess *Session

	buffers [][]byte
	heads   [][]byte // slice heads kept for recycle

	bufferLock sync.Mutex
	frameSize  int

	// notify a read event
	chReadEvent chan struct{}

	// flag the stream has closed
	die     chan struct{}
	dieOnce sync.Once

	// FIN
	chFinEvent   chan struct{}
	finEventOnce sync.Once

	// deadlines
	readDeadline  atomic.Value
	writeDeadline atomic.Value

	// count writes
	numWrite uint64
}

// newStream initiates a Stream struct
func newStream(id uint32, frameSize int, sess *Session) *Stream {
	s := new(Stream)
	s.id = id
	s.chReadEvent = make(chan struct{}, 1)
	s.frameSize = frameSize
	s.sess = sess
	s.die = make(chan struct{})
	s.chFinEvent = make(chan struct{})
	return s
}

// ID returns the unique stream ID.
func (s *Stream) ID() uint32 {
	return s.id
}

// Read implements net.Conn
func (s *Stream) Read(b []byte) (n int, err error) {
	if len(b) == 0 {
		return 0, nil
	}

	for {
		s.bufferLock.Lock()
		if len(s.buffers) > 0 {
			n = copy(b, s.buffers[0])
			s.buffers[0] = s.buffers[0][n:]
			if len(s.buffers[0]) == 0 {
				s.buffers[0] = nil
				s.buffers = s.buffers[1:]
				// full recycle
				defaultAllocator.Put(s.heads[0])
				s.heads = s.heads[1:]
			}
		}
		s.bufferLock.Unlock()

		if n > 0 {
			s.sess.returnTokens(n)
			return n, nil
		}

		if ew := s.waitRead(); ew != nil {
			return 0, ew
		}
	}
}

// WriteTo implements io.WriteTo
func (s *Stream) WriteTo(w io.Writer) (n int64, err error) {
	for {
		var buf []byte
		s.bufferLock.Lock()
		if len(s.buffers) > 0 {
			buf = s.buffers[0]
			s.buffers = s.buffers[1:]
			s.heads = s.heads[1:]
		}
		s.bufferLock.Unlock()

		if buf != nil {
			nw, ew := w.Write(buf)
			defaultAllocator.Put(buf)
			if nw > 0 {
				s.sess.returnTokens(nw)
				n += int64(nw)
			}

			if ew != nil {
				return n, ew
			}
		} else if ew := s.waitRead(); ew != nil {
			return n, ew
		}
	}
}

func (s *Stream) waitRead() error {
	var timer *time.Timer
	var deadline <-chan time.Time
	if d, ok := s.readDeadline.Load().(time.Time); ok && !d.IsZero() {
		timer = time.NewTimer(time.Until(d))
		defer timer.Stop()
		deadline = timer.C
	}

	select {
	case <-s.chReadEvent:
		return nil
	case <-s.chFinEvent:
		return errors.WithStack(io.EOF)
	case <-s.sess.chSocketReadError:
		return s.sess.socketReadError.Load().(error)
	case <-s.sess.chProtoError:
		return s.sess.protoError.Load().(error)
	case <-deadline:
		return errors.WithStack(ErrTimeout)
	case <-s.die:
		return errors.WithStack(io.ErrClosedPipe)
	}

}

// Write implements net.Conn
func (s *Stream) Write(b []byte) (n int, err error) {
	var deadline <-chan time.Time
	if d, ok := s.writeDeadline.Load().(time.Time); ok && !d.IsZero() {
		timer := time.NewTimer(time.Until(d))
		defer timer.Stop()
		deadline = timer.C
	}

	// check if stream has closed
	select {
	case <-s.die:
		return 0, errors.WithStack(io.ErrClosedPipe)
	default:
	}

	// frame split and transmit
	sent := 0
	frame := newFrame(cmdPSH, s.id)
	bts := b
	for len(bts) > 0 {
		sz := len(bts)
		if sz > s.frameSize {
			sz = s.frameSize
		}
		frame.data = bts[:sz]
		bts = bts[sz:]
		n, err := s.sess.writeFrameInternal(frame, deadline, s.numWrite)
		s.numWrite++
		sent += n
		if err != nil {
			return sent, errors.WithStack(err)
		}
	}

	return sent, nil
}

// Close implements net.Conn
func (s *Stream) Close() error {
	var once bool
	var err error
	s.dieOnce.Do(func() {
		close(s.die)
		once = true
	})

	if once {
		_, err = s.sess.writeFrame(newFrame(cmdFIN, s.id))
		s.sess.streamClosed(s.id)
		return err
	} else {
		return errors.WithStack(io.ErrClosedPipe)
	}
}

// GetDieCh returns a readonly chan which can be readable
// when the stream is to be closed.
func (s *Stream) GetDieCh() <-chan struct{} {
	return s.die
}

// SetReadDeadline sets the read deadline as defined by
// net.Conn.SetReadDeadline.
// A zero time value disables the deadline.
func (s *Stream) SetReadDeadline(t time.Time) error {
	s.readDeadline.Store(t)
	return nil
}

// SetWriteDeadline sets the write deadline as defined by
// net.Conn.SetWriteDeadline.
// A zero time value disables the deadline.
func (s *Stream) SetWriteDeadline(t time.Time) error {
	s.writeDeadline.Store(t)
	return nil
}

// SetDeadline sets both read and write deadlines as defined by
// net.Conn.SetDeadline.
// A zero time value disables the deadlines.
func (s *Stream) SetDeadline(t time.Time) error {
	if err := s.SetReadDeadline(t); err != nil {
		return errors.WithStack(err)
	}
	if err := s.SetWriteDeadline(t); err != nil {
		return errors.WithStack(err)
	}
	return nil
}

// session closes
func (s *Stream) sessionClose() { s.dieOnce.Do(func() { close(s.die) }) }

// LocalAddr satisfies net.Conn interface
func (s *Stream) LocalAddr() net.Addr {
	if ts, ok := s.sess.conn.(interface {
		LocalAddr() net.Addr
	}); ok {
		return ts.LocalAddr()
	}
	return nil
}

// RemoteAddr satisfies net.Conn interface
func (s *Stream) RemoteAddr() net.Addr {
	if ts, ok := s.sess.conn.(interface {
		RemoteAddr() net.Addr
	}); ok {
		return ts.RemoteAddr()
	}
	return nil
}

// pushBytes append buf to buffers
func (s *Stream) pushBytes(buf []byte) (written int, err error) {
	s.bufferLock.Lock()
	s.buffers = append(s.buffers, buf)
	s.heads = append(s.heads, buf)
	s.bufferLock.Unlock()
	return
}

// recycleTokens transform remaining bytes to tokens(will truncate buffer)
func (s *Stream) recycleTokens() (n int) {
	s.bufferLock.Lock()
	for k := range s.buffers {
		n += len(s.buffers[k])
		defaultAllocator.Put(s.heads[k])
	}
	s.buffers = nil
	s.heads = nil
	s.bufferLock.Unlock()
	return
}

// notify read event
func (s *Stream) notifyReadEvent() {
	select {
	case s.chReadEvent <- struct{}{}:
	default:
	}
}

// mark this stream has been closed in protocol
func (s *Stream) fin() {
	s.finEventOnce.Do(func() {
		close(s.chFinEvent)
	})
}
<commit_msg>should return all tokens<commit_after>package smux

import (
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
)

// Stream implements net.Conn
type Stream struct {
	id   uint32
	sess *Session

	buffers [][]byte
	heads   [][]byte // slice heads kept for recycle

	bufferLock sync.Mutex
	frameSize  int

	// notify a read event
	chReadEvent chan struct{}

	// flag the stream has closed
	die     chan struct{}
	dieOnce sync.Once

	// FIN
	chFinEvent   chan struct{}
	finEventOnce sync.Once

	// deadlines
	readDeadline  atomic.Value
	writeDeadline atomic.Value

	// count writes
	numWrite uint64
}

// newStream initiates a Stream struct
func newStream(id uint32, frameSize int, sess *Session) *Stream {
	s := new(Stream)
	s.id = id
	s.chReadEvent = make(chan struct{}, 1)
	s.frameSize = frameSize
	s.sess = sess
	s.die = make(chan struct{})
	s.chFinEvent = make(chan struct{})
	return s
}

// ID returns the unique stream ID.
func (s *Stream) ID() uint32 {
	return s.id
}

// Read implements net.Conn
func (s *Stream) Read(b []byte) (n int, err error) {
	if len(b) == 0 {
		return 0, nil
	}

	for {
		s.bufferLock.Lock()
		if len(s.buffers) > 0 {
			n = copy(b, s.buffers[0])
			s.buffers[0] = s.buffers[0][n:]
			if len(s.buffers[0]) == 0 {
				s.buffers[0] = nil
				s.buffers = s.buffers[1:]
				// full recycle
				defaultAllocator.Put(s.heads[0])
				s.heads = s.heads[1:]
			}
		}
		s.bufferLock.Unlock()

		if n > 0 {
			s.sess.returnTokens(n)
			return n, nil
		}

		if ew := s.waitRead(); ew != nil {
			return 0, ew
		}
	}
}

// WriteTo implements io.WriteTo
func (s *Stream) WriteTo(w io.Writer) (n int64, err error) {
	for {
		var buf []byte
		s.bufferLock.Lock()
		if len(s.buffers) > 0 {
			buf = s.buffers[0]
			s.buffers = s.buffers[1:]
			s.heads = s.heads[1:]
		}
		s.bufferLock.Unlock()

		if buf != nil {
			nw, ew := w.Write(buf)
			defaultAllocator.Put(buf)
			s.sess.returnTokens(len(buf))
			if nw > 0 {
				n += int64(nw)
			}

			if ew != nil {
				return n, ew
			}
		} else if ew := s.waitRead(); ew != nil {
			return n, ew
		}
	}
}

func (s *Stream) waitRead() error {
	var timer *time.Timer
	var deadline <-chan time.Time
	if d, ok := s.readDeadline.Load().(time.Time); ok && !d.IsZero() {
		timer = time.NewTimer(time.Until(d))
		defer timer.Stop()
		deadline = timer.C
	}

	select {
	case <-s.chReadEvent:
		return nil
	case <-s.chFinEvent:
		return errors.WithStack(io.EOF)
	case <-s.sess.chSocketReadError:
		return s.sess.socketReadError.Load().(error)
	case <-s.sess.chProtoError:
		return s.sess.protoError.Load().(error)
	case <-deadline:
		return errors.WithStack(ErrTimeout)
	case <-s.die:
		return errors.WithStack(io.ErrClosedPipe)
	}

}

// Write implements net.Conn
func (s *Stream) Write(b []byte) (n int, err error) {
	var deadline <-chan time.Time
	if d, ok := s.writeDeadline.Load().(time.Time); ok && !d.IsZero() {
		timer := time.NewTimer(time.Until(d))
		defer timer.Stop()
		deadline = timer.C
	}

	// check if stream has closed
	select {
	case <-s.die:
		return 0, errors.WithStack(io.ErrClosedPipe)
	default:
	}

	// frame split and transmit
	sent := 0
	frame := newFrame(cmdPSH, s.id)
	bts := b
	for len(bts) > 0 {
		sz := len(bts)
		if sz > s.frameSize {
			sz = s.frameSize
		}
		frame.data = bts[:sz]
		bts = bts[sz:]
		n, err := s.sess.writeFrameInternal(frame, deadline, s.numWrite)
		s.numWrite++
		sent += n
		if err != nil {
			return sent, errors.WithStack(err)
		}
	}

	return sent, nil
}

// Close implements net.Conn
func (s *Stream) Close() error {
	var once bool
	var err error
	s.dieOnce.Do(func() {
		close(s.die)
		once = true
	})

	if once {
		_, err = s.sess.writeFrame(newFrame(cmdFIN, s.id))
		s.sess.streamClosed(s.id)
		return err
	} else {
		return errors.WithStack(io.ErrClosedPipe)
	}
}

// GetDieCh returns a readonly chan which can be readable
// when the stream is to be closed.
func (s *Stream) GetDieCh() <-chan struct{} {
	return s.die
}

// SetReadDeadline sets the read deadline as defined by
// net.Conn.SetReadDeadline.
// A zero time value disables the deadline.
func (s *Stream) SetReadDeadline(t time.Time) error {
	s.readDeadline.Store(t)
	return nil
}

// SetWriteDeadline sets the write deadline as defined by
// net.Conn.SetWriteDeadline.
// A zero time value disables the deadline.
func (s *Stream) SetWriteDeadline(t time.Time) error {
	s.writeDeadline.Store(t)
	return nil
}

// SetDeadline sets both read and write deadlines as defined by
// net.Conn.SetDeadline.
// A zero time value disables the deadlines.
func (s *Stream) SetDeadline(t time.Time) error {
	if err := s.SetReadDeadline(t); err != nil {
		return errors.WithStack(err)
	}
	if err := s.SetWriteDeadline(t); err != nil {
		return errors.WithStack(err)
	}
	return nil
}

// session closes
func (s *Stream) sessionClose() { s.dieOnce.Do(func() { close(s.die) }) }

// LocalAddr satisfies net.Conn interface
func (s *Stream) LocalAddr() net.Addr {
	if ts, ok := s.sess.conn.(interface {
		LocalAddr() net.Addr
	}); ok {
		return ts.LocalAddr()
	}
	return nil
}

// RemoteAddr satisfies net.Conn interface
func (s *Stream) RemoteAddr() net.Addr {
	if ts, ok := s.sess.conn.(interface {
		RemoteAddr() net.Addr
	}); ok {
		return ts.RemoteAddr()
	}
	return nil
}

// pushBytes append buf to buffers
func (s *Stream) pushBytes(buf []byte) (written int, err error) {
	s.bufferLock.Lock()
	s.buffers = append(s.buffers, buf)
	s.heads = append(s.heads, buf)
	s.bufferLock.Unlock()
	return
}

// recycleTokens transform remaining bytes to tokens(will truncate buffer)
func (s *Stream) recycleTokens() (n int) {
	s.bufferLock.Lock()
	for k := range s.buffers {
		n += len(s.buffers[k])
		defaultAllocator.Put(s.heads[k])
	}
	s.buffers = nil
	s.heads = nil
	s.bufferLock.Unlock()
	return
}

// notify read event
func (s *Stream) notifyReadEvent() {
	select {
	case s.chReadEvent <- struct{}{}:
	default:
	}
}

// mark this stream has been closed in protocol
func (s *Stream) fin() {
	s.finEventOnce.Do(func() {
		close(s.chFinEvent)
	})
}
<|endoftext|>


<commit_before>package gofakeit

import (
	rand "math/rand"
	"reflect"
	"strconv"
	"strings"
)

// Struct fills in exported fields of a struct with random data
// based on the value of `fake` tag of exported fields.
// Use `fake:"skip"` to explicitly skip an element.
// All built-in types are supported, with templating support
// for string types.
func Struct(v interface{}) { structFunc(globalFaker.Rand, v) }

// Struct fills in exported fields of a struct with random data
// based on the value of `fake` tag of exported fields.
// Use `fake:"skip"` to explicitly skip an element.
// All built-in types are supported, with templating support
// for string types.
func (f *Faker) Struct(v interface{}) { structFunc(f.Rand, v) }

func structFunc(ra *rand.Rand, v interface{}) {
	r(ra, reflect.TypeOf(v), reflect.ValueOf(v), "", 0)
}

func r(ra *rand.Rand, t reflect.Type, v reflect.Value, function string, size int) {
	switch t.Kind() {
	case reflect.Ptr:
		rPointer(ra, t, v, function)
	case reflect.Struct:
		rStruct(ra, t, v)
	case reflect.String:
		rString(ra, t, v, function)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		rUint(ra, t, v, function)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		rInt(ra, t, v, function)
	case reflect.Float32, reflect.Float64:
		rFloat(ra, t, v, function)
	case reflect.Bool:
		rBool(ra, t, v, function)
	case reflect.Array, reflect.Slice:
		rSlice(ra, t, v, function, size)
	}
}

func rStruct(ra *rand.Rand, t reflect.Type, v reflect.Value) {
	n := t.NumField()
	for i := 0; i < n; i++ {
		elementT := t.Field(i)
		elementV := v.Field(i)
		t, ok := elementT.Tag.Lookup("fake")
		if ok && t == "skip" {
			// Do nothing, skip it
		} else if elementV.CanSet() || elementT.Anonymous {
			// Check if fakesize is set
			size := -1 // Set to -1 to indicate fakesize was not set
			fs, ok := elementT.Tag.Lookup("fakesize")
			if ok {
				var err error
				size, err = strconv.Atoi(fs)
				if err != nil {
					size = number(ra, 1, 10)
				}
			}
			r(ra, elementT.Type, elementV, t, size)
		}
	}
}

func rPointer(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	elemT := t.Elem()
	if v.IsNil() {
		nv := reflect.New(elemT)
		r(ra, elemT, nv.Elem(), function, 0)
		v.Set(nv)
	} else {
		r(ra, elemT, v.Elem(), function, 0)
	}
}

func rSlice(ra *rand.Rand, t reflect.Type, v reflect.Value, function string, size int) {
	// If you cant even set it dont even try
	if !v.CanSet() {
		return
	}

	// Grab original size to use if needed for sub arrays
	ogSize := size

	// If the value has a cap and is less than the size
	// use that instead of the requested size
	elemCap := v.Cap()
	if elemCap == 0 && size == -1 {
		size = number(ra, 1, 10)
	} else if elemCap != 0 && (size == -1 || elemCap < size) {
		size = elemCap
	}

	// Get the element type
	elemT := t.Elem()

	// If values are already set fill them up, otherwise append
	if v.Len() != 0 {
		// Loop through the elements length and set based upon the index
		for i := 0; i < size; i++ {
			nv := reflect.New(elemT)
			r(ra, elemT, nv.Elem(), function, ogSize)
			v.Index(i).Set(reflect.Indirect(nv))
		}
	} else {
		// Loop through the size and append and set
		for i := 0; i < size; i++ {
			nv := reflect.New(elemT)
			r(ra, elemT, nv.Elem(), function, ogSize)
			v.Set(reflect.Append(reflect.Indirect(v), reflect.Indirect(nv)))
		}
	}
}

func rString(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		v.SetString(generate(ra, function))
	} else {
		v.SetString(generate(ra, strings.Repeat("?", number(ra, 4, 10))))
	}
}

func rInt(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		i, err := strconv.ParseInt(generate(ra, function), 10, 64)
		if err == nil {
			v.SetInt(i)
			return
		}
	}

	// If no function or error converting to int, set with random value
	switch t.Kind() {
	case reflect.Int:
		v.SetInt(int64Func(ra))
	case reflect.Int8:
		v.SetInt(int64(int8Func(ra)))
	case reflect.Int16:
		v.SetInt(int64(int16Func(ra)))
	case reflect.Int32:
		v.SetInt(int64(int32Func(ra)))
	case reflect.Int64:
		v.SetInt(int64Func(ra))
	}
}

func rUint(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		u, err := strconv.ParseUint(generate(ra, function), 10, 64)
		if err == nil {
			v.SetUint(u)
			return
		}
	}

	// If no function or error converting to uint, set with random value
	switch t.Kind() {
	case reflect.Uint:
		v.SetUint(uint64Func(ra))
	case reflect.Uint8:
		v.SetUint(uint64(uint8Func(ra)))
	case reflect.Uint16:
		v.SetUint(uint64(uint16Func(ra)))
	case reflect.Uint32:
		v.SetUint(uint64(uint32Func(ra)))
	case reflect.Uint64:
		v.SetUint(uint64Func(ra))
	}
}

func rFloat(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		f, err := strconv.ParseFloat(generate(ra, function), 64)
		if err == nil {
			v.SetFloat(f)
			return
		}
	}

	// If no function or error converting to float, set with random value
	switch t.Kind() {
	case reflect.Float64:
		v.SetFloat(float64Func(ra))
	case reflect.Float32:
		v.SetFloat(float64(float32Func(ra)))
	}
}

func rBool(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		b, err := strconv.ParseBool(generate(ra, function))
		if err == nil {
			v.SetBool(b)
			return
		}
	}

	// If no function or error converting to boolean, set with random value
	v.SetBool(boolFunc(ra))
}
<commit_msg>struct - made it so you can pass in a size for rPointer to pass along<commit_after>package gofakeit

import (
	rand "math/rand"
	"reflect"
	"strconv"
	"strings"
)

// Struct fills in exported fields of a struct with random data
// based on the value of `fake` tag of exported fields.
// Use `fake:"skip"` to explicitly skip an element.
// All built-in types are supported, with templating support
// for string types.
func Struct(v interface{}) { structFunc(globalFaker.Rand, v) }

// Struct fills in exported fields of a struct with random data
// based on the value of `fake` tag of exported fields.
// Use `fake:"skip"` to explicitly skip an element.
// All built-in types are supported, with templating support
// for string types.
func (f *Faker) Struct(v interface{}) { structFunc(f.Rand, v) }

func structFunc(ra *rand.Rand, v interface{}) {
	r(ra, reflect.TypeOf(v), reflect.ValueOf(v), "", 0)
}

func r(ra *rand.Rand, t reflect.Type, v reflect.Value, function string, size int) {
	switch t.Kind() {
	case reflect.Ptr:
		rPointer(ra, t, v, function, size)
	case reflect.Struct:
		rStruct(ra, t, v)
	case reflect.String:
		rString(ra, t, v, function)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		rUint(ra, t, v, function)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		rInt(ra, t, v, function)
	case reflect.Float32, reflect.Float64:
		rFloat(ra, t, v, function)
	case reflect.Bool:
		rBool(ra, t, v, function)
	case reflect.Array, reflect.Slice:
		rSlice(ra, t, v, function, size)
	}
}

func rStruct(ra *rand.Rand, t reflect.Type, v reflect.Value) {
	n := t.NumField()
	for i := 0; i < n; i++ {
		elementT := t.Field(i)
		elementV := v.Field(i)
		t, ok := elementT.Tag.Lookup("fake")
		if ok && t == "skip" {
			// Do nothing, skip it
		} else if elementV.CanSet() || elementT.Anonymous {
			// Check if fakesize is set
			size := -1 // Set to -1 to indicate fakesize was not set
			fs, ok := elementT.Tag.Lookup("fakesize")
			if ok {
				var err error
				size, err = strconv.Atoi(fs)
				if err != nil {
					size = number(ra, 1, 10)
				}
			}
			r(ra, elementT.Type, elementV, t, size)
		}
	}
}

func rPointer(ra *rand.Rand, t reflect.Type, v reflect.Value, function string, size int) {
	elemT := t.Elem()
	if v.IsNil() {
		nv := reflect.New(elemT)
		r(ra, elemT, nv.Elem(), function, size)
		v.Set(nv)
	} else {
		r(ra, elemT, v.Elem(), function, size)
	}
}

func rSlice(ra *rand.Rand, t reflect.Type, v reflect.Value, function string, size int) {
	// If you cant even set it dont even try
	if !v.CanSet() {
		return
	}

	// Grab original size to use if needed for sub arrays
	ogSize := size

	// If the value has a cap and is less than the size
	// use that instead of the requested size
	elemCap := v.Cap()
	if elemCap == 0 && size == -1 {
		size = number(ra, 1, 10)
	} else if elemCap != 0 && (size == -1 || elemCap < size) {
		size = elemCap
	}

	// Get the element type
	elemT := t.Elem()

	// If values are already set fill them up, otherwise append
	if v.Len() != 0 {
		// Loop through the elements length and set based upon the index
		for i := 0; i < size; i++ {
			nv := reflect.New(elemT)
			r(ra, elemT, nv.Elem(), function, ogSize)
			v.Index(i).Set(reflect.Indirect(nv))
		}
	} else {
		// Loop through the size and append and set
		for i := 0; i < size; i++ {
			nv := reflect.New(elemT)
			r(ra, elemT, nv.Elem(), function, ogSize)
			v.Set(reflect.Append(reflect.Indirect(v), reflect.Indirect(nv)))
		}
	}
}

func rString(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		v.SetString(generate(ra, function))
	} else {
		v.SetString(generate(ra, strings.Repeat("?", number(ra, 4, 10))))
	}
}

func rInt(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		i, err := strconv.ParseInt(generate(ra, function), 10, 64)
		if err == nil {
			v.SetInt(i)
			return
		}
	}

	// If no function or error converting to int, set with random value
	switch t.Kind() {
	case reflect.Int:
		v.SetInt(int64Func(ra))
	case reflect.Int8:
		v.SetInt(int64(int8Func(ra)))
	case reflect.Int16:
		v.SetInt(int64(int16Func(ra)))
	case reflect.Int32:
		v.SetInt(int64(int32Func(ra)))
	case reflect.Int64:
		v.SetInt(int64Func(ra))
	}
}

func rUint(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		u, err := strconv.ParseUint(generate(ra, function), 10, 64)
		if err == nil {
			v.SetUint(u)
			return
		}
	}

	// If no function or error converting to uint, set with random value
	switch t.Kind() {
	case reflect.Uint:
		v.SetUint(uint64Func(ra))
	case reflect.Uint8:
		v.SetUint(uint64(uint8Func(ra)))
	case reflect.Uint16:
		v.SetUint(uint64(uint16Func(ra)))
	case reflect.Uint32:
		v.SetUint(uint64(uint32Func(ra)))
	case reflect.Uint64:
		v.SetUint(uint64Func(ra))
	}
}

func rFloat(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		f, err := strconv.ParseFloat(generate(ra, function), 64)
		if err == nil {
			v.SetFloat(f)
			return
		}
	}

	// If no function or error converting to float, set with random value
	switch t.Kind() {
	case reflect.Float64:
		v.SetFloat(float64Func(ra))
	case reflect.Float32:
		v.SetFloat(float64(float32Func(ra)))
	}
}

func rBool(ra *rand.Rand, t reflect.Type, v reflect.Value, function string) {
	if function != "" {
		b, err := strconv.ParseBool(generate(ra, function))
		if err == nil {
			v.SetBool(b)
			return
		}
	}

	// If no function or error converting to boolean, set with random value
	v.SetBool(boolFunc(ra))
}
<|endoftext|>


<commit_before>package svgxml

import (
    "encoding/xml"
    "fmt"
    "os"
    s "strings"
)

type PathDef struct {
    D       string `xml:"d,attr"`
    Id      string `xml:"id,attr"`
    Style   string `xml:"style,attr"`
}

type GroupDef struct {
    Path []PathDef  `xml:"path"`
    Id   string     `xml:"id,attr"`
}

type DefsDef struct {
    Id  string  `xml:"id,attr"`
}

type SVG struct {
    XMLName     xml.Name    `xml:"svg"`
    XMLNS       string      `xml:"xmlns,attr"`
    Width       string      `xml:"width,attr"`
    Height      string      `xml:"height,attr"`
    Id          string      `xml:"id,attr"`
    G           []GroupDef  `xml:"g"`
    Defs        DefsDef     `xml:"defs"`
    Version     string      `xml:"version,attr"`
}

func XML2SVG(svg_xml []byte)(*SVG) {

    svg_obj := SVG{}
    err := xml.Unmarshal([]byte(svg_xml), &svg_obj)
    if err == nil {
        return &svg_obj
    } else {
        fmt.Fprintf(os.Stderr, "error: %v", err)
        return nil
    }
}

func SVG2XML(imgxml *SVG, multi_line bool)([]byte) {

    var xml_txt []byte
    var err     error

    if multi_line {
        xml_txt, err = xml.MarshalIndent(imgxml, "", "    ")
    } else {
        xml_txt, err = xml.Marshal(imgxml)
    }
    if err != nil {
        fmt.Fprintf(os.Stderr, "error: %v", err)
        return nil
    }

    svgtag_end := s.Index(string(xml_txt), "<svg") + 4
    xmlout := []byte(
        xml.Header +
        string(xml_txt[:svgtag_end]) +
        ` xmlns:svg="http://www.w3.org/2000/svg" xml:space="preserve"` +
        s.Replace(
            s.Replace(
                string(xml_txt[svgtag_end:]),
                "></path", " /",
                -1),
            "></defs", " /",
            -1))
    return xmlout
}

func FindPathById(mapsvg_obj *SVG, id string)(*PathDef) {
    for g_ind, group := range mapsvg_obj.G {
        for p_ind, path := range group.Path {
            if path.Id == id {
                return &(mapsvg_obj.G[g_ind].Path[p_ind])
            }
        }
    }
    return nil
}
<commit_msg>picked up some missing attributes for groups and added path defs to top-level SVG tag<commit_after>package svgxml

import (
    "encoding/xml"
    "fmt"
    "os"
    s "strings"
)

type PathDef struct {
    D       string  `xml:"d,attr"`
    Id      string  `xml:"id,attr"`
    Style   string  `xml:"style,attr"`
}

type GroupDef struct {
    Path    []PathDef   `xml:"path"`
    Id      string      `xml:"id,attr"`
    Xform   string      `xml:"transform,attr"`
    Style   string      `xml:"style,attr"`
}

type DefsDef struct {
    Id  string  `xml:"id,attr"`
}

type SVG struct {
    XMLName xml.Name    `xml:"svg"`
    XMLNS   string      `xml:"xmlns,attr"`
    Width   string      `xml:"width,attr"`
    Height  string      `xml:"height,attr"`
    Id      string      `xml:"id,attr"`
    G       []GroupDef  `xml:"g"`
    Path    []PathDef   `xml:"path"`
    Defs    DefsDef     `xml:"defs"`
    Version string      `xml:"version,attr"`
}

func XML2SVG(svg_xml []byte)(*SVG) {

    svg_obj := SVG{}
    err := xml.Unmarshal([]byte(svg_xml), &svg_obj)
    if err == nil {
        return &svg_obj
    } else {
        fmt.Fprintf(os.Stderr, "error: %v", err)
        return nil
    }
}

func SVG2XML(imgxml *SVG, multi_line bool)([]byte) {

    var xml_txt []byte
    var err     error

    if multi_line {
        xml_txt, err = xml.MarshalIndent(imgxml, "", "    ")
    } else {
        xml_txt, err = xml.Marshal(imgxml)
    }
    if err != nil {
        fmt.Fprintf(os.Stderr, "error: %v", err)
        return nil
    }

    svgtag_end := s.Index(string(xml_txt), "<svg") + 4
    xmlout := []byte(
        xml.Header +
        string(xml_txt[:svgtag_end]) +
        ` xmlns:svg="http://www.w3.org/2000/svg" xml:space="preserve"` +
        s.Replace(
            s.Replace(
                string(xml_txt[svgtag_end:]),
                "></path", " /",
                -1),
            "></defs", " /",
            -1))
    return xmlout
}

func FindPathById(mapsvg_obj *SVG, id string)(*PathDef) {
    for g_ind, group := range mapsvg_obj.G {
        for p_ind, path := range group.Path {
            if path.Id == id {
                return &(mapsvg_obj.G[g_ind].Path[p_ind])
            }
        }
    }
    return nil
}
<|endoftext|>


<commit_before>package tarpan

import (
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/soniah/gosnmp"
)

// Tarpan constants
const (
	ConcurrentProcesses = 5
)

// Exit code
const (
	ExitCodeOK = iota
	ExitCodeConfError
	ExitCodeParseFlagError
)

// SNMP constants
const (
	community = "public"
	port      = 161
	timeout   = 2
	retry     = 3
	//MaximumPDUBytes = 512
)

// JSON structure definition
type DataSet struct {
	Default Defaults `json:"defaults"`
	Targets []Target `json:"targets"`
}

type Defaults struct {
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
}

type Target struct {
	Name      string `json:"name"`
	Address   string `json:"address"`
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
	OIDs      []OID  `json:"oids"`
}

type OID struct {
	OID         string `json:"oid"`
	Description string `json:"description"`
}

// Tarpan result structures
type TarpanResult struct {
	Name      string    `json:"name"`
	Address   string    `json:"address"`
	Port      uint16    `json:"port"`
	Version   string    `json:"version"`
	Community string    `json:"community"`
	VarBinds  []VarBind `json:"varbinds"`
}

type VarBind struct {
	Description string         `json:"description"`
	OID         string         `json:"oid"`
	Type        gosnmp.Asn1BER `json:"type"`
	Value       interface{}    `json:"value"`
	Time        int64          `json:"time"`
}

type Tarpan interface {
	makeRequestBody()
	SetManager()
	SetParams()
	Get(params map[string]string, oids []string) []gosnmp.SnmpPDU
	Run()
}

type TarpanManager struct {
	snmp *gosnmp.GoSNMP
}

type SnmpResult struct {
	target  Target
	results []gosnmp.SnmpPDU
	time    int64
}

type Channels struct {
	semaphoe chan int
	results  chan SnmpResult
}

func (t *TarpanManager) Get(params map[string]string,
	oids []string) []gosnmp.SnmpPDU {
	t.SetParams(params)
	connection_err := t.snmp.Connect()
	if connection_err != nil {
		log.Fatalf("Connection error: %")
	}
	result, request_err := t.snmp.Get(oids)
	if request_err != nil {
		log.Println("Get() error: %v", request_err)
	}
	t.snmp.Conn.Close()

	return result.Variables
}

func (t *TarpanManager) SetManager(manager *gosnmp.GoSNMP) {
	t.snmp = manager
	return
}

func (t *TarpanManager) SetParams(p map[string]string) {
	if value, ok := p["target"]; ok {
		t.snmp.Target = value
	}
	if value, ok := p["port"]; ok {
		port, _ := strconv.ParseUint(value, 10, 16)
		t.snmp.Port = uint16(port)
	} else {
		t.snmp.Port = uint16(port)
	}
	if value, ok := p["community"]; ok {
		t.snmp.Community = value
	}
	if value, ok := p["version"]; ok {
		if value == "2c" {
			t.snmp.Version = gosnmp.Version2c
		}
	}
	if value, ok := p["timeout"]; ok {
		sec, _ := strconv.Atoi(value)
		t.snmp.Timeout = time.Duration(sec) * time.Second
	} else {
		t.snmp.Timeout = time.Duration(timeout) * time.Second
	}
	if value, ok := p["retries"]; ok {
		retries, _ := strconv.Atoi(value)
		t.snmp.Retries = retries
	} else {
		t.snmp.Retries = retry

	}

	return
}

func makeRequestBody(ds *DataSet, idx int) (map[string]string, error) {
	address := ""
	err := validateIP(ds.Targets[idx].Address)
	if err == nil {
		address = ds.Targets[idx].Address
	} else {
		return map[string]string{}, err
	}
	community := ds.Targets[idx].Community
	if community == "" {
		community = ds.Default.Community
	}
	version := ds.Targets[idx].Version
	if version == "" {
		version = ds.Default.Version
	}
	request_body := map[string]string{
		"target":    address,
		"community": community,
		"version":   version,
	}

	return request_body, err
}

func makeChannel(result_buffer_length int) *Channels {
	channel := &Channels{
		semaphoe: make(chan int, ConcurrentProcesses),
		results:  make(chan SnmpResult, result_buffer_length),
	}

	return channel
}

func makeTarpanResult(c *Channels) []*TarpanResult {
	var tarpanResults []*TarpanResult
	var varbinds []VarBind
	var result SnmpResult
	length := len(c.results)
	for i := 0; i < length; i++ {
		varbinds = []VarBind{}
		result = <-c.results
		for _, val := range result.results {
			for _, o := range result.target.OIDs {
				n := strings.TrimLeft(val.Name, ".")
				if o.OID == n {
					v := VarBind{
						Description: o.Description,
						OID:         val.Name,
						Type:        val.Type,
						Value:       val.Value,
						Time:        result.time,
					}
					varbinds = append(varbinds, v)
					break
				}
			}
		}
		log.Println(result.target)
		tr := &TarpanResult{
			Name:      result.target.Name,
			Address:   result.target.Address,
			Port:      result.target.Port,
			Version:   result.target.Version,
			Community: result.target.Community,
			VarBinds:  varbinds,
		}
		tarpanResults = append(tarpanResults, tr)
	}

	return tarpanResults
}

func Collect(dataset *DataSet) []*TarpanResult {
	var waitGroup sync.WaitGroup
	var tm TarpanManager
	var channels *Channels
	var oids []string

	channels = makeChannel(len(dataset.Targets))
	for index := range dataset.Targets {
		tm = TarpanManager{}
		tm.SetManager(new(gosnmp.GoSNMP))

		// TODO: split oid slice according to PDU size
		oids = []string{}
		for oid_idx := range dataset.Targets[index].OIDs {
			oids = append(oids, dataset.Targets[index].OIDs[oid_idx].OID)
		}

		waitGroup.Add(1)
		go func(ds *DataSet, idx int, o []string, c *Channels) {
			c.semaphoe <- 0
			req_body, req_body_err := makeRequestBody(ds, idx)
			if req_body_err != nil {
				log.Print(req_body_err)
				<-c.semaphoe
				return
			}
			t := time.Now()
			results := tm.Get(req_body, o)
			ret := SnmpResult{
				target:  ds.Targets[idx],
				results: results,
				time:    t.Unix(),
			}
			c.results <- ret
			<-c.semaphoe
			waitGroup.Done()
		}(dataset, index, oids, channels)
	}
	waitGroup.Wait()
	tr := makeTarpanResult(channels)

	return tr
}

func Run(target string) (int, error) {
	var err error

	// Load targets
	dataset, loadErr := loadConfig(target)
	if loadErr != nil {
		return ExitCodeConfError, loadErr
	}

	// collect data
	tr := Collect(dataset)

	// output
	Write("aa", tr)

	return ExitCodeOK, err
}
<commit_msg>delete debug print<commit_after>package tarpan

import (
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/soniah/gosnmp"
)

// Tarpan constants
const (
	ConcurrentProcesses = 5
)

// Exit code
const (
	ExitCodeOK = iota
	ExitCodeConfError
	ExitCodeParseFlagError
)

// SNMP constants
const (
	community = "public"
	port      = 161
	timeout   = 2
	retry     = 3
	//MaximumPDUBytes = 512
)

// JSON structure definition
type DataSet struct {
	Default Defaults `json:"defaults"`
	Targets []Target `json:"targets"`
}

type Defaults struct {
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
}

type Target struct {
	Name      string `json:"name"`
	Address   string `json:"address"`
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
	OIDs      []OID  `json:"oids"`
}

type OID struct {
	OID         string `json:"oid"`
	Description string `json:"description"`
}

// Tarpan result structures
type TarpanResult struct {
	Name      string    `json:"name"`
	Address   string    `json:"address"`
	Port      uint16    `json:"port"`
	Version   string    `json:"version"`
	Community string    `json:"community"`
	VarBinds  []VarBind `json:"varbinds"`
}

type VarBind struct {
	Description string         `json:"description"`
	OID         string         `json:"oid"`
	Type        gosnmp.Asn1BER `json:"type"`
	Value       interface{}    `json:"value"`
	Time        int64          `json:"time"`
}

type Tarpan interface {
	makeRequestBody()
	SetManager()
	SetParams()
	Get(params map[string]string, oids []string) []gosnmp.SnmpPDU
	Run()
}

type TarpanManager struct {
	snmp *gosnmp.GoSNMP
}

type SnmpResult struct {
	target  Target
	results []gosnmp.SnmpPDU
	time    int64
}

type Channels struct {
	semaphoe chan int
	results  chan SnmpResult
}

func (t *TarpanManager) Get(params map[string]string,
	oids []string) []gosnmp.SnmpPDU {
	t.SetParams(params)
	connection_err := t.snmp.Connect()
	if connection_err != nil {
		log.Fatalf("Connection error: %")
	}
	result, request_err := t.snmp.Get(oids)
	if request_err != nil {
		log.Println("Get() error: %v", request_err)
	}
	t.snmp.Conn.Close()

	return result.Variables
}

func (t *TarpanManager) SetManager(manager *gosnmp.GoSNMP) {
	t.snmp = manager
	return
}

func (t *TarpanManager) SetParams(p map[string]string) {
	if value, ok := p["target"]; ok {
		t.snmp.Target = value
	}
	if value, ok := p["port"]; ok {
		port, _ := strconv.ParseUint(value, 10, 16)
		t.snmp.Port = uint16(port)
	} else {
		t.snmp.Port = uint16(port)
	}
	if value, ok := p["community"]; ok {
		t.snmp.Community = value
	}
	if value, ok := p["version"]; ok {
		if value == "2c" {
			t.snmp.Version = gosnmp.Version2c
		}
	}
	if value, ok := p["timeout"]; ok {
		sec, _ := strconv.Atoi(value)
		t.snmp.Timeout = time.Duration(sec) * time.Second
	} else {
		t.snmp.Timeout = time.Duration(timeout) * time.Second
	}
	if value, ok := p["retries"]; ok {
		retries, _ := strconv.Atoi(value)
		t.snmp.Retries = retries
	} else {
		t.snmp.Retries = retry

	}

	return
}

func makeRequestBody(ds *DataSet, idx int) (map[string]string, error) {
	address := ""
	err := validateIP(ds.Targets[idx].Address)
	if err == nil {
		address = ds.Targets[idx].Address
	} else {
		return map[string]string{}, err
	}
	community := ds.Targets[idx].Community
	if community == "" {
		community = ds.Default.Community
	}
	version := ds.Targets[idx].Version
	if version == "" {
		version = ds.Default.Version
	}
	request_body := map[string]string{
		"target":    address,
		"community": community,
		"version":   version,
	}

	return request_body, err
}

func makeChannel(result_buffer_length int) *Channels {
	channel := &Channels{
		semaphoe: make(chan int, ConcurrentProcesses),
		results:  make(chan SnmpResult, result_buffer_length),
	}

	return channel
}

func makeTarpanResult(c *Channels) []*TarpanResult {
	var tarpanResults []*TarpanResult
	var varbinds []VarBind
	var result SnmpResult
	length := len(c.results)
	for i := 0; i < length; i++ {
		varbinds = []VarBind{}
		result = <-c.results
		for _, val := range result.results {
			for _, o := range result.target.OIDs {
				n := strings.TrimLeft(val.Name, ".")
				if o.OID == n {
					v := VarBind{
						Description: o.Description,
						OID:         val.Name,
						Type:        val.Type,
						Value:       val.Value,
						Time:        result.time,
					}
					varbinds = append(varbinds, v)
					break
				}
			}
		}
		tr := &TarpanResult{
			Name:      result.target.Name,
			Address:   result.target.Address,
			Port:      result.target.Port,
			Version:   result.target.Version,
			Community: result.target.Community,
			VarBinds:  varbinds,
		}
		tarpanResults = append(tarpanResults, tr)
	}

	return tarpanResults
}

func Collect(dataset *DataSet) []*TarpanResult {
	var waitGroup sync.WaitGroup
	var tm TarpanManager
	var channels *Channels
	var oids []string

	channels = makeChannel(len(dataset.Targets))
	for index := range dataset.Targets {
		tm = TarpanManager{}
		tm.SetManager(new(gosnmp.GoSNMP))

		// TODO: split oid slice according to PDU size
		oids = []string{}
		for oid_idx := range dataset.Targets[index].OIDs {
			oids = append(oids, dataset.Targets[index].OIDs[oid_idx].OID)
		}

		waitGroup.Add(1)
		go func(ds *DataSet, idx int, o []string, c *Channels) {
			c.semaphoe <- 0
			req_body, req_body_err := makeRequestBody(ds, idx)
			if req_body_err != nil {
				log.Print(req_body_err)
				<-c.semaphoe
				return
			}
			t := time.Now()
			results := tm.Get(req_body, o)
			ret := SnmpResult{
				target:  ds.Targets[idx],
				results: results,
				time:    t.Unix(),
			}
			c.results <- ret
			<-c.semaphoe
			waitGroup.Done()
		}(dataset, index, oids, channels)
	}
	waitGroup.Wait()
	tr := makeTarpanResult(channels)

	return tr
}

func Run(target string) (int, error) {
	var err error

	// Load targets
	dataset, loadErr := loadConfig(target)
	if loadErr != nil {
		return ExitCodeConfError, loadErr
	}

	// collect data
	tr := Collect(dataset)

	// output
	Write("aa", tr)

	return ExitCodeOK, err
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/thoas/stats"
	"github.com/unrolled/render"
)

func createContextForTestSetup() appContext {
	testVersion := "0.0.0"
	db := createMockDatabase()
	ctx := appContext{
		metrics: stats.New(),
		render:  render.New(),
		version: testVersion,
		env:     "LOCAL",
		port:    "3001",
		db:      db,
	}
	return ctx
}

func createMockDatabase() *MockDB {
	list := make(map[int]User)
	dt, _ := time.Parse(time.RFC3339, "1985-12-31T00:00:00Z")
	list[0] = User{0, "John", "Doe", dt, "London"}
	dt, _ = time.Parse(time.RFC3339, "1992-01-01T00:00:00Z")
	list[1] = User{1, "Jane", "Doe", dt, "Milton Keynes"}
	return &MockDB{list, 1}
}

func createTimestampForTest() time.Time {
	timestamp, _ := time.Parse(time.RFC3339, "2016-03-15T14:44:27Z")
	return timestamp
}
<commit_msg>removing unused timestamp generator<commit_after>package main

import (
	"time"

	"github.com/thoas/stats"
	"github.com/unrolled/render"
)

func createContextForTestSetup() appContext {
	testVersion := "0.0.0"
	db := createMockDatabase()
	ctx := appContext{
		metrics: stats.New(),
		render:  render.New(),
		version: testVersion,
		env:     "LOCAL",
		port:    "3001",
		db:      db,
	}
	return ctx
}

func createMockDatabase() *MockDB {
	list := make(map[int]User)
	dt, _ := time.Parse(time.RFC3339, "1985-12-31T00:00:00Z")
	list[0] = User{0, "John", "Doe", dt, "London"}
	dt, _ = time.Parse(time.RFC3339, "1992-01-01T00:00:00Z")
	list[1] = User{1, "Jane", "Doe", dt, "Milton Keynes"}
	return &MockDB{list, 1}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"strings"
)

const (
	// Default stylesheet filename (GCSS)
	defaultStyleFilename = "style.gcss"

	// highlight.js style for custom CSS
	defaultCustomCodeStyle = "github"
)

var (
	// The available built-in CSS themes. Corresponds with the font themes below.
	builtinThemes = map[string]string{
		"gray":   "@import url(//fonts.googleapis.com/css?family=Lato:300); body { background-color: #e7eaed; color: #0b0b0b; font-family: 'Lato', sans-serif; font-weight: 300;  margin: 4.5em; font-size: 1em; } a { color: #401010; font-family: courier; } a:hover { color: #801010; } a:active { color: yellow; } h1 { color: #101010; } img { max-width: 100%; } table { font-family: Helvetica, sans-serif; border-collapse: collapse; width: 100%; font-weight: bold; } table td, table th { border: 1px solid #ccc; padding: 0.45em 0.55em 0.45em 0.55em; } table tr:nth-child(even){background-color: #efefef; } table tr:nth-child(odd) { background-color: #fff; } table tr:hover { background-color: #e0f0ff; color: black; } table th { padding-top: 12px; padding-bottom: 12px; text-align: left; background-color: #666; color: white; }",
		"dark":   "@import url(//fonts.googleapis.com/css?family=Lato:400); body { background-color: #101010; color: #f0f0f0; font-family: 'Lato', sans-serif; font-weight: 400;  margin: 4.5em; font-size: 1em; } a { color: #c0a0a0; font-family: courier; } a:hover { color: #f0a0a0; } a:active { color: yellow; } h1 { color: #f0f0f0; } img { max-width: 100%; } table { font-family: Helvetica, sans-serif; border-collapse: collapse; width: 100%; font-weight: bold; } table td, table th { border: 1px solid #999; padding: 0.45em 0.55em 0.45em 0.55em; } table tr:nth-child(even){background-color: #222; } table tr:nth-child(odd) { background-color: #333; } table tr:hover { background-color: #333; color: #ff8060; } table th { padding-top: 12px; padding-bottom: 12px; text-align: left; background-color: #d7d7d7; color: #292929; }",
		"redbox": "@import url(//fonts.googleapis.com/css?family=Monoton|Monofett);html{background-color:#222;}body{color:#111;background-color:#999;font-family:Impact,'Arial Black',sans-serif;font-size:1.7em;margin:2.7em;padding:0 5em 1em 2em;border-radius:50px;border:solid 10px #a00;box-shadow:10px 10px 16px black, 6px 6px 8px #222 inset;}h2{font-family:Monoton,cursive;color:black;}ul{margin-left:1em;}a{text-decoration:none;color:#b00;}a:hover{color:#dc0;}code{font-family:Monofett,cursive;} img { max-width: 100%; }",
	}

	// Built in themes corresponding to highlight.js styles
	// See https://github.com/isagalaev/highlight.js/tree/master/src/styles for more styles
	defaultCodeStyles = map[string]string{"gray": "color-brewer", "dark": "ocean", "redbox": "railscasts"}

	// A selection of allowed keywords for the HTML meta tag
	metaKeywords = []string{"application-name", "author", "description", "generator", "keywords", "robots", "language", "googlebot", "Slurp", "bingbot", "geo.position", "geo.placename", "geo.region", "ICBM", "viewport"}
)

// Easy way to output a HTML page
func easyPage(title, body, theme string) string {
	return fmt.Sprintf("<!doctype html><title>%s</title>%s
%s
%s", title, "", builtinThemes[theme], title, body)
}

// Easy way to build links to directories
func easyLink(text, url string, isDirectory bool) string {
	// Add a final slash, if needed
	if isDirectory {
		text += "/"
		url += "/"
	}
	return "" + text + "
"
}

// HTML to be added for enabling highlighting.
func highlightHTML(codeStyle string) string {
	return `<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.6.0/styles/` + codeStyle + `.min.css"><script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.6.0/highlight.min.js"></script><script>hljs.initHighlightingOnLoad();</script>`
}

// Add a link to a stylesheet in the given Amber code
func linkToStyle(amberdata *[]byte, url string) {
	// If the given url is not already mentioned and the data contains "body"
	if !bytes.Contains(*amberdata, []byte(url)) && bytes.Contains(*amberdata, []byte("html")) && bytes.Contains(*amberdata, []byte("body")) {
		// Extract one level of indendation
		whitespace := oneLevelOfIndentation(amberdata, "body")
		// Check if there already is a head section
		if bytes.Contains(*amberdata, []byte("head")) {
			// Add a link to the stylesheet
			*amberdata = bytes.Replace(*amberdata, []byte("head\n"), []byte("head\n"+whitespace+whitespace+`link[href="`+url+`"][rel="stylesheet"][type="text/css"]`+"\n"), 1)

		} else if bytes.Contains(*amberdata, []byte("body")) {

			// Add a link to the stylesheet
			*amberdata = bytes.Replace(*amberdata, []byte("html\n"), []byte("html\n"+whitespace+"head\n"+whitespace+whitespace+`link[href="`+url+`"][rel="stylesheet"][type="text/css"]`+"\n"), 1)
		}
	}
}

// Add a link to a stylesheet in the given HTML code
func linkToStyleHTML(htmldata *[]byte, url string) {
	// If the given url is not already mentioned and the data contains "body"
	if !bytes.Contains(*htmldata, []byte(url)) && bytes.Contains(*htmldata, []byte("body")) {
		if bytes.Contains(*htmldata, []byte("")) {
			*htmldata = bytes.Replace(*htmldata, []byte(""), []byte("  <link rel=\"stylesheet\" href=\""+url+"\">\n  "), 1)
		} else if bytes.Contains(*htmldata, []byte("")) {
			*htmldata = bytes.Replace(*htmldata, []byte(""), []byte("  \n  <link rel=\"stylesheet\" href=\""+url+"\">\n  \n  "), 1)
		}
	}
}

// Add meta tag names to the given map
func addMetaKeywords(keywords map[string]string) {
	for _, keyword := range metaKeywords {
		keywords[keyword] = ""
	}
}

// Insert doctype in HTML, if missing
// Does not check if the given data is HTML. Assumes it to be HTML.
func insertDoctype(htmldata []byte) []byte {
	// If there are more than two lines
	if bytes.Count(htmldata, []byte("\n")) > 2 {
		fields := bytes.SplitN(htmldata, []byte("\n"), 3)
		line1 := strings.ToLower(string(fields[0]))
		line2 := strings.ToLower(string(fields[1]))
		if strings.Contains(line1, "doctype") || strings.Contains(line2, "doctype") {
			return htmldata
		}
		// Doctype is missing from the first two lines, add it
		return []byte("<!doctype html>\n" + string(htmldata))
	}
	return htmldata
}
<commit_msg>Nicer style for  tags in rendered Markdown<commit_after>package main

import (
	"bytes"
	"fmt"
	"strings"
)

const (
	// Default stylesheet filename (GCSS)
	defaultStyleFilename = "style.gcss"

	// highlight.js style for custom CSS
	defaultCustomCodeStyle = "github"
)

var (
	// The available built-in CSS themes. Corresponds with the font themes below.
	builtinThemes = map[string]string{
		"gray":   "@import url(//fonts.googleapis.com/css?family=Lato:300); body { background-color: #e7eaed; color: #0b0b0b; font-family: 'Lato', sans-serif; font-weight: 300;  margin: 4.5em; font-size: 1em; } a { color: #401010; font-family: courier; } a:hover { color: #801010; } a:active { color: yellow; } h1 { color: #101010; } img { max-width: 100%; } table { font-family: Helvetica, sans-serif; border-collapse: collapse; width: 100%; font-weight: bold; } table td, table th { border: 1px solid #ccc; padding: 0.45em 0.55em 0.45em 0.55em; } table tr:nth-child(even){background-color: #efefef; } table tr:nth-child(odd) { background-color: #fff; } table tr:hover { background-color: #e0f0ff; color: black; } table th { padding-top: 12px; padding-bottom: 12px; text-align: left; background-color: #666; color: white; } code { background-color: rgba(50, 50, 50, 0.1); color: #202020; border-radius: 0.3em; padding: 0.1em 0.4em 0.1em 0.4em; font-size: 90%; word-wrap: break-word; font-family: Menlo, Monaco, Consolas, \"Courier New\", monospace; }",
		"dark":   "@import url(//fonts.googleapis.com/css?family=Lato:400); body { background-color: #101010; color: #f0f0f0; font-family: 'Lato', sans-serif; font-weight: 400;  margin: 4.5em; font-size: 1em; } a { color: #c0a0a0; font-family: courier; } a:hover { color: #f0a0a0; } a:active { color: yellow; } h1 { color: #f0f0f0; } img { max-width: 100%; } table { font-family: Helvetica, sans-serif; border-collapse: collapse; width: 100%; font-weight: bold; } table td, table th { border: 1px solid #999; padding: 0.45em 0.55em 0.45em 0.55em; } table tr:nth-child(even){background-color: #222; } table tr:nth-child(odd) { background-color: #333; } table tr:hover { background-color: #333; color: #ff8060; } table th { padding-top: 12px; padding-bottom: 12px; text-align: left; background-color: #d7d7d7; color: #292929; } code { background-color: rgba(255, 255, 255, 0.18); color: #e0e0e0; border-radius: 0.3em; padding: 0.1em 0.4em 0.1em 0.4em; font-size: 90%; word-wrap: break-word; font-family: Menlo, Monaco, Consolas, \"Courier New\", monospace; }",
		"redbox": "@import url(//fonts.googleapis.com/css?family=Monoton|Monofett);html{background-color:#222;}body{color:#111;background-color:#999;font-family:Impact,'Arial Black',sans-serif;font-size:1.7em;margin:2.7em;padding:0 5em 1em 2em;border-radius:50px;border:solid 10px #a00;box-shadow:10px 10px 16px black, 6px 6px 8px #222 inset;}h2{font-family:Monoton,cursive;color:black;}ul{margin-left:1em;}a{text-decoration:none;color:#b00;}a:hover{color:#dc0;}code{font-family:Monofett,cursive;} img { max-width: 100%; } code { background-color: rgba(255, 255, 255, 0.18); color: #e0e0e0; border-radius: 0.3em; padding: 0.1em 0.4em 0.1em 0.4em; font-size: 90%; word-wrap: break-word; font-family: Menlo, Monaco, Consolas, \"Courier New\", monospace; }",
	}

	// Built in themes corresponding to highlight.js styles
	// See https://github.com/isagalaev/highlight.js/tree/master/src/styles for more styles
	defaultCodeStyles = map[string]string{"gray": "color-brewer", "dark": "ocean", "redbox": "railscasts"}

	// A selection of allowed keywords for the HTML meta tag
	metaKeywords = []string{"application-name", "author", "description", "generator", "keywords", "robots", "language", "googlebot", "Slurp", "bingbot", "geo.position", "geo.placename", "geo.region", "ICBM", "viewport"}
)

// Easy way to output a HTML page
func easyPage(title, body, theme string) string {
	return fmt.Sprintf("<!doctype html><title>%s</title>%s
%s
%s", title, "", builtinThemes[theme], title, body)
}

// Easy way to build links to directories
func easyLink(text, url string, isDirectory bool) string {
	// Add a final slash, if needed
	if isDirectory {
		text += "/"
		url += "/"
	}
	return "" + text + "
"
}

// HTML to be added for enabling highlighting.
func highlightHTML(codeStyle string) string {
	return `<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.6.0/styles/` + codeStyle + `.min.css"><script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.6.0/highlight.min.js"></script><script>hljs.initHighlightingOnLoad();</script>`
}

// Add a link to a stylesheet in the given Amber code
func linkToStyle(amberdata *[]byte, url string) {
	// If the given url is not already mentioned and the data contains "body"
	if !bytes.Contains(*amberdata, []byte(url)) && bytes.Contains(*amberdata, []byte("html")) && bytes.Contains(*amberdata, []byte("body")) {
		// Extract one level of indendation
		whitespace := oneLevelOfIndentation(amberdata, "body")
		// Check if there already is a head section
		if bytes.Contains(*amberdata, []byte("head")) {
			// Add a link to the stylesheet
			*amberdata = bytes.Replace(*amberdata, []byte("head\n"), []byte("head\n"+whitespace+whitespace+`link[href="`+url+`"][rel="stylesheet"][type="text/css"]`+"\n"), 1)

		} else if bytes.Contains(*amberdata, []byte("body")) {

			// Add a link to the stylesheet
			*amberdata = bytes.Replace(*amberdata, []byte("html\n"), []byte("html\n"+whitespace+"head\n"+whitespace+whitespace+`link[href="`+url+`"][rel="stylesheet"][type="text/css"]`+"\n"), 1)
		}
	}
}

// Add a link to a stylesheet in the given HTML code
func linkToStyleHTML(htmldata *[]byte, url string) {
	// If the given url is not already mentioned and the data contains "body"
	if !bytes.Contains(*htmldata, []byte(url)) && bytes.Contains(*htmldata, []byte("body")) {
		if bytes.Contains(*htmldata, []byte("")) {
			*htmldata = bytes.Replace(*htmldata, []byte(""), []byte("  <link rel=\"stylesheet\" href=\""+url+"\">\n  "), 1)
		} else if bytes.Contains(*htmldata, []byte("")) {
			*htmldata = bytes.Replace(*htmldata, []byte(""), []byte("  \n  <link rel=\"stylesheet\" href=\""+url+"\">\n  \n  "), 1)
		}
	}
}

// Add meta tag names to the given map
func addMetaKeywords(keywords map[string]string) {
	for _, keyword := range metaKeywords {
		keywords[keyword] = ""
	}
}

// Insert doctype in HTML, if missing
// Does not check if the given data is HTML. Assumes it to be HTML.
func insertDoctype(htmldata []byte) []byte {
	// If there are more than two lines
	if bytes.Count(htmldata, []byte("\n")) > 2 {
		fields := bytes.SplitN(htmldata, []byte("\n"), 3)
		line1 := strings.ToLower(string(fields[0]))
		line2 := strings.ToLower(string(fields[1]))
		if strings.Contains(line1, "doctype") || strings.Contains(line2, "doctype") {
			return htmldata
		}
		// Doctype is missing from the first two lines, add it
		return []byte("<!doctype html>\n" + string(htmldata))
	}
	return htmldata
}
<|endoftext|>


<commit_before>package backoff

import (
	"runtime"
	"sync"
	"time"
)

// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff.
//
// Ticks will continue to arrive when the previous operation is still running,
// so operations that take a while to fail could run in quick succession.
type Ticker struct {
	C        <-chan time.Time
	c        chan time.Time
	b        BackOffContext
	stop     chan struct{}
	stopOnce sync.Once
}

// NewTicker returns a new Ticker containing a channel that will send
// the time at times specified by the BackOff argument. Ticker is
// guaranteed to tick at least once.  The channel is closed when Stop
// method is called or BackOff stops. It is not safe to manipulate the
// provided backoff policy (notably calling NextBackOff or Reset)
// while the ticker is running.
func NewTicker(b BackOff) *Ticker {
	c := make(chan time.Time)
	t := &Ticker{
		C:    c,
		c:    c,
		b:    ensureContext(b),
		stop: make(chan struct{}),
	}
	t.b.Reset()
	go t.run()
	runtime.SetFinalizer(t, (*Ticker).Stop)
	return t
}

// Stop turns off a ticker. After Stop, no more ticks will be sent.
func (t *Ticker) Stop() {
	t.stopOnce.Do(func() { close(t.stop) })
}

func (t *Ticker) run() {
	c := t.c
	defer close(c)

	// Ticker is guaranteed to tick at least once.
	afterC := t.send(time.Now())

	for {
		if afterC == nil {
			return
		}

		select {
		case tick := <-afterC:
			afterC = t.send(tick)
		case <-t.stop:
			t.c = nil // Prevent future ticks from being sent to the channel.
			return
		case <-t.b.Context().Done():
			return
		}
	}
}

func (t *Ticker) send(tick time.Time) <-chan time.Time {
	select {
	case t.c <- tick:
	case <-t.stop:
		return nil
	}

	next := t.b.NextBackOff()
	if next == Stop {
		t.Stop()
		return nil
	}

	return time.After(next)
}
<commit_msg>Remove SetFinalizer call<commit_after>package backoff

import (
	"runtime"
	"sync"
	"time"
)

// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff.
//
// Ticks will continue to arrive when the previous operation is still running,
// so operations that take a while to fail could run in quick succession.
type Ticker struct {
	C        <-chan time.Time
	c        chan time.Time
	b        BackOffContext
	stop     chan struct{}
	stopOnce sync.Once
}

// NewTicker returns a new Ticker containing a channel that will send
// the time at times specified by the BackOff argument. Ticker is
// guaranteed to tick at least once.  The channel is closed when Stop
// method is called or BackOff stops. It is not safe to manipulate the
// provided backoff policy (notably calling NextBackOff or Reset)
// while the ticker is running.
func NewTicker(b BackOff) *Ticker {
	c := make(chan time.Time)
	t := &Ticker{
		C:    c,
		c:    c,
		b:    ensureContext(b),
		stop: make(chan struct{}),
	}
	t.b.Reset()
	go t.run()
	return t
}

// Stop turns off a ticker. After Stop, no more ticks will be sent.
func (t *Ticker) Stop() {
	t.stopOnce.Do(func() { close(t.stop) })
}

func (t *Ticker) run() {
	c := t.c
	defer close(c)

	// Ticker is guaranteed to tick at least once.
	afterC := t.send(time.Now())

	for {
		if afterC == nil {
			return
		}

		select {
		case tick := <-afterC:
			afterC = t.send(tick)
		case <-t.stop:
			t.c = nil // Prevent future ticks from being sent to the channel.
			return
		case <-t.b.Context().Done():
			return
		}
	}
}

func (t *Ticker) send(tick time.Time) <-chan time.Time {
	select {
	case t.c <- tick:
	case <-t.stop:
		return nil
	}

	next := t.b.NextBackOff()
	if next == Stop {
		t.Stop()
		return nil
	}

	return time.After(next)
}
<|endoftext|>


<commit_before>// Contains the external API to the whisper sub-protocol.

package xeth

import (
	"fmt"
	"time"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/whisper"
)

var qlogger = logger.NewLogger("XSHH")

// Whisper represents the API wrapper around the internal whisper implementation.
type Whisper struct {
	*whisper.Whisper
}

// NewWhisper wraps an internal whisper client into an external API version.
func NewWhisper(w *whisper.Whisper) *Whisper {
	return &Whisper{w}
}

// NewIdentity generates a new cryptographic identity for the client, and injects
// it into the known identities for message decryption.
func (self *Whisper) NewIdentity() string {
	identity := self.Whisper.NewIdentity()
	return common.ToHex(crypto.FromECDSAPub(&identity.PublicKey))
}

// HasIdentity checks if the the whisper node is configured with the private key
// of the specified public pair.
func (self *Whisper) HasIdentity(key string) bool {
	return self.Whisper.HasIdentity(crypto.ToECDSAPub(common.FromHex(key)))
}

// Post injects a message into the whisper network for distribution.
func (self *Whisper) Post(payload string, to, from string, topics []string, priority, ttl uint32) error {
	// Construct the whisper message and transmission options
	message := whisper.NewMessage(common.FromHex(payload))
	options := whisper.Options{
		To:     crypto.ToECDSAPub(common.FromHex(to)),
		TTL:    time.Duration(ttl) * time.Second,
		Topics: whisper.NewTopicsFromStrings(topics...),
	}
	if len(from) != 0 {
		if key := self.Whisper.GetIdentity(crypto.ToECDSAPub(common.FromHex(from))); key != nil {
			options.From = key
		} else {
			return fmt.Errorf("unknown identity to send from: %s", from)
		}
	}
	// Wrap and send the message
	pow := time.Duration(priority) * time.Millisecond
	envelope, err := message.Wrap(pow, options)
	if err != nil {
		return err
	}
	if err := self.Whisper.Send(envelope); err != nil {
		return err
	}
	return nil
}

// Watch installs a new message handler to run in case a matching packet arrives
// from the whisper network.
func (self *Whisper) Watch(to, from string, topics [][]string, fn func(WhisperMessage)) int {
	filter := whisper.Filter{
		To:     crypto.ToECDSAPub(common.FromHex(to)),
		From:   crypto.ToECDSAPub(common.FromHex(from)),
		Topics: whisper.NewFilterTopicsFromStrings(topics...),
	}
	filter.Fn = func(message *whisper.Message) {
		fn(NewWhisperMessage(message))
	}
	return self.Whisper.Watch(filter)
}

// Messages retrieves all the currently pooled messages matching a filter id.
func (self *Whisper) Messages(id int) []WhisperMessage {
	pool := self.Whisper.Messages(id)

	messages := make([]WhisperMessage, len(pool))
	for i, message := range pool {
		messages[i] = NewWhisperMessage(message)
	}
	return messages
}
<commit_msg>xeth: fix un-decoded whisper RPC topic string bug<commit_after>// Contains the external API to the whisper sub-protocol.

package xeth

import (
	"fmt"
	"time"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/whisper"
)

var qlogger = logger.NewLogger("XSHH")

// Whisper represents the API wrapper around the internal whisper implementation.
type Whisper struct {
	*whisper.Whisper
}

// NewWhisper wraps an internal whisper client into an external API version.
func NewWhisper(w *whisper.Whisper) *Whisper {
	return &Whisper{w}
}

// NewIdentity generates a new cryptographic identity for the client, and injects
// it into the known identities for message decryption.
func (self *Whisper) NewIdentity() string {
	identity := self.Whisper.NewIdentity()
	return common.ToHex(crypto.FromECDSAPub(&identity.PublicKey))
}

// HasIdentity checks if the the whisper node is configured with the private key
// of the specified public pair.
func (self *Whisper) HasIdentity(key string) bool {
	return self.Whisper.HasIdentity(crypto.ToECDSAPub(common.FromHex(key)))
}

// Post injects a message into the whisper network for distribution.
func (self *Whisper) Post(payload string, to, from string, topics []string, priority, ttl uint32) error {
	// Decode the topic strings
	topicsDecoded := make([][]byte, len(topics))
	for i, topic := range topics {
		topicsDecoded[i] = common.FromHex(topic)
	}
	// Construct the whisper message and transmission options
	message := whisper.NewMessage(common.FromHex(payload))
	options := whisper.Options{
		To:     crypto.ToECDSAPub(common.FromHex(to)),
		TTL:    time.Duration(ttl) * time.Second,
		Topics: whisper.NewTopics(topicsDecoded...),
	}
	if len(from) != 0 {
		if key := self.Whisper.GetIdentity(crypto.ToECDSAPub(common.FromHex(from))); key != nil {
			options.From = key
		} else {
			return fmt.Errorf("unknown identity to send from: %s", from)
		}
	}
	// Wrap and send the message
	pow := time.Duration(priority) * time.Millisecond
	envelope, err := message.Wrap(pow, options)
	if err != nil {
		return err
	}
	if err := self.Whisper.Send(envelope); err != nil {
		return err
	}
	return nil
}

// Watch installs a new message handler to run in case a matching packet arrives
// from the whisper network.
func (self *Whisper) Watch(to, from string, topics [][]string, fn func(WhisperMessage)) int {
	// Decode the topic strings
	topicsDecoded := make([][][]byte, len(topics))
	for i, condition := range topics {
		topicsDecoded[i] = make([][]byte, len(condition))
		for j, topic := range condition {
			topicsDecoded[i][j] = common.FromHex(topic)
		}
	}
	// Assemble and inject the filter into the whisper client
	filter := whisper.Filter{
		To:     crypto.ToECDSAPub(common.FromHex(to)),
		From:   crypto.ToECDSAPub(common.FromHex(from)),
		Topics: whisper.NewFilterTopics(topicsDecoded...),
	}
	filter.Fn = func(message *whisper.Message) {
		fn(NewWhisperMessage(message))
	}
	return self.Whisper.Watch(filter)
}

// Messages retrieves all the currently pooled messages matching a filter id.
func (self *Whisper) Messages(id int) []WhisperMessage {
	pool := self.Whisper.Messages(id)

	messages := make([]WhisperMessage, len(pool))
	for i, message := range pool {
		messages[i] = NewWhisperMessage(message)
	}
	return messages
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"testing"
)

func TestRedis(t *testing.T) {
	r, _ := NewRedis(":6379", 10)
	r.Set("test", []byte("yup"))
	b, err := r.Get("test")
	if err != nil {
		t.Fatal(err)
	}
	fmt.Println(string(b))
}
<commit_msg>added benchmark test for set on redis server<commit_after>package storage

import "testing"

// TestSetGet must have a redis server running on localhost:6379
func TestSetGet(t *testing.T) {
	expected := "test"
	r, _ := NewRedis(":6379", 10)
	r.Set("test", []byte(expected))
	b, err := r.Get("test")
	if err != nil {
		t.Fatal(err)
	}

	// test that b comes back with the correct value
	if string(b) != expected {
		t.Fatal("Expected: " + expected + " got: " + string(b))
	}
}

// BenchmarkSet benchmark the SET command to the redis server
func BenchmarkSetPoolsize10(b *testing.B) {
	setValue := []byte("set-test-value")
	r, _ := NewRedis(":6379", 100)
	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			r.Set("set", setValue)
		}
	})

}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/Preetam/onecontactlink/middleware"
	"github.com/VividCortex/siesta"
)

func servePostLogin(c siesta.Context, w http.ResponseWriter, r *http.Request) {
	requestData := c.Get(middleware.RequestDataKey).(*middleware.RequestData)
	captchaResult := extractCaptchaResult(c)
	switch captchaResult {
	case captchaResultOK:
		// nothing to do
	case captchaResultFail:
		requestData.StatusCode = http.StatusBadRequest
		templ.ExecuteTemplate(w, "in", map[string]string{
			"Error": "Invalid CAPTCHA.",
		})
		return
	case captchaResultError:
		requestData.StatusCode = http.StatusInternalServerError
		templ.ExecuteTemplate(w, "login", map[string]string{
			"Error": "Something went wrong. Please try again.",
		})
		return
	}
	params := &siesta.Params{}
	emailStr := params.String("email", "", "email")
	err := params.Parse(r.Form)
	if err != nil {
		requestData.StatusCode = http.StatusBadRequest
		templ.ExecuteTemplate(w, "login", map[string]string{
			"Error": "Invalid parameters.",
		})
		return
	}

	internalAPIClient.SendAuth(*emailStr)
	templ.ExecuteTemplate(w, "success", map[string]string{
		"Info": "We've sent a login link to '" + *emailStr +
			"' if it's associated with a valid account.",
	})
}
<commit_msg>fix post login<commit_after>package main

import (
	"net/http"

	"github.com/Preetam/onecontactlink/middleware"
	"github.com/VividCortex/siesta"
)

func servePostLogin(c siesta.Context, w http.ResponseWriter, r *http.Request) {
	requestData := c.Get(middleware.RequestDataKey).(*middleware.RequestData)
	captchaResult := extractCaptchaResult(c)
	switch captchaResult {
	case captchaResultOK:
		// nothing to do
	case captchaResultFail:
		requestData.StatusCode = http.StatusBadRequest
		templ.ExecuteTemplate(w, "invalid", map[string]string{
			"Error": "Invalid CAPTCHA.",
		})
		return
	case captchaResultError:
		requestData.StatusCode = http.StatusInternalServerError
		templ.ExecuteTemplate(w, "login", map[string]string{
			"Error": "Something went wrong. Please try again.",
		})
		return
	}
	params := &siesta.Params{}
	emailStr := params.String("email", "", "email")
	err := params.Parse(r.Form)
	if err != nil {
		requestData.StatusCode = http.StatusBadRequest
		templ.ExecuteTemplate(w, "login", map[string]string{
			"Error": "Invalid parameters.",
		})
		return
	}

	internalAPIClient.SendAuth(*emailStr)
	templ.ExecuteTemplate(w, "success", map[string]string{
		"Info": "We've sent a login link to '" + *emailStr +
			"' if it's associated with a valid account.",
	})
}
<|endoftext|>


<commit_before>package web

import (
	"bytes"
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"net/url"
	"sort"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/sql"
	"github.com/gorilla/mux"
	"github.com/retailnext/hllpp"
)

const (
	pauseTime    = 250 * time.Millisecond
	shortTimeout = 5 * time.Second
	longTimeout  = 1000 * time.Hour
)

type QueryResult struct {
	SQL                string
	Permalink          string
	TS                 int64
	TSCardinality      uint64
	Fields             []string
	FieldCardinalities []uint64
	Dims               []string
	DimCardinalities   []uint64
	Rows               []*ResultRow
	Stats              *common.QueryStats
}

type ResultRow struct {
	TS   int64
	Key  map[string]interface{}
	Vals []float64
}

type query struct {
	sqlString string
	parsed    *sql.Query
	immediate bool
	ce        cacheEntry
}

func (h *handler) runQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, longTimeout, false)
}

func (h *handler) asyncQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, shortTimeout, false)
}

func (h *handler) immediateQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, shortTimeout, true)
}

func (h *handler) cachedQuery(resp http.ResponseWriter, req *http.Request) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	log.Debug(req.URL)
	permalink := mux.Vars(req)["permalink"]
	ce, err := h.cache.getByPermalink(permalink)
	if ce == nil {
		http.NotFound(resp, req)
		return
	}
	h.respondWithCacheEntry(resp, req, ce, err, shortTimeout)
}

func (h *handler) sqlQuery(resp http.ResponseWriter, req *http.Request, timeout time.Duration, immediate bool) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	log.Debug(req.URL)
	sqlString, _ := url.QueryUnescape(req.URL.RawQuery)

	ce, err := h.query(req, sqlString, immediate)
	h.respondWithCacheEntry(resp, req, ce, err, timeout)
}

func (h *handler) respondWithCacheEntry(resp http.ResponseWriter, req *http.Request, ce cacheEntry, err error, timeout time.Duration) {
	limit := int(timeout / pauseTime)
	for i := 0; i < limit; i++ {
		if err != nil {
			log.Error(err)
			resp.WriteHeader(http.StatusInternalServerError)
			fmt.Fprint(resp, err.Error())
			return
		}
		switch ce.status() {
		case statusSuccess:
			h.respondSuccess(resp, req, ce)
			return
		case statusError:
			h.respondError(resp, req, ce)
			return
		case statusPending:
			// Pause a little bit and try again
			time.Sleep(pauseTime)
			ce, err = h.cache.getByPermalink(ce.permalink())
		}
	}
	// Let the client know that we're still working on it
	resp.WriteHeader(http.StatusAccepted)
	fmt.Fprintf(resp, "/cached/%v", ce.permalink())
}

func (h *handler) respondSuccess(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.Header().Set("Content-Type", "application/json")
	resp.Header().Set("Expires", "0")
	resp.Header().Set("Cache-control", "no-cache, no-store, must-revalidate")
	resp.Header().Set("Content-Encoding", "gzip")
	resp.WriteHeader(http.StatusOK)
	resp.Write(ce.data())
}

func (h *handler) respondError(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.WriteHeader(http.StatusInternalServerError)
	resp.Write(ce.error())
}

func (h *handler) query(req *http.Request, sqlString string, immediate bool) (ce cacheEntry, err error) {
	parsed, parseErr := sql.Parse(sqlString)
	if parseErr != nil {
		return nil, parseErr
	}

	if req.Header.Get("Cache-control") == "no-cache" {
		ce, err = h.cache.begin(sqlString)
		if err != nil {
			return
		}
	} else {
		var created bool
		ce, created, err = h.cache.getOrBegin(sqlString)
		if err != nil || !created {
			return
		}
		if ce.status() != statusPending {
			log.Debugf("Found results for %v in cache", sqlString)
			return
		}
	}

	// Request query to run in background
	h.queries <- &query{sqlString, parsed, immediate, ce}

	return
}

func (h *handler) coalesceQueries() {
	for q := range h.queries {
		coalescedQueries := append([]*query(nil), q)
		var remainingQueries []*query
		table := q.parsed.From
	coalesceLoop:
		for {
			select {
			case query := <-h.queries:
				if table == query.parsed.From {
					coalescedQueries = append(coalescedQueries, query)
				} else {
					remainingQueries = append(remainingQueries, query)
				}
			case <-time.After(15 * time.Second):
				break coalesceLoop
			}
		}

		log.Debugf("Coalesced %d queries to %v", len(coalescedQueries), table)
		// re-queue queries that weren't included in this run
		for _, query := range remainingQueries {
			// TODO: this could theoretically deadlock if h.queries fills up, since
			// this goroutine is the only that reads from it.
			h.queries <- query
		}
		h.coalescedQueries <- coalescedQueries
	}
}

func (h *handler) processQueries() {
	for queries := range h.coalescedQueries {
		var wg sync.WaitGroup
		wg.Add(len(queries))
		for _, query := range queries {
			go h.execQuery(&wg, query)
		}
		wg.Wait()
	}
}

func (h *handler) execQuery(wg *sync.WaitGroup, query *query) {
	defer wg.Done()
	sqlString := query.sqlString
	ce := query.ce
	result, err := h.doQuery(sqlString, ce.permalink())
	if err != nil {
		err = fmt.Errorf("Unable to query: %v", err)
		log.Error(err)
		ce = ce.fail(err)
	} else {
		resultBytes, err := compress(json.Marshal(result))
		if err != nil {
			err = fmt.Errorf("Unable to marshal result: %v", err)
			log.Error(err)
			ce = ce.fail(err)
		} else if len(resultBytes) > h.MaxResponseBytes {
			err = fmt.Errorf("Query result size %v exceeded limit of %v", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(h.MaxResponseBytes)))
			log.Error(err)
			ce = ce.fail(err)
		} else {
			ce = ce.succeed(resultBytes)
		}
	}
	h.cache.put(sqlString, ce)
	log.Debugf("Cached results for %v", sqlString)
}

func compress(resultBytes []byte, err error) ([]byte, error) {
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(make([]byte, 0, len(resultBytes)))
	gw, err := gzip.NewWriterLevel(buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	_, err = gw.Write(resultBytes)
	if err != nil {
		return nil, err
	}
	err = gw.Close()
	if err != nil {
		return nil, err
	}
	compressed := buf.Bytes()
	log.Debugf("Compressed result from %v down to %v using gzip", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(len(compressed))))
	return compressed, nil
}

func (h *handler) doQuery(sqlString string, permalink string) (*QueryResult, error) {
	rs, err := h.db.Query(sqlString, false, nil, false)
	if err != nil {
		log.Errorf("Error running query: %v", err)
		return nil, err
	}

	var addDim func(dim string)

	result := &QueryResult{
		SQL:       sqlString,
		Permalink: permalink,
		TS:        common.TimeToMillis(time.Now()),
	}
	groupBy := rs.GetGroupBy()
	if len(groupBy) > 0 {
		addDim = func(dim string) {
			// noop
		}
		for _, gb := range groupBy {
			result.Dims = append(result.Dims, gb.Name)
		}
	} else {
		addDim = func(dim string) {
			found := false
			for _, existing := range result.Dims {
				if existing == dim {
					found = true
					break
				}
			}
			if !found {
				result.Dims = append(result.Dims, dim)
			}
		}
	}

	var fields core.Fields
	var fieldCardinalities []*hllpp.HLLPP
	dimCardinalities := make(map[string]*hllpp.HLLPP)
	tsCardinality := hllpp.New()
	cbytes := make([]byte, 8)

	estimatedResultBytes := 0
	var mx sync.Mutex
	ctx, cancel := context.WithTimeout(context.Background(), h.QueryTimeout)
	defer cancel()
	stats, _ := rs.Iterate(ctx, func(inFields core.Fields) error {
		fields = inFields
		for _, field := range fields {
			result.Fields = append(result.Fields, field.Name)
			fieldCardinalities = append(fieldCardinalities, hllpp.New())
		}
		return nil
	}, func(row *core.FlatRow) (bool, error) {
		mx.Lock()
		key := make(map[string]interface{}, 10)
		row.Key.Iterate(true, true, func(dim string, value interface{}, valueBytes []byte) bool {
			key[dim] = value
			addDim(dim)
			hlp := dimCardinalities[dim]
			if hlp == nil {
				hlp = hllpp.New()
				dimCardinalities[dim] = hlp
			}
			hlp.Add(valueBytes)
			estimatedResultBytes += len(dim) + len(valueBytes)
			return true
		})

		estimatedResultBytes += 8 * len(row.Values)
		if estimatedResultBytes > h.MaxResponseBytes {
			mx.Unlock()
			// Note - the estimated size here is always an underestimate of the final
			// JSON size, so this is a conservative way to check. The final check
			// after generating the JSON may sometimes catch things that slipped
			// through here.
			return false, fmt.Errorf("Estimated query result size %v exceeded limit of %v", humanize.Bytes(uint64(estimatedResultBytes)), humanize.Bytes(uint64(h.MaxResponseBytes)))
		}

		encoding.Binary.PutUint64(cbytes, uint64(row.TS))
		tsCardinality.Add(cbytes)

		resultRow := &ResultRow{
			TS:   common.NanosToMillis(row.TS),
			Key:  key,
			Vals: make([]float64, 0, len(row.Values)),
		}

		for i, value := range row.Values {
			resultRow.Vals = append(resultRow.Vals, value)
			encoding.Binary.PutUint64(cbytes, math.Float64bits(value))
			fieldCardinalities[i].Add(cbytes)
		}
		result.Rows = append(result.Rows, resultRow)
		mx.Unlock()
		return true, nil
	})

	result.TSCardinality = tsCardinality.Count()
	result.Dims = make([]string, 0, len(dimCardinalities))
	for dim := range dimCardinalities {
		result.Dims = append(result.Dims, dim)
	}
	sort.Strings(result.Dims)
	for _, dim := range result.Dims {
		result.DimCardinalities = append(result.DimCardinalities, dimCardinalities[dim].Count())
	}

	result.FieldCardinalities = make([]uint64, 0, len(fieldCardinalities))
	for _, fieldCardinality := range fieldCardinalities {
		result.FieldCardinalities = append(result.FieldCardinalities, fieldCardinality.Count())
	}

	if stats != nil {
		result.Stats = stats.(*common.QueryStats)
	}

	return result, nil
}

func intToBytes(i uint64) []byte {
	b := make([]byte, 8)
	encoding.Binary.PutUint64(b, i)
	return b
}
<commit_msg>Another fix to immediate query logic<commit_after>package web

import (
	"bytes"
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"net/url"
	"sort"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/sql"
	"github.com/gorilla/mux"
	"github.com/retailnext/hllpp"
)

const (
	pauseTime    = 250 * time.Millisecond
	shortTimeout = 5 * time.Second
	longTimeout  = 1000 * time.Hour
)

type QueryResult struct {
	SQL                string
	Permalink          string
	TS                 int64
	TSCardinality      uint64
	Fields             []string
	FieldCardinalities []uint64
	Dims               []string
	DimCardinalities   []uint64
	Rows               []*ResultRow
	Stats              *common.QueryStats
}

type ResultRow struct {
	TS   int64
	Key  map[string]interface{}
	Vals []float64
}

type query struct {
	sqlString string
	parsed    *sql.Query
	immediate bool
	ce        cacheEntry
}

func (h *handler) runQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, longTimeout, false)
}

func (h *handler) asyncQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, shortTimeout, false)
}

func (h *handler) immediateQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, shortTimeout, true)
}

func (h *handler) cachedQuery(resp http.ResponseWriter, req *http.Request) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	log.Debug(req.URL)
	permalink := mux.Vars(req)["permalink"]
	ce, err := h.cache.getByPermalink(permalink)
	if ce == nil {
		http.NotFound(resp, req)
		return
	}
	h.respondWithCacheEntry(resp, req, ce, err, shortTimeout)
}

func (h *handler) sqlQuery(resp http.ResponseWriter, req *http.Request, timeout time.Duration, immediate bool) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	log.Debug(req.URL)
	sqlString, _ := url.QueryUnescape(req.URL.RawQuery)

	ce, err := h.query(req, sqlString, immediate)
	h.respondWithCacheEntry(resp, req, ce, err, timeout)
}

func (h *handler) respondWithCacheEntry(resp http.ResponseWriter, req *http.Request, ce cacheEntry, err error, timeout time.Duration) {
	limit := int(timeout / pauseTime)
	for i := 0; i < limit; i++ {
		if err != nil {
			log.Error(err)
			resp.WriteHeader(http.StatusInternalServerError)
			fmt.Fprint(resp, err.Error())
			return
		}
		switch ce.status() {
		case statusSuccess:
			h.respondSuccess(resp, req, ce)
			return
		case statusError:
			h.respondError(resp, req, ce)
			return
		case statusPending:
			// Pause a little bit and try again
			time.Sleep(pauseTime)
			ce, err = h.cache.getByPermalink(ce.permalink())
		}
	}
	// Let the client know that we're still working on it
	resp.WriteHeader(http.StatusAccepted)
	fmt.Fprintf(resp, "/cached/%v", ce.permalink())
}

func (h *handler) respondSuccess(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.Header().Set("Content-Type", "application/json")
	resp.Header().Set("Expires", "0")
	resp.Header().Set("Cache-control", "no-cache, no-store, must-revalidate")
	resp.Header().Set("Content-Encoding", "gzip")
	resp.WriteHeader(http.StatusOK)
	resp.Write(ce.data())
}

func (h *handler) respondError(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.WriteHeader(http.StatusInternalServerError)
	resp.Write(ce.error())
}

func (h *handler) query(req *http.Request, sqlString string, immediate bool) (ce cacheEntry, err error) {
	parsed, parseErr := sql.Parse(sqlString)
	if parseErr != nil {
		return nil, parseErr
	}

	if req.Header.Get("Cache-control") == "no-cache" {
		ce, err = h.cache.begin(sqlString)
		if err != nil {
			return
		}
	} else {
		var created bool
		ce, created, err = h.cache.getOrBegin(sqlString)
		if err != nil || !created {
			return
		}
		if ce.status() != statusPending {
			log.Debugf("Found results for %v in cache", sqlString)
			return
		}
	}

	// Request query to run in background
	h.queries <- &query{sqlString, parsed, immediate, ce}

	return
}

func (h *handler) coalesceQueries() {
	for q := range h.queries {
		coalescedQueries := append([]*query(nil), q)
		var remainingQueries []*query
		table := q.parsed.From
	coalesceLoop:
		for {
			select {
			case query := <-h.queries:
				if table == query.parsed.From {
					coalescedQueries = append(coalescedQueries, query)
				} else {
					remainingQueries = append(remainingQueries, query)
				}
				if query.immediate {
					break coalesceLoop
				}
			case <-time.After(15 * time.Second):
				break coalesceLoop
			}
		}

		log.Debugf("Coalesced %d queries to %v", len(coalescedQueries), table)
		// re-queue queries that weren't included in this run
		for _, query := range remainingQueries {
			// TODO: this could theoretically deadlock if h.queries fills up, since
			// this goroutine is the only that reads from it.
			h.queries <- query
		}
		h.coalescedQueries <- coalescedQueries
	}
}

func (h *handler) processQueries() {
	for queries := range h.coalescedQueries {
		var wg sync.WaitGroup
		wg.Add(len(queries))
		for _, query := range queries {
			go h.execQuery(&wg, query)
		}
		wg.Wait()
	}
}

func (h *handler) execQuery(wg *sync.WaitGroup, query *query) {
	defer wg.Done()
	sqlString := query.sqlString
	ce := query.ce
	result, err := h.doQuery(sqlString, ce.permalink())
	if err != nil {
		err = fmt.Errorf("Unable to query: %v", err)
		log.Error(err)
		ce = ce.fail(err)
	} else {
		resultBytes, err := compress(json.Marshal(result))
		if err != nil {
			err = fmt.Errorf("Unable to marshal result: %v", err)
			log.Error(err)
			ce = ce.fail(err)
		} else if len(resultBytes) > h.MaxResponseBytes {
			err = fmt.Errorf("Query result size %v exceeded limit of %v", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(h.MaxResponseBytes)))
			log.Error(err)
			ce = ce.fail(err)
		} else {
			ce = ce.succeed(resultBytes)
		}
	}
	h.cache.put(sqlString, ce)
	log.Debugf("Cached results for %v", sqlString)
}

func compress(resultBytes []byte, err error) ([]byte, error) {
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(make([]byte, 0, len(resultBytes)))
	gw, err := gzip.NewWriterLevel(buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	_, err = gw.Write(resultBytes)
	if err != nil {
		return nil, err
	}
	err = gw.Close()
	if err != nil {
		return nil, err
	}
	compressed := buf.Bytes()
	log.Debugf("Compressed result from %v down to %v using gzip", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(len(compressed))))
	return compressed, nil
}

func (h *handler) doQuery(sqlString string, permalink string) (*QueryResult, error) {
	rs, err := h.db.Query(sqlString, false, nil, false)
	if err != nil {
		log.Errorf("Error running query: %v", err)
		return nil, err
	}

	var addDim func(dim string)

	result := &QueryResult{
		SQL:       sqlString,
		Permalink: permalink,
		TS:        common.TimeToMillis(time.Now()),
	}
	groupBy := rs.GetGroupBy()
	if len(groupBy) > 0 {
		addDim = func(dim string) {
			// noop
		}
		for _, gb := range groupBy {
			result.Dims = append(result.Dims, gb.Name)
		}
	} else {
		addDim = func(dim string) {
			found := false
			for _, existing := range result.Dims {
				if existing == dim {
					found = true
					break
				}
			}
			if !found {
				result.Dims = append(result.Dims, dim)
			}
		}
	}

	var fields core.Fields
	var fieldCardinalities []*hllpp.HLLPP
	dimCardinalities := make(map[string]*hllpp.HLLPP)
	tsCardinality := hllpp.New()
	cbytes := make([]byte, 8)

	estimatedResultBytes := 0
	var mx sync.Mutex
	ctx, cancel := context.WithTimeout(context.Background(), h.QueryTimeout)
	defer cancel()
	stats, _ := rs.Iterate(ctx, func(inFields core.Fields) error {
		fields = inFields
		for _, field := range fields {
			result.Fields = append(result.Fields, field.Name)
			fieldCardinalities = append(fieldCardinalities, hllpp.New())
		}
		return nil
	}, func(row *core.FlatRow) (bool, error) {
		mx.Lock()
		key := make(map[string]interface{}, 10)
		row.Key.Iterate(true, true, func(dim string, value interface{}, valueBytes []byte) bool {
			key[dim] = value
			addDim(dim)
			hlp := dimCardinalities[dim]
			if hlp == nil {
				hlp = hllpp.New()
				dimCardinalities[dim] = hlp
			}
			hlp.Add(valueBytes)
			estimatedResultBytes += len(dim) + len(valueBytes)
			return true
		})

		estimatedResultBytes += 8 * len(row.Values)
		if estimatedResultBytes > h.MaxResponseBytes {
			mx.Unlock()
			// Note - the estimated size here is always an underestimate of the final
			// JSON size, so this is a conservative way to check. The final check
			// after generating the JSON may sometimes catch things that slipped
			// through here.
			return false, fmt.Errorf("Estimated query result size %v exceeded limit of %v", humanize.Bytes(uint64(estimatedResultBytes)), humanize.Bytes(uint64(h.MaxResponseBytes)))
		}

		encoding.Binary.PutUint64(cbytes, uint64(row.TS))
		tsCardinality.Add(cbytes)

		resultRow := &ResultRow{
			TS:   common.NanosToMillis(row.TS),
			Key:  key,
			Vals: make([]float64, 0, len(row.Values)),
		}

		for i, value := range row.Values {
			resultRow.Vals = append(resultRow.Vals, value)
			encoding.Binary.PutUint64(cbytes, math.Float64bits(value))
			fieldCardinalities[i].Add(cbytes)
		}
		result.Rows = append(result.Rows, resultRow)
		mx.Unlock()
		return true, nil
	})

	result.TSCardinality = tsCardinality.Count()
	result.Dims = make([]string, 0, len(dimCardinalities))
	for dim := range dimCardinalities {
		result.Dims = append(result.Dims, dim)
	}
	sort.Strings(result.Dims)
	for _, dim := range result.Dims {
		result.DimCardinalities = append(result.DimCardinalities, dimCardinalities[dim].Count())
	}

	result.FieldCardinalities = make([]uint64, 0, len(fieldCardinalities))
	for _, fieldCardinality := range fieldCardinalities {
		result.FieldCardinalities = append(result.FieldCardinalities, fieldCardinality.Count())
	}

	if stats != nil {
		result.Stats = stats.(*common.QueryStats)
	}

	return result, nil
}

func intToBytes(i uint64) []byte {
	b := make([]byte, 8)
	encoding.Binary.PutUint64(b, i)
	return b
}
<|endoftext|>


<commit_before>// HTTP router
package violetear

import (
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"
	"sync/atomic"
	"time"
)

type Router struct {
	// Routes to be matched
	routes *Trie

	// dynamicRoutes map of dynamic routes and regular expresions
	dynamicRoutes dynamicSet

	// logRequests yes or no
	LogRequests bool

	// NotFoundHandler configurable http.Handler which is called when no matching
	// route is found. If it is not set, http.NotFound is used.
	NotFoundHandler http.Handler

	// NotAllowedHandler configurable http.Handler which is called when method not allowed.
	NotAllowedHandler http.Handler

	// request-id to use
	Request_ID string

	// extraHeaders adds exta headers to the response
	extraHeaders map[string]string

	// count counter for hits
	count int64

	// Verbose
	Verbose bool

	// Function to handle panics recovered from http handlers.
	PanicHandler func(http.ResponseWriter, *http.Request)
}

var split_path_rx = regexp.MustCompile(`[^/ ]+`)

// New returns a new initialized router.
func New() *Router {
	return &Router{
		routes:        NewTrie(),
		dynamicRoutes: make(dynamicSet),
		extraHeaders:  make(map[string]string),
		Verbose:       true,
	}
}

// Run violetear as an HTTP server.
// The addr string takes the same format as http.ListenAndServe.
func (v *Router) Run(addr string) {
	if v.Verbose {
		log.Printf("Router listening on %s", addr)
	}
	log.Fatal(http.ListenAndServe(addr, v))
}

// SetHeader adds extra headers to the response
func (v *Router) SetHeader(key, value string) {
	v.extraHeaders[key] = value
}

// HandleFunc add a route to the router (path, HandlerFunc, methods)
func (v *Router) HandleFunc(path string, handler http.HandlerFunc, http_methods ...string) error {
	path_parts := v.splitPath(path)

	// search for dynamic routes
	for _, p := range path_parts {
		if strings.HasPrefix(p, ":") {
			if _, ok := v.dynamicRoutes[p]; !ok {
				log.Fatalf("[%s] not found, need to add it using AddRegex(\"%s\", `your regex`)", p, p)
			}
		}
	}

	// if no methods, accept ALL
	methods := "ALL"
	if len(http_methods) > 0 {
		methods = http_methods[0]
	}

	if v.Verbose {
		log.Printf("Adding path: %s [%s]", path, methods)
	}
	if err := v.routes.Set(path_parts, handler, methods); err != nil {
		log.Fatal(err)
	}
	return nil
}

// AddRegex adds a ":named" regular expression to the dynamicRoutes
func (v *Router) AddRegex(name string, regex string) error {
	return v.dynamicRoutes.Set(name, regex)
}

// MethodNotAllowed default handler for 405
func (v *Router) MethodNotAllowed() http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w,
			http.StatusText(http.StatusMethodNotAllowed),
			http.StatusMethodNotAllowed,
		)
	})
}

// ServerHTTP dispatches the handler registered in the matched path
func (v *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	atomic.AddInt64(&v.count, 1)
	lw := NewResponseWriter(w)

	node, path, leaf, err := v.routes.Get(v.splitPath(r.URL.Path))

	if err != nil {
		log.Fatal(err)
	}

	// checkMethod check if method is allowed or not
	checkMethod := func(node *Trie, method string) http.Handler {
		if h, ok := node.Handler[method]; ok {
			return h
		} else if h, ok := node.Handler["ALL"]; ok {
			return h
		}
		if v.NotAllowedHandler != nil {
			return v.NotAllowedHandler
		} else {
			return v.MethodNotAllowed()
		}
	}

	var match func(node *Trie, path []string, leaf bool) http.Handler

	// match find a handler for the request
	match = func(node *Trie, path []string, leaf bool) http.Handler {
		if len(node.Handler) > 0 && leaf {
			return checkMethod(node, r.Method)
		} else if node.HasRegex {
			for k, _ := range node.Node {
				if strings.HasPrefix(k, ":") {
					rx := v.dynamicRoutes[k]
					if rx.MatchString(path[0]) {
						path[0] = k
						if leaf {
							return checkMethod(node.Node[k], r.Method)
						} else {
							node, path, leaf, _ := node.Get(path)
							return match(node, path, leaf)
						}
					}
				}
			}
			if node.HasCatchall {
				return checkMethod(node.Node["*"], r.Method)
			}
		} else if node.HasCatchall {
			return checkMethod(node.Node["*"], r.Method)
		}
		// NotFound
		if v.NotFoundHandler != nil {
			return v.NotFoundHandler
		}
		return http.NotFoundHandler()
	}

	// rid Set Request-ID
	rid := r.Header.Get(v.Request_ID)
	if rid != "" {
		w.Header().Set(v.Request_ID, rid)
	} else {
		rid = fmt.Sprintf("%s-%d-%d", r.Method, time.Now().UnixNano(), atomic.LoadInt64(&v.count))
		w.Header().Set("Request-ID", rid)
	}

	// set extra headers
	for k, v := range v.extraHeaders {
		w.Header().Set(k, v)
	}

	//h http.Handler
	h := match(node, path, leaf)
	h.ServeHTTP(lw, r)

	if v.LogRequests {
		log.Printf("%s [%s] %d %d %v %s", r.RemoteAddr, r.URL, lw.Status(), lw.Size(), time.Since(start), rid)
	}
	return
}

// splitPath returns an slice of the path
func (v *Router) splitPath(p string) []string {
	path_parts := split_path_rx.FindAllString(p, -1)

	// root (empty slice)
	if len(path_parts) == 0 {
		path_parts = append(path_parts, "/")
	}

	return path_parts
}
<commit_msg>	modified:   violetear.go<commit_after>// HTTP router
//
// Basic example:
// package main
//
// import (
//	  "fmt"
//	  "github.com/nbari/violetear"
//	  "net/http"
// )
//
// func catchAll(w http.ResponseWriter, r *http.Request) {
//	   fmt.Fprintf(w, r.URL.Path[1:])
// }
//
// func helloWorld(w http.ResponseWriter, r *http.Request) {
//     fmt.Fprintf(w, r.URL.Path[1:])
// }
//
// func handleUUID(w http.ResponseWriter, r *http.Request) {
//     fmt.Fprintf(w, r.URL.Path[1:])
// }
//
// func main() {
//     router := violetear.New()
//     router.LogRequests = true
//     router.Request_ID = "REQUEST_LOG_ID"
//
//	   router.AddRegex(":uuid", `[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
//
//     router.HandleFunc("*", catchAll)
//     router.HandleFunc("/hello/", helloWorld, "GET,HEAD")
//     router.HandleFunc("/root/:uuid/item", handleUUID, "POST,PUT")
//
//     router.SetHeader("X-app-version", "1.1")
//
//     router.Run(":8080")
// }

package violetear

import (
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"
	"sync/atomic"
	"time"
)

type Router struct {
	// Routes to be matched
	routes *Trie

	// dynamicRoutes map of dynamic routes and regular expresions
	dynamicRoutes dynamicSet

	// logRequests yes or no
	LogRequests bool

	// NotFoundHandler configurable http.Handler which is called when no matching
	// route is found. If it is not set, http.NotFound is used.
	NotFoundHandler http.Handler

	// NotAllowedHandler configurable http.Handler which is called when method not allowed.
	NotAllowedHandler http.Handler

	// request-id to use
	Request_ID string

	// extraHeaders adds exta headers to the response
	extraHeaders map[string]string

	// count counter for hits
	count int64

	// Verbose
	Verbose bool

	// Function to handle panics recovered from http handlers.
	PanicHandler func(http.ResponseWriter, *http.Request)
}

var split_path_rx = regexp.MustCompile(`[^/ ]+`)

// New returns a new initialized router.
func New() *Router {
	return &Router{
		routes:        NewTrie(),
		dynamicRoutes: make(dynamicSet),
		extraHeaders:  make(map[string]string),
		Verbose:       true,
	}
}

// Run violetear as an HTTP server.
// The addr string takes the same format as http.ListenAndServe.
func (v *Router) Run(addr string) {
	if v.Verbose {
		log.Printf("Router listening on %s", addr)
	}
	log.Fatal(http.ListenAndServe(addr, v))
}

// SetHeader adds extra headers to the response
func (v *Router) SetHeader(key, value string) {
	v.extraHeaders[key] = value
}

// HandleFunc add a route to the router (path, HandlerFunc, methods)
func (v *Router) HandleFunc(path string, handler http.HandlerFunc, http_methods ...string) error {
	path_parts := v.splitPath(path)

	// search for dynamic routes
	for _, p := range path_parts {
		if strings.HasPrefix(p, ":") {
			if _, ok := v.dynamicRoutes[p]; !ok {
				log.Fatalf("[%s] not found, need to add it using AddRegex(\"%s\", `your regex`)", p, p)
			}
		}
	}

	// if no methods, accept ALL
	methods := "ALL"
	if len(http_methods) > 0 {
		methods = http_methods[0]
	}

	if v.Verbose {
		log.Printf("Adding path: %s [%s]", path, methods)
	}
	if err := v.routes.Set(path_parts, handler, methods); err != nil {
		log.Fatal(err)
	}
	return nil
}

// AddRegex adds a ":named" regular expression to the dynamicRoutes
func (v *Router) AddRegex(name string, regex string) error {
	return v.dynamicRoutes.Set(name, regex)
}

// MethodNotAllowed default handler for 405
func (v *Router) MethodNotAllowed() http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w,
			http.StatusText(http.StatusMethodNotAllowed),
			http.StatusMethodNotAllowed,
		)
	})
}

// ServerHTTP dispatches the handler registered in the matched path
func (v *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	atomic.AddInt64(&v.count, 1)
	lw := NewResponseWriter(w)

	node, path, leaf, err := v.routes.Get(v.splitPath(r.URL.Path))

	if err != nil {
		log.Fatal(err)
	}

	// checkMethod check if method is allowed or not
	checkMethod := func(node *Trie, method string) http.Handler {
		if h, ok := node.Handler[method]; ok {
			return h
		} else if h, ok := node.Handler["ALL"]; ok {
			return h
		}
		if v.NotAllowedHandler != nil {
			return v.NotAllowedHandler
		} else {
			return v.MethodNotAllowed()
		}
	}

	var match func(node *Trie, path []string, leaf bool) http.Handler

	// match find a handler for the request
	match = func(node *Trie, path []string, leaf bool) http.Handler {
		if len(node.Handler) > 0 && leaf {
			return checkMethod(node, r.Method)
		} else if node.HasRegex {
			for k, _ := range node.Node {
				if strings.HasPrefix(k, ":") {
					rx := v.dynamicRoutes[k]
					if rx.MatchString(path[0]) {
						path[0] = k
						if leaf {
							return checkMethod(node.Node[k], r.Method)
						} else {
							node, path, leaf, _ := node.Get(path)
							return match(node, path, leaf)
						}
					}
				}
			}
			if node.HasCatchall {
				return checkMethod(node.Node["*"], r.Method)
			}
		} else if node.HasCatchall {
			return checkMethod(node.Node["*"], r.Method)
		}
		// NotFound
		if v.NotFoundHandler != nil {
			return v.NotFoundHandler
		}
		return http.NotFoundHandler()
	}

	// rid Set Request-ID
	rid := r.Header.Get(v.Request_ID)
	if rid != "" {
		w.Header().Set(v.Request_ID, rid)
	} else {
		rid = fmt.Sprintf("%s-%d-%d", r.Method, time.Now().UnixNano(), atomic.LoadInt64(&v.count))
		w.Header().Set("Request-ID", rid)
	}

	// set extra headers
	for k, v := range v.extraHeaders {
		w.Header().Set(k, v)
	}

	//h http.Handler
	h := match(node, path, leaf)
	h.ServeHTTP(lw, r)

	if v.LogRequests {
		log.Printf("%s [%s] %d %d %v %s", r.RemoteAddr, r.URL, lw.Status(), lw.Size(), time.Since(start), rid)
	}
	return
}

// splitPath returns an slice of the path
func (v *Router) splitPath(p string) []string {
	path_parts := split_path_rx.FindAllString(p, -1)

	// root (empty slice)
	if len(path_parts) == 0 {
		path_parts = append(path_parts, "/")
	}

	return path_parts
}
<|endoftext|>