<commit_before>package transmit

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/binary"
	"errors"
	"hash/adler32"
	"io"
	"io/ioutil"
	"log"
	"net"
	"sync"
	"sync/atomic"

	"github.com/midbel/uuid"
)

var ErrClosed = errors.New("closed")

var (
	ErrCorrupted = errors.New("packet corrupted")
	ErrUnknownId = errors.New("unknown packet id")
)

const (
	Size    = 26
	Padding = 512
)

const (
	Bind uint16 = iota
	Accept
	Reject
	Copy
	Abort
	Done
)

type Route struct {
	Id   string `json:"id"`
	Addr string `json:"addr"`
	Eth  string `json:"ifi"`
}

func Subscribe(a, n string) (net.Conn, error) {
	addr, err := net.ResolveUDPAddr("udp", a)
	if err != nil {
		return nil, err
	}
	i, err := net.InterfaceByName(n)
	if err != nil && len(n) > 0 {
		return nil, err
	}
	c, err := net.ListenMulticastUDP("udp", i, addr)
	if err != nil {
		return nil, err
	}
	return &subscriber{c}, nil
}

func Dispatch(a string) (net.Conn, error) {
	c, err := net.Dial("udp", a)
	if err != nil {
		return nil, err
	}
	return &subscriber{c}, nil
}

func Forward(a, s string) (net.Conn, error) {
	c, err := net.Dial("tcp", a)
	if err != nil {
		return nil, err
	}
	id, _ := uuid.UUID5(uuid.URL, []byte(s))

	f := &forwarder{
		Conn:    c,
		id:      id.Bytes(),
		padding: Padding,
		reader:  bufio.NewReaderSize(rand.Reader, 4096),
	}
	if _, err := f.Write(f.id); err != nil {
		f.Close()
		return nil, err
	}
	return f, nil
}

type Router struct {
	net.Listener

	routes map[string]*pool
}

func (r *Router) Accept() (net.Conn, net.Conn, error) {
	c, err := r.Listener.Accept()
	if err != nil {
		return nil, nil, err
	}
	id := make([]byte, Size+uuid.Size)
	if _, err := io.ReadFull(c, id); err != nil {
		c.Close()
		return nil, nil, err
	} else {
		id = id[:uuid.Size]
	}
	p, ok := r.routes[string(id)]
	if !ok {
		c.Close()
		return nil, nil, ErrUnknownId
	}
	io.CopyN(ioutil.Discard, c, Padding)
	w, err := p.Acquire()
	if err != nil {
		return nil, nil, err
	}
	return &forwarder{Conn: c, id: id}, w, nil
}

func (r *Router) Close() error {
	err := r.Listener.Close()
	for _, p := range r.routes {
		if e := p.Close(); err == nil && e != nil {
			err = e
		}
	}
	return err
}

func NewRouter(a string, rs []Route) (*Router, error) {
	l, err := net.Listen("tcp", a)
	if err != nil {
		return nil, err
	}
	gs := make(map[string]*pool)
	for _, r := range rs {
		id, _ := uuid.UUID5(uuid.URL, []byte(r.Id))
		p := &pool{
			c: make(chan net.Conn, 5),
			a: r.Addr,
		}
		gs[string(id.Bytes())] = p
	}
	return &Router{Listener: l, routes: gs}, nil
}

type subscriber struct {
	net.Conn
}

func (s *subscriber) Read(b []byte) (int, error) {
	d := make([]byte, len(b))
	r, err := s.Conn.Read(d)
	if err != nil && r == 0 {
		return r, err
	} else {
		d = d[:r]
		go log.Printf("%d bytes read from %s (%x)", r, s.LocalAddr(), md5.Sum(d))
	}
	sum := make([]byte, 4)
	binary.BigEndian.PutUint32(sum, adler32.Checksum(d))
	d = append(d, sum...)

	return copy(b, d), err
}

func (s *subscriber) Write(b []byte) (int, error) {
	d, sum := b[:len(b)-adler32.Size], b[len(b)-adler32.Size:]
	if a := adler32.Checksum(d); a != binary.BigEndian.Uint32(sum) {
		return len(b), ErrCorrupted
	}
	_, err := s.Conn.Write(d)
	if err == nil {
		go log.Printf("%d bytes written to %s (%x)", len(d), s.RemoteAddr(), md5.Sum(d))
	}
	return len(b), err
}

type forwarder struct {
	net.Conn

	id []byte

	sequence uint32
	padding  uint16

	reader io.Reader
}

func (f *forwarder) Read(b []byte) (int, error) {
	d, err := readFrom(f.Conn, f.id)
	go log.Printf("%d bytes received from %s", len(d), f.RemoteAddr())
	return copy(b, d), err
}

func (f *forwarder) Write(b []byte) (int, error) {
	s := atomic.AddUint32(&f.sequence, 1)
	err := writeTo(f.Conn, s, f.id, b)
	go log.Printf("%d bytes sent to %s", len(b), f.RemoteAddr())
	return len(b), err
}

type block struct {
	Id       [uuid.Size]byte
	Length   uint16
	Sequence uint32
	N        uint8
	C        uint8
	R        uint16
}

func readFrom(r io.Reader, i []byte) ([]byte, error) {
	w := new(bytes.Buffer)

	for {
		v := new(block)
		if err := binary.Read(r, binary.BigEndian, v); err != nil {
			return nil, err
		}
		if _, err := io.CopyN(w, r, int64(v.R)); err != nil {
			return nil, err
		}
		if v.R < Padding {
			io.CopyN(ioutil.Discard, r, Padding)
		}
		if !bytes.Equal(v.Id[:], i) {
			return nil, ErrUnknownId
		}
		if v.N == v.C {
			break
		}
	}
	return w.Bytes(), nil
}

func writeTo(w io.Writer, s uint32, id, b []byte) error {
	const size = 1024
	r := bytes.NewReader(b)

	n := r.Size() / size
	/*if m := r.Size() % size; n > 0 && m != 0 {
		n += 1
	}*/
	var j uint8
	for r.Len() > 0 {
		buf := new(bytes.Buffer)

		buf.Write(id)
		binary.Write(buf, binary.BigEndian, s)
		binary.Write(buf, binary.BigEndian, uint16(len(b)))
		binary.Write(buf, binary.BigEndian, uint8(n))
		binary.Write(buf, binary.BigEndian, uint8(j))

		count := size
		if r.Len() < size {
			count = r.Len()
		}
		binary.Write(buf, binary.BigEndian, uint16(count))

		io.CopyN(buf, r, int64(count))
		if count < Padding {
			io.CopyN(buf, rand.Reader, Padding)
		}
		if _, err := io.Copy(w, buf); err != nil {
			return err
		}
		j++
	}
	return nil
}

type pool struct {
	a string
	c chan net.Conn

	closed bool
	mu     sync.Mutex
}

func (p *pool) Acquire() (net.Conn, error) {
	select {
	case c, ok := <-p.c:
		if !ok {
			return nil, ErrClosed
		}
		return c, nil
	default:
		return Dispatch(p.a)
	}
}

func (p *pool) Release(c net.Conn) error {
	if p.closed {
		return ErrClosed
	}
	select {
	case p.c <- c:
		return nil
	default:
		return c.Close()
	}
}

func (p *pool) Close() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.closed {
		return ErrClosed
	}
	var err error
	for c := range p.c {
		if e := c.Close(); err == nil && e != nil {
			err = e
		}
	}
	close(p.c)
	return err
}
<commit_msg>manage empty packet received by forwarder<commit_after>package transmit

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/binary"
	"errors"
	"fmt"
	"hash/adler32"
	"io"
	"io/ioutil"
	"log"
	"net"
	"sync"
	"sync/atomic"

	"github.com/midbel/uuid"
)

var ErrClosed = errors.New("closed")

var (
	ErrEmpty = errors.New("empty")
	ErrCorrupted = errors.New("packet corrupted")
	ErrUnknownId = errors.New("unknown packet id")
)

const (
	Size    = 26
	Padding = 512
)

const (
	Bind uint16 = iota
	Accept
	Reject
	Copy
	Abort
	Done
)

type Route struct {
	Id   string `json:"id"`
	Addr string `json:"addr"`
	Eth  string `json:"ifi"`
}

func Subscribe(a, n string) (net.Conn, error) {
	addr, err := net.ResolveUDPAddr("udp", a)
	if err != nil {
		return nil, err
	}
	i, err := net.InterfaceByName(n)
	if err != nil && len(n) > 0 {
		return nil, err
	}
	c, err := net.ListenMulticastUDP("udp", i, addr)
	if err != nil {
		return nil, err
	}
	return &subscriber{c}, nil
}

func Dispatch(a string) (net.Conn, error) {
	c, err := net.Dial("udp", a)
	if err != nil {
		return nil, err
	}
	return &subscriber{c}, nil
}

func Forward(a, s string) (net.Conn, error) {
	c, err := net.Dial("tcp", a)
	if err != nil {
		return nil, err
	}
	id, _ := uuid.UUID5(uuid.URL, []byte(s))

	f := &forwarder{
		Conn:    c,
		id:      id.Bytes(),
		padding: Padding,
		reader:  bufio.NewReaderSize(rand.Reader, 4096),
	}
	if _, err := f.Write(f.id); err != nil {
		f.Close()
		return nil, err
	}
	return f, nil
}

type Router struct {
	net.Listener

	routes map[string]*pool
}

func (r *Router) Accept() (net.Conn, net.Conn, error) {
	c, err := r.Listener.Accept()
	if err != nil {
		return nil, nil, err
	}
	id := make([]byte, Size+uuid.Size)
	if _, err := io.ReadFull(c, id); err != nil {
		c.Close()
		return nil, nil, err
	} else {
		id = id[:uuid.Size]
	}
	p, ok := r.routes[string(id)]
	if !ok {
		c.Close()
		return nil, nil, ErrUnknownId
	}
	io.CopyN(ioutil.Discard, c, Padding)
	w, err := p.Acquire()
	if err != nil {
		return nil, nil, err
	}
	return &forwarder{Conn: c, id: id}, w, nil
}

func (r *Router) Close() error {
	err := r.Listener.Close()
	for _, p := range r.routes {
		if e := p.Close(); err == nil && e != nil {
			err = e
		}
	}
	return err
}

func NewRouter(a string, rs []Route) (*Router, error) {
	l, err := net.Listen("tcp", a)
	if err != nil {
		return nil, err
	}
	gs := make(map[string]*pool)
	for _, r := range rs {
		id, _ := uuid.UUID5(uuid.URL, []byte(r.Id))
		p := &pool{
			c: make(chan net.Conn, 5),
			a: r.Addr,
		}
		gs[string(id.Bytes())] = p
	}
	return &Router{Listener: l, routes: gs}, nil
}

type subscriber struct {
	net.Conn
}

func (s *subscriber) Read(b []byte) (int, error) {
	d := make([]byte, len(b))
	r, err := s.Conn.Read(d)
	if err != nil && r == 0 {
		return r, err
	} else {
		d = d[:r]
		go log.Printf("%d bytes read from %s (%x)", r, s.LocalAddr(), md5.Sum(d))
	}
	sum := make([]byte, 4)
	binary.BigEndian.PutUint32(sum, adler32.Checksum(d))
	d = append(d, sum...)

	return copy(b, d), err
}

func (s *subscriber) Write(b []byte) (int, error) {
	d, sum := b[:len(b)-adler32.Size], b[len(b)-adler32.Size:]
	if a := adler32.Checksum(d); a != binary.BigEndian.Uint32(sum) {
		return len(b), ErrCorrupted
	}
	_, err := s.Conn.Write(d)
	if err == nil {
		go log.Printf("%d bytes written to %s (%x)", len(d), s.RemoteAddr(), md5.Sum(d))
	}
	return len(b), err
}

type forwarder struct {
	net.Conn

	id []byte

	sequence uint32
	padding  uint16

	reader io.Reader
}

func (f *forwarder) Read(b []byte) (int, error) {
	d, err := readFrom(f.Conn, f.id)
	if len(d) == 0 {
		return 0, ErrEmpty
	}
	go log.Printf("%d bytes received from %s", len(d), f.RemoteAddr())
	return copy(b, d), err
}

func (f *forwarder) Write(b []byte) (int, error) {
	s := atomic.AddUint32(&f.sequence, 1)
	err := writeTo(f.Conn, s, f.id, b)
	go log.Printf("%d bytes sent to %s", len(b), f.RemoteAddr())
	return len(b), err
}

type block struct {
	Id       [uuid.Size]byte
	Length   uint16
	Sequence uint32
	N        uint8
	C        uint8
	R        uint16
}

func readFrom(r io.Reader, i []byte) ([]byte, error) {
	w := new(bytes.Buffer)

	for {
		v := new(block)
		if err := binary.Read(r, binary.BigEndian, v); err != nil {
			return nil, err
		}
		if _, err := io.CopyN(w, r, int64(v.R)); err != nil {
			return nil, err
		}
		if v.R < Padding {
			io.CopyN(ioutil.Discard, r, Padding)
		}
		if !bytes.Equal(v.Id[:], i) {
			return nil, ErrUnknownId
		}
		if v.N == v.C {
			break
		}
	}
	return w.Bytes(), nil
}

func writeTo(w io.Writer, s uint32, id, b []byte) error {
	const size = 1024
	r := bytes.NewReader(b)

	n := r.Size() / size
	var j uint8
	for r.Len() > 0 {
		buf := new(bytes.Buffer)

		buf.Write(id)
		binary.Write(buf, binary.BigEndian, s)
		binary.Write(buf, binary.BigEndian, uint16(len(b)))
		binary.Write(buf, binary.BigEndian, uint8(n))
		binary.Write(buf, binary.BigEndian, uint8(j))

		count := size
		if r.Len() < size {
			count = r.Len()
		}
		binary.Write(buf, binary.BigEndian, uint16(count))

		io.CopyN(buf, r, int64(count))
		if count < Padding {
			io.CopyN(buf, rand.Reader, Padding)
		}
		if _, err := io.Copy(w, buf); err != nil {
			return err
		}
		j++
	}
	return nil
}

type pool struct {
	a string
	c chan net.Conn

	closed bool
	mu     sync.Mutex
}

func (p *pool) Acquire() (net.Conn, error) {
	select {
	case c, ok := <-p.c:
		if !ok {
			return nil, ErrClosed
		}
		return c, nil
	default:
		return Dispatch(p.a)
	}
}

func (p *pool) Release(c net.Conn) error {
	if p.closed {
		return ErrClosed
	}
	select {
	case p.c <- c:
		return nil
	default:
		return c.Close()
	}
}

func (p *pool) Close() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.closed {
		return ErrClosed
	}
	var err error
	for c := range p.c {
		if e := c.Close(); err == nil && e != nil {
			err = e
		}
	}
	close(p.c)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state_test

import (
	"fmt"

	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	"github.com/juju/juju/state"
	"github.com/juju/juju/testing/factory"
)

type EnvUserSuite struct {
	ConnSuite
}

var _ = gc.Suite(&EnvUserSuite{})

func (s *EnvUserSuite) TestAddEnvironmentUser(c *gc.C) {
	now := state.NowToTheSecond()
	user := s.factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	createdBy := s.factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	envUser, err := s.State.AddEnvironmentUser(user.UserTag(), createdBy.UserTag(), "display-name")
	c.Assert(err, gc.IsNil)

	c.Assert(envUser.ID(), gc.Equals, fmt.Sprintf("%s:validusername@local", s.envTag.Id()))
	c.Assert(envUser.EnvironmentTag(), gc.Equals, s.envTag)
	c.Assert(envUser.UserName(), gc.Equals, "validusername@local")
	c.Assert(envUser.DisplayName(), gc.Equals, "display-name")
	c.Assert(envUser.CreatedBy(), gc.Equals, "createdby@local")
	c.Assert(envUser.DateCreated().Equal(now), jc.IsTrue)
	c.Assert(envUser.LastConnection(), gc.IsNil)

	envUser, err = s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, gc.IsNil)
	c.Assert(envUser.ID(), gc.Equals, fmt.Sprintf("%s:validusername@local", s.envTag.Id()))
	c.Assert(envUser.EnvironmentTag(), gc.Equals, s.envTag)
	c.Assert(envUser.UserName(), gc.Equals, "validusername@local")
	c.Assert(envUser.CreatedBy(), gc.Equals, "createdby@local")
	c.Assert(envUser.DateCreated().Equal(now), jc.IsTrue)
	c.Assert(envUser.LastConnection(), gc.IsNil)
}

func (s *EnvUserSuite) TestAddEnvironmentNoUserFails(c *gc.C) {
	createdBy := s.factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	_, err := s.State.AddEnvironmentUser(names.NewUserTag("validusername"), createdBy.UserTag(), "display-name")
	c.Assert(err, gc.ErrorMatches, `user "validusername" does not exist locally: user "validusername" not found`)
}

func (s *EnvUserSuite) TestAddEnvironmentNoCreatedByUserFails(c *gc.C) {
	user := s.factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	_, err := s.State.AddEnvironmentUser(user.UserTag(), names.NewUserTag("createdby"), user.DisplayName())
	c.Assert(err, gc.ErrorMatches, `createdBy user "createdby" does not exist locally: user "createdby" not found`)
}

func (s *EnvUserSuite) TestUpdateLastConnection(c *gc.C) {
	now := state.NowToTheSecond()
	createdBy := s.factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	user := s.factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	envUser, err := s.State.AddEnvironmentUser(user.UserTag(), createdBy.UserTag(), user.DisplayName())
	c.Assert(err, gc.IsNil)
	err = envUser.UpdateLastConnection()
	c.Assert(err, gc.IsNil)
	c.Assert(envUser.LastConnection().Equal(now), jc.IsTrue)
}
<commit_msg>Fix race in test.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state_test

import (
	"fmt"

	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	"github.com/juju/juju/state"
	"github.com/juju/juju/testing/factory"
)

type EnvUserSuite struct {
	ConnSuite
}

var _ = gc.Suite(&EnvUserSuite{})

func (s *EnvUserSuite) TestAddEnvironmentUser(c *gc.C) {
	now := state.NowToTheSecond()
	user := s.factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	createdBy := s.factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	envUser, err := s.State.AddEnvironmentUser(user.UserTag(), createdBy.UserTag(), "display-name")
	c.Assert(err, gc.IsNil)

	c.Assert(envUser.ID(), gc.Equals, fmt.Sprintf("%s:validusername@local", s.envTag.Id()))
	c.Assert(envUser.EnvironmentTag(), gc.Equals, s.envTag)
	c.Assert(envUser.UserName(), gc.Equals, "validusername@local")
	c.Assert(envUser.DisplayName(), gc.Equals, "display-name")
	c.Assert(envUser.CreatedBy(), gc.Equals, "createdby@local")
	c.Assert(envUser.DateCreated().Equal(now), jc.IsTrue)
	c.Assert(envUser.LastConnection(), gc.IsNil)

	envUser, err = s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, gc.IsNil)
	c.Assert(envUser.ID(), gc.Equals, fmt.Sprintf("%s:validusername@local", s.envTag.Id()))
	c.Assert(envUser.EnvironmentTag(), gc.Equals, s.envTag)
	c.Assert(envUser.UserName(), gc.Equals, "validusername@local")
	c.Assert(envUser.CreatedBy(), gc.Equals, "createdby@local")
	c.Assert(envUser.DateCreated().Equal(now), jc.IsTrue)
	c.Assert(envUser.LastConnection(), gc.IsNil)
}

func (s *EnvUserSuite) TestAddEnvironmentNoUserFails(c *gc.C) {
	createdBy := s.factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	_, err := s.State.AddEnvironmentUser(names.NewUserTag("validusername"), createdBy.UserTag(), "display-name")
	c.Assert(err, gc.ErrorMatches, `user "validusername" does not exist locally: user "validusername" not found`)
}

func (s *EnvUserSuite) TestAddEnvironmentNoCreatedByUserFails(c *gc.C) {
	user := s.factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	_, err := s.State.AddEnvironmentUser(user.UserTag(), names.NewUserTag("createdby"), user.DisplayName())
	c.Assert(err, gc.ErrorMatches, `createdBy user "createdby" does not exist locally: user "createdby" not found`)
}

func (s *EnvUserSuite) TestUpdateLastConnection(c *gc.C) {
	now := state.NowToTheSecond()
	createdBy := s.factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	user := s.factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	envUser, err := s.State.AddEnvironmentUser(user.UserTag(), createdBy.UserTag(), user.DisplayName())
	c.Assert(err, gc.IsNil)
	err = envUser.UpdateLastConnection()
	c.Assert(err, gc.IsNil)
	// It is possible that the update is done over a second boundary, so we need
	// to check for after now as well as equal.
	c.Assert(envUser.LastConnection().After(now) ||
		envUser.LastConnection().Equal(now), jc.IsTrue)
}
<|endoftext|>


<commit_before>// Copyright 2018 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package client implements the XRootD client following protocol from http://xrootd.org.

The NewClient function connects to a server:

	ctx := context.Background()

	client, err := NewClient(ctx, *Addr)
	if err != nil {
		// handle error
	}

	// ...

	if err := client.Close(); err != nil {
		// handle error
	}
*/
package client // import "go-hep.org/x/hep/xrootd/client"

import (
	"context"
	"io"
	"net"

	"go-hep.org/x/hep/xrootd/encoder"
	"go-hep.org/x/hep/xrootd/internal/mux"
	"go-hep.org/x/hep/xrootd/protocol"
)

// A Client to xrootd server which allows to send requests and receive responses.
// Concurrent requests are supported.
// Zero value is invalid, Client should be instantiated using NewClient.
type Client struct {
	cancel          context.CancelFunc
	conn            net.Conn
	mux             *mux.Mux
	protocolVersion int32
}

// NewClient creates a new xrootd client that connects to the given address.
// When the context expires, a response handling is stopped, however, it is
// necessary to call Cancel to correctly free resources.
func NewClient(ctx context.Context, address string) (*Client, error) {
	ctx, cancel := context.WithCancel(ctx)

	var d net.Dialer
	conn, err := d.DialContext(ctx, "tcp", address)
	if err != nil {
		cancel()
		return nil, err
	}

	client := &Client{cancel, conn, mux.New(), 0}

	go client.consume(ctx)

	if err := client.handshake(ctx); err != nil {
		client.Close()
		return nil, err
	}

	return client, nil
}

// Close closes the connection. Any blocked operation will be unblocked and return error.
func (client *Client) Close() error {
	client.cancel()

	client.mux.Close()
	return client.conn.Close()
}

func (client *Client) consume(ctx context.Context) {
	var header protocol.ResponseHeader
	var headerBytes = make([]byte, protocol.ResponseHeaderLength)

	for {
		select {
		case <-ctx.Done():
			// TODO: Should wait for active requests to be completed?
			return
		default:
			if _, err := io.ReadFull(client.conn, headerBytes); err != nil {
				if ctx.Err() != nil {
					// something happened to the context.
					// ignore this error.
					continue
				}
				panic(err)
				// TODO: handle EOF by redirection as specified at http://xrootd.org/doc/dev45/XRdv310.pdf, page 11
			}

			if err := encoder.Unmarshal(headerBytes, &header); err != nil {
				if ctx.Err() != nil {
					// something happened to the context.
					// ignore this error.
					continue
				}
				panic(err)
				// TODO: should redirect in case if is not possible to decode a header as well?
			}

			resp := mux.ServerResponse{Data: make([]byte, header.DataLength)}
			if _, err := io.ReadFull(client.conn, resp.Data); err != nil {
				if ctx.Err() != nil {
					// something happened to the context.
					// ignore this error.
					continue
				}
				resp.Err = err
			} else if header.Status != protocol.Ok {
				resp.Err = header.Error(resp.Data)
			}

			if err := client.mux.SendData(header.StreamID, resp); err != nil {
				if ctx.Err() != nil {
					// something happened to the context.
					// ignore this error.
					continue
				}
				panic(err)
				// TODO: should we just ignore responses to unclaimed stream IDs?
			}

			if header.Status != protocol.OkSoFar {
				client.mux.Unclaim(header.StreamID)
			}
		}
	}
}

func (client *Client) send(ctx context.Context, responseChannel mux.DataRecvChan, request []byte) ([]byte, error) {
	if _, err := client.conn.Write(request); err != nil {
		return nil, err
	}

	more := true
	var resp mux.ServerResponse
	var data []byte

	for more {
		select {
		case resp, more = <-responseChannel:
			data = append(data, resp.Data...)
			if resp.Err != nil {
				return nil, resp.Err
			}
		case <-ctx.Done():
			if err := ctx.Err(); err != nil {
				return nil, err
			}
		}
	}

	return data, nil
}

func (client *Client) call(ctx context.Context, requestID uint16, request interface{}) ([]byte, error) {
	streamID, responseChannel, err := client.mux.Claim()
	if err != nil {
		return nil, err
	}

	data, err := encoder.MarshalRequest(requestID, streamID, request)
	if err != nil {
		return nil, err
	}

	return client.send(ctx, responseChannel, data)
}
<commit_msg>xrootd/client: fix 'more' usage in channel reading<commit_after>// Copyright 2018 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package client implements the XRootD client following protocol from http://xrootd.org.

The NewClient function connects to a server:

	ctx := context.Background()

	client, err := NewClient(ctx, *Addr)
	if err != nil {
		// handle error
	}

	// ...

	if err := client.Close(); err != nil {
		// handle error
	}
*/
package client // import "go-hep.org/x/hep/xrootd/client"

import (
	"context"
	"io"
	"net"

	"go-hep.org/x/hep/xrootd/encoder"
	"go-hep.org/x/hep/xrootd/internal/mux"
	"go-hep.org/x/hep/xrootd/protocol"
)

// A Client to xrootd server which allows to send requests and receive responses.
// Concurrent requests are supported.
// Zero value is invalid, Client should be instantiated using NewClient.
type Client struct {
	cancel          context.CancelFunc
	conn            net.Conn
	mux             *mux.Mux
	protocolVersion int32
}

// NewClient creates a new xrootd client that connects to the given address.
// When the context expires, a response handling is stopped, however, it is
// necessary to call Cancel to correctly free resources.
func NewClient(ctx context.Context, address string) (*Client, error) {
	ctx, cancel := context.WithCancel(ctx)

	var d net.Dialer
	conn, err := d.DialContext(ctx, "tcp", address)
	if err != nil {
		cancel()
		return nil, err
	}

	client := &Client{cancel, conn, mux.New(), 0}

	go client.consume(ctx)

	if err := client.handshake(ctx); err != nil {
		client.Close()
		return nil, err
	}

	return client, nil
}

// Close closes the connection. Any blocked operation will be unblocked and return error.
func (client *Client) Close() error {
	client.cancel()

	client.mux.Close()
	return client.conn.Close()
}

func (client *Client) consume(ctx context.Context) {
	var header protocol.ResponseHeader
	var headerBytes = make([]byte, protocol.ResponseHeaderLength)

	for {
		select {
		case <-ctx.Done():
			// TODO: Should wait for active requests to be completed?
			return
		default:
			if _, err := io.ReadFull(client.conn, headerBytes); err != nil {
				if ctx.Err() != nil {
					// something happened to the context.
					// ignore this error.
					continue
				}
				panic(err)
				// TODO: handle EOF by redirection as specified at http://xrootd.org/doc/dev45/XRdv310.pdf, page 11
			}

			if err := encoder.Unmarshal(headerBytes, &header); err != nil {
				if ctx.Err() != nil {
					// something happened to the context.
					// ignore this error.
					continue
				}
				panic(err)
				// TODO: should redirect in case if is not possible to decode a header as well?
			}

			resp := mux.ServerResponse{Data: make([]byte, header.DataLength)}
			if _, err := io.ReadFull(client.conn, resp.Data); err != nil {
				if ctx.Err() != nil {
					// something happened to the context.
					// ignore this error.
					continue
				}
				resp.Err = err
			} else if header.Status != protocol.Ok {
				resp.Err = header.Error(resp.Data)
			}

			if err := client.mux.SendData(header.StreamID, resp); err != nil {
				if ctx.Err() != nil {
					// something happened to the context.
					// ignore this error.
					continue
				}
				panic(err)
				// TODO: should we just ignore responses to unclaimed stream IDs?
			}

			if header.Status != protocol.OkSoFar {
				client.mux.Unclaim(header.StreamID)
			}
		}
	}
}

func (client *Client) send(ctx context.Context, responseChannel mux.DataRecvChan, request []byte) ([]byte, error) {
	if _, err := client.conn.Write(request); err != nil {
		return nil, err
	}

	var data []byte

	for {
		select {
		case resp, more := <-responseChannel:
			if !more {
				return data, nil
			}

			if resp.Err != nil {
				return nil, resp.Err
			}

			data = append(data, resp.Data...)
		case <-ctx.Done():
			if err := ctx.Err(); err != nil {
				return nil, err
			}
		}
	}
	panic("unreachable")
}

func (client *Client) call(ctx context.Context, requestID uint16, request interface{}) ([]byte, error) {
	streamID, responseChannel, err := client.mux.Claim()
	if err != nil {
		return nil, err
	}

	data, err := encoder.MarshalRequest(requestID, streamID, request)
	if err != nil {
		return nil, err
	}

	return client.send(ctx, responseChannel, data)
}
<|endoftext|>


<commit_before>package main

import (
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/gce"
	"go.skia.org/infra/go/gce/server"
)

func PushBase(name string) *gce.Instance {
	vm := server.Server20170928(name)
	vm.DataDisks = nil
	vm.MachineType = gce.MACHINE_TYPE_STANDARD_1
	vm.Metadata["owner_primary"] = "jcgregorio"

	vm.Scopes = append(vm.Scopes, auth.SCOPE_COMPUTE_READ_ONLY)
	return vm
}

func Prod() *gce.Instance {
	return PushBase("skia-push")
}

func main() {
	server.Main(gce.ZONE_DEFAULT, map[string]*gce.Instance{
		"prod": Prod(),
	})
}
<commit_msg>Rebuild skia-push.<commit_after>package main

import (
	"go.skia.org/infra/go/gce"
	"go.skia.org/infra/go/gce/server"
)

func PushBase(name string) *gce.Instance {
	vm := server.Server20170928(name)
	vm.DataDisks = nil
	vm.MachineType = gce.MACHINE_TYPE_STANDARD_1
	vm.Metadata["owner_primary"] = "jcgregorio"
	vm.ServiceAccount = "skia-push@skia-buildbots.google.com.iam.gserviceaccount.com"
	return vm
}

func Prod() *gce.Instance {
	return PushBase("skia-push")
}

func main() {
	server.Main(gce.ZONE_DEFAULT, map[string]*gce.Instance{
		"prod": Prod(),
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 Daniel Pupius

// Package push2ws contains a web server which will act as a PubsubHubBub to
// WebSocket bridge. WebSocket clients can connect to /bridge?feed=FEED_URL in
// order to be notified of updates published to the feed's hub.
//
// <pre>
//   var conn = new WebSocket('ws://my.bridge.com/bridge?feed=' + encodeURIComponent(feedURL));
// </pre>
//
// To function, the server needs to be publicly accessible in order for the PuSH
// hub to send updates.
package push2ws

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/dpup/gohubbub"
	"github.com/gorilla/websocket"
)

// TODO(dan): Store history and optionally send to new clients.

// Bridge broadcasts messages received via PuSH to Web Socket clients.
type Bridge struct {
	pushClient       *gohubbub.Client
	conns            map[*conn]int
	mu               sync.Mutex // protects conns
	connCounter      int
	allowCrossOrigin bool
}

// NewBridge creates a new PubSubHubbub to Web Socket bridge.
func NewBridge(allowCrossOrigin bool) *Bridge {
	return &Bridge{
		conns:            make(map[*conn]int),
		allowCrossOrigin: allowCrossOrigin,
	}
}

// Start registers handles on DefaultServeMux and starts up a HTTP server using
// ListenAndServe.
// 'self' the host and ip that remote clients should connect to.
// 'addr' ip address that the server should listen on, "" is default.
// 'port' that the server should listen on.
func (b *Bridge) Start(self string, addr string, port int) {
	log.Printf("Setting up server on %s:%d", addr, port)
	b.pushClient = gohubbub.NewClient(self, "strimmer")
	b.pushClient.RegisterHandler(http.DefaultServeMux)
	go b.pushClient.Start()
	b.watchSignals()
	http.HandleFunc("/bridge", b.HandleConnection)
	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", addr, port), nil))
}

// Shutdown closes websockets and removes PuSH subscriptions.
func (b *Bridge) Shutdown() {
	log.Println("Shutting down...")
	for c, _ := range b.conns {
		b.remove(c)
	}
}

// HandleConnection responds to a HTTP request and attempts to upgrade the
// request into a websocket connection.
func (b *Bridge) HandleConnection(w http.ResponseWriter, r *http.Request) {
	feedUrl := r.URL.Query().Get("feed")
	log.Println(r.URL, feedUrl)

	if !b.pushClient.HasSubscription(feedUrl) {
		err := b.pushClient.DiscoverAndSubscribe(feedUrl, func(contentType string, body []byte) {
			b.broadcast(feedUrl, body)
		})
		if err != nil {
			log.Println("Error subscribing to hub,", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	} else {
		log.Println("Subscription already exists for", feedUrl)
	}

	socket, err, statusCode := b.upgrade(w, r)
	if err != nil {
		log.Println("Error upgrading connection,", err)
		http.Error(w, err.Error(), statusCode)
		return
	}

	conn := &conn{
		feedUrl: feedUrl,
		output:  make(chan []byte, 256),
		socket:  socket,
		created: time.Now(),
	}

	b.add(conn)

	defer func() { b.remove(conn) }()
	conn.wait()
}

func (b *Bridge) upgrade(w http.ResponseWriter, r *http.Request) (*websocket.Conn, error, int) {
	if r.Method != "GET" {
		return nil, errors.New("method not allowed"), http.StatusMethodNotAllowed
	}

	if !b.allowCrossOrigin && r.Header.Get("Origin") != "http://"+r.Host {
		return nil, errors.New("origin not allowed"), http.StatusMethodNotAllowed
	}

	socket, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		return nil, errors.New("not a websocket handshake"), http.StatusBadRequest
	}

	if err != nil {
		return nil, fmt.Errorf("error upgrading socket %s", err), http.StatusInternalServerError
	}

	return socket, nil, http.StatusOK
}

// broadcast sends data to each matching connection.
func (b *Bridge) broadcast(feedUrl string, data []byte) {
	// TODO: Remove this lock, need to copy conns and then have
	// a separate cleanup pass for dead subscriptions.
	defer b.lock()()

	feed, err := XMLToFeed(data)
	if err != nil {
		log.Println("Error processing hub response", err)
	} else {

		jsonFeed, _ := json.Marshal(feed)

		count := 0
		for c, id := range b.conns {
			// TODO(dan): Use a map to look up connections that match a feed.
			if c.feedUrl == feedUrl {
				select {
				case c.output <- jsonFeed:
					count++
				default:
					log.Println("Connection closed :", id)
					delete(b.conns, c)
					close(c.output)
				}
			}
		}

		for _, entry := range feed.Entries {
			log.Printf("Broadcasted to %d clients : %s", count, entry.Title)
		}
	}

	b.checkFeedHasClients(feedUrl)
}

// add a new connection.
func (b *Bridge) add(c *conn) {
	defer b.lock()()
	b.connCounter++
	b.conns[c] = b.connCounter
	log.Printf("Connection %d registered", b.conns[c])
}

// remove a connection from the hub and closes its channel.
func (b *Bridge) remove(c *conn) {
	defer b.lock()()
	if id, exists := b.conns[c]; exists {
		delete(b.conns, c)
		close(c.output)
		log.Printf("Connection %d (%s) removed", id, c.feedUrl)
	}
	b.checkFeedHasClients(c.feedUrl)
}

// checkFeedHasClients will unsubscribe from a feed if all the connections have
// been removed. Should be called within a lock.
func (b *Bridge) checkFeedHasClients(feedUrl string) {
	for c, _ := range b.conns {
		if c.feedUrl == feedUrl {
			return
		}
	}
	if b.pushClient.HasSubscription(feedUrl) {
		b.pushClient.Unsubscribe(feedUrl)
	}
}

// watchSignals handles various signals and gracefully shuts down the bridge by
// disconnecting clients and unsubscribing from the hub.
func (b *Bridge) watchSignals() {
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGHUP,
		syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		sig := <-c
		log.Printf("Captured signal %v", sig)

		// If shutdown takes more than 10s, just give up.
		time.AfterFunc(10*time.Second, func() {
			log.Println("Graceful shutdown failed")
			os.Exit(2)
		})

		b.Shutdown()
		os.Exit(1)
	}()
}

// lock locks the mutex and returns unlock as a function for use with defer.
func (b *Bridge) lock() func() {
	b.mu.Lock()
	return func() { b.mu.Unlock() }
}
<commit_msg>Update docs<commit_after>// Copyright 2014 Daniel Pupius

// Package push2ws contains a web server which will act as a PubsubHubBub to
// WebSocket bridge. WebSocket clients can connect to /bridge?feed=FEED_URL in
// order to be notified of updates published to the feed's hub.
//
//   var conn = new WebSocket('ws://my.bridge.com/bridge?feed=' + feedURL);
//
// To function, the server needs to be publicly accessible in order for the PuSH
// hub to send updates.
package push2ws

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/dpup/gohubbub"
	"github.com/gorilla/websocket"
)

// TODO(dan): Store history and optionally send to new clients.

// Bridge broadcasts messages received via PuSH to Web Socket clients.
type Bridge struct {
	pushClient       *gohubbub.Client
	conns            map[*conn]int
	mu               sync.Mutex // protects conns
	connCounter      int
	allowCrossOrigin bool
}

// NewBridge creates a new PubSubHubbub to Web Socket bridge.
func NewBridge(allowCrossOrigin bool) *Bridge {
	return &Bridge{
		conns:            make(map[*conn]int),
		allowCrossOrigin: allowCrossOrigin,
	}
}

// Start registers handles on DefaultServeMux and starts up a HTTP server using
// ListenAndServe.
//   'self' the host and ip that remote clients should connect to.
//   'addr' ip address that the server should listen on, "" is default.
//   'port' that the server should listen on.
func (b *Bridge) Start(self string, addr string, port int) {
	log.Printf("Setting up server on %s:%d", addr, port)
	b.pushClient = gohubbub.NewClient(self, "strimmer")
	b.pushClient.RegisterHandler(http.DefaultServeMux)
	go b.pushClient.Start()
	b.watchSignals()
	http.HandleFunc("/bridge", b.HandleConnection)
	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", addr, port), nil))
}

// Shutdown closes websockets and removes PuSH subscriptions.
func (b *Bridge) Shutdown() {
	log.Println("Shutting down...")
	for c, _ := range b.conns {
		b.remove(c)
	}
}

// HandleConnection responds to a HTTP request and attempts to upgrade the
// request into a websocket connection.
func (b *Bridge) HandleConnection(w http.ResponseWriter, r *http.Request) {
	feedUrl := r.URL.Query().Get("feed")
	log.Println(r.URL, feedUrl)

	if !b.pushClient.HasSubscription(feedUrl) {
		err := b.pushClient.DiscoverAndSubscribe(feedUrl, func(contentType string, body []byte) {
			b.broadcast(feedUrl, body)
		})
		if err != nil {
			log.Println("Error subscribing to hub,", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	} else {
		log.Println("Subscription already exists for", feedUrl)
	}

	socket, err, statusCode := b.upgrade(w, r)
	if err != nil {
		log.Println("Error upgrading connection,", err)
		http.Error(w, err.Error(), statusCode)
		return
	}

	conn := &conn{
		feedUrl: feedUrl,
		output:  make(chan []byte, 256),
		socket:  socket,
		created: time.Now(),
	}

	b.add(conn)

	defer func() { b.remove(conn) }()
	conn.wait()
}

func (b *Bridge) upgrade(w http.ResponseWriter, r *http.Request) (*websocket.Conn, error, int) {
	if r.Method != "GET" {
		return nil, errors.New("method not allowed"), http.StatusMethodNotAllowed
	}

	if !b.allowCrossOrigin && r.Header.Get("Origin") != "http://"+r.Host {
		return nil, errors.New("origin not allowed"), http.StatusMethodNotAllowed
	}

	socket, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		return nil, errors.New("not a websocket handshake"), http.StatusBadRequest
	}

	if err != nil {
		return nil, fmt.Errorf("error upgrading socket %s", err), http.StatusInternalServerError
	}

	return socket, nil, http.StatusOK
}

// broadcast sends data to each matching connection.
func (b *Bridge) broadcast(feedUrl string, data []byte) {
	// TODO: Remove this lock, need to copy conns and then have
	// a separate cleanup pass for dead subscriptions.
	defer b.lock()()

	feed, err := XMLToFeed(data)
	if err != nil {
		log.Println("Error processing hub response", err)
	} else {

		jsonFeed, _ := json.Marshal(feed)

		count := 0
		for c, id := range b.conns {
			// TODO(dan): Use a map to look up connections that match a feed.
			if c.feedUrl == feedUrl {
				select {
				case c.output <- jsonFeed:
					count++
				default:
					log.Println("Connection closed :", id)
					delete(b.conns, c)
					close(c.output)
				}
			}
		}

		for _, entry := range feed.Entries {
			log.Printf("Broadcasted to %d clients : %s", count, entry.Title)
		}
	}

	b.checkFeedHasClients(feedUrl)
}

// add a new connection.
func (b *Bridge) add(c *conn) {
	defer b.lock()()
	b.connCounter++
	b.conns[c] = b.connCounter
	log.Printf("Connection %d registered", b.conns[c])
}

// remove a connection from the hub and closes its channel.
func (b *Bridge) remove(c *conn) {
	defer b.lock()()
	if id, exists := b.conns[c]; exists {
		delete(b.conns, c)
		close(c.output)
		log.Printf("Connection %d (%s) removed", id, c.feedUrl)
	}
	b.checkFeedHasClients(c.feedUrl)
}

// checkFeedHasClients will unsubscribe from a feed if all the connections have
// been removed. Should be called within a lock.
func (b *Bridge) checkFeedHasClients(feedUrl string) {
	for c, _ := range b.conns {
		if c.feedUrl == feedUrl {
			return
		}
	}
	if b.pushClient.HasSubscription(feedUrl) {
		b.pushClient.Unsubscribe(feedUrl)
	}
}

// watchSignals handles various signals and gracefully shuts down the bridge by
// disconnecting clients and unsubscribing from the hub.
func (b *Bridge) watchSignals() {
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGHUP,
		syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		sig := <-c
		log.Printf("Captured signal %v", sig)

		// If shutdown takes more than 10s, just give up.
		time.AfterFunc(10*time.Second, func() {
			log.Println("Graceful shutdown failed")
			os.Exit(2)
		})

		b.Shutdown()
		os.Exit(1)
	}()
}

// lock locks the mutex and returns unlock as a function for use with defer.
func (b *Bridge) lock() func() {
	b.mu.Lock()
	return func() { b.mu.Unlock() }
}
<|endoftext|>


<commit_before>// None objects

package py

type NoneType struct{}

var (
	NoneTypeType = NewType("NoneType", "")
	// And the ubiquitous
	None = NoneType(struct{}{})
)

// Type of this object
func (s NoneType) Type() *Type {
	return NoneTypeType
}

func (a NoneType) M__bool__() Object {
	return False
}

// Check interface is satisfied
var _ I__bool__ = None
<commit_msg>Add __str__ for None<commit_after>// None objects

package py

type NoneType struct{}

var (
	NoneTypeType = NewType("NoneType", "")
	// And the ubiquitous
	None = NoneType(struct{}{})
)

// Type of this object
func (s NoneType) Type() *Type {
	return NoneTypeType
}

func (a NoneType) M__bool__() Object {
	return False
}

func (a NoneType) M__str__() Object {
	return String("None")
}

// Check interface is satisfied
var _ I__bool__ = None
var _ I__str__ = None
<|endoftext|>


<commit_before>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package util

// VERSION identifies Khan's current version
var VERSION = "4.3.10"
<commit_msg>Bump API version to 4.3.12<commit_after>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package util

// VERSION identifies Khan's current version
var VERSION = "4.3.12"
<|endoftext|>


<commit_before>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package frankenstein

import (
	"sort"
	"time"

	"github.com/prometheus/common/model"
	"golang.org/x/net/context"

	"github.com/prometheus/prometheus/storage/local"
	"github.com/prometheus/prometheus/storage/metric"
)

// A Querier allows querying all samples in a given time range that match a set
// of label matchers.
type Querier interface {
	Query(ctx context.Context, from, to model.Time, matchers ...*metric.LabelMatcher) (model.Matrix, error)
}

// A ChunkQuerier is a Querier that fetches samples from a ChunkStore.
type ChunkQuerier struct {
	ctx   context.Context // TODO initialise this to something reasonable.
	Store ChunkStore
}

// Query implements Querier and transforms a list of chunks into sample
// matrices.
func (q *ChunkQuerier) Query(_ context.Context, from, to model.Time, matchers ...*metric.LabelMatcher) (model.Matrix, error) {
	// Get chunks for all matching series from ChunkStore.
	chunks, err := q.Store.Get(q.ctx, from, to, matchers...)
	if err != nil {
		return nil, err
	}

	// Group chunks by series, sort and dedupe samples.
	sampleStreams := map[model.Fingerprint]*model.SampleStream{}

	for _, c := range chunks {
		fp := c.Metric.Fingerprint()
		ss, ok := sampleStreams[fp]
		if !ok {
			ss = &model.SampleStream{
				Metric: c.Metric,
			}
			sampleStreams[fp] = ss
		}
		ss.Values = append(ss.Values, local.DecodeDoubleDeltaChunk(c.Data)...)
	}

	for _, ss := range sampleStreams {
		sort.Sort(timeSortableSamplePairs(ss.Values))
		// TODO: should we also dedupe samples here or leave that to the upper layers?
	}

	matrix := make(model.Matrix, 0, len(sampleStreams))
	for _, ss := range sampleStreams {
		matrix = append(matrix, ss)
	}

	return matrix, nil
}

type timeSortableSamplePairs []model.SamplePair

func (ts timeSortableSamplePairs) Len() int {
	return len(ts)
}

func (ts timeSortableSamplePairs) Less(i, j int) bool {
	return ts[i].Timestamp < ts[j].Timestamp
}

func (ts timeSortableSamplePairs) Swap(i, j int) {
	ts[i], ts[j] = ts[j], ts[i]
}

// A MergeQuerier is a promql.Querier that merges the results of multiple
// frankenstein.Queriers for the same query.
type MergeQuerier struct {
	ctx      context.Context
	Queriers []Querier
}

// QueryRange fetches series for a given time range and label matchers from multiple
// promql.Queriers and returns the merged results as a map of series iterators.
func (qm MergeQuerier) QueryRange(from, to model.Time, matchers ...*metric.LabelMatcher) ([]local.SeriesIterator, error) {
	fpToIt := map[model.Fingerprint]local.SeriesIterator{}

	// Fetch samples from all queriers and group them by fingerprint (unsorted
	// and with overlap).
	for _, q := range qm.Queriers {
		matrix, err := q.Query(qm.ctx, from, to, matchers...)
		if err != nil {
			return nil, err
		}

		for _, ss := range matrix {
			fp := ss.Metric.Fingerprint()
			if it, ok := fpToIt[fp]; !ok {
				fpToIt[fp] = sampleStreamIterator{
					ss: ss,
				}
			} else {
				ssIt := it.(sampleStreamIterator)
				ssIt.ss.Values = append(ssIt.ss.Values, ss.Values...)
			}
		}
	}

	// Sort and dedupe samples.
	for _, it := range fpToIt {
		sortable := timeSortableSamplePairs(it.(sampleStreamIterator).ss.Values)
		sort.Sort(sortable)
		// TODO: Dedupe samples. Not strictly necessary.
	}

	iterators := make([]local.SeriesIterator, 0, len(fpToIt))
	for _, it := range fpToIt {
		iterators = append(iterators, it)
	}

	return iterators, nil
}

// QueryInstant fetches series for a given instant and label matchers from multiple
// promql.Queriers and returns the merged results as a map of series iterators.
func (qm MergeQuerier) QueryInstant(ts model.Time, stalenessDelta time.Duration, matchers ...*metric.LabelMatcher) ([]local.SeriesIterator, error) {
	// For now, just fall back to QueryRange, as QueryInstant is merely allows
	// for instant-specific optimization.
	return qm.QueryRange(ts.Add(-stalenessDelta), ts, matchers...)
}
<commit_msg>Review feedback<commit_after>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package frankenstein

import (
	"sort"
	"time"

	"github.com/prometheus/common/model"
	"golang.org/x/net/context"

	"github.com/prometheus/prometheus/storage/local"
	"github.com/prometheus/prometheus/storage/metric"
)

// A Querier allows querying all samples in a given time range that match a set
// of label matchers.
type Querier interface {
	Query(ctx context.Context, from, to model.Time, matchers ...*metric.LabelMatcher) (model.Matrix, error)
}

// A ChunkQuerier is a Querier that fetches samples from a ChunkStore.
type ChunkQuerier struct {
	Store ChunkStore
}

// Query implements Querier and transforms a list of chunks into sample
// matrices.
func (q *ChunkQuerier) Query(ctx context.Context, from, to model.Time, matchers ...*metric.LabelMatcher) (model.Matrix, error) {
	// Get chunks for all matching series from ChunkStore.
	chunks, err := q.Store.Get(ctx, from, to, matchers...)
	if err != nil {
		return nil, err
	}

	// Group chunks by series, sort and dedupe samples.
	sampleStreams := map[model.Fingerprint]*model.SampleStream{}

	for _, c := range chunks {
		fp := c.Metric.Fingerprint()
		ss, ok := sampleStreams[fp]
		if !ok {
			ss = &model.SampleStream{
				Metric: c.Metric,
			}
			sampleStreams[fp] = ss
		}
		ss.Values = append(ss.Values, local.DecodeDoubleDeltaChunk(c.Data)...)
	}

	for _, ss := range sampleStreams {
		sort.Sort(timeSortableSamplePairs(ss.Values))
		// TODO: should we also dedupe samples here or leave that to the upper layers?
	}

	matrix := make(model.Matrix, 0, len(sampleStreams))
	for _, ss := range sampleStreams {
		matrix = append(matrix, ss)
	}

	return matrix, nil
}

type timeSortableSamplePairs []model.SamplePair

func (ts timeSortableSamplePairs) Len() int {
	return len(ts)
}

func (ts timeSortableSamplePairs) Less(i, j int) bool {
	return ts[i].Timestamp < ts[j].Timestamp
}

func (ts timeSortableSamplePairs) Swap(i, j int) {
	ts[i], ts[j] = ts[j], ts[i]
}

// A MergeQuerier is a promql.Querier that merges the results of multiple
// frankenstein.Queriers for the same query.
type MergeQuerier struct {
	ctx      context.Context
	Queriers []Querier
}

// QueryRange fetches series for a given time range and label matchers from multiple
// promql.Queriers and returns the merged results as a map of series iterators.
func (qm MergeQuerier) QueryRange(from, to model.Time, matchers ...*metric.LabelMatcher) ([]local.SeriesIterator, error) {
	fpToIt := map[model.Fingerprint]local.SeriesIterator{}

	// Fetch samples from all queriers and group them by fingerprint (unsorted
	// and with overlap).
	for _, q := range qm.Queriers {
		matrix, err := q.Query(qm.ctx, from, to, matchers...)
		if err != nil {
			return nil, err
		}

		for _, ss := range matrix {
			fp := ss.Metric.Fingerprint()
			if it, ok := fpToIt[fp]; !ok {
				fpToIt[fp] = sampleStreamIterator{
					ss: ss,
				}
			} else {
				ssIt := it.(sampleStreamIterator)
				ssIt.ss.Values = append(ssIt.ss.Values, ss.Values...)
			}
		}
	}

	// Sort and dedupe samples.
	for _, it := range fpToIt {
		sortable := timeSortableSamplePairs(it.(sampleStreamIterator).ss.Values)
		sort.Sort(sortable)
		// TODO: Dedupe samples. Not strictly necessary.
	}

	iterators := make([]local.SeriesIterator, 0, len(fpToIt))
	for _, it := range fpToIt {
		iterators = append(iterators, it)
	}

	return iterators, nil
}

// QueryInstant fetches series for a given instant and label matchers from multiple
// promql.Queriers and returns the merged results as a map of series iterators.
func (qm MergeQuerier) QueryInstant(ts model.Time, stalenessDelta time.Duration, matchers ...*metric.LabelMatcher) ([]local.SeriesIterator, error) {
	// For now, just fall back to QueryRange, as QueryInstant is merely allows
	// for instant-specific optimization.
	return qm.QueryRange(ts.Add(-stalenessDelta), ts, matchers...)
}
<|endoftext|>


<commit_before>package utils

// Record data-type defines ReqMgr2 data record
type Record map[string]interface{}

// Convert2Record converts given interface to Record data type
func Convert2Record(item interface{}) Record {
	switch r := item.(type) {
	case map[string]interface{}:
		rec := make(Record)
		for kkk, vvv := range r {
			rec[kkk] = vvv
		}
		return rec
	case Record:
		return r
	}
	return nil
}
<commit_msg>Add String to Record<commit_after>package utils

import (
	"encoding/json"

	log "github.com/sirupsen/logrus"
)

// Record data-type defines ReqMgr2 data record
type Record map[string]interface{}

// String function implements Stringer interface
func (r Record) String() string {
	rec, err := json.Marshal(r)
	if err != nil {
		log.Fatal(err)
	}
	return string(rec)
}

// Convert2Record converts given interface to Record data type
func Convert2Record(item interface{}) Record {
	switch r := item.(type) {
	case map[string]interface{}:
		rec := make(Record)
		for kkk, vvv := range r {
			rec[kkk] = vvv
		}
		return rec
	case Record:
		return r
	}
	return nil
}
<|endoftext|>


<commit_before>package lclip

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

var tempPath string

func TestMain(m *testing.M) {
	f, err := ioutil.TempFile(os.TempDir(), "test")
	if err != nil {
		fmt.Fprintln(os.Stderr, "lclip_test:", err)
		return
	}
	f.Close()
	tempPath = f.Name()

	e := m.Run()
	defer os.Exit(e)

	os.Remove(tempPath)
}

func TestGetText(t *testing.T) {
	tempData := []byte(`
{"foo": "bar"}
`[1:])
	if err := ioutil.WriteFile(tempPath, tempData, 0644); err != nil {
		t.Fatal(err)
	}

	c, err := NewClipboard(tempPath)
	if err != nil {
		t.Errorf("NewClipboard returns %q; want nil", err)
	}
	src := "foo"
	actual := c.Get(src)
	expect := "bar"
	if actual != expect {
		t.Errorf("Get(%q) = %q; want %q",
			src, actual, expect)
	}
}
<commit_msg>Add triangulation test<commit_after>package lclip

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

var tempPath string

func TestMain(m *testing.M) {
	f, err := ioutil.TempFile(os.TempDir(), "test")
	if err != nil {
		fmt.Fprintln(os.Stderr, "lclip_test:", err)
		return
	}
	f.Close()
	tempPath = f.Name()

	e := m.Run()
	defer os.Exit(e)

	os.Remove(tempPath)
}

func TestGetText(t *testing.T) {
	tempData := []byte(`
{"foo": "bar", "hoge": "piyo"}
`[1:])
	if err := ioutil.WriteFile(tempPath, tempData, 0644); err != nil {
		t.Fatal(err)
	}

	c, err := NewClipboard(tempPath)
	if err != nil {
		t.Errorf("NewClipboard returns %q; want nil", err)
	}
	src := "foo"
	actual := c.Get(src)
	expect := "bar"
	if actual != expect {
		t.Errorf("Get(%q) = %q; want %q",
			src, actual, expect)
	}

	src = "hoge"
	actual = c.Get(src)
	expect = "piyo"
	if actual != expect {
		t.Errorf("Get(%q) = %q; want %q",
			src, actual, expect)
	}
}
<|endoftext|>


<commit_before>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

/*
#cgo LDFLAGS: -Llibsass -lsass -lstdc++
#cgo CFLAGS: -Ilibsass

#include <stdlib.h>
#include <sass_interface.h>
*/
import "C"

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"

	sprite "github.com/drewwells/sprite_sass"
)

const version = `v0.1.0`

var (
	Dir, Gen, Input, Output, Includes string
	MainFile, Style                   string
	Comments                          bool
	cpuprofile                        string
	ShowVersion                       bool
	BuildDir                          string
)

func init() {
	flag.StringVar(&Output, "output", "", "Output file")
	flag.StringVar(&Output, "o", "", "Output file")
	flag.StringVar(&BuildDir, "b", "", "Build Directory")
	flag.StringVar(&Includes, "p", "", "SASS import path")
	flag.StringVar(&Dir, "dir", "", "Image directory")
	flag.StringVar(&Dir, "d", "", "Image directory")
	flag.StringVar(&Gen, "gen", ".", "Directory for generated images")
	flag.StringVar(&Style, "style", "nested", "CSS nested style")
	flag.StringVar(&Style, "s", "nested", "CSS nested style")
	flag.BoolVar(&Comments, "comment", true, "Turn on source comments")
	flag.BoolVar(&Comments, "c", true, "Turn on source comments")
	flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
	flag.BoolVar(&ShowVersion, "version", false, "Show the app version")
}

func main() {
	flag.Parse()

	if ShowVersion {
		fmt.Println(version)
		os.Exit(0)
	}

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer func() {
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			pprof.StopCPUProfile()
		}()
	}

	for _, v := range flag.Args() {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if len(flag.Args()) == 0 {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		flag.PrintDefaults()
		return
	}

	style, ok := sprite.Style[Style]

	if !ok {
		style = sprite.NESTED_STYLE
	}

	for _, f := range flag.Args() {
		// Remove partials
		if strings.HasPrefix(filepath.Base(f), "_") {
			continue
		}
		log.Println("Open:", f)
		// If no imagedir specified, assume relative to the input file
		if Dir == "" {
			Dir = filepath.Dir(f)
		}
		rel, _ := filepath.Rel(Includes, filepath.Dir(f))
		filename := strings.Replace(filepath.Base(f), ".scss", ".css", 1)
		output := filepath.Join(BuildDir, rel, filename)
		ctx := sprite.Context{
			OutputStyle: style,
			ImageDir:    Dir,
			// Assumption that output is a file
			BuildDir:     filepath.Dir(output),
			GenImgDir:    Gen,
			MainFile:     f,
			Comments:     Comments,
			IncludePaths: []string{filepath.Dir(f)},
		}
		if Includes != "" {
			ctx.IncludePaths = append(ctx.IncludePaths,
				strings.Split(Includes, ",")...)
		}
		fRead, err := os.Open(f)
		if err != nil {
			panic(err)
		}

		var out io.WriteCloser

		if output == "" {
			out = os.Stdout
		} else {
			dir := filepath.Dir(output)
			err := os.MkdirAll(dir, 0755)
			if err != nil {
				log.Fatalf("Failed to create directory: %s", dir)
			}

			out, err = os.Create(output)
			log.Println("Created:", output)
			if err != nil {
				log.Fatalf("Failed to create file: %s", Output)
			}
		}

		err = ctx.Run(fRead, out, filepath.Dir(Input))
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>more gently fail on bad runs<commit_after>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

/*
#cgo LDFLAGS: -Llibsass -lsass -lstdc++
#cgo CFLAGS: -Ilibsass

#include <stdlib.h>
#include <sass_interface.h>
*/
import "C"

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"

	sprite "github.com/drewwells/sprite_sass"
)

const version = `v0.1.0`

var (
	Dir, Gen, Input, Output, Includes string
	MainFile, Style                   string
	Comments                          bool
	cpuprofile                        string
	ShowVersion                       bool
	BuildDir                          string
)

func init() {
	flag.StringVar(&Output, "output", "", "Output file")
	flag.StringVar(&Output, "o", "", "Output file")
	flag.StringVar(&BuildDir, "b", "", "Build Directory")
	flag.StringVar(&Includes, "p", "", "SASS import path")
	flag.StringVar(&Dir, "dir", "", "Image directory")
	flag.StringVar(&Dir, "d", "", "Image directory")
	flag.StringVar(&Gen, "gen", ".", "Directory for generated images")
	flag.StringVar(&Style, "style", "nested", "CSS nested style")
	flag.StringVar(&Style, "s", "nested", "CSS nested style")
	flag.BoolVar(&Comments, "comment", true, "Turn on source comments")
	flag.BoolVar(&Comments, "c", true, "Turn on source comments")
	flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
	flag.BoolVar(&ShowVersion, "version", false, "Show the app version")
}

func main() {
	flag.Parse()

	if ShowVersion {
		fmt.Println(version)
		os.Exit(0)
	}

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer func() {
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			pprof.StopCPUProfile()
		}()
	}

	for _, v := range flag.Args() {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if len(flag.Args()) == 0 {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		flag.PrintDefaults()
		return
	}

	style, ok := sprite.Style[Style]

	if !ok {
		style = sprite.NESTED_STYLE
	}
	for _, f := range flag.Args() {
		// Remove partials
		if strings.HasPrefix(filepath.Base(f), "_") {
			continue
		}
		log.Println("Open:", f)

		// If no imagedir specified, assume relative to the input file
		if Dir == "" {
			Dir = filepath.Dir(f)
		}
		rel, _ := filepath.Rel(Includes, filepath.Dir(f))
		filename := strings.Replace(filepath.Base(f), ".scss", ".css", 1)
		output := filepath.Join(BuildDir, rel, filename)
		ctx := sprite.Context{
			OutputStyle: style,
			ImageDir:    Dir,
			// Assumption that output is a file
			BuildDir:     filepath.Dir(output),
			GenImgDir:    Gen,
			MainFile:     f,
			Comments:     Comments,
			IncludePaths: []string{filepath.Dir(f)},
		}
		if Includes != "" {
			ctx.IncludePaths = append(ctx.IncludePaths,
				strings.Split(Includes, ",")...)
		}
		fRead, err := os.Open(f)
		if err != nil {
			panic(err)
		}

		var out io.WriteCloser

		if output == "" {
			out = os.Stdout
		} else {
			dir := filepath.Dir(output)
			err := os.MkdirAll(dir, 0755)
			if err != nil {
				log.Fatalf("Failed to create directory: %s", dir)
			}

			out, err = os.Create(output)
			log.Println("Created:", output)
			if err != nil {
				log.Fatalf("Failed to create file: %s", Output)
			}
		}

		err = ctx.Run(fRead, out, filepath.Dir(Input))
		if err != nil {
			log.Println(err)
		}
	}
}
<|endoftext|>


<commit_before>package sqldb

import (
	"crypto/rand"
	"database/sql"
	"fmt"
	"log"
)

func genUUIDv4() string {
	u := make([]byte, 16)
	rand.Read(u)
	//Set the version to 4
	u[6] = (u[6] | 0x40) & 0x4F
	u[8] = (u[8] | 0x80) & 0xBF
	return fmt.Sprintf("%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:])
}

type SqlDB struct {
	Db *sql.DB
}

func NewSqlDB(db *sql.DB) *SqlDB {
	sdb := new(SqlDB)
	sdb.Db = db
	db.Begin()

	check(db.Exec("CREATE TABLE IF NOT EXISTS clients (id TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL, secret TEXT NOT NULL, type TEXT NOT NULL, internal INTEGER NOT NULL DEFAULT 0, redirecturis TEXT NOT NULL)"))
	check(db.Exec("CREATE TABLE IF NOT EXISTS users (id TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL, json TEXT)"))
	check(db.Exec("CREATE TABLE IF NOT EXISTS auth (userid TEXT NOT NULL PRIMARY KEY, username TEXT NOT NULL UNIQUE, password TEXT NOT NULL, salt TEXT NOT NULL, FOREIGN KEY userid REFERENCES users(id) ON DELETE CASCADE"))
	check(db.Exec("CREATE TABLE IF NOT EXISTS tokens (id TEXT NOT NULL PRIMARY KEY, type TEXT NOT NULL, userid TEXT NOT NULL, clientid TEXT NOT NULL, expires DATETIME NOT NULL, scope TEXT NOT NULL, accesstype TEXT NOT NULL, refreshtokenid TEXT NOT NULL, FOREIGN KEY userid REFERENCES users(id) ON DELETE CASCADE, FOREIGN KEY clientid REFERENCES clients(id) ON DELETE CASCADE, FOREIGN KEY refreshtokenid REFERENCES tokens(id) ON DELETE CASCADE)"))
	check(db.Exec("CREATE TABLE IF NOT EXISTS concents (userid TEXT NOT NULL, clientid TEXT NOT NULL, concent TEXT NOT NULL, PRIMARY KEY(userid,clientid,concent), FOREIGN KEY userid REFERENCES users(id) ON DELETE CASCADE, FOREIGN KEY clientid REFERENCES clients(id) ON DELETE CASCADE)"))

	return sdb
}

func check(r sql.Result, err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Fixing some sql<commit_after>package sqldb

import (
	"crypto/rand"
	"database/sql"
	"fmt"
	"log"
)

func genUUIDv4() string {
	u := make([]byte, 16)
	rand.Read(u)
	//Set the version to 4
	u[6] = (u[6] | 0x40) & 0x4F
	u[8] = (u[8] | 0x80) & 0xBF
	return fmt.Sprintf("%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:])
}

type SqlDB struct {
	Db *sql.DB
}

func NewSqlDB(db *sql.DB) *SqlDB {
	sdb := new(SqlDB)
	sdb.Db = db
	db.Begin()

	check(db.Exec("CREATE TABLE IF NOT EXISTS clients (id TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL, secret TEXT NOT NULL, type TEXT NOT NULL, internal INTEGER NOT NULL DEFAULT 0, redirecturis TEXT NOT NULL)"))
	check(db.Exec("CREATE TABLE IF NOT EXISTS users (id TEXT NOT NULL PRIMARY KEY, name TEXT NOT NULL, json TEXT)"))
	check(db.Exec("CREATE TABLE IF NOT EXISTS auth (userid TEXT NOT NULL PRIMARY KEY, username TEXT NOT NULL UNIQUE, password TEXT NOT NULL, salt TEXT NOT NULL, FOREIGN KEY (userid) REFERENCES users(id) ON DELETE CASCADE)"))
	check(db.Exec("CREATE TABLE IF NOT EXISTS tokens (id TEXT NOT NULL PRIMARY KEY, type TEXT NOT NULL, userid TEXT NOT NULL, clientid TEXT NOT NULL, expires DATETIME NOT NULL, scope TEXT NOT NULL, accesstype TEXT NOT NULL, refreshtokenid TEXT NOT NULL, FOREIGN KEY (userid) REFERENCES users(id) ON DELETE CASCADE, FOREIGN KEY (clientid) REFERENCES clients(id) ON DELETE CASCADE, FOREIGN KEY (refreshtokenid) REFERENCES tokens(id) ON DELETE CASCADE)"))
	check(db.Exec("CREATE TABLE IF NOT EXISTS concents (userid TEXT NOT NULL, clientid TEXT NOT NULL, concent TEXT NOT NULL, PRIMARY KEY(userid,clientid,concent), FOREIGN KEY (userid) REFERENCES users(id) ON DELETE CASCADE, FOREIGN KEY (clientid) REFERENCES clients(id) ON DELETE CASCADE)"))

	return sdb
}

func check(r sql.Result, err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package tfdiags

import (
	"bytes"
	"fmt"
	"strconv"

	"github.com/zclconf/go-cty/cty"
)

// FormatCtyPath is a helper function to produce a user-friendly string
// representation of a cty.Path. The result uses a syntax similar to the
// HCL expression language in the hope of it being familiar to users.
func FormatCtyPath(path cty.Path) string {
	var buf bytes.Buffer
	for _, step := range path {
		switch ts := step.(type) {
		case cty.GetAttrStep:
			fmt.Fprintf(&buf, ".%s", ts.Name)
		case cty.IndexStep:
			buf.WriteByte('[')
			key := ts.Key
			keyTy := key.Type()
			switch {
			case key.IsNull():
				buf.WriteString("null")
			case !key.IsKnown():
				buf.WriteString("(not yet known)")
			case keyTy == cty.Number:
				bf := key.AsBigFloat()
				buf.WriteString(bf.Text('g', -1))
			case keyTy == cty.String:
				buf.WriteString(strconv.Quote(key.AsString()))
			default:
				buf.WriteString("...")
			}
			buf.WriteByte(']')
		}
	}
	return buf.String()
}

// FormatError is a helper function to produce a user-friendly string
// representation of certain special error types that we might want to
// include in diagnostic messages.
//
// This currently has special behavior only for cty.PathError, where a
// non-empty path is rendered in a HCL-like syntax as context.
func FormatError(err error) string {
	perr, ok := err.(cty.PathError)
	if !ok || len(perr.Path) == 0 {
		return err.Error()
	}

	return fmt.Sprintf("%s: %s", FormatCtyPath(perr.Path), perr.Error())
}
<commit_msg>tfdiags: FormatErrorPrefixed<commit_after>package tfdiags

import (
	"bytes"
	"fmt"
	"strconv"

	"github.com/zclconf/go-cty/cty"
)

// FormatCtyPath is a helper function to produce a user-friendly string
// representation of a cty.Path. The result uses a syntax similar to the
// HCL expression language in the hope of it being familiar to users.
func FormatCtyPath(path cty.Path) string {
	var buf bytes.Buffer
	for _, step := range path {
		switch ts := step.(type) {
		case cty.GetAttrStep:
			fmt.Fprintf(&buf, ".%s", ts.Name)
		case cty.IndexStep:
			buf.WriteByte('[')
			key := ts.Key
			keyTy := key.Type()
			switch {
			case key.IsNull():
				buf.WriteString("null")
			case !key.IsKnown():
				buf.WriteString("(not yet known)")
			case keyTy == cty.Number:
				bf := key.AsBigFloat()
				buf.WriteString(bf.Text('g', -1))
			case keyTy == cty.String:
				buf.WriteString(strconv.Quote(key.AsString()))
			default:
				buf.WriteString("...")
			}
			buf.WriteByte(']')
		}
	}
	return buf.String()
}

// FormatError is a helper function to produce a user-friendly string
// representation of certain special error types that we might want to
// include in diagnostic messages.
//
// This currently has special behavior only for cty.PathError, where a
// non-empty path is rendered in a HCL-like syntax as context.
func FormatError(err error) string {
	perr, ok := err.(cty.PathError)
	if !ok || len(perr.Path) == 0 {
		return err.Error()
	}

	return fmt.Sprintf("%s: %s", FormatCtyPath(perr.Path), perr.Error())
}

// FormatErrorPrefixed is like FormatError except that it presents any path
// information after the given prefix string, which is assumed to contain
// an HCL syntax representation of the value that errors are relative to.
func FormatErrorPrefixed(err error, prefix string) string {
	perr, ok := err.(cty.PathError)
	if !ok || len(perr.Path) == 0 {
		return fmt.Sprintf("%s: %s", prefix, err.Error())
	}

	return fmt.Sprintf("%s%s: %s", prefix, FormatCtyPath(perr.Path), perr.Error())
}
<|endoftext|>


<commit_before>package golog

import "testing"
import "github.com/mndrix/golog/read"
import "github.com/mndrix/golog/term"

func BenchmarkTrue(b *testing.B) {
    m := NewMachine()
    g := read.Term_(`true.`)
    for i := 0; i < b.N; i++ {
        _ = m.ProveAll(g)
    }
}

func BenchmarkAppend(b *testing.B) {
    m := NewMachine().Consult(`
        append([], A, A).   % test same variable name as other clauses
        append([A|B], C, [A|D]) :-
            append(B, C, D).
    `)
    g := read.Term_(`append([a,b,c], [d,e], List).`)

    for i := 0; i < b.N; i++ {
        _ = m.ProveAll(g)
    }
}

// unify two compounds terms with deep structure. unification succeeds
func BenchmarkUnifyDeep(b *testing.B) {
    x := read.Term_(`a(b(c(d(e(f(g(h(i(j))))))))).`)
    y := read.Term_(`a(b(c(d(e(f(g(h(i(X))))))))).`)

    env := term.NewBindings()
    for i := 0; i < b.N; i++ {
        _, _ = x.Unify(env, y)
    }
}

// unify two compounds terms with deep structure. unificatin fails
func BenchmarkUnifyDeepFail(b *testing.B) {
    x := read.Term_(`a(b(c(d(e(f(g(h(i(j))))))))).`)
    y := read.Term_(`a(b(c(d(e(f(g(h(i(x))))))))).`)

    env := term.NewBindings()
    for i := 0; i < b.N; i++ {
        _, _ = x.Unify(env, y)
    }
}

// test performance of a standard maplist implementation
func BenchmarkMaplist(b *testing.B) {
    m := NewMachine().Consult(`
        always_a(_, a).

        maplist(C, A, B) :-
            maplist_(A, B, C).

        maplist_([], [], _).
        maplist_([B|D], [C|E], A) :-
            call(A, B, C),
            maplist_(D, E, A).
    `)
    g := read.Term_(`maplist(always_a, [1,2,3,4,5], As).`)

    for i := 0; i < b.N; i++ {
        _ = m.ProveAll(g)
    }
}

// traditional, naive reverse benchmark
// The Art of Prolog by Sterling, etal says that reversing a 30 element
// list using this technique does 496 reductions.  From this we can
// calculate a rough measure of Golog's LIPS.
func BenchmarkNaiveReverse(b *testing.B) {
    m := NewMachine().Consult(`
        append([], A, A).
        append([A|B], C, [A|D]) :-
            append(B, C, D).

        reverse([],[]).
        reverse([X|Xs], Zs) :-
            reverse(Xs, Ys),
            append(Ys, [X], Zs).
    `)
    g := read.Term_(`reverse([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30], As).`)

    for i := 0; i < b.N; i++ {
        _ = m.ProveAll(g)
    }
}

func BenchmarkRead(b *testing.B) {
    for i := 0; i < b.N; i++ {
        _ = read.Term_(`reverse([1,2,3,4,5,6,7], Xs).`)
    }
}

// two small benchmarks for integer and string comparison
func BenchmarkCompareUint64(b *testing.B) {
    var nintendo uint64 = 282429536481
    var other uint64 = 387429489
    for i := 0; i < b.N; i++ {
        if nintendo == other {
            // do nothing
        }
    }
}
func BenchmarkCompareString(b *testing.B) {
    nintendo := "nintendo"
    other := "other"
    for i := 0; i < b.N; i++ {
        if nintendo == other {
            // do nothing
        }
    }
}
<commit_msg>More low-level benchmarks<commit_after>package golog

import "fmt"
import "strconv"
import "testing"
import "github.com/mndrix/golog/read"
import "github.com/mndrix/golog/term"

func BenchmarkTrue(b *testing.B) {
    m := NewMachine()
    g := read.Term_(`true.`)
    for i := 0; i < b.N; i++ {
        _ = m.ProveAll(g)
    }
}

func BenchmarkAppend(b *testing.B) {
    m := NewMachine().Consult(`
        append([], A, A).   % test same variable name as other clauses
        append([A|B], C, [A|D]) :-
            append(B, C, D).
    `)
    g := read.Term_(`append([a,b,c], [d,e], List).`)

    for i := 0; i < b.N; i++ {
        _ = m.ProveAll(g)
    }
}

// unify two compounds terms with deep structure. unification succeeds
func BenchmarkUnifyDeep(b *testing.B) {
    x := read.Term_(`a(b(c(d(e(f(g(h(i(j))))))))).`)
    y := read.Term_(`a(b(c(d(e(f(g(h(i(X))))))))).`)

    env := term.NewBindings()
    for i := 0; i < b.N; i++ {
        _, _ = x.Unify(env, y)
    }
}

// unify two compounds terms with deep structure. unification fails
func BenchmarkUnifyDeepFail(b *testing.B) {
    x := read.Term_(`a(b(c(d(e(f(g(h(i(j))))))))).`)
    y := read.Term_(`a(b(c(d(e(f(g(h(i(x))))))))).`)

    env := term.NewBindings()
    for i := 0; i < b.N; i++ {
        _, _ = x.Unify(env, y)
    }
}

// test performance of a standard maplist implementation
func BenchmarkMaplist(b *testing.B) {
    m := NewMachine().Consult(`
        always_a(_, a).

        maplist(C, A, B) :-
            maplist_(A, B, C).

        maplist_([], [], _).
        maplist_([B|D], [C|E], A) :-
            call(A, B, C),
            maplist_(D, E, A).
    `)
    g := read.Term_(`maplist(always_a, [1,2,3,4,5], As).`)

    for i := 0; i < b.N; i++ {
        _ = m.ProveAll(g)
    }
}

// traditional, naive reverse benchmark
// The Art of Prolog by Sterling, etal says that reversing a 30 element
// list using this technique does 496 reductions.  From this we can
// calculate a rough measure of Golog's LIPS.
func BenchmarkNaiveReverse(b *testing.B) {
    m := NewMachine().Consult(`
        append([], A, A).
        append([A|B], C, [A|D]) :-
            append(B, C, D).

        reverse([],[]).
        reverse([X|Xs], Zs) :-
            reverse(Xs, Ys),
            append(Ys, [X], Zs).
    `)
    g := read.Term_(`reverse([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30], As).`)

    for i := 0; i < b.N; i++ {
        _ = m.ProveAll(g)
    }
}

func BenchmarkRead(b *testing.B) {
    for i := 0; i < b.N; i++ {
        _ = read.Term_(`reverse([1,2,3,4,5,6,7], Xs).`)
    }
}

// Low level benchmarks to test Go's implementation
func init() {   // avoid import errors when low level benchmarks comment out
    _ = fmt.Sprintf("")
    _ = strconv.Itoa(1)
}
func BenchmarkLowLevelCompareUint64(b *testing.B) {
    var nintendo uint64 = 282429536481
    var other uint64 = 387429489
    for i := 0; i < b.N; i++ {
        if nintendo == other {
            // do nothing
        }
    }
}
func BenchmarkLowLevelCompareString(b *testing.B) {
    nintendo := "nintendo"
    other := "other"
    for i := 0; i < b.N; i++ {
        if nintendo == other {
            // do nothing
        }
    }
}
func BenchmarkLowLevelBitwise(b *testing.B) {
    var nintendo uint64 = 282429536481
    var other uint64 = 387429489
    for i := 0; i < b.N; i++ {
        if nintendo & other == nintendo {
            // do nothing
        }
    }
}
func BenchmarkLowLevelFloatBinaryExponent(b *testing.B) {
	f := 3.1415
    for i := 0; i < b.N; i++ {
        _ = strconv.FormatFloat(f, 'b', 0, 64)
    }
}
func BenchmarkLowLevelFloatDecimalExponent(b *testing.B) {
	f := 3.1415
    for i := 0; i < b.N; i++ {
        _ = strconv.FormatFloat(f, 'e', 64, 64)
    }
}
func BenchmarkLowLevelIntDecimal(b *testing.B) {
	var x uint64 = 1967
    for i := 0; i < b.N; i++ {
        _ = fmt.Sprintf("%d", x)
    }
}
func BenchmarkLowLevelIntHex(b *testing.B) {
	var x uint64 = 1967
    for i := 0; i < b.N; i++ {
        _ = fmt.Sprintf("%x", x)
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"io/ioutil"
	"testing"
)

func BenchmarkParseAllFrames(b *testing.B) {
	if err := resetMP3Tag(); err != nil {
		b.Fatal("Error while reseting mp3 file:", err)
	}
	b.ResetTimer()

	for n := 0; n < b.N; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		if err = tag.Close(); err != nil {
			b.Error("Error while closing a tag:", err)
		}
	}
}

func BenchmarkParseAndWriteCommonCase(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}

	// We use b.N+1, because in first iteration we just resetting tag
	// and setting common frames. Also timer will be resetted.
	for n := 0; n < b.N+1; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}

		// Delete all frames in first iteration
		if n == 0 {
			tag.DeleteAllFrames()
		}

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetYear("2016")

		// Set front cover
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		if err = tag.Save(); err != nil {
			b.Error("Error while saving a tag:", err)
		}
		if err = tag.Close(); err != nil {
			b.Error("Error while closing a tag:", err)
		}

		// Reset timer because we just resetting tag in first iteration
		if n == 0 {
			b.ResetTimer()
		}
	}
}

func BenchmarkParseAndWriteManyFrames(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}

	// We use b.N+1, because in first iteration we just resetting tag
	// and setting many frames. Also timer will be resetted.
	for n := 0; n < b.N+1; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}

		// Delete all frames in first iteration
		if n == 0 {
			tag.DeleteAllFrames()
		}

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetAlbum("Album")
		tag.SetYear("2016")
		tag.SetGenre("Genre")

		// Set front cover
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		// Set USLT
		uslt := UnsynchronisedLyricsFrame{
			Encoding:          ENUTF8,
			Language:          "eng",
			ContentDescriptor: "Content descriptor",
			Lyrics:            "bogem/id3v2",
		}
		tag.AddUnsynchronisedLyricsFrame(uslt)

		// Set comment
		comm := CommentFrame{
			Encoding:    ENUTF8,
			Language:    "eng",
			Description: "Short description",
			Text:        "The actual text",
		}
		tag.AddCommentFrame(comm)

		if err = tag.Save(); err != nil {
			b.Error("Error while saving a tag:", err)
		}
		if err = tag.Close(); err != nil {
			b.Error("Error while closing a tag:", err)
		}

		// Reset timer because we just resetting tag in first iteration
		if n == 0 {
			b.ResetTimer()
		}
	}
}
<commit_msg>Add BenchmarkReuseTag<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"io/ioutil"
	"testing"
)

func BenchmarkParseAllFrames(b *testing.B) {
	if err := resetMP3Tag(); err != nil {
		b.Fatal("Error while reseting mp3 file:", err)
	}
	b.ResetTimer()

	for n := 0; n < b.N; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		if err = tag.Close(); err != nil {
			b.Error("Error while closing a tag:", err)
		}
	}
}

func BenchmarkParseAndWriteCommonCase(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	b.ResetTimer()

	// We use b.N+1, because in first iteration we just reset tag
	// and set common frames. Also timer will be resetted.
	for n := 0; n < b.N+1; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		defer tag.Close()

		// Delete all frames in first iteration
		if n == 0 {
			tag.DeleteAllFrames()
		}

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetYear("2016")

		// Set front cover
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		if err = tag.Save(); err != nil {
			b.Error("Error while saving a tag:", err)
		}

		// Reset timer because we just resetting tag in first iteration
		if n == 0 {
			b.ResetTimer()
		}
	}
}

func BenchmarkParseAndWriteManyFrames(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	b.ResetTimer()

	// We use b.N+1, because in first iteration we just reset tag
	// and set many frames. Also timer will be resetted.
	for n := 0; n < b.N+1; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		defer tag.Close()

		// Delete all frames in first iteration.
		if n == 0 {
			tag.DeleteAllFrames()
		}

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetAlbum("Album")
		tag.SetYear("2016")
		tag.SetGenre("Genre")

		// Set front cover.
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		// Set USLT.
		uslt := UnsynchronisedLyricsFrame{
			Encoding:          ENUTF8,
			Language:          "eng",
			ContentDescriptor: "Content descriptor",
			Lyrics:            "bogem/id3v2",
		}
		tag.AddUnsynchronisedLyricsFrame(uslt)

		// Set comment.
		comm := CommentFrame{
			Encoding:    ENUTF8,
			Language:    "eng",
			Description: "Short description",
			Text:        "The actual text",
		}
		tag.AddCommentFrame(comm)

		if err = tag.Save(); err != nil {
			b.Error("Error while saving a tag:", err)
		}

		// Reset timer because we just reset file in first iteration.
		if n == 0 {
			b.ResetTimer()
		}
	}
}

func BenchmarkReuseTag(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	tag := NewEmptyTag()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tag.DeleteAllFrames()

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetAlbum("Album")
		tag.SetYear("2016")
		tag.SetGenre("Genre")

		// Set front cover.
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		// Set USLT.
		uslt := UnsynchronisedLyricsFrame{
			Encoding:          ENUTF8,
			Language:          "eng",
			ContentDescriptor: "Content descriptor",
			Lyrics:            "bogem/id3v2",
		}
		tag.AddUnsynchronisedLyricsFrame(uslt)

		// Set comment.
		comm := CommentFrame{
			Encoding:    ENUTF8,
			Language:    "eng",
			Description: "Short description",
			Text:        "The actual text",
		}
		tag.AddCommentFrame(comm)
	}
}
<|endoftext|>


<commit_before>package dskvs

import (
	"fmt"
	"os"
	"strconv"
	"testing"
)

func setUpBench(b *testing.B) *Store {
	store, err := NewStore("./db")
	if err != nil {
		b.Fatalf("Error creating store, %v", err)
	}

	err = store.Load()
	if err != nil {
		b.Fatalf("Error loading store, %v", err)
	}
	return store
}

func tearDownBench(store *Store, b *testing.B) {
	err := store.Close()
	if err != nil {
		b.Fatalf("Error closing store, %v", err)
	}
	err = os.RemoveAll(store.storagePath)
	if err != nil {
		b.Fatalf("Error deleting storage path, %v", err)
	}
}

func benchPut(valSize int, b *testing.B) {
	if testing.Short() {
		b.Skip("Skipping put benchmarks")
	}

	b.ReportAllocs()
	store := setUpBench(b)
	defer tearDownBench(store, b)
	baseKey := "hello/hello"
	val := make([]byte, valSize)
	for i := int(0); i < valSize; i++ {
		val[i] = uint8(i % 256)
	}
	keyList := make([]string, b.N)
	for i := 0; i < b.N; i++ {
		key := baseKey + strconv.Itoa(i)
		keyList = append(keyList, key)
	}
	b.ResetTimer()
	for _, key := range keyList {
		err := store.Put(key, val)
		if err != nil {
			b.Errorf("Get has error %v", err)
		}
	}
	b.StopTimer()
}

func BenchmarkPut100B(b *testing.B) {
	benchPut(100, b)
}

func BenchmarkPut400B(b *testing.B) {
	benchPut(400, b)
}

func BenchmarkPut1KB(b *testing.B) {
	benchPut(1000, b)
}

func BenchmarkPut4KB(b *testing.B) {
	benchPut(4000, b)
}

func BenchmarkPut10KB(b *testing.B) {
	benchPut(10000, b)
}

func BenchmarkPut40KB(b *testing.B) {
	benchPut(40000, b)
}

func BenchmarkPut100KB(b *testing.B) {
	benchPut(100000, b)
}

func BenchmarkPut400KB(b *testing.B) {
	benchPut(400000, b)
}

func BenchmarkPut1MB(b *testing.B) {
	benchPut(1000000, b)
}

func benchGet(valSize int, b *testing.B) {
	if testing.Short() {
		b.Skip("Skipping get benchmarks")
	}

	fmt.Printf("N=%d\n", b.N)

	b.ReportAllocs()
	store := setUpBench(b)
	defer tearDownBench(store, b)
	baseKey := "hello/hello"
	val := make([]byte, valSize)
	for i := int(0); i < valSize; i++ {
		val[i] = uint8(i % 256)
	}

	keyList := make([]string, b.N)
	for i := 0; i < b.N; i++ {
		key := baseKey + strconv.Itoa(i)
		keyList[i] = key
		store.Put(key, val)
	}
	b.ResetTimer()
	for _, key := range keyList {
		_, err := store.Get(key)
		if err != nil {
			b.Errorf("Get has error %v", err)
		}
	}
	b.StopTimer()
}

func BenchmarkGet100B(b *testing.B) {
	benchGet(100, b)
}

func BenchmarkGet400B(b *testing.B) {
	benchGet(400, b)
}

func BenchmarkGet1KB(b *testing.B) {
	benchGet(1000, b)
}

func BenchmarkGet4KB(b *testing.B) {
	benchGet(4000, b)
}

func BenchmarkGet10KB(b *testing.B) {
	benchGet(10000, b)
}

func BenchmarkGet40KB(b *testing.B) {
	benchGet(40000, b)
}

func BenchmarkGet100KB(b *testing.B) {
	benchGet(100000, b)
}

func BenchmarkGet400KB(b *testing.B) {
	benchGet(400000, b)
}

func BenchmarkGet1MB(b *testing.B) {
	benchGet(1000000, b)
}
<commit_msg>Add Delete benchmark.<commit_after>package dskvs

import (
	"fmt"
	"os"
	"strconv"
	"testing"
)

func setUpBench(b *testing.B) *Store {
	store, err := NewStore("./db")
	if err != nil {
		b.Fatalf("Error creating store, %v", err)
	}

	err = store.Load()
	if err != nil {
		b.Fatalf("Error loading store, %v", err)
	}
	return store
}

func tearDownBench(store *Store, b *testing.B) {
	err := store.Close()
	if err != nil {
		b.Fatalf("Error closing store, %v", err)
	}
	err = os.RemoveAll(store.storagePath)
	if err != nil {
		b.Fatalf("Error deleting storage path, %v", err)
	}
}

func benchPut(valSize int, b *testing.B) {
	if testing.Short() && valSize > 100 {
		b.Skip("Skipping put benchmarks")
		return
	}

	b.ReportAllocs()
	store := setUpBench(b)
	defer tearDownBench(store, b)
	baseKey := "hello/hello"
	val := make([]byte, valSize)
	for i := int(0); i < valSize; i++ {
		val[i] = uint8(i % 256)
	}
	keyList := make([]string, b.N)
	for i := 0; i < b.N; i++ {
		key := baseKey + strconv.Itoa(i)
		keyList[i] = key
	}
	b.ResetTimer()
	for _, key := range keyList {
		err := store.Put(key, val)
		if err != nil {
			b.Errorf("Get has error %v", err)
		}
	}
	b.StopTimer()
}

func BenchmarkPut100B(b *testing.B) {
	benchPut(100, b)
}

func BenchmarkPut400B(b *testing.B) {
	benchPut(400, b)
}

func BenchmarkPut1KB(b *testing.B) {
	benchPut(1000, b)
}

func BenchmarkPut4KB(b *testing.B) {
	benchPut(4000, b)
}

func BenchmarkPut10KB(b *testing.B) {
	benchPut(10000, b)
}

func BenchmarkPut40KB(b *testing.B) {
	benchPut(40000, b)
}

func BenchmarkPut100KB(b *testing.B) {
	benchPut(100000, b)
}

func BenchmarkPut400KB(b *testing.B) {
	benchPut(400000, b)
}

func BenchmarkPut1MB(b *testing.B) {
	benchPut(1000000, b)
}

func benchGet(valSize int, b *testing.B) {
	if testing.Short() && valSize > 100 {
		b.Skip("Skipping get benchmarks")
		return
	}

	fmt.Printf("N=%d\n", b.N)

	b.ReportAllocs()
	store := setUpBench(b)
	defer tearDownBench(store, b)
	baseKey := "hello/hello"
	val := make([]byte, valSize)
	for i := int(0); i < valSize; i++ {
		val[i] = uint8(i % 256)
	}

	keyList := make([]string, b.N)
	for i := 0; i < b.N; i++ {
		key := baseKey + strconv.Itoa(i)
		keyList[i] = key
		store.Put(key, val)
	}
	b.ResetTimer()
	for _, key := range keyList {
		_, err := store.Get(key)
		if err != nil {
			b.Errorf("Get has error %v", err)
		}
	}
	b.StopTimer()
}

func BenchmarkGet100B(b *testing.B) {
	benchGet(100, b)
}

func BenchmarkGet400B(b *testing.B) {
	benchGet(400, b)
}

func BenchmarkGet1KB(b *testing.B) {
	benchGet(1000, b)
}

func BenchmarkGet4KB(b *testing.B) {
	benchGet(4000, b)
}

func BenchmarkGet10KB(b *testing.B) {
	benchGet(10000, b)
}

func BenchmarkGet40KB(b *testing.B) {
	benchGet(40000, b)
}

func BenchmarkGet100KB(b *testing.B) {
	benchGet(100000, b)
}

func BenchmarkGet400KB(b *testing.B) {
	benchGet(400000, b)
}

func BenchmarkGet1MB(b *testing.B) {
	benchGet(1000000, b)
}

func benchDelete(valSize int, b *testing.B) {
	if testing.Short() && valSize > 100 {
		b.Skip("Skipping delete benchmarks")
		return
	}

	fmt.Printf("N=%d\n", b.N)

	b.ReportAllocs()
	store := setUpBench(b)
	defer tearDownBench(store, b)
	baseKey := "hello/hello"
	val := make([]byte, valSize)
	for i := int(0); i < valSize; i++ {
		val[i] = uint8(i % 256)
	}

	keyList := make([]string, b.N)
	for i := 0; i < b.N; i++ {
		key := baseKey + strconv.Itoa(i)
		keyList[i] = key
		store.Put(key, val)
	}
	b.ResetTimer()
	for _, key := range keyList {
		err := store.Delete(key)
		if err != nil {
			b.Errorf("Get has error %v", err)
		}
	}
	b.StopTimer()
}

func BenchmarkDelete100B(b *testing.B) {
	benchDelete(100, b)
}

func BenchmarkDelete400B(b *testing.B) {
	benchDelete(400, b)
}

func BenchmarkDelete1KB(b *testing.B) {
	benchDelete(1000, b)
}

func BenchmarkDelete4KB(b *testing.B) {
	benchDelete(4000, b)
}

func BenchmarkDelete10KB(b *testing.B) {
	benchDelete(10000, b)
}

func BenchmarkDelete40KB(b *testing.B) {
	benchDelete(40000, b)
}

func BenchmarkDelete100KB(b *testing.B) {
	benchDelete(100000, b)
}

func BenchmarkDelete400KB(b *testing.B) {
	benchDelete(400000, b)
}

func BenchmarkDelete1MB(b *testing.B) {
	benchDelete(1000000, b)
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package cmd

import (
	"encoding/binary"
	"hash/crc32"
	"io"
	"log"
	"os"

	"github.com/jessevdk/go-flags"
	"my/errs"

	"my/itto/verify/anal"
	"my/itto/verify/efhsim"
	"my/itto/verify/rec"
)

type cmdEfhsim struct {
	InputFileName           string `long:"input" short:"i" required:"y" value-name:"PCAP_FILE" description:"input pcap file to read"`
	SubscriptionFileName    string `long:"subscribe" short:"s" value-name:"SUBSCRIPTION_FILE" description:"read subscriptions from file"`
	OutputFileNameSimOrders string `long:"output-sim-orders" value-name:"FILE" description:"output file for hw simulator"`
	OutputFileNameSimQuotes string `long:"output-sim-quotes" value-name:"FILE" description:"output file for hw simulator"`
	OutputFileNameEfhOrders string `long:"output-efh-orders" value-name:"FILE" description:"output file for EFH order messages"`
	OutputFileNameEfhQuotes string `long:"output-efh-quotes" value-name:"FILE" description:"output file for EFH quote messages"`
	OutputFileNameAvt       string `long:"output-avt" value-name:"FILE" description:"output file for AVT CSV"`
	InputFileNameAvtDict    string `long:"avt-dict" value-name:"DICT" description:"read dictionary for AVT CSV output"`
	OutputDirStats          string `long:"output-stats" value-name:"DIR" description:"output dir for stats"`
	PacketNumLimit          int    `long:"count" short:"c" value-name:"NUM" description:"limit number of input packets"`
	shouldExecute           bool
	outFiles                []io.Closer
}

func (c *cmdEfhsim) Execute(args []string) error {
	c.shouldExecute = true
	return nil
}

func (c *cmdEfhsim) ConfigParser(parser *flags.Parser) {
	_, err := parser.AddCommand("efhsim", "simulate efh", "", c)
	if err != nil {
		log.Fatalln(err)
	}
}

func (c *cmdEfhsim) ParsingFinished() {
	if !c.shouldExecute {
		return
	}
	defer func() {
		for _, f := range c.outFiles {
			f.Close()
		}
	}()
	efh := efhsim.NewEfhSim()
	efh.SetInput(c.InputFileName, c.PacketNumLimit)
	if c.SubscriptionFileName != "" {
		file, err := os.Open(c.SubscriptionFileName)
		errs.CheckE(err)
		errs.CheckE(efh.SubscribeFromReader(file))
		file.Close()
	}
	efh.AddLogger(rec.NewHwLimChecker())
	c.addOut(c.OutputFileNameSimOrders, func(w io.Writer) error {
		logger := rec.NewSimLogger(w)
		logger.SetOutputMode(rec.EfhLoggerOutputOrders)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameSimQuotes, func(w io.Writer) error {
		logger := rec.NewSimLogger(w)
		logger.SetOutputMode(rec.EfhLoggerOutputQuotes)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameEfhOrders, func(w io.Writer) error {
		logger := rec.NewEfhLogger(rec.NewTestefhPrinter(w))
		logger.SetOutputMode(rec.EfhLoggerOutputOrders)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameEfhQuotes, func(w io.Writer) error {
		logger := rec.NewEfhLogger(rec.NewTestefhPrinter(w))
		logger.SetOutputMode(rec.EfhLoggerOutputQuotes)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameAvt, func(w io.Writer) (err error) {
		var dict io.ReadCloser
		if c.InputFileNameAvtDict != "" {
			if dict, err = os.Open(c.InputFileNameAvtDict); err != nil {
				return
			} else {
				c.outFiles = append(c.outFiles, dict)
			}
		}
		return efh.AddLogger(rec.NewAvtLogger(w, dict))
	})

	reporter := c.addAnalyzer(efh)
	errs.CheckE(efh.AnalyzeInput())
	if reporter != nil {
		reporter.SaveAll()
	}
}

func (c *cmdEfhsim) addOut(fileName string, setOut func(io.Writer) error) {
	if fileName == "" {
		return
	}
	file, err := os.Create(fileName)
	if err != nil {
		log.Fatalln(err)
	}
	if err := setOut(file); err != nil {
		file.Close()
		log.Fatalln(err)
	}
	c.outFiles = append(c.outFiles, file)
}
func (c *cmdEfhsim) addAnalyzer(efh *efhsim.EfhSim) *anal.Reporter {
	if c.OutputDirStats == "" {
		return nil
	}
	hashFn := func(v uint64) uint64 {
		data := make([]byte, 8)
		binary.BigEndian.PutUint64(data, v)
		h := crc32.ChecksumIEEE(data)
		return uint64(h & (1<<24 - 1))
	}
	moduloFn := func(v uint64) uint64 {
		return v & (1<<24 - 1)
	}
	analyzer := anal.NewAnalyzer()
	analyzer.AddOrderHashFunction(hashFn)
	analyzer.AddOrderHashFunction(moduloFn)
	efh.AddLogger(analyzer.Observer())
	reporter := anal.NewReporter()
	reporter.SetAnalyzer(analyzer)
	reporter.SetOutputDir(c.OutputDirStats)
	return reporter
}

func init() {
	var c cmdEfhsim
	Registry.Register(&c)
}
<commit_msg>cmd:efhsim: add --no-hw-lim option<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package cmd

import (
	"encoding/binary"
	"hash/crc32"
	"io"
	"log"
	"os"

	"github.com/jessevdk/go-flags"
	"my/errs"

	"my/itto/verify/anal"
	"my/itto/verify/efhsim"
	"my/itto/verify/rec"
)

type cmdEfhsim struct {
	InputFileName           string `long:"input" short:"i" required:"y" value-name:"PCAP_FILE" description:"input pcap file to read"`
	SubscriptionFileName    string `long:"subscribe" short:"s" value-name:"SUBSCRIPTION_FILE" description:"read subscriptions from file"`
	OutputFileNameSimOrders string `long:"output-sim-orders" value-name:"FILE" description:"output file for hw simulator"`
	OutputFileNameSimQuotes string `long:"output-sim-quotes" value-name:"FILE" description:"output file for hw simulator"`
	OutputFileNameEfhOrders string `long:"output-efh-orders" value-name:"FILE" description:"output file for EFH order messages"`
	OutputFileNameEfhQuotes string `long:"output-efh-quotes" value-name:"FILE" description:"output file for EFH quote messages"`
	OutputFileNameAvt       string `long:"output-avt" value-name:"FILE" description:"output file for AVT CSV"`
	InputFileNameAvtDict    string `long:"avt-dict" value-name:"DICT" description:"read dictionary for AVT CSV output"`
	OutputDirStats          string `long:"output-stats" value-name:"DIR" description:"output dir for stats"`
	PacketNumLimit          int    `long:"count" short:"c" value-name:"NUM" description:"limit number of input packets"`
	NoHwLim                 bool   `long:"no-hw-lim" description:"do not enforce HW limits"`
	shouldExecute           bool
	outFiles                []io.Closer
}

func (c *cmdEfhsim) Execute(args []string) error {
	c.shouldExecute = true
	return nil
}

func (c *cmdEfhsim) ConfigParser(parser *flags.Parser) {
	_, err := parser.AddCommand("efhsim", "simulate efh", "", c)
	if err != nil {
		log.Fatalln(err)
	}
}

func (c *cmdEfhsim) ParsingFinished() {
	if !c.shouldExecute {
		return
	}
	defer func() {
		for _, f := range c.outFiles {
			f.Close()
		}
	}()
	efh := efhsim.NewEfhSim()
	efh.SetInput(c.InputFileName, c.PacketNumLimit)
	if c.SubscriptionFileName != "" {
		file, err := os.Open(c.SubscriptionFileName)
		errs.CheckE(err)
		errs.CheckE(efh.SubscribeFromReader(file))
		file.Close()
	}
	if !c.NoHwLim {
		efh.AddLogger(rec.NewHwLimChecker())
	}
	c.addOut(c.OutputFileNameSimOrders, func(w io.Writer) error {
		logger := rec.NewSimLogger(w)
		logger.SetOutputMode(rec.EfhLoggerOutputOrders)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameSimQuotes, func(w io.Writer) error {
		logger := rec.NewSimLogger(w)
		logger.SetOutputMode(rec.EfhLoggerOutputQuotes)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameEfhOrders, func(w io.Writer) error {
		logger := rec.NewEfhLogger(rec.NewTestefhPrinter(w))
		logger.SetOutputMode(rec.EfhLoggerOutputOrders)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameEfhQuotes, func(w io.Writer) error {
		logger := rec.NewEfhLogger(rec.NewTestefhPrinter(w))
		logger.SetOutputMode(rec.EfhLoggerOutputQuotes)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameAvt, func(w io.Writer) (err error) {
		var dict io.ReadCloser
		if c.InputFileNameAvtDict != "" {
			if dict, err = os.Open(c.InputFileNameAvtDict); err != nil {
				return
			} else {
				c.outFiles = append(c.outFiles, dict)
			}
		}
		return efh.AddLogger(rec.NewAvtLogger(w, dict))
	})

	reporter := c.addAnalyzer(efh)
	errs.CheckE(efh.AnalyzeInput())
	if reporter != nil {
		reporter.SaveAll()
	}
}

func (c *cmdEfhsim) addOut(fileName string, setOut func(io.Writer) error) {
	if fileName == "" {
		return
	}
	file, err := os.Create(fileName)
	if err != nil {
		log.Fatalln(err)
	}
	if err := setOut(file); err != nil {
		file.Close()
		log.Fatalln(err)
	}
	c.outFiles = append(c.outFiles, file)
}
func (c *cmdEfhsim) addAnalyzer(efh *efhsim.EfhSim) *anal.Reporter {
	if c.OutputDirStats == "" {
		return nil
	}
	hashFn := func(v uint64) uint64 {
		data := make([]byte, 8)
		binary.BigEndian.PutUint64(data, v)
		h := crc32.ChecksumIEEE(data)
		return uint64(h & (1<<24 - 1))
	}
	moduloFn := func(v uint64) uint64 {
		return v & (1<<24 - 1)
	}
	analyzer := anal.NewAnalyzer()
	analyzer.AddOrderHashFunction(hashFn)
	analyzer.AddOrderHashFunction(moduloFn)
	efh.AddLogger(analyzer.Observer())
	reporter := anal.NewReporter()
	reporter.SetAnalyzer(analyzer)
	reporter.SetOutputDir(c.OutputDirStats)
	return reporter
}

func init() {
	var c cmdEfhsim
	Registry.Register(&c)
}
<|endoftext|>


<commit_before>package alidayu

import (
	"testing"
)

// 构建回调接口
type CallBack struct {
	t *testing.T
}

func (cb *CallBack) CALLBACK_Check(msg interface{}) bool {
	//cb.t.Logf("CALLBACK_Check: %#v", msg)
	return true
}

func (cb *CallBack) CALLBACK_Request(msg interface{}) bool {
	//cb.t.Logf("CALLBACK_Request: %#v", msg)
	return true
}

func (cb *CallBack) CALLBACK_Response(msg interface{}, ok bool, result string) {
	cb.t.Logf("CALLBACK_Response: %#v", msg)
	cb.t.Logf("CALLBACK_Response: %v", ok)
	cb.t.Logf("CALLBACK_Response: %v", result)
}

func (cb *CallBack) CALLBACK_Close() {
	cb.t.Log("CALLBACK_Close")
}
<commit_msg>style: codes<commit_after>package alidayu

import "testing"

// 构建回调接口
type CallBack struct {
	t *testing.T
}

func (cb *CallBack) CALLBACK_Check(msg interface{}) bool {
	//cb.t.Logf("CALLBACK_Check: %#v", msg)
	return true
}

func (cb *CallBack) CALLBACK_Request(msg interface{}) bool {
	//cb.t.Logf("CALLBACK_Request: %#v", msg)
	return true
}

func (cb *CallBack) CALLBACK_Response(msg interface{}, ok bool, result string) {
	cb.t.Logf("CALLBACK_Response: Msg > %#v", msg)
	cb.t.Logf("CALLBACK_Response: OK > %v", ok)
	cb.t.Logf("CALLBACK_Response: Result > %v", result)
}

func (cb *CallBack) CALLBACK_Close() {
	cb.t.Log("CALLBACK_Close")
}
<|endoftext|>