<commit_before>package conn

import (
	"errors"
	"sync"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
	ctxc "github.com/jbenet/go-ipfs/util/ctxcloser"
)

// MultiConnMap is for shorthand
type MultiConnMap map[u.Key]*MultiConn

// MultiConn represents a single connection to another Peer (IPFS Node).
type MultiConn struct {

	// connections, mapped by a string, which uniquely identifies the connection.
	// this string is:  /addr1/peer1/addr2/peer2 (peers ordered lexicographically)
	conns map[string]Conn

	local  peer.Peer
	remote peer.Peer

	// fan-in
	fanIn chan []byte

	// for adding/removing connections concurrently
	sync.RWMutex
	ctxc.ContextCloser
}

// NewMultiConn constructs a new connection
func NewMultiConn(ctx context.Context, local, remote peer.Peer, conns []Conn) (*MultiConn, error) {

	c := &MultiConn{
		local:  local,
		remote: remote,
		conns:  map[string]Conn{},
		fanIn:  make(chan []byte),
	}

	// must happen before Adds / fanOut
	c.ContextCloser = ctxc.NewContextCloser(ctx, c.close)

	if conns != nil && len(conns) > 0 {
		c.Add(conns...)
	}

	return c, nil
}

// Add adds given Conn instances to multiconn.
func (c *MultiConn) Add(conns ...Conn) {
	c.Lock()
	defer c.Unlock()

	for _, c2 := range conns {
		log.Debugf("MultiConn: adding %s", c2)
		if c.LocalPeer() != c2.LocalPeer() || c.RemotePeer() != c2.RemotePeer() {
			log.Error(c2)
			c.Unlock() // ok to unlock (to log). panicing.
			log.Error(c)
			// log.Errorf("c.LocalPeer: %s %p", c.LocalPeer(), c.LocalPeer())
			// log.Errorf("c2.LocalPeer: %s %p", c2.LocalPeer(), c2.LocalPeer())
			// log.Errorf("c.RemotePeer: %s %p", c.RemotePeer(), c.RemotePeer())
			// log.Errorf("c2.RemotePeer: %s %p", c2.RemotePeer(), c2.RemotePeer())
			c.Lock() // gotta relock to avoid lock panic from deferring.
			panic("connection addresses mismatch")
		}

		c.conns[c2.ID()] = c2
		c.Children().Add(1)
		c2.Children().Add(1) // yep, on the child too.
		go c.fanInSingle(c2)
		log.Debugf("MultiConn: added %s", c2)
	}
}

// Remove removes given Conn instances from multiconn.
func (c *MultiConn) Remove(conns ...Conn) {

	// first remove them to avoid sending any more messages through it.
	{
		c.Lock()
		for _, c1 := range conns {
			c2, found := c.conns[c1.ID()]
			if !found {
				panic("Conn not in MultiConn")
			}
			if c1 != c2 {
				panic("different Conn objects for same id.")
			}

			delete(c.conns, c2.ID())
		}
		c.Unlock()
	}

	// close all in parallel, but wait for all to be done closing.
	CloseConns(conns...)
}

// CloseConns closes multiple connections in parallel, and waits for all
// to finish closing.
func CloseConns(conns ...Conn) {
	var wg sync.WaitGroup
	for _, child := range conns {

		select {
		case <-child.Closed(): // if already closed, continue
			continue
		default:
		}

		wg.Add(1)
		go func(child Conn) {
			child.Close()
			wg.Done()
		}(child)
	}
	wg.Wait()
}

// fanInSingle Reads from a connection, and sends to the fanIn.
// waits for child to close and reclaims resources
func (c *MultiConn) fanInSingle(child Conn) {
	// cleanup all data associated with this child Connection.
	defer func() {
		log.Debugf("closing: %s", child)

		// in case it still is in the map, remove it.
		c.Lock()
		delete(c.conns, child.ID())
		connLen := len(c.conns)
		c.Unlock()

		c.Children().Done()
		child.Children().Done()

		if connLen == 0 {
			c.Close() // close self if all underlying children are gone?
		}
	}()

	for {
		msg, err := child.ReadMsg()
		if err != nil {
			log.Warning(err)
			return
		}

		select {
		case <-c.Closing(): // multiconn closing
			return

		case <-child.Closing(): // child closing
			return

		case c.fanIn <- msg:
		}
	}
}

// close is the internal close function, called by ContextCloser.Close
func (c *MultiConn) close() error {
	log.Debugf("%s closing Conn with %s", c.local, c.remote)

	// get connections
	c.RLock()
	conns := make([]Conn, 0, len(c.conns))
	for _, c := range c.conns {
		conns = append(conns, c)
	}
	c.RUnlock()

	// close underlying connections
	CloseConns(conns...)
	return nil
}

// BestConn is the best connection in this MultiConn
func (c *MultiConn) BestConn() Conn {
	c.RLock()
	defer c.RUnlock()

	var id1 string
	var c1 Conn
	for id2, c2 := range c.conns {
		if id1 == "" || id1 < id2 {
			id1 = id2
			c1 = c2
		}
	}
	return c1
}

// ID is an identifier unique to this connection.
// In MultiConn, this is all the children IDs XORed together.
func (c *MultiConn) ID() string {
	c.RLock()
	defer c.RUnlock()

	ids := []byte(nil)
	for i := range c.conns {
		if ids == nil {
			ids = []byte(i)
		} else {
			ids = u.XOR(ids, []byte(i))
		}
	}

	return string(ids)
}

func (c *MultiConn) getConns() []Conn {
	c.RLock()
	defer c.RUnlock()
	var conns []Conn
	for _, c := range c.conns {
		conns = append(conns, c)
	}
	return conns
}

func (c *MultiConn) String() string {
	return String(c, "MultiConn")
}

// LocalMultiaddr is the Multiaddr on this side
func (c *MultiConn) LocalMultiaddr() ma.Multiaddr {
	return c.BestConn().LocalMultiaddr()
}

// RemoteMultiaddr is the Multiaddr on the remote side
func (c *MultiConn) RemoteMultiaddr() ma.Multiaddr {
	return c.BestConn().RemoteMultiaddr()
}

// LocalPeer is the Peer on this side
func (c *MultiConn) LocalPeer() peer.Peer {
	return c.local
}

// RemotePeer is the Peer on the remote side
func (c *MultiConn) RemotePeer() peer.Peer {
	return c.remote
}

// Read reads data, net.Conn style
func (c *MultiConn) Read(buf []byte) (int, error) {
	return 0, errors.New("multiconn does not support Read. use ReadMsg")
}

// Write writes data, net.Conn style
func (c *MultiConn) Write(buf []byte) (int, error) {
	bc := c.BestConn()
	if bc == nil {
		return 0, errors.New("no best connection")
	}
	return bc.Write(buf)
}

// ReadMsg reads data, net.Conn style
func (c *MultiConn) ReadMsg() ([]byte, error) {
	next := <-c.fanIn
	return next, nil
}

// WriteMsg writes data, net.Conn style
func (c *MultiConn) WriteMsg(buf []byte) error {
	bc := c.BestConn()
	if bc == nil {
		return errors.New("no best connection")
	}
	return bc.WriteMsg(buf)
}

// ReleaseMsg releases a buffer
func (c *MultiConn) ReleaseMsg(m []byte) {
	for _, c := range c.getConns() {
		c.ReleaseMsg(m)
	}
}
<commit_msg>multiconn: close fanIn + error<commit_after>package conn

import (
	"errors"
	"fmt"
	"sync"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
	ctxc "github.com/jbenet/go-ipfs/util/ctxcloser"
)

// MultiConnMap is for shorthand
type MultiConnMap map[u.Key]*MultiConn

// MultiConn represents a single connection to another Peer (IPFS Node).
type MultiConn struct {

	// connections, mapped by a string, which uniquely identifies the connection.
	// this string is:  /addr1/peer1/addr2/peer2 (peers ordered lexicographically)
	conns map[string]Conn

	local  peer.Peer
	remote peer.Peer

	// fan-in
	fanIn chan []byte

	// for adding/removing connections concurrently
	sync.RWMutex
	ctxc.ContextCloser
}

// NewMultiConn constructs a new connection
func NewMultiConn(ctx context.Context, local, remote peer.Peer, conns []Conn) (*MultiConn, error) {

	c := &MultiConn{
		local:  local,
		remote: remote,
		conns:  map[string]Conn{},
		fanIn:  make(chan []byte),
	}

	// must happen before Adds / fanOut
	c.ContextCloser = ctxc.NewContextCloser(ctx, c.close)

	if conns != nil && len(conns) > 0 {
		c.Add(conns...)
	}

	return c, nil
}

// Add adds given Conn instances to multiconn.
func (c *MultiConn) Add(conns ...Conn) {
	c.Lock()
	defer c.Unlock()

	for _, c2 := range conns {
		log.Debugf("MultiConn: adding %s", c2)
		if c.LocalPeer() != c2.LocalPeer() || c.RemotePeer() != c2.RemotePeer() {
			log.Error(c2)
			c.Unlock() // ok to unlock (to log). panicing.
			log.Error(c)
			// log.Errorf("c.LocalPeer: %s %p", c.LocalPeer(), c.LocalPeer())
			// log.Errorf("c2.LocalPeer: %s %p", c2.LocalPeer(), c2.LocalPeer())
			// log.Errorf("c.RemotePeer: %s %p", c.RemotePeer(), c.RemotePeer())
			// log.Errorf("c2.RemotePeer: %s %p", c2.RemotePeer(), c2.RemotePeer())
			c.Lock() // gotta relock to avoid lock panic from deferring.
			panic("connection addresses mismatch")
		}

		c.conns[c2.ID()] = c2
		c.Children().Add(1)
		c2.Children().Add(1) // yep, on the child too.
		go c.fanInSingle(c2)
		log.Debugf("MultiConn: added %s", c2)
	}
}

// Remove removes given Conn instances from multiconn.
func (c *MultiConn) Remove(conns ...Conn) {

	// first remove them to avoid sending any more messages through it.
	{
		c.Lock()
		for _, c1 := range conns {
			c2, found := c.conns[c1.ID()]
			if !found {
				panic("Conn not in MultiConn")
			}
			if c1 != c2 {
				panic("different Conn objects for same id.")
			}

			delete(c.conns, c2.ID())
		}
		c.Unlock()
	}

	// close all in parallel, but wait for all to be done closing.
	CloseConns(conns...)
}

// CloseConns closes multiple connections in parallel, and waits for all
// to finish closing.
func CloseConns(conns ...Conn) {
	var wg sync.WaitGroup
	for _, child := range conns {

		select {
		case <-child.Closed(): // if already closed, continue
			continue
		default:
		}

		wg.Add(1)
		go func(child Conn) {
			child.Close()
			wg.Done()
		}(child)
	}
	wg.Wait()
}

// fanInSingle Reads from a connection, and sends to the fanIn.
// waits for child to close and reclaims resources
func (c *MultiConn) fanInSingle(child Conn) {
	// cleanup all data associated with this child Connection.
	defer func() {
		log.Debugf("closing: %s", child)

		// in case it still is in the map, remove it.
		c.Lock()
		delete(c.conns, child.ID())
		connLen := len(c.conns)
		c.Unlock()

		c.Children().Done()
		child.Children().Done()

		if connLen == 0 {
			c.Close() // close self if all underlying children are gone?
		}
	}()

	for {
		msg, err := child.ReadMsg()
		if err != nil {
			log.Warning(err)
			return
		}

		select {
		case <-c.Closing(): // multiconn closing
			return

		case <-child.Closing(): // child closing
			return

		case c.fanIn <- msg:
		}
	}
}

// close is the internal close function, called by ContextCloser.Close
func (c *MultiConn) close() error {
	log.Debugf("%s closing Conn with %s", c.local, c.remote)

	// get connections
	c.RLock()
	conns := make([]Conn, 0, len(c.conns))
	for _, c := range c.conns {
		conns = append(conns, c)
	}
	c.RUnlock()

	// close underlying connections
	CloseConns(conns...)
	close(c.fanIn)
	return nil
}

// BestConn is the best connection in this MultiConn
func (c *MultiConn) BestConn() Conn {
	c.RLock()
	defer c.RUnlock()

	var id1 string
	var c1 Conn
	for id2, c2 := range c.conns {
		if id1 == "" || id1 < id2 {
			id1 = id2
			c1 = c2
		}
	}
	return c1
}

// ID is an identifier unique to this connection.
// In MultiConn, this is all the children IDs XORed together.
func (c *MultiConn) ID() string {
	c.RLock()
	defer c.RUnlock()

	ids := []byte(nil)
	for i := range c.conns {
		if ids == nil {
			ids = []byte(i)
		} else {
			ids = u.XOR(ids, []byte(i))
		}
	}

	return string(ids)
}

func (c *MultiConn) getConns() []Conn {
	c.RLock()
	defer c.RUnlock()
	var conns []Conn
	for _, c := range c.conns {
		conns = append(conns, c)
	}
	return conns
}

func (c *MultiConn) String() string {
	return String(c, "MultiConn")
}

// LocalMultiaddr is the Multiaddr on this side
func (c *MultiConn) LocalMultiaddr() ma.Multiaddr {
	return c.BestConn().LocalMultiaddr()
}

// RemoteMultiaddr is the Multiaddr on the remote side
func (c *MultiConn) RemoteMultiaddr() ma.Multiaddr {
	return c.BestConn().RemoteMultiaddr()
}

// LocalPeer is the Peer on this side
func (c *MultiConn) LocalPeer() peer.Peer {
	return c.local
}

// RemotePeer is the Peer on the remote side
func (c *MultiConn) RemotePeer() peer.Peer {
	return c.remote
}

// Read reads data, net.Conn style
func (c *MultiConn) Read(buf []byte) (int, error) {
	return 0, errors.New("multiconn does not support Read. use ReadMsg")
}

// Write writes data, net.Conn style
func (c *MultiConn) Write(buf []byte) (int, error) {
	bc := c.BestConn()
	if bc == nil {
		return 0, errors.New("no best connection")
	}
	return bc.Write(buf)
}

// ReadMsg reads data, net.Conn style
func (c *MultiConn) ReadMsg() ([]byte, error) {
	next, ok := <-c.fanIn
	if !ok {
		return nil, fmt.Errorf("multiconn closed")
	}
	return next, nil
}

// WriteMsg writes data, net.Conn style
func (c *MultiConn) WriteMsg(buf []byte) error {
	bc := c.BestConn()
	if bc == nil {
		return errors.New("no best connection")
	}
	return bc.WriteMsg(buf)
}

// ReleaseMsg releases a buffer
func (c *MultiConn) ReleaseMsg(m []byte) {
	// here, we dont know where it came from. hm.
	for _, c := range c.getConns() {
		c.ReleaseMsg(m)
	}
}
<|endoftext|>


<commit_before>package dockerutil

import (
	"fmt"
	"path/filepath"
	"strings"

	digest "github.com/opencontainers/go-digest"
)

type ConfigType struct {
	MediaType string
	Size      int
	Digest    string
}

type Layer struct {
	MediaType string
	Size      int
	Digest    string
}

type Manifest struct {
	SchemaVersion int
	MediaType     string
	Config        ConfigType
	Layers        []Layer
}

type ThinImageLayer struct {
	Digest string `json:"digest"`
	Url    string `json:"url,omitempty"`
}

type ThinImage struct {
	Version    string           `json:"version"`
	MinVersion string           `json:"min_version,omitempty"`
	Origin     string           `json:"origin,omitempty"`
	Layers     []ThinImageLayer `json:"layers"`
	Comment    string           `json:"comment,omitempty"`
}

var thinImageVersion = "1.0"

// m is the manifest of the original image
// repoLocation is where inside the repo we saved the several layers
// origin is an ecoding fo the original referencese and original registry
// I believe origin is quite useless but maybe is better to preserv it for
// ergonomic reasons.
func MakeThinImage(m Manifest, layersMapping map[string]string, origin string) (ThinImage, error) {
	layers := make([]ThinImageLayer, len(m.Layers))

	url_base := "cvmfs://"

	for i, layer := range m.Layers {
		digest := strings.Split(layer.Digest, ":")[1]
		location, ok := layersMapping[layer.Digest]
		if !ok {
			err := fmt.Errorf("Impossible to create thin image, missing layer")
			return ThinImage{}, err
		}
		// the location comes as /cvmfs/$reponame/$path
		// we need to remove the /cvmfs/ part, which are 7 chars
		url := url_base + location[7:]
		layers[i] = ThinImageLayer{Digest: digest, Url: url}
	}

	return ThinImage{Layers: layers,
		Origin:  origin,
		Version: thinImageVersion}, nil
}

func (m Manifest) GetSingularityPath() string {
	digest := strings.Split(m.Config.Digest, ":")[1]
	return filepath.Join(".flat", digest[0:2], digest)
}

// please note how we use the simple digest from the layers, it is not
// striclty correct, since we would need the digest of the uncompressed
// layer, that can be found in the Config file of the image.
// For our purposes, however, this is good enough.
func (m Manifest) GetChainIDs() []digest.Digest {
	result := make([]digest.Digest, len(m.Layers))
	for i, l := range m.Layers {
		if i == 0 {
			result = append(result, digest.Digest(l.Digest))
			continue
		}
		digest := digest.FromString(result[i-1].String() + " " + l.Digest)
		result = append(result, digest)
	}
	return result
}
<commit_msg>fix implementation of computing chainIds<commit_after>package dockerutil

import (
	"fmt"
	"path/filepath"
	"strings"

	digest "github.com/opencontainers/go-digest"
)

type ConfigType struct {
	MediaType string
	Size      int
	Digest    string
}

type Layer struct {
	MediaType string
	Size      int
	Digest    string
}

type Manifest struct {
	SchemaVersion int
	MediaType     string
	Config        ConfigType
	Layers        []Layer
}

type ThinImageLayer struct {
	Digest string `json:"digest"`
	Url    string `json:"url,omitempty"`
}

type ThinImage struct {
	Version    string           `json:"version"`
	MinVersion string           `json:"min_version,omitempty"`
	Origin     string           `json:"origin,omitempty"`
	Layers     []ThinImageLayer `json:"layers"`
	Comment    string           `json:"comment,omitempty"`
}

var thinImageVersion = "1.0"

// m is the manifest of the original image
// repoLocation is where inside the repo we saved the several layers
// origin is an ecoding fo the original referencese and original registry
// I believe origin is quite useless but maybe is better to preserv it for
// ergonomic reasons.
func MakeThinImage(m Manifest, layersMapping map[string]string, origin string) (ThinImage, error) {
	layers := make([]ThinImageLayer, len(m.Layers))

	url_base := "cvmfs://"

	for i, layer := range m.Layers {
		digest := strings.Split(layer.Digest, ":")[1]
		location, ok := layersMapping[layer.Digest]
		if !ok {
			err := fmt.Errorf("Impossible to create thin image, missing layer")
			return ThinImage{}, err
		}
		// the location comes as /cvmfs/$reponame/$path
		// we need to remove the /cvmfs/ part, which are 7 chars
		url := url_base + location[7:]
		layers[i] = ThinImageLayer{Digest: digest, Url: url}
	}

	return ThinImage{Layers: layers,
		Origin:  origin,
		Version: thinImageVersion}, nil
}

func (m Manifest) GetSingularityPath() string {
	digest := strings.Split(m.Config.Digest, ":")[1]
	return filepath.Join(".flat", digest[0:2], digest)
}

// please note how we use the simple digest from the layers, it is not
// striclty correct, since we would need the digest of the uncompressed
// layer, that can be found in the Config file of the image.
// For our purposes, however, this is good enough.
func (m Manifest) GetChainIDs() []digest.Digest {
	result := []digest.Digest{}
	for i, l := range m.Layers {
		if i == 0 {
			d := digest.FromString(l.Digest)
			result = append(result, d)
			continue
		}
		digest := digest.FromString(result[i-1].String() + " " + l.Digest)
		result = append(result, digest)
	}
	return result
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"testing"
)

func TestBufReader(t *testing.T) {
	reader, writer := io.Pipe()
	bufreader := NewBufReader(reader)

	// Write everything down to a Pipe
	// Usually, a pipe should block but because of the buffered reader,
	// the writes will go through
	done := make(chan bool)
	go func() {
		writer.Write([]byte("hello world"))
		writer.Close()
		done <- true
	}()

	// Drain the reader *after* everything has been written, just to verify
	// it is indeed buffering
	<-done
	output, err := ioutil.ReadAll(bufreader)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(output, []byte("hello world")) {
		t.Error(string(output))
	}
}

type dummyWriter struct {
	buffer      bytes.Buffer
	failOnWrite bool
}

func (dw *dummyWriter) Write(p []byte) (n int, err error) {
	if dw.failOnWrite {
		return 0, errors.New("Fake fail")
	}
	return dw.buffer.Write(p)
}

func (dw *dummyWriter) String() string {
	return dw.buffer.String()
}

func (dw *dummyWriter) Close() error {
	return nil
}

func TestWriteBroadcaster(t *testing.T) {
	writer := NewWriteBroadcaster()

	// Test 1: Both bufferA and bufferB should contain "foo"
	bufferA := &dummyWriter{}
	writer.AddWriter(bufferA)
	bufferB := &dummyWriter{}
	writer.AddWriter(bufferB)
	writer.Write([]byte("foo"))

	if bufferA.String() != "foo" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}

	if bufferB.String() != "foo" {
		t.Errorf("Buffer contains %v", bufferB.String())
	}

	// Test2: bufferA and bufferB should contain "foobar",
	// while bufferC should only contain "bar"
	bufferC := &dummyWriter{}
	writer.AddWriter(bufferC)
	writer.Write([]byte("bar"))

	if bufferA.String() != "foobar" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}

	if bufferB.String() != "foobar" {
		t.Errorf("Buffer contains %v", bufferB.String())
	}

	if bufferC.String() != "bar" {
		t.Errorf("Buffer contains %v", bufferC.String())
	}

	// Test3: Test removal
	writer.RemoveWriter(bufferB)
	writer.Write([]byte("42"))
	if bufferA.String() != "foobar42" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}
	if bufferB.String() != "foobar" {
		t.Errorf("Buffer contains %v", bufferB.String())
	}
	if bufferC.String() != "bar42" {
		t.Errorf("Buffer contains %v", bufferC.String())
	}

	// Test4: Test eviction on failure
	bufferA.failOnWrite = true
	writer.Write([]byte("fail"))
	if bufferA.String() != "foobar42" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}
	if bufferC.String() != "bar42fail" {
		t.Errorf("Buffer contains %v", bufferC.String())
	}
	// Even though we reset the flag, no more writes should go in there
	bufferA.failOnWrite = false
	writer.Write([]byte("test"))
	if bufferA.String() != "foobar42" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}
	if bufferC.String() != "bar42failtest" {
		t.Errorf("Buffer contains %v", bufferC.String())
	}

	writer.CloseWriters()
}

type devNullCloser int

func (d devNullCloser) Close() error {
	return nil
}

func (d devNullCloser) Write(buf []byte) (int, error) {
	return len(buf), nil
}

// This test checks for races. It is only useful when run with the race detector.
func TestRaceWriteBroadcaster(t *testing.T) {
	writer := NewWriteBroadcaster()
	c := make(chan bool)
	go func() {
		writer.AddWriter(devNullCloser(0))
		c <- true
	}()
	writer.Write([]byte("hello"))
	<-c
}

// Test the behavior of TruncIndex, an index for querying IDs from a non-conflicting prefix.
func TestTruncIndex(t *testing.T) {
	index := NewTruncIndex()
	// Get on an empty index
	if _, err := index.Get("foobar"); err == nil {
		t.Fatal("Get on an empty index should return an error")
	}

	// Spaces should be illegal in an id
	if err := index.Add("I have a space"); err == nil {
		t.Fatalf("Adding an id with ' ' should return an error")
	}

	id := "99b36c2c326ccc11e726eee6ee78a0baf166ef96"
	// Add an id
	if err := index.Add(id); err != nil {
		t.Fatal(err)
	}
	// Get a non-existing id
	assertIndexGet(t, index, "abracadabra", "", true)
	// Get the exact id
	assertIndexGet(t, index, id, id, false)
	// The first letter should match
	assertIndexGet(t, index, id[:1], id, false)
	// The first half should match
	assertIndexGet(t, index, id[:len(id)/2], id, false)
	// The second half should NOT match
	assertIndexGet(t, index, id[len(id)/2:], "", true)

	id2 := id[:6] + "blabla"
	// Add an id
	if err := index.Add(id2); err != nil {
		t.Fatal(err)
	}
	// Both exact IDs should work
	assertIndexGet(t, index, id, id, false)
	assertIndexGet(t, index, id2, id2, false)

	// 6 characters or less should conflict
	assertIndexGet(t, index, id[:6], "", true)
	assertIndexGet(t, index, id[:4], "", true)
	assertIndexGet(t, index, id[:1], "", true)

	// 7 characters should NOT conflict
	assertIndexGet(t, index, id[:7], id, false)
	assertIndexGet(t, index, id2[:7], id2, false)

	// Deleting a non-existing id should return an error
	if err := index.Delete("non-existing"); err == nil {
		t.Fatalf("Deleting a non-existing id should return an error")
	}

	// Deleting id2 should remove conflicts
	if err := index.Delete(id2); err != nil {
		t.Fatal(err)
	}
	// id2 should no longer work
	assertIndexGet(t, index, id2, "", true)
	assertIndexGet(t, index, id2[:7], "", true)
	assertIndexGet(t, index, id2[:11], "", true)

	// conflicts between id and id2 should be gone
	assertIndexGet(t, index, id[:6], id, false)
	assertIndexGet(t, index, id[:4], id, false)
	assertIndexGet(t, index, id[:1], id, false)

	// non-conflicting substrings should still not conflict
	assertIndexGet(t, index, id[:7], id, false)
	assertIndexGet(t, index, id[:15], id, false)
	assertIndexGet(t, index, id, id, false)
}

func assertIndexGet(t *testing.T, index *TruncIndex, input, expectedResult string, expectError bool) {
	if result, err := index.Get(input); err != nil && !expectError {
		t.Fatalf("Unexpected error getting '%s': %s", input, err)
	} else if err == nil && expectError {
		t.Fatalf("Getting '%s' should return an error", input)
	} else if result != expectedResult {
		t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult)
	}
}

func assertKernelVersion(t *testing.T, a, b *KernelVersionInfo, result int) {
	if r := CompareKernelVersion(a, b); r != result {
		t.Fatalf("Unepected kernel version comparaison result. Found %d, expected %d", r, result)
	}
}

func TestCompareKernelVersion(t *testing.T) {
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		0)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		-1)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		&KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0},
		1)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "0"},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "16"},
		0)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 5},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		1)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 0, Minor: 20, Flavor: "25"},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "0"},
		-1)
}

func TestHumanSize(t *testing.T) {

	size1000 := HumanSize(1000)
	if size1000 != "1 kB" {
		t.Errorf("1000 -> expected 1 kB, got %s", size1000)
	}

	size1024 := HumanSize(1024)
	if size1024 != "1.024 kB" {
		t.Errorf("1024 -> expected 1.024 kB, got %s", size1024)
	}
}
<commit_msg>add tests<commit_after>package utils

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"testing"
)

func TestBufReader(t *testing.T) {
	reader, writer := io.Pipe()
	bufreader := NewBufReader(reader)

	// Write everything down to a Pipe
	// Usually, a pipe should block but because of the buffered reader,
	// the writes will go through
	done := make(chan bool)
	go func() {
		writer.Write([]byte("hello world"))
		writer.Close()
		done <- true
	}()

	// Drain the reader *after* everything has been written, just to verify
	// it is indeed buffering
	<-done
	output, err := ioutil.ReadAll(bufreader)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(output, []byte("hello world")) {
		t.Error(string(output))
	}
}

type dummyWriter struct {
	buffer      bytes.Buffer
	failOnWrite bool
}

func (dw *dummyWriter) Write(p []byte) (n int, err error) {
	if dw.failOnWrite {
		return 0, errors.New("Fake fail")
	}
	return dw.buffer.Write(p)
}

func (dw *dummyWriter) String() string {
	return dw.buffer.String()
}

func (dw *dummyWriter) Close() error {
	return nil
}

func TestWriteBroadcaster(t *testing.T) {
	writer := NewWriteBroadcaster()

	// Test 1: Both bufferA and bufferB should contain "foo"
	bufferA := &dummyWriter{}
	writer.AddWriter(bufferA)
	bufferB := &dummyWriter{}
	writer.AddWriter(bufferB)
	writer.Write([]byte("foo"))

	if bufferA.String() != "foo" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}

	if bufferB.String() != "foo" {
		t.Errorf("Buffer contains %v", bufferB.String())
	}

	// Test2: bufferA and bufferB should contain "foobar",
	// while bufferC should only contain "bar"
	bufferC := &dummyWriter{}
	writer.AddWriter(bufferC)
	writer.Write([]byte("bar"))

	if bufferA.String() != "foobar" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}

	if bufferB.String() != "foobar" {
		t.Errorf("Buffer contains %v", bufferB.String())
	}

	if bufferC.String() != "bar" {
		t.Errorf("Buffer contains %v", bufferC.String())
	}

	// Test3: Test removal
	writer.RemoveWriter(bufferB)
	writer.Write([]byte("42"))
	if bufferA.String() != "foobar42" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}
	if bufferB.String() != "foobar" {
		t.Errorf("Buffer contains %v", bufferB.String())
	}
	if bufferC.String() != "bar42" {
		t.Errorf("Buffer contains %v", bufferC.String())
	}

	// Test4: Test eviction on failure
	bufferA.failOnWrite = true
	writer.Write([]byte("fail"))
	if bufferA.String() != "foobar42" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}
	if bufferC.String() != "bar42fail" {
		t.Errorf("Buffer contains %v", bufferC.String())
	}
	// Even though we reset the flag, no more writes should go in there
	bufferA.failOnWrite = false
	writer.Write([]byte("test"))
	if bufferA.String() != "foobar42" {
		t.Errorf("Buffer contains %v", bufferA.String())
	}
	if bufferC.String() != "bar42failtest" {
		t.Errorf("Buffer contains %v", bufferC.String())
	}

	writer.CloseWriters()
}

type devNullCloser int

func (d devNullCloser) Close() error {
	return nil
}

func (d devNullCloser) Write(buf []byte) (int, error) {
	return len(buf), nil
}

// This test checks for races. It is only useful when run with the race detector.
func TestRaceWriteBroadcaster(t *testing.T) {
	writer := NewWriteBroadcaster()
	c := make(chan bool)
	go func() {
		writer.AddWriter(devNullCloser(0))
		c <- true
	}()
	writer.Write([]byte("hello"))
	<-c
}

// Test the behavior of TruncIndex, an index for querying IDs from a non-conflicting prefix.
func TestTruncIndex(t *testing.T) {
	index := NewTruncIndex()
	// Get on an empty index
	if _, err := index.Get("foobar"); err == nil {
		t.Fatal("Get on an empty index should return an error")
	}

	// Spaces should be illegal in an id
	if err := index.Add("I have a space"); err == nil {
		t.Fatalf("Adding an id with ' ' should return an error")
	}

	id := "99b36c2c326ccc11e726eee6ee78a0baf166ef96"
	// Add an id
	if err := index.Add(id); err != nil {
		t.Fatal(err)
	}
	// Get a non-existing id
	assertIndexGet(t, index, "abracadabra", "", true)
	// Get the exact id
	assertIndexGet(t, index, id, id, false)
	// The first letter should match
	assertIndexGet(t, index, id[:1], id, false)
	// The first half should match
	assertIndexGet(t, index, id[:len(id)/2], id, false)
	// The second half should NOT match
	assertIndexGet(t, index, id[len(id)/2:], "", true)

	id2 := id[:6] + "blabla"
	// Add an id
	if err := index.Add(id2); err != nil {
		t.Fatal(err)
	}
	// Both exact IDs should work
	assertIndexGet(t, index, id, id, false)
	assertIndexGet(t, index, id2, id2, false)

	// 6 characters or less should conflict
	assertIndexGet(t, index, id[:6], "", true)
	assertIndexGet(t, index, id[:4], "", true)
	assertIndexGet(t, index, id[:1], "", true)

	// 7 characters should NOT conflict
	assertIndexGet(t, index, id[:7], id, false)
	assertIndexGet(t, index, id2[:7], id2, false)

	// Deleting a non-existing id should return an error
	if err := index.Delete("non-existing"); err == nil {
		t.Fatalf("Deleting a non-existing id should return an error")
	}

	// Deleting id2 should remove conflicts
	if err := index.Delete(id2); err != nil {
		t.Fatal(err)
	}
	// id2 should no longer work
	assertIndexGet(t, index, id2, "", true)
	assertIndexGet(t, index, id2[:7], "", true)
	assertIndexGet(t, index, id2[:11], "", true)

	// conflicts between id and id2 should be gone
	assertIndexGet(t, index, id[:6], id, false)
	assertIndexGet(t, index, id[:4], id, false)
	assertIndexGet(t, index, id[:1], id, false)

	// non-conflicting substrings should still not conflict
	assertIndexGet(t, index, id[:7], id, false)
	assertIndexGet(t, index, id[:15], id, false)
	assertIndexGet(t, index, id, id, false)
}

func assertIndexGet(t *testing.T, index *TruncIndex, input, expectedResult string, expectError bool) {
	if result, err := index.Get(input); err != nil && !expectError {
		t.Fatalf("Unexpected error getting '%s': %s", input, err)
	} else if err == nil && expectError {
		t.Fatalf("Getting '%s' should return an error", input)
	} else if result != expectedResult {
		t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult)
	}
}

func assertKernelVersion(t *testing.T, a, b *KernelVersionInfo, result int) {
	if r := CompareKernelVersion(a, b); r != result {
		t.Fatalf("Unepected kernel version comparaison result. Found %d, expected %d", r, result)
	}
}

func TestCompareKernelVersion(t *testing.T) {
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		0)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		-1)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		&KernelVersionInfo{Kernel: 2, Major: 6, Minor: 0},
		1)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "0"},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "16"},
		0)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 5},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0},
		1)
	assertKernelVersion(t,
		&KernelVersionInfo{Kernel: 3, Major: 0, Minor: 20, Flavor: "25"},
		&KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0, Flavor: "0"},
		-1)
}

func TestHumanSize(t *testing.T) {

	size1000 := HumanSize(1000)
	if size1000 != "1 kB" {
		t.Errorf("1000 -> expected 1 kB, got %s", size1000)
	}

	size1024 := HumanSize(1024)
	if size1024 != "1.024 kB" {
		t.Errorf("1024 -> expected 1.024 kB, got %s", size1024)
	}
}

func TestParseHost(t *testing.T) {
	if addr := ParseHost("127.0.0.1", 4243, "0.0.0.0"); addr != "tcp://0.0.0.0:4243" {
		t.Errorf("0.0.0.0 -> expected tcp://0.0.0.0:4243, got %s", addr)
	}
	if addr := ParseHost("127.0.0.1", 4243, "0.0.0.1:5555"); addr != "tcp://0.0.0.1:5555" {
		t.Errorf("0.0.0.1:5555 -> expected tcp://0.0.0.1:5555, got %s", addr)
	}
	if addr := ParseHost("127.0.0.1", 4243, ":6666"); addr != "tcp://127.0.0.1:6666" {
		t.Errorf(":6666 -> expected tcp://127.0.0.1:6666, got %s", addr)
	}
	if addr := ParseHost("127.0.0.1", 4243, "tcp://:7777"); addr != "tcp://127.0.0.1:7777" {
		t.Errorf("tcp://:7777 -> expected tcp://127.0.0.1:7777, got %s", addr)
	}
	if addr := ParseHost("127.0.0.1", 4243, "unix:///var/run/docker.sock"); addr != "unix:///var/run/docker.sock" {
		t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr)
	}
}
<|endoftext|>


<commit_before>package brokers

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/signatures"

	"github.com/garyburd/redigo/redis"

	"strings"
)

// RedisBroker represents a Redis broker
type RedisBroker struct {
	config   *config.Config
	stopChan chan int
}

// NewRedisBroker creates new RedisBroker instance
func NewRedisBroker(cnf *config.Config) Broker {
	return Broker(&RedisBroker{
		config: cnf,
	})
}

// StartConsuming enters a loop and waits for incoming messages
func (redisBroker *RedisBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
	conn, err := openRedisConn(redisBroker.config)
	if err != nil {
		return true, err // retry true
	}

	defer conn.Close()

	redisBroker.stopChan = make(chan int)

	psc := redis.PubSubConn{Conn: conn}
	if err := psc.Subscribe(redisBroker.config.DefaultQueue); err != nil {
		return true, err // retry true
	}
	// Unsubscribe from all connections. This will cause the receiving
	// goroutine to exit.
	defer psc.Unsubscribe()

	deliveries := make(chan signatures.TaskSignature)
	errors := make(chan error)

	log.Print("[*] Waiting for messages. To exit press CTRL+C")

	go func() {
		for {
			switch n := psc.Receive().(type) {
			case redis.Message:
				log.Printf("Received new message: %s", n.Data)

				signature := signatures.TaskSignature{}
				if err := json.Unmarshal(n.Data, &signature); err != nil {
					errors <- err
					continue
				}

				deliveries <- signature
			case redis.Subscription:
				if n.Count == 0 {
					return
				}
			case error:
				log.Print(n)
			}
		}
	}()

	for {
		select {
		case signature := <-deliveries:
			taskProcessor.Process(&signature)
		case err := <-errors:
			// Unsubscribe from all connections. This will cause the receiving
			// goroutine to exit.
			psc.Unsubscribe()
			return true, err
		case <-redisBroker.stopChan:
			// Unsubscribe from all connections. This will cause the receiving
			// goroutine to exit.
			psc.Unsubscribe()
			return false, nil
		}
	}
}

// StopConsuming quits the loop
func (redisBroker *RedisBroker) StopConsuming() {
	// Notifying the quit channel stops consuming of messages
	redisBroker.stopChan <- 1
}

// Publish places a new message on the default queue
func (redisBroker *RedisBroker) Publish(signature *signatures.TaskSignature) error {
	conn, err := openRedisConn(redisBroker.config)
	if err != nil {
		return err
	}

	defer conn.Close()

	message, err := json.Marshal(signature)
	if err != nil {
		return fmt.Errorf("JSON Encode Message: %v", err)
	}

	conn.Do("PUBLISH", redisBroker.config.DefaultQueue, message)
	return conn.Flush()
}

func openRedisConn(cnf *config.Config) (redis.Conn, error) {
	network := "tcp"
	address := strings.Split(cnf.Broker, "redis://")[1]
	return redis.Dial(network, address)
}
<commit_msg>Refactoring Redis broker<commit_after>package brokers

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/signatures"

	"github.com/garyburd/redigo/redis"

	"strings"
)

// RedisBroker represents a Redis broker
type RedisBroker struct {
	config   *config.Config
	stopChan chan int
}

// NewRedisBroker creates new RedisBroker instance
func NewRedisBroker(cnf *config.Config) Broker {
	return Broker(&RedisBroker{
		config: cnf,
	})
}

// StartConsuming enters a loop and waits for incoming messages
func (redisBroker *RedisBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
	conn, err := redisBroker.open()
	if err != nil {
		return true, fmt.Errorf("Dial: %s", err) // retry true
	}

	defer redisBroker.closeConn(conn)

	psc := redis.PubSubConn{Conn: conn}
	if err := psc.Subscribe(redisBroker.config.DefaultQueue); err != nil {
		return true, err // retry true
	}
	defer redisBroker.closePubSub(psc)

	redisBroker.stopChan = make(chan int)
	deliveries := make(chan signatures.TaskSignature)
	errors := make(chan error)

	log.Print("[*] Waiting for messages. To exit press CTRL+C")

	// Receving goroutine
	go func() {
		for {
			switch n := psc.Receive().(type) {
			case redis.Message:
				log.Printf("Received new message: %s", n.Data)

				signature := signatures.TaskSignature{}
				if err := json.Unmarshal(n.Data, &signature); err != nil {
					errors <- err
					continue
				}

				deliveries <- signature
			case redis.Subscription:
				if n.Count == 0 {
					return
				}
			case error:
				log.Print(n)
			}
		}
	}()

	// Iterate over delivered tasks and process them
	for {
		select {
		case signature := <-deliveries:
			taskProcessor.Process(&signature)
		case err := <-errors:
			return true, err // retry true
		case <-redisBroker.stopChan:
			return false, nil // retry false
		}
	}
}

// StopConsuming quits the loop
func (redisBroker *RedisBroker) StopConsuming() {
	// Notifying the quit channel stops consuming of messages
	redisBroker.stopChan <- 1
}

// Publish places a new message on the default queue
func (redisBroker *RedisBroker) Publish(signature *signatures.TaskSignature) error {
	conn, err := redisBroker.open()
	if err != nil {
		fmt.Errorf("Dial: %s", err)
	}

	defer redisBroker.closeConn(conn)

	message, err := json.Marshal(signature)
	if err != nil {
		return fmt.Errorf("JSON Encode Message: %v", err)
	}

	conn.Do("PUBLISH", redisBroker.config.DefaultQueue, message)
	return conn.Flush()
}

func (redisBroker *RedisBroker) open() (redis.Conn, error) {
	network := "tcp"
	address := strings.Split(redisBroker.config.Broker, "redis://")[1]
	return redis.Dial(network, address)
}

func (redisBroker *RedisBroker) closeConn(conn redis.Conn) error {
	if err := conn.Close(); err != nil {
		return fmt.Errorf("Connection Close: %s", err)
	}

	return nil
}

func (redisBroker *RedisBroker) closePubSub(psc redis.PubSubConn) error {
	// Unsubscribe from all connections. This will cause the receiving
	// goroutine to exit.
	if err := psc.Unsubscribe(); err != nil {
		return fmt.Errorf("PubSub Unsubscribe: %s", err)
	}

	if err := psc.Close(); err != nil {
		return fmt.Errorf("PubSub Close: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package network

import (
    "fmt"
    "net"
    "bufio"
    "encoding/binary"
    "time"

)

const (
    ConnStateIn    = iota
    ConnStateDisc  = iota

    MAX_LEN_HEAD   = 1024 * 4
)

type ClientConnection struct {
    conn        net.Conn
    reader      *bufio.Reader
    writer      *bufio.Writer
    connState   int
}

func NewClientConnection(c net.Conn) *ClientConnection {
    cliConn := new(ClientConnection)
    cliConn.conn = c
    cliConn.reader = bufio.NewReader(c)
    cliConn.writer = bufio.NewWriter(c)
    return cliConn
}

func (this *ClientConnection) Send(buf []byte) {
    if this.connState == ConnStateDisc { return }

    head := make([]byte, 4)
    binary.LittleEndian.PutUint32(head, uint32(len(buf)))
    buf = append(head, buf...)

    if _, err := this.writer.Write(buf); err != nil {
        fmt.Println("Send err:", err)
    }
}

func (this *ClientConnection) sendall() bool {
    if err := this.writer.Flush(); err != nil {
        if this.connState != ConnStateDisc {
            this.connState = ConnStateDisc
        }
        fmt.Println("send err:", err)
        return false
    }
    return true
}

func (this *ClientConnection) duplexRead(buff []byte) bool {
    var read_size int
    for {
        //// write
        //if !this.sendall() {
        //    return false
        //}

        // read
        this.conn.SetReadDeadline(time.Now().Add(1e8))
        n, err := this.reader.Read(buff[read_size:])
        if err != nil {
            if e, ok := err.(*net.OpError); ok && e.Temporary() {
                read_size = n
                continue
            } else {
                fmt.Println("read err, disconnect", err)
                return false
            }
        }

        if n == 0 { return true }
        if n < len(buff) {
            read_size += n
            continue
        }
        return true
    }
    return false
}

func (this *ClientConnection) duplexReadBody() (ret []byte,  ok bool) {
    buff_head := make([]byte, 4)
    if !this.duplexRead(buff_head) {
        return
    }
    len_head := binary.LittleEndian.Uint32(buff_head)
    if len_head > MAX_LEN_HEAD {
        fmt.Println("message len too long", len_head)
        return
    }
    ret = make([]byte, len_head)
    if !this.duplexRead(ret) {
        return
    }
    ok = true
    return
}

func (this *ClientConnection) Close() {
    this.connState  = ConnStateDisc
    this.conn.Close()
}

<commit_msg>send all issuse<commit_after>package network

import (
    "fmt"
    "net"
    "bufio"
    "encoding/binary"
    "time"

)

const (
    ConnStateIn    = iota
    ConnStateDisc  = iota

    MAX_LEN_HEAD   = 1024 * 4
)

type ClientConnection struct {
    conn        net.Conn
    reader      *bufio.Reader
    writer      *bufio.Writer
    sendchan    chan []byte
    connState   int
}

func NewClientConnection(c net.Conn) *ClientConnection {
    cliConn := new(ClientConnection)
    cliConn.conn = c
    cliConn.reader = bufio.NewReader(c)
    cliConn.writer = bufio.NewWriter(c)
    cliConn.sendchan = make(chan []byte, 64)
    return cliConn
}

func (this *ClientConnection) Send(buf []byte) {
    if this.connState == ConnStateDisc { return }

    head := make([]byte, 4)
    binary.LittleEndian.PutUint32(head, uint32(len(buf)))
    buf = append(head, buf...)

    select {
        case this.sendchan <- buf:

        default:
            fmt.Println("send chan overflow")
    }
}

func (this *ClientConnection) sendall() bool {
    for more := true; more; {
        select {
            case b := <-this.sendchan:
                if _, err := this.writer.Write(b); err != nil {
                    fmt.Println("write err:", err)
                    return false
                }
            default:
                more = false
        }
    }

    if err := this.writer.Flush(); err != nil {
        fmt.Println("flush err:", err)
        return false
    }
    return true
}

func (this *ClientConnection) duplexRead(buff []byte) bool {
    var read_size int
    for {
        // write
        if !this.sendall() {
            this.connState = ConnStateDisc
            return false
        }

        // read
        this.conn.SetReadDeadline(time.Now().Add(1e8))
        n, err := this.reader.Read(buff[read_size:])
        if err != nil {
            if e, ok := err.(*net.OpError); ok && e.Temporary() {
                read_size = n
                continue
            } else {
                fmt.Println("read err, disconnect", err)
                return false
            }
        }

        if n == 0 { return true }
        if n < len(buff) {
            read_size += n
            continue
        }
        return true
    }
    return false
}

func (this *ClientConnection) duplexReadBody() (ret []byte,  ok bool) {
    buff_head := make([]byte, 4)
    if !this.duplexRead(buff_head) {
        return
    }
    len_head := binary.LittleEndian.Uint32(buff_head)
    if len_head > MAX_LEN_HEAD {
        fmt.Println("message len too long", len_head)
        return
    }
    ret = make([]byte, len_head)
    if !this.duplexRead(ret) {
        return
    }
    ok = true
    return
}

func (this *ClientConnection) Close() {
    this.connState  = ConnStateDisc
    this.conn.Close()
}

<|endoftext|>


<commit_before>// Copyright 2010 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import (
	"syscall"
	"unsafe"
)

import . "github.com/lxn/go-winapi"

const (
	lineEditMinChars    = 10 // number of characters needed to make a LineEdit usable
	lineEditGreedyLimit = 80 // fields with MaxLength larger than this will be greedy (default length is 32767)
)

type LineEdit struct {
	WidgetBase
	editingFinishedPublisher EventPublisher
	readOnlyChangedPublisher EventPublisher
	textChangedPublisher     EventPublisher
	charWidthFont            *Font
	charWidth                int
}

func newLineEdit(parent Widget) (*LineEdit, error) {
	le := new(LineEdit)

	if err := InitWidget(
		le,
		parent,
		"EDIT",
		WS_CHILD|WS_TABSTOP|WS_VISIBLE|ES_AUTOHSCROLL,
		WS_EX_CLIENTEDGE); err != nil {
		return nil, err
	}

	le.MustRegisterProperty("ReadOnly", NewProperty(
		func() interface{} {
			return le.ReadOnly()
		},
		func(v interface{}) error {
			return le.SetReadOnly(v.(bool))
		},
		le.readOnlyChangedPublisher.Event()))

	le.MustRegisterProperty("Text", NewProperty(
		func() interface{} {
			return le.Text()
		},
		func(v interface{}) error {
			return le.SetText(v.(string))
		},
		le.textChangedPublisher.Event()))

	return le, nil
}

func NewLineEdit(parent Container) (*LineEdit, error) {
	if parent == nil {
		return nil, newError("parent cannot be nil")
	}

	le, err := newLineEdit(parent)
	if err != nil {
		return nil, err
	}

	var succeeded bool
	defer func() {
		if !succeeded {
			le.Dispose()
		}
	}()

	le.parent = parent
	if err = parent.Children().Add(le); err != nil {
		return nil, err
	}

	succeeded = true

	return le, nil
}

func (le *LineEdit) CueBanner() string {
	buf := make([]uint16, 128)
	if FALSE == le.SendMessage(EM_GETCUEBANNER, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) {
		newError("EM_GETCUEBANNER failed")
		return ""
	}

	return syscall.UTF16ToString(buf)
}

func (le *LineEdit) SetCueBanner(value string) error {
	if FALSE == le.SendMessage(EM_SETCUEBANNER, FALSE, uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(value)))) {
		return newError("EM_SETCUEBANNER failed")
	}

	return nil
}

func (le *LineEdit) MaxLength() int {
	return int(le.SendMessage(EM_GETLIMITTEXT, 0, 0))
}

func (le *LineEdit) SetMaxLength(value int) {
	le.SendMessage(EM_LIMITTEXT, uintptr(value), 0)
}

func (le *LineEdit) Text() string {
	return widgetText(le.hWnd)
}

func (le *LineEdit) SetText(value string) error {
	return setWidgetText(le.hWnd, value)
}

func (le *LineEdit) TextSelection() (start, end int) {
	le.SendMessage(EM_GETSEL, uintptr(unsafe.Pointer(&start)), uintptr(unsafe.Pointer(&end)))
	return
}

func (le *LineEdit) SetTextSelection(start, end int) {
	le.SendMessage(EM_SETSEL, uintptr(start), uintptr(end))
}

func (le *LineEdit) PasswordMode() bool {
	return le.SendMessage(EM_GETPASSWORDCHAR, 0, 0) != 0
}

func (le *LineEdit) SetPasswordMode(value bool) {
	var c uintptr
	if value {
		c = uintptr('*')
	}

	le.SendMessage(EM_SETPASSWORDCHAR, c, 0)
}

func (le *LineEdit) ReadOnly() bool {
	return le.hasStyleBits(ES_READONLY)
}

func (le *LineEdit) SetReadOnly(readOnly bool) error {
	if 0 == le.SendMessage(EM_SETREADONLY, uintptr(BoolToBOOL(readOnly)), 0) {
		return newError("SendMessage(EM_SETREADONLY)")
	}

	le.readOnlyChangedPublisher.Publish()

	return nil
}

func (le *LineEdit) LayoutFlags() (lf LayoutFlags) {
	lf = ShrinkableHorz | GrowableHorz
	if le.MaxLength() > lineEditGreedyLimit {
		lf |= GreedyHorz
	}
	return
}

func (le *LineEdit) MinSizeHint() Size {
	return le.sizeHintForLimit(lineEditMinChars)
}

func (le *LineEdit) SizeHint() (size Size) {
	return le.sizeHintForLimit(lineEditGreedyLimit)
}

func (le *LineEdit) sizeHintForLimit(limit int) (size Size) {
	size = le.dialogBaseUnitsToPixels(Size{50, 12})
	le.initCharWidth()
	n := le.MaxLength()
	if n > limit {
		n = limit
	}
	size.Width = le.charWidth * (n + 1)
	return
}

func (le *LineEdit) initCharWidth() {

	font := le.Font()
	if font == le.charWidthFont {
		return
	}
	le.charWidthFont = font
	le.charWidth = 8

	hdc := GetDC(le.hWnd)
	if hdc == 0 {
		newError("GetDC failed")
		return
	}
	defer ReleaseDC(le.hWnd, hdc)

	defer SelectObject(hdc, SelectObject(hdc, HGDIOBJ(font.handleForDPI(0))))

	buf := []uint16{'M'}

	var s SIZE
	if !GetTextExtentPoint32(hdc, &buf[0], int32(len(buf)), &s) {
		newError("GetTextExtentPoint32 failed")
		return
	}
	le.charWidth = int(s.CX)
}

func (le *LineEdit) EditingFinished() *Event {
	return le.editingFinishedPublisher.Event()
}

func (le *LineEdit) TextChanged() *Event {
	return le.textChangedPublisher.Event()
}

func (le *LineEdit) WndProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	switch msg {
	case WM_COMMAND:
		switch HIWORD(uint32(wParam)) {
		case EN_CHANGE:
			le.textChangedPublisher.Publish()
		}

	case WM_GETDLGCODE:
		if root := rootWidget(le); root != nil {
			if dlg, ok := root.(dialogish); ok {
				if dlg.DefaultButton() != nil {
					// If the LineEdit lives in a Dialog that has a DefaultButton,
					// we won't swallow the return key.
					break
				}
			}
		}

		if wParam == VK_RETURN {
			return DLGC_WANTALLKEYS
		}

	case WM_KEYDOWN:
		if wParam == VK_RETURN {
			le.editingFinishedPublisher.Publish()
		}

	case WM_KILLFOCUS:
		// FIXME: This may be dangerous, see remarks section:
		// http://msdn.microsoft.com/en-us/library/ms646282(v=vs.85).aspx
		le.editingFinishedPublisher.Publish()
	}

	return le.WidgetBase.WndProc(hwnd, msg, wParam, lParam)
}
<commit_msg>LineEdit: Make select all (Ctrl+A) work again<commit_after>// Copyright 2010 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import (
	"syscall"
	"unsafe"
)

import . "github.com/lxn/go-winapi"

const (
	lineEditMinChars    = 10 // number of characters needed to make a LineEdit usable
	lineEditGreedyLimit = 80 // fields with MaxLength larger than this will be greedy (default length is 32767)
)

type LineEdit struct {
	WidgetBase
	editingFinishedPublisher EventPublisher
	readOnlyChangedPublisher EventPublisher
	textChangedPublisher     EventPublisher
	charWidthFont            *Font
	charWidth                int
}

func newLineEdit(parent Widget) (*LineEdit, error) {
	le := new(LineEdit)

	if err := InitWidget(
		le,
		parent,
		"EDIT",
		WS_CHILD|WS_TABSTOP|WS_VISIBLE|ES_AUTOHSCROLL,
		WS_EX_CLIENTEDGE); err != nil {
		return nil, err
	}

	le.MustRegisterProperty("ReadOnly", NewProperty(
		func() interface{} {
			return le.ReadOnly()
		},
		func(v interface{}) error {
			return le.SetReadOnly(v.(bool))
		},
		le.readOnlyChangedPublisher.Event()))

	le.MustRegisterProperty("Text", NewProperty(
		func() interface{} {
			return le.Text()
		},
		func(v interface{}) error {
			return le.SetText(v.(string))
		},
		le.textChangedPublisher.Event()))

	return le, nil
}

func NewLineEdit(parent Container) (*LineEdit, error) {
	if parent == nil {
		return nil, newError("parent cannot be nil")
	}

	le, err := newLineEdit(parent)
	if err != nil {
		return nil, err
	}

	var succeeded bool
	defer func() {
		if !succeeded {
			le.Dispose()
		}
	}()

	le.parent = parent
	if err = parent.Children().Add(le); err != nil {
		return nil, err
	}

	succeeded = true

	return le, nil
}

func (le *LineEdit) CueBanner() string {
	buf := make([]uint16, 128)
	if FALSE == le.SendMessage(EM_GETCUEBANNER, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) {
		newError("EM_GETCUEBANNER failed")
		return ""
	}

	return syscall.UTF16ToString(buf)
}

func (le *LineEdit) SetCueBanner(value string) error {
	if FALSE == le.SendMessage(EM_SETCUEBANNER, FALSE, uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(value)))) {
		return newError("EM_SETCUEBANNER failed")
	}

	return nil
}

func (le *LineEdit) MaxLength() int {
	return int(le.SendMessage(EM_GETLIMITTEXT, 0, 0))
}

func (le *LineEdit) SetMaxLength(value int) {
	le.SendMessage(EM_LIMITTEXT, uintptr(value), 0)
}

func (le *LineEdit) Text() string {
	return widgetText(le.hWnd)
}

func (le *LineEdit) SetText(value string) error {
	return setWidgetText(le.hWnd, value)
}

func (le *LineEdit) TextSelection() (start, end int) {
	le.SendMessage(EM_GETSEL, uintptr(unsafe.Pointer(&start)), uintptr(unsafe.Pointer(&end)))
	return
}

func (le *LineEdit) SetTextSelection(start, end int) {
	le.SendMessage(EM_SETSEL, uintptr(start), uintptr(end))
}

func (le *LineEdit) PasswordMode() bool {
	return le.SendMessage(EM_GETPASSWORDCHAR, 0, 0) != 0
}

func (le *LineEdit) SetPasswordMode(value bool) {
	var c uintptr
	if value {
		c = uintptr('*')
	}

	le.SendMessage(EM_SETPASSWORDCHAR, c, 0)
}

func (le *LineEdit) ReadOnly() bool {
	return le.hasStyleBits(ES_READONLY)
}

func (le *LineEdit) SetReadOnly(readOnly bool) error {
	if 0 == le.SendMessage(EM_SETREADONLY, uintptr(BoolToBOOL(readOnly)), 0) {
		return newError("SendMessage(EM_SETREADONLY)")
	}

	le.readOnlyChangedPublisher.Publish()

	return nil
}

func (le *LineEdit) LayoutFlags() (lf LayoutFlags) {
	lf = ShrinkableHorz | GrowableHorz
	if le.MaxLength() > lineEditGreedyLimit {
		lf |= GreedyHorz
	}
	return
}

func (le *LineEdit) MinSizeHint() Size {
	return le.sizeHintForLimit(lineEditMinChars)
}

func (le *LineEdit) SizeHint() (size Size) {
	return le.sizeHintForLimit(lineEditGreedyLimit)
}

func (le *LineEdit) sizeHintForLimit(limit int) (size Size) {
	size = le.dialogBaseUnitsToPixels(Size{50, 12})
	le.initCharWidth()
	n := le.MaxLength()
	if n > limit {
		n = limit
	}
	size.Width = le.charWidth * (n + 1)
	return
}

func (le *LineEdit) initCharWidth() {

	font := le.Font()
	if font == le.charWidthFont {
		return
	}
	le.charWidthFont = font
	le.charWidth = 8

	hdc := GetDC(le.hWnd)
	if hdc == 0 {
		newError("GetDC failed")
		return
	}
	defer ReleaseDC(le.hWnd, hdc)

	defer SelectObject(hdc, SelectObject(hdc, HGDIOBJ(font.handleForDPI(0))))

	buf := []uint16{'M'}

	var s SIZE
	if !GetTextExtentPoint32(hdc, &buf[0], int32(len(buf)), &s) {
		newError("GetTextExtentPoint32 failed")
		return
	}
	le.charWidth = int(s.CX)
}

func (le *LineEdit) EditingFinished() *Event {
	return le.editingFinishedPublisher.Event()
}

func (le *LineEdit) TextChanged() *Event {
	return le.textChangedPublisher.Event()
}

func (le *LineEdit) WndProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	switch msg {
	case WM_COMMAND:
		switch HIWORD(uint32(wParam)) {
		case EN_CHANGE:
			le.textChangedPublisher.Publish()
		}

	case WM_GETDLGCODE:
		if root := rootWidget(le); root != nil {
			if dlg, ok := root.(dialogish); ok {
				if dlg.DefaultButton() != nil {
					// If the LineEdit lives in a Dialog that has a DefaultButton,
					// we won't swallow the return key.
					break
				}
			}
		}

		if wParam == VK_RETURN {
			return DLGC_WANTALLKEYS
		}

	case WM_KEYDOWN:
		switch Key(wParam) {
		case KeyA:
			if ControlDown() {
				le.SetTextSelection(0, -1)
			}

		case KeyReturn:
			le.editingFinishedPublisher.Publish()
		}

	case WM_KILLFOCUS:
		// FIXME: This may be dangerous, see remarks section:
		// http://msdn.microsoft.com/en-us/library/ms646282(v=vs.85).aspx
		le.editingFinishedPublisher.Publish()
	}

	return le.WidgetBase.WndProc(hwnd, msg, wParam, lParam)
}
<|endoftext|>


<commit_before>// Code generated by go generate. DO NOT EDIT.
// Source: vendor/istio.io/api/mixer/v1/global_dictionary.yaml

package attribute

func GlobalList() []string {
	tmp := make([]string, len(globalList))
	copy(tmp, globalList)
	return tmp
}

var (
	globalList = []string{
		"source.ip",
		"source.port",
		"source.name",
		"source.uid",
		"source.namespace",
		"source.labels",
		"source.user",
		"target.ip",
		"target.port",
		"target.service",
		"target.name",
		"target.uid",
		"target.namespace",
		"target.labels",
		"target.user",
		"request.headers",
		"request.id",
		"request.path",
		"request.host",
		"request.method",
		"request.reason",
		"request.referer",
		"request.scheme",
		"request.size",
		"request.time",
		"request.useragent",
		"response.headers",
		"response.size",
		"response.time",
		"response.duration",
		"response.code",
		":authority",
		":method",
		":path",
		":scheme",
		":status",
		"access-control-allow-origin",
		"access-control-allow-methods",
		"access-control-allow-headers",
		"access-control-max-age",
		"access-control-request-method",
		"access-control-request-headers",
		"accept-charset",
		"accept-encoding",
		"accept-language",
		"accept-ranges",
		"accept",
		"access-control-allow",
		"age",
		"allow",
		"authorization",
		"cache-control",
		"content-disposition",
		"content-encoding",
		"content-language",
		"content-length",
		"content-location",
		"content-range",
		"content-type",
		"cookie",
		"date",
		"etag",
		"expect",
		"expires",
		"from",
		"host",
		"if-match",
		"if-modified-since",
		"if-none-match",
		"if-range",
		"if-unmodified-since",
		"keep-alive",
		"last-modified",
		"link",
		"location",
		"max-forwards",
		"proxy-authenticate",
		"proxy-authorization",
		"range",
		"referer",
		"refresh",
		"retry-after",
		"server",
		"set-cookie",
		"strict-transport-sec",
		"transfer-encoding",
		"user-agent",
		"vary",
		"via",
		"www-authenticate",
		"GET",
		"POST",
		"http",
		"envoy",
		"'200'",
		"Keep-Alive",
		"chunked",
		"x-envoy-service-time",
		"x-forwarded-for",
		"x-forwarded-host",
		"x-forwarded-proto",
		"x-http-method-override",
		"x-request-id",
		"x-requested-with",
		"application/json",
		"application/xml",
		"gzip",
		"text/html",
		"text/html; charset=utf-8",
		"text/plain",
		"text/plain; charset=utf-8",
		"'0'",
		"'1'",
		"true",
		"false",
		"gzip, deflate",
		"max-age=0",
		"x-envoy-upstream-service-time",
		"x-envoy-internal",
		"x-envoy-expected-rq-timeout-ms",
		"x-ot-span-context",
		"x-b3-traceid",
		"x-b3-sampled",
		"x-b3-spanid",
		"tcp",
		"connection.id",
		"connection.received.bytes",
		"connection.received.bytes_total",
		"connection.sent.bytes",
		"connection.sent.bytes_total",
		"connection.duration",
		"context.protocol",
		"context.timestamp",
		"context.time",
		"0",
		"1",
		"200",
		"302",
		"400",
		"401",
		"403",
		"404",
		"409",
		"429",
		"499",
		"500",
		"501",
		"502",
		"503",
		"504",
		"destination.ip",
		"destination.port",
		"destination.service",
		"destination.name",
		"destination.uid",
		"destination.namespace",
		"destination.labels",
		"destination.user",
		"source.service",
		"api.service",
		"api.version",
		"api.operation",
		"api.protocol",
		"request.auth.principal",
		"request.auth.audiences",
		"request.auth.presenter",
		"request.api_key",
		"check.error_code",
		"check.error_message",
	}
)
<commit_msg>Templates istio.mixer.v1.config.descriptor.ValueType to istio.mixer.v1.template.Value (#2595)<commit_after>// Code generated by go generate. DO NOT EDIT.
// Source: vendor/istio.io/api/mixer/v1/global_dictionary.yaml

package attribute

func GlobalList() ([]string) {
    tmp := make([]string, len(globalList))
    copy(tmp, globalList)
    return tmp
}

var (
    globalList = []string{
		"source.ip",
		"source.port",
		"source.name",
		"source.uid",
		"source.namespace",
		"source.labels",
		"source.user",
		"target.ip",
		"target.port",
		"target.service",
		"target.name",
		"target.uid",
		"target.namespace",
		"target.labels",
		"target.user",
		"request.headers",
		"request.id",
		"request.path",
		"request.host",
		"request.method",
		"request.reason",
		"request.referer",
		"request.scheme",
		"request.size",
		"request.time",
		"request.useragent",
		"response.headers",
		"response.size",
		"response.time",
		"response.duration",
		"response.code",
		":authority",
		":method",
		":path",
		":scheme",
		":status",
		"access-control-allow-origin",
		"access-control-allow-methods",
		"access-control-allow-headers",
		"access-control-max-age",
		"access-control-request-method",
		"access-control-request-headers",
		"accept-charset",
		"accept-encoding",
		"accept-language",
		"accept-ranges",
		"accept",
		"access-control-allow",
		"age",
		"allow",
		"authorization",
		"cache-control",
		"content-disposition",
		"content-encoding",
		"content-language",
		"content-length",
		"content-location",
		"content-range",
		"content-type",
		"cookie",
		"date",
		"etag",
		"expect",
		"expires",
		"from",
		"host",
		"if-match",
		"if-modified-since",
		"if-none-match",
		"if-range",
		"if-unmodified-since",
		"keep-alive",
		"last-modified",
		"link",
		"location",
		"max-forwards",
		"proxy-authenticate",
		"proxy-authorization",
		"range",
		"referer",
		"refresh",
		"retry-after",
		"server",
		"set-cookie",
		"strict-transport-sec",
		"transfer-encoding",
		"user-agent",
		"vary",
		"via",
		"www-authenticate",
		"GET",
		"POST",
		"http",
		"envoy",
		"'200'",
		"Keep-Alive",
		"chunked",
		"x-envoy-service-time",
		"x-forwarded-for",
		"x-forwarded-host",
		"x-forwarded-proto",
		"x-http-method-override",
		"x-request-id",
		"x-requested-with",
		"application/json",
		"application/xml",
		"gzip",
		"text/html",
		"text/html; charset=utf-8",
		"text/plain",
		"text/plain; charset=utf-8",
		"'0'",
		"'1'",
		"true",
		"false",
		"gzip, deflate",
		"max-age=0",
		"x-envoy-upstream-service-time",
		"x-envoy-internal",
		"x-envoy-expected-rq-timeout-ms",
		"x-ot-span-context",
		"x-b3-traceid",
		"x-b3-sampled",
		"x-b3-spanid",
		"tcp",
		"connection.id",
		"connection.received.bytes",
		"connection.received.bytes_total",
		"connection.sent.bytes",
		"connection.sent.bytes_total",
		"connection.duration",
		"context.protocol",
		"context.timestamp",
		"context.time",
		"0",
		"1",
		"200",
		"302",
		"400",
		"401",
		"403",
		"404",
		"409",
		"429",
		"499",
		"500",
		"501",
		"502",
		"503",
		"504",
		"destination.ip",
		"destination.port",
		"destination.service",
		"destination.name",
		"destination.uid",
		"destination.namespace",
		"destination.labels",
		"destination.user",
		"source.service",
		"api.service",
		"api.version",
		"api.operation",
		"api.protocol",
		"request.auth.principal",
		"request.auth.audiences",
		"request.auth.presenter",
		"request.api_key",
		"check.error_code",
		"check.error_message",
    }
)
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"regexp"
	"testing"
	"time"
)

func validate(key string, logFunc func(msg ...interface{}), valueExpected string, msg ...interface{}) (err error) {
	rescueStdout := os.Stdout
	defer func() { os.Stdout = rescueStdout }()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	logFunc(msg...)

	err = w.Close()
	if err != nil {
		return
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	if string(out) != valueExpected {
		err = fmt.Errorf("Error, '%s' printed %q, expected %q", key, string(out), valueExpected)
	}
	return
}

func TestLog(t *testing.T) {
	now = func() time.Time { return time.Unix(1498405744, 0) }
	DebugMode = false

	data := []struct {
		key           string
		logFunc       func(msg ...interface{})
		expectedValue string
	}{
		{"Println", Println, "\x1b[37m2017/06/25 15:49:04 [msg] log test\x1b[0;00m\n"},
		{"Errorln", Errorln, "\x1b[91m2017/06/25 15:49:04 [error] log test\x1b[0;00m\n"},
		{"Warningln", Warningln, "\x1b[93m2017/06/25 15:49:04 [warning] log test\x1b[0;00m\n"},
		{"Debugln", Debugln, ""},
	}
	formatedData := []struct {
		key           string
		logFunc       func(msg ...interface{})
		expectedValue string
	}{
		{"Println", Println, "\x1b[37m2017/06/25 15:49:04 [msg] formated log 1.12\x1b[0;00m\n"},
		{"Errorln", Errorln, "\x1b[91m2017/06/25 15:49:04 [error] formated log 1.12\x1b[0;00m\n"},
		{"Warningln", Warningln, "\x1b[93m2017/06/25 15:49:04 [warning] formated log 1.12\x1b[0;00m\n"},
	}
	for _, v := range data {
		err := validate(v.key, v.logFunc, v.expectedValue, "log test")
		if err != nil {
			t.Fatal(err.Error())
		}
	}
	for _, v := range formatedData {
		err := validate(v.key, v.logFunc, v.expectedValue, "%s %s %.2f", "formated", "log", 1.1234)
		if err != nil {
			t.Fatal(err.Error())
		}
	}

	DebugMode = true

	rescueStdout := os.Stdout
	defer func() {
		os.Stdout = rescueStdout
		DebugMode = false
	}()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	Debugln("log test")

	os.Stdout = rescueStdout

	err = w.Close()
	if err != nil {
		t.Fatal(err.Error())
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatal(err.Error())
	}
	rstr := "\x1b\\[96m2017/06/25 15:49:04 \\[debug\\] log_test.go:\\d+ log test\x1b\\[0;00m\n"
	match, err := regexp.Match(rstr, out)
	if err != nil {
		t.Fatal(err.Error())
	}

	if !match {
		t.Fatalf("Error, 'Debugln' printed %q, not match with expected", string(out))
	}

}

func TestHTTPError(t *testing.T) {
	now = func() time.Time { return time.Unix(1498405744, 0) }

	rescueStdout := os.Stdout
	DebugMode = false
	defer func() { os.Stdout = rescueStdout }()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	handler := func(w http.ResponseWriter, r *http.Request) {
		HTTPError(w, http.StatusBadRequest)
	}

	req := httptest.NewRequest("GET", "http://example.com/foo", nil)
	httpw := httptest.NewRecorder()
	handler(httpw, req)

	resp := httpw.Result()
	body, _ := ioutil.ReadAll(resp.Body)

	os.Stdout = rescueStdout
	err = w.Close()
	if err != nil {
		return
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	valueExpected := "\x1b[91m2017/06/25 15:49:04 [error] Bad Request\x1b[0;00m\n"
	if string(out) != valueExpected {
		t.Fatalf("Error, 'HTTPError' printed %q, expected %q", string(out), valueExpected)
	}

	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("Error, 'HTTPError' status code %v, expected 400", resp.StatusCode)
	}

	valueExpected = "{\n\t\"error\": \"Bad Request\",\n\t\"status\": \"error\"\n}\n"
	if string(body) != valueExpected {
		t.Fatalf("Error, 'HTTPError' write to client %q, expected %q", string(body), valueExpected)
	}

}
<commit_msg>Fixing tests of formatted logs<commit_after>package log

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"regexp"
	"testing"
	"time"
)

func validate(key string, logFunc func(msg ...interface{}), valueExpected string, msg ...interface{}) (err error) {
	rescueStdout := os.Stdout
	defer func() { os.Stdout = rescueStdout }()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	logFunc(msg...)

	err = w.Close()
	if err != nil {
		return
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	if string(out) != valueExpected {
		err = fmt.Errorf("Error, '%s' printed %q, expected %q", key, string(out), valueExpected)
	}
	return
}

func TestLog(t *testing.T) {
	now = func() time.Time { return time.Unix(1498405744, 0) }
	DebugMode = false

	data := []struct {
		key           string
		logFunc       func(msg ...interface{})
		expectedValue string
	}{
		{"Println", Println, "\x1b[37m2017/06/25 15:49:04 [msg] log test\x1b[0;00m\n"},
		{"Errorln", Errorln, "\x1b[91m2017/06/25 15:49:04 [error] log test\x1b[0;00m\n"},
		{"Warningln", Warningln, "\x1b[93m2017/06/25 15:49:04 [warning] log test\x1b[0;00m\n"},
		{"Debugln", Debugln, ""},
	}
	formatedData := []struct {
		key           string
		logFunc       func(msg ...interface{})
		expectedValue string
	}{
		{"Printf", Printf, "\x1b[37m2017/06/25 15:49:04 [msg] formated log 1.12\x1b[0;00m"},
		{"Errorf", Errorf, "\x1b[91m2017/06/25 15:49:04 [error] formated log 1.12\x1b[0;00m"},
		{"Warningf", Warningf, "\x1b[93m2017/06/25 15:49:04 [warning] formated log 1.12\x1b[0;00m"},
		{"Debugf", Debugf, ""},
	}
	for _, v := range data {
		err := validate(v.key, v.logFunc, v.expectedValue, "log test")
		if err != nil {
			t.Fatal(err.Error())
		}
	}
	for _, v := range formatedData {
		err := validate(v.key, v.logFunc, v.expectedValue, "%s %s %.2f", "formated", "log", 1.1234)
		if err != nil {
			t.Fatal(err.Error())
		}
	}

	DebugMode = true

	rescueStdout := os.Stdout
	defer func() {
		os.Stdout = rescueStdout
		DebugMode = false
	}()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	Debugln("log test")

	os.Stdout = rescueStdout

	err = w.Close()
	if err != nil {
		t.Fatal(err.Error())
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatal(err.Error())
	}
	rstr := "\x1b\\[96m2017/06/25 15:49:04 \\[debug\\] log_test.go:\\d+ log test\x1b\\[0;00m\n"
	match, err := regexp.Match(rstr, out)
	if err != nil {
		t.Fatal(err.Error())
	}

	if !match {
		t.Fatalf("Error, 'Debugln' printed %q, not match with expected", string(out))
	}

}

func TestHTTPError(t *testing.T) {
	now = func() time.Time { return time.Unix(1498405744, 0) }

	rescueStdout := os.Stdout
	DebugMode = false
	defer func() { os.Stdout = rescueStdout }()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	handler := func(w http.ResponseWriter, r *http.Request) {
		HTTPError(w, http.StatusBadRequest)
	}

	req := httptest.NewRequest("GET", "http://example.com/foo", nil)
	httpw := httptest.NewRecorder()
	handler(httpw, req)

	resp := httpw.Result()
	body, _ := ioutil.ReadAll(resp.Body)

	os.Stdout = rescueStdout
	err = w.Close()
	if err != nil {
		return
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	valueExpected := "\x1b[91m2017/06/25 15:49:04 [error] Bad Request\x1b[0;00m\n"
	if string(out) != valueExpected {
		t.Fatalf("Error, 'HTTPError' printed %q, expected %q", string(out), valueExpected)
	}

	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("Error, 'HTTPError' status code %v, expected 400", resp.StatusCode)
	}

	valueExpected = "{\n\t\"error\": \"Bad Request\",\n\t\"status\": \"error\"\n}\n"
	if string(body) != valueExpected {
		t.Fatalf("Error, 'HTTPError' write to client %q, expected %q", string(body), valueExpected)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"log"
	"net"
	"strconv"
	"time"
)

var (
	ping   = []byte("ping")
	closed = []byte("close")
)

// Listener accepts connections from devices.
type Listener struct {
	debug    bool
	listener net.Listener
}

// Listen creates a TCP listener with the given PEM encoded X.509 certificate and the private key on the local network address laddr.
// Debug mode logs all server activity.
func Listen(cert, privKey []byte, laddr string, debug bool) (*Listener, error) {
	tlsCert, err := tls.X509KeyPair(cert, privKey)
	pool := x509.NewCertPool()
	ok := pool.AppendCertsFromPEM(cert)
	if err != nil || !ok {
		return nil, fmt.Errorf("failed to parse the certificate or the private key with error: %v", err)
	}

	config := tls.Config{
		Certificates: []tls.Certificate{tlsCert},
		ClientCAs:    pool,
	}

	listener, err := tls.Listen("tcp", laddr, &config)
	if err != nil {
		return nil, err
	}

	if debug {
		log.Printf("Listener created with local network address: %v\n", laddr)
	}

	return &Listener{
		debug:    debug,
		listener: listener,
	}, nil
}

// Accept waits for incoming connections and forwards incoming messages to handleMsg in a new goroutine.
// This function never returns, unless there is an error while accepting a new connection.
func (l *Listener) Accept(handleMsg func(msg []byte)) error {
	for {
		conn, err := l.listener.Accept()
		if err != nil {
			return fmt.Errorf("error while accepting a new connection from a client: %v", err)
			// todo: it might not be appropriate to break the loop on recoverable errors (like client disconnect during handshake)
			// the underlying fd.accept() does some basic recovery though we might need more: http://golang.org/src/net/fd_unix.go
		}

		log.Println("Client connected: listening for messages from client IP:", conn.RemoteAddr())
		go handleConn(conn, handleMsg)
	}
}

func handleConn(conn net.Conn, handleMsg func(msg []byte)) {
	defer conn.Close()
	defer log.Println("Closed connection to client with IP:", conn.RemoteAddr())
	header := make([]byte, 4) // so max message size is 9999 bytes for now
	for {
		err := conn.SetReadDeadline(time.Now().Add(time.Minute * 5))
		// read the content length header
		n, err := conn.Read(header)
		if err != nil || n == 0 {
			log.Fatalln("Client read error: ", err)
			break
		}
		// calculate the content length
		th := bytes.TrimRight(header, " ")
		n, err = strconv.Atoi(string(th))
		if err != nil || n == 0 {
			log.Fatalln("Client read error: invalid content lenght header sent or content lenght mismatch: ", err)
			break
		}
		log.Println("Starting to read message content of bytes: ", n)
		// read the message content
		msg := make([]byte, n)
		n, err = conn.Read(msg)
		if err != nil || n == 0 {
			log.Fatalln("Client read error: ", err)
			break
		}

		log.Printf("Read %v bytes message '%v' from client with IP: %v\n", n, string(msg), conn.RemoteAddr())

		if n == 4 && bytes.Equal(msg, ping) {
			continue
		}

		go handleMsg(msg)
		if n == 5 && bytes.Equal(msg, closed) {
			return
		}
	}
}

// Close closes the listener.
func (l *Listener) Close() error {
	defer log.Println("Listener was closed on local network address:", l.listener.Addr())
	return l.listener.Close()
}
<commit_msg>conditional logging<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"log"
	"net"
	"strconv"
	"time"
)

var (
	ping   = []byte("ping")
	closed = []byte("close")
)

// Listener accepts connections from devices.
type Listener struct {
	debug    bool
	listener net.Listener
}

// Listen creates a TCP listener with the given PEM encoded X.509 certificate and the private key on the local network address laddr.
// Debug mode logs all server activity.
func Listen(cert, privKey []byte, laddr string, debug bool) (*Listener, error) {
	tlsCert, err := tls.X509KeyPair(cert, privKey)
	pool := x509.NewCertPool()
	ok := pool.AppendCertsFromPEM(cert)
	if err != nil || !ok {
		return nil, fmt.Errorf("failed to parse the certificate or the private key with error: %v", err)
	}

	config := tls.Config{
		Certificates: []tls.Certificate{tlsCert},
		ClientCAs:    pool,
	}

	listener, err := tls.Listen("tcp", laddr, &config)
	if err != nil {
		return nil, err
	}

	if debug {
		log.Printf("Listener created with local network address: %v\n", laddr)
	}

	return &Listener{
		debug:    debug,
		listener: listener,
	}, nil
}

// Accept waits for incoming connections and forwards incoming messages to handleMsg in a new goroutine.
// This function never returns, unless there is an error while accepting a new connection.
func (l *Listener) Accept(handleMsg func(msg []byte)) error {
	for {
		conn, err := l.listener.Accept()
		if err != nil {
			return fmt.Errorf("error while accepting a new connection from a client: %v", err)
			// todo: it might not be appropriate to break the loop on recoverable errors (like client disconnect during handshake)
			// the underlying fd.accept() does some basic recovery though we might need more: http://golang.org/src/net/fd_unix.go
		}

		if l.debug {
			log.Println("Client connected: listening for messages from client IP:", conn.RemoteAddr())
		}
		go handleConn(conn, l.debug, handleMsg)
	}
}

func handleConn(conn net.Conn, debug bool, handleMsg func(msg []byte)) {
	defer conn.Close()
	if debug {
		defer log.Println("Closed connection to client with IP:", conn.RemoteAddr())
	}
	header := make([]byte, 4) // so max message size is 9999 bytes for now
	for {
		err := conn.SetReadDeadline(time.Now().Add(time.Minute * 5))
		// read the content length header
		n, err := conn.Read(header)
		if err != nil || n == 0 {
			log.Fatalln("Client read error: ", err)
			break
		}
		// calculate the content length
		th := bytes.TrimRight(header, " ")
		n, err = strconv.Atoi(string(th))
		if err != nil || n == 0 {
			log.Fatalln("Client read error: invalid content lenght header sent or content lenght mismatch: ", err)
			break
		}
		// read the message content
		if debug {
			log.Println("Starting to read message content of bytes: ", n)
		}
		msg := make([]byte, n)
		n, err = conn.Read(msg)
		if err != nil || n == 0 {
			log.Fatalln("Client read error: ", err)
			break
		}
		if debug {
			log.Printf("Read %v bytes message '%v' from client with IP: %v\n", n, string(msg), conn.RemoteAddr())
		}

		if n == 4 && bytes.Equal(msg, ping) {
			continue
		}

		go handleMsg(msg)
		if n == 5 && bytes.Equal(msg, closed) {
			return
		}
	}
}

// Close closes the listener.
func (l *Listener) Close() error {
	if l.debug {
		defer log.Println("Listener was closed on local network address:", l.listener.Addr())
	}
	return l.listener.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"

	baps3 "github.com/UniversityRadioYork/baps3-go"
)

// Maintains communications with the connector and connected clients.
// Also does any processing needed with the commands.
type hub struct {
	// All current clients.
	clients map[*Client]bool

	// Version string from the connector (playd)
	cVersion string

	// For communication with the connector.
	cReqCh chan<- baps3.Message
	cResCh <-chan baps3.Message

	// Where new requests from clients come through.
	reqCh chan baps3.Message

	// Handlers for adding/removing connections.
	addCh chan *Client
	rmCh  chan *Client
	Quit  chan bool
}

var h = hub{
	clients: make(map[*Client]bool),

	reqCh: make(chan baps3.Message),

	addCh: make(chan *Client),
	rmCh:  make(chan *Client),
	Quit:  make(chan bool),
}

// Handles a new client connection.
// conn is the new connection object.
func (h *hub) handleNewConnection(conn net.Conn) {
	defer conn.Close()
	client := &Client{
		conn:  conn,
		resCh: make(chan baps3.Message),
		tok:   baps3.NewTokeniser(),
	}

	// Register user
	h.addCh <- client

	go client.Read(h.reqCh, h.rmCh)
	client.Write(client.resCh, h.rmCh)
}

// Appends the downstream service's version (from the OHAI) to the listd version.
func makeWelcomeMsg() *baps3.Message {
	return baps3.NewMessage(baps3.RsOhai).AddArg("listd " + LD_VERSION + "/" + h.downstreamVersion)
}

func makeFeaturesMsg() *baps3.Message {
	// TODO: Implement actual features
	return baps3.NewMessage(baps3.RsFeatures)
}

// Handles a request from a client.
// Falls through to the connector cReqCh if command is "not understood".
func (h *hub) processRequest(req baps3.Message) {
	// TODO: Do something else
	log.Println("New request:", req.String())
	h.cReqCh <- req
}

// Processes a response from the connector.
func (h *hub) processResponse(res baps3.Message) {
	// TODO: Do something else
	log.Println("New response:", res.String())
	switch res.Word() {
	case baps3.RsOhai:
		h.cVersion, _ = res.Arg(0)
	default:
		h.broadcast(res)
	}
}

// Send a response message to all clients.
func (h *hub) broadcast(res baps3.Message) {
	for c, _ := range h.clients {
		c.resCh <- res
	}
}

// Listens for new connections on addr:port and spins up the relevant goroutines.
func (h *hub) runListener(addr string, port string) {
	netListener, err := net.Listen("tcp", addr+":"+port)
	if err != nil {
		log.Println("Listening error:", err.Error())
		return
	}

	// Get new connections
	go func() {
		for {
			conn, err := netListener.Accept()
			if err != nil {
				log.Println("Error accepting connection:", err.Error())
				continue
			}

			go h.handleNewConnection(conn)
		}
	}()

	for {
		select {
		case msg := <-h.cResCh:
			h.processResponse(msg)
		case msg := <-h.reqCh:
			h.processRequest(msg)
		case client := <-h.addCh:
			h.clients[client] = true
			client.resCh <- *makeWelcomeMsg()
			client.resCh <- *makeFeaturesMsg()
			log.Println("New connection from", client.conn.RemoteAddr())
		case client := <-h.rmCh:
			close(client.resCh)
			delete(h.clients, client)
			log.Println("Closed connection from", client.conn.RemoteAddr())
		case <-h.Quit:
			log.Println("Closing all connections")
			for c, _ := range h.clients {
				close(c.resCh)
				delete(h.clients, c)
			}
			//			h.Quit <- true
		}
	}
}

// Sets up the connector channels for the hub object.
func (h *hub) setConnector(cReqCh chan<- baps3.Message, cResCh <-chan baps3.Message) {
	h.cReqCh = cReqCh
	h.cResCh = cResCh
}
<commit_msg>Add feature handling<commit_after>package main

import (
	"log"
	"net"

	baps3 "github.com/UniversityRadioYork/baps3-go"
)

// Maintains communications with the connector and connected clients.
// Also does any processing needed with the commands.
type hub struct {
	// All current clients.
	clients map[*Client]bool

	// Dump state from the downstream service (playd)
	downstreamVersion  string
	downstreamFeatures baps3.FeatureSet

	// For communication with the connector.
	cReqCh chan<- baps3.Message
	cResCh <-chan baps3.Message

	// Where new requests from clients come through.
	reqCh chan baps3.Message

	// Handlers for adding/removing connections.
	addCh chan *Client
	rmCh  chan *Client
	Quit  chan bool
}

var h = hub{
	clients: make(map[*Client]bool),

	downstreamFeatures: make(baps3.FeatureSet),

	reqCh: make(chan baps3.Message),

	addCh: make(chan *Client),
	rmCh:  make(chan *Client),
	Quit:  make(chan bool),
}

// Handles a new client connection.
// conn is the new connection object.
func (h *hub) handleNewConnection(conn net.Conn) {
	defer conn.Close()
	client := &Client{
		conn:  conn,
		resCh: make(chan baps3.Message),
		tok:   baps3.NewTokeniser(),
	}

	// Register user
	h.addCh <- client

	go client.Read(h.reqCh, h.rmCh)
	client.Write(client.resCh, h.rmCh)
}

// Appends the downstream service's version (from the OHAI) to the listd version.
func makeWelcomeMsg() *baps3.Message {
	return baps3.NewMessage(baps3.RsOhai).AddArg("listd " + LD_VERSION + "/" + h.downstreamVersion)
}

// Crafts the features message by adding listd's features to the downstream service's and removing
// features listd intercepts.
func makeFeaturesMsg() (msg *baps3.Message) {
	features := h.downstreamFeatures
	features.DelFeature(baps3.FtFileLoad) // 'Mask' the features listd intercepts
	features.AddFeature(baps3.FtPlaylist)
	features.AddFeature(baps3.FtPlaylistTextItems)
	msg = features.ToMessage()
	return
}

// Handles a request from a client.
// Falls through to the connector cReqCh if command is "not understood".
func (h *hub) processRequest(req baps3.Message) {
	// TODO: Do something else
	log.Println("New request:", req.String())
	h.cReqCh <- req
}

// Processes a response from the connector.
func (h *hub) processResponse(res baps3.Message) {
	// TODO: Do something else
	log.Println("New response:", res.String())
	switch res.Word() {
	case baps3.RsOhai:
		h.downstreamVersion, _ = res.Arg(0)
	case baps3.RsFeatures:
		fs, err := baps3.FeatureSetFromMsg(&res)
		if err != nil {
			log.Fatal("Error reading features: " + err.Error())
		}
		h.downstreamFeatures = fs
	default:
		h.broadcast(res)
	}
}

// Send a response message to all clients.
func (h *hub) broadcast(res baps3.Message) {
	for c, _ := range h.clients {
		c.resCh <- res
	}
}

// Listens for new connections on addr:port and spins up the relevant goroutines.
func (h *hub) runListener(addr string, port string) {
	netListener, err := net.Listen("tcp", addr+":"+port)
	if err != nil {
		log.Println("Listening error:", err.Error())
		return
	}

	// Get new connections
	go func() {
		for {
			conn, err := netListener.Accept()
			if err != nil {
				log.Println("Error accepting connection:", err.Error())
				continue
			}

			go h.handleNewConnection(conn)
		}
	}()

	for {
		select {
		case msg := <-h.cResCh:
			h.processResponse(msg)
		case msg := <-h.reqCh:
			h.processRequest(msg)
		case client := <-h.addCh:
			h.clients[client] = true
			client.resCh <- *makeWelcomeMsg()
			client.resCh <- *makeFeaturesMsg()
			log.Println("New connection from", client.conn.RemoteAddr())
		case client := <-h.rmCh:
			close(client.resCh)
			delete(h.clients, client)
			log.Println("Closed connection from", client.conn.RemoteAddr())
		case <-h.Quit:
			log.Println("Closing all connections")
			for c, _ := range h.clients {
				close(c.resCh)
				delete(h.clients, c)
			}
			//			h.Quit <- true
		}
	}
}

// Sets up the connector channels for the hub object.
func (h *hub) setConnector(cReqCh chan<- baps3.Message, cResCh <-chan baps3.Message) {
	h.cReqCh = cReqCh
	h.cResCh = cResCh
}
<|endoftext|>