<commit_before>package dht

import (
	"testing"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	ci "github.com/jbenet/go-ipfs/crypto"
	spipe "github.com/jbenet/go-ipfs/crypto/spipe"
	inet "github.com/jbenet/go-ipfs/net"
	mux "github.com/jbenet/go-ipfs/net/mux"
	netservice "github.com/jbenet/go-ipfs/net/service"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	"bytes"
	"fmt"
	"time"
)

func setupDHT(t *testing.T, p *peer.Peer) *IpfsDHT {
	ctx := context.TODO()

	peerstore := peer.NewPeerstore()

	ctx, _ = context.WithCancel(ctx)
	dhts := netservice.NewService(nil) // nil handler for now, need to patch it
	if err := dhts.Start(ctx); err != nil {
		t.Fatal(err)
	}

	net, err := inet.NewIpfsNetwork(context.TODO(), p, &mux.ProtocolMap{
		mux.ProtocolID_Routing: dhts,
	})
	if err != nil {
		t.Fatal(err)
	}

	d := NewDHT(p, peerstore, net, dhts, ds.NewMapDatastore())
	dhts.Handler = d
	return d
}

func setupDHTS(n int, t *testing.T) ([]*ma.Multiaddr, []*peer.Peer, []*IpfsDHT) {
	var addrs []*ma.Multiaddr
	for i := 0; i < 4; i++ {
		a, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5000+i))
		if err != nil {
			t.Fatal(err)
		}
		addrs = append(addrs, a)
	}

	var peers []*peer.Peer
	for i := 0; i < 4; i++ {
		p := new(peer.Peer)
		p.AddAddress(addrs[i])
		sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
		if err != nil {
			panic(err)
		}
		p.PubKey = pk
		p.PrivKey = sk
		id, err := spipe.IDFromPubKey(pk)
		if err != nil {
			panic(err)
		}
		p.ID = id
		peers = append(peers, p)
	}

	var dhts []*IpfsDHT
	for i := 0; i < 4; i++ {
		dhts[i] = setupDHT(t, peers[i])
	}

	return addrs, peers, dhts
}

func makePeer(addr *ma.Multiaddr) *peer.Peer {
	p := new(peer.Peer)
	p.AddAddress(addr)
	sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
	if err != nil {
		panic(err)
	}
	p.PrivKey = sk
	p.PubKey = pk
	id, err := spipe.IDFromPubKey(pk)
	if err != nil {
		panic(err)
	}

	p.ID = id
	return p
}

func TestPing(t *testing.T) {
	u.Debug = true
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/2222")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5678")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(t, peerA)
	dhtB := setupDHT(t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()

	_, err = dhtA.Connect(addrB)
	if err != nil {
		t.Fatal(err)
	}

	//Test that we can ping the node
	err = dhtA.Ping(peerB, time.Second*2)
	if err != nil {
		t.Fatal(err)
	}
}

func TestValueGetSet(t *testing.T) {
	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1235")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5679")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(t, peerA)
	dhtB := setupDHT(t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()

	_, err = dhtA.Connect(addrB)
	if err != nil {
		t.Fatal(err)
	}

	dhtA.PutValue("hello", []byte("world"))

	val, err := dhtA.GetValue("hello", time.Second*2)
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}

}

func TestProvides(t *testing.T) {
	u.Debug = false

	addrs, _, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
		}
	}()

	_, err := dhts[0].Connect(addrs[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(addrs[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(addrs[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	provs, err := dhts[0].FindProviders(u.Key("hello"), time.Second)
	if err != nil {
		t.Fatal(err)
	}

	if len(provs) != 1 {
		t.Fatal("Didnt get back providers")
	}
}

func TestLayeredGet(t *testing.T) {
	u.Debug = false
	addrs, _, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
		}
	}()

	_, err := dhts[0].Connect(addrs[1])
	if err != nil {
		t.Fatalf("Failed to connect: %s", err)
	}

	_, err = dhts[1].Connect(addrs[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(addrs[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].Provide(u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	val, err := dhts[0].GetValue(u.Key("hello"), time.Second)
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatal("Got incorrect value.")
	}

}

func TestFindPeer(t *testing.T) {
	u.Debug = false

	addrs, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
		}
	}()

	_, err := dhts[0].Connect(addrs[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(addrs[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(addrs[3])
	if err != nil {
		t.Fatal(err)
	}

	p, err := dhts[0].FindPeer(peers[2].ID, time.Second)
	if err != nil {
		t.Fatal(err)
	}

	if p == nil {
		t.Fatal("Failed to find peer.")
	}

	if !p.ID.Equal(peers[2].ID) {
		t.Fatal("Didnt find expected peer.")
	}
}
<commit_msg>comment out dht_test for now.<commit_after>package dht

// import (
// 	"testing"
//
// 	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
//
// 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
// 	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
//
// 	ci "github.com/jbenet/go-ipfs/crypto"
// 	spipe "github.com/jbenet/go-ipfs/crypto/spipe"
// 	inet "github.com/jbenet/go-ipfs/net"
// 	mux "github.com/jbenet/go-ipfs/net/mux"
// 	netservice "github.com/jbenet/go-ipfs/net/service"
// 	peer "github.com/jbenet/go-ipfs/peer"
// 	u "github.com/jbenet/go-ipfs/util"
//
// 	"bytes"
// 	"fmt"
// 	"time"
// )
//
// func setupDHT(t *testing.T, p *peer.Peer) *IpfsDHT {
// 	ctx := context.TODO()
//
// 	peerstore := peer.NewPeerstore()
//
// 	ctx, _ = context.WithCancel(ctx)
// 	dhts := netservice.NewService(nil) // nil handler for now, need to patch it
// 	if err := dhts.Start(ctx); err != nil {
// 		t.Fatal(err)
// 	}
//
// 	net, err := inet.NewIpfsNetwork(context.TODO(), p, &mux.ProtocolMap{
// 		mux.ProtocolID_Routing: dhts,
// 	})
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	d := NewDHT(p, peerstore, net, dhts, ds.NewMapDatastore())
// 	dhts.Handler = d
// 	return d
// }
//
// func setupDHTS(n int, t *testing.T) ([]*ma.Multiaddr, []*peer.Peer, []*IpfsDHT) {
// 	var addrs []*ma.Multiaddr
// 	for i := 0; i < 4; i++ {
// 		a, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5000+i))
// 		if err != nil {
// 			t.Fatal(err)
// 		}
// 		addrs = append(addrs, a)
// 	}
//
// 	var peers []*peer.Peer
// 	for i := 0; i < 4; i++ {
// 		p := new(peer.Peer)
// 		p.AddAddress(addrs[i])
// 		sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
// 		if err != nil {
// 			panic(err)
// 		}
// 		p.PubKey = pk
// 		p.PrivKey = sk
// 		id, err := spipe.IDFromPubKey(pk)
// 		if err != nil {
// 			panic(err)
// 		}
// 		p.ID = id
// 		peers = append(peers, p)
// 	}
//
// 	var dhts []*IpfsDHT
// 	for i := 0; i < 4; i++ {
// 		dhts[i] = setupDHT(t, peers[i])
// 	}
//
// 	return addrs, peers, dhts
// }
//
// func makePeer(addr *ma.Multiaddr) *peer.Peer {
// 	p := new(peer.Peer)
// 	p.AddAddress(addr)
// 	sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
// 	if err != nil {
// 		panic(err)
// 	}
// 	p.PrivKey = sk
// 	p.PubKey = pk
// 	id, err := spipe.IDFromPubKey(pk)
// 	if err != nil {
// 		panic(err)
// 	}
//
// 	p.ID = id
// 	return p
// }
//
// func TestPing(t *testing.T) {
// 	u.Debug = true
// 	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/2222")
// 	if err != nil {
// 		t.Fatal(err)
// 	}
// 	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5678")
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	peerA := makePeer(addrA)
// 	peerB := makePeer(addrB)
//
// 	dhtA := setupDHT(t, peerA)
// 	dhtB := setupDHT(t, peerB)
//
// 	defer dhtA.Halt()
// 	defer dhtB.Halt()
//
// 	_, err = dhtA.Connect(addrB)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	//Test that we can ping the node
// 	err = dhtA.Ping(peerB, time.Second*2)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
// }
//
// func TestValueGetSet(t *testing.T) {
// 	u.Debug = false
// 	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1235")
// 	if err != nil {
// 		t.Fatal(err)
// 	}
// 	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5679")
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	peerA := makePeer(addrA)
// 	peerB := makePeer(addrB)
//
// 	dhtA := setupDHT(t, peerA)
// 	dhtB := setupDHT(t, peerB)
//
// 	defer dhtA.Halt()
// 	defer dhtB.Halt()
//
// 	_, err = dhtA.Connect(addrB)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	dhtA.PutValue("hello", []byte("world"))
//
// 	val, err := dhtA.GetValue("hello", time.Second*2)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	if string(val) != "world" {
// 		t.Fatalf("Expected 'world' got '%s'", string(val))
// 	}
//
// }
//
// func TestProvides(t *testing.T) {
// 	u.Debug = false
//
// 	addrs, _, dhts := setupDHTS(4, t)
// 	defer func() {
// 		for i := 0; i < 4; i++ {
// 			dhts[i].Halt()
// 		}
// 	}()
//
// 	_, err := dhts[0].Connect(addrs[1])
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	_, err = dhts[1].Connect(addrs[2])
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	_, err = dhts[1].Connect(addrs[3])
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	bits, err := dhts[3].getLocal(u.Key("hello"))
// 	if err != nil && bytes.Equal(bits, []byte("world")) {
// 		t.Fatal(err)
// 	}
//
// 	err = dhts[3].Provide(u.Key("hello"))
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	time.Sleep(time.Millisecond * 60)
//
// 	provs, err := dhts[0].FindProviders(u.Key("hello"), time.Second)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	if len(provs) != 1 {
// 		t.Fatal("Didnt get back providers")
// 	}
// }
//
// func TestLayeredGet(t *testing.T) {
// 	u.Debug = false
// 	addrs, _, dhts := setupDHTS(4, t)
// 	defer func() {
// 		for i := 0; i < 4; i++ {
// 			dhts[i].Halt()
// 		}
// 	}()
//
// 	_, err := dhts[0].Connect(addrs[1])
// 	if err != nil {
// 		t.Fatalf("Failed to connect: %s", err)
// 	}
//
// 	_, err = dhts[1].Connect(addrs[2])
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	_, err = dhts[1].Connect(addrs[3])
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	err = dhts[3].Provide(u.Key("hello"))
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	time.Sleep(time.Millisecond * 60)
//
// 	val, err := dhts[0].GetValue(u.Key("hello"), time.Second)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	if string(val) != "world" {
// 		t.Fatal("Got incorrect value.")
// 	}
//
// }
//
// func TestFindPeer(t *testing.T) {
// 	u.Debug = false
//
// 	addrs, peers, dhts := setupDHTS(4, t)
// 	go func() {
// 		for i := 0; i < 4; i++ {
// 			dhts[i].Halt()
// 		}
// 	}()
//
// 	_, err := dhts[0].Connect(addrs[1])
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	_, err = dhts[1].Connect(addrs[2])
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	_, err = dhts[1].Connect(addrs[3])
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	p, err := dhts[0].FindPeer(peers[2].ID, time.Second)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
//
// 	if p == nil {
// 		t.Fatal("Failed to find peer.")
// 	}
//
// 	if !p.ID.Equal(peers[2].ID) {
// 		t.Fatal("Didnt find expected peer.")
// 	}
// }
<|endoftext|>


<commit_before>package dht

import (
	"testing"

	crand "crypto/rand"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	msg "github.com/jbenet/go-ipfs/net/message"
	mux "github.com/jbenet/go-ipfs/net/mux"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	"time"
)

// mesHandleFunc is a function that takes in outgoing messages
// and can respond to them, simulating other peers on the network.
// returning nil will chose not to respond and pass the message onto the
// next registered handler
type mesHandleFunc func(msg.NetMessage) msg.NetMessage

// fauxNet is a standin for a swarm.Network in order to more easily recreate
// different testing scenarios
type fauxSender struct {
	handlers []mesHandleFunc
}

func (f *fauxSender) AddHandler(fn func(msg.NetMessage) msg.NetMessage) {
	f.handlers = append(f.handlers, fn)
}

func (f *fauxSender) SendRequest(ctx context.Context, m msg.NetMessage) (msg.NetMessage, error) {
	for _, h := range f.handlers {
		reply := h(m)
		if reply != nil {
			return reply, nil
		}
	}

	// no reply? ok force a timeout
	select {
	case <-ctx.Done():
	}

	return nil, ctx.Err()
}

func (f *fauxSender) SendMessage(ctx context.Context, m msg.NetMessage) error {
	for _, h := range f.handlers {
		reply := h(m)
		if reply != nil {
			return nil
		}
	}
	return nil
}

// fauxNet is a standin for a swarm.Network in order to more easily recreate
// different testing scenarios
type fauxNet struct {
}

// DialPeer attempts to establish a connection to a given peer
func (f *fauxNet) DialPeer(*peer.Peer) error {
	return nil
}

// ClosePeer connection to peer
func (f *fauxNet) ClosePeer(*peer.Peer) error {
	return nil
}

// IsConnected returns whether a connection to given peer exists.
func (f *fauxNet) IsConnected(*peer.Peer) (bool, error) {
	return true, nil
}

// GetProtocols returns the protocols registered in the network.
func (f *fauxNet) GetProtocols() *mux.ProtocolMap { return nil }

// SendMessage sends given Message out
func (f *fauxNet) SendMessage(msg.NetMessage) error {
	return nil
}

// Close terminates all network operation
func (f *fauxNet) Close() error { return nil }

func TestGetFailures(t *testing.T) {
	// t.Skip("skipping test because it makes a lot of output")

	ctx := context.Background()
	fn := &fauxNet{}
	fs := &fauxSender{}

	peerstore := peer.NewPeerstore()
	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")

	d := NewDHT(local, peerstore, fn, fs, ds.NewMapDatastore())
	other := &peer.Peer{ID: peer.ID("other_peer")}
	d.Update(other)

	// This one should time out
	// u.POut("Timout Test\n")
	ctx1, _ := context.WithTimeout(context.Background(), time.Second)
	_, err := d.GetValue(ctx1, u.Key("test"))
	if err != nil {
		if err != context.DeadlineExceeded {
			t.Fatal("Got different error than we expected", err)
		}
	} else {
		t.Fatal("Did not get expected error!")
	}

	// u.POut("NotFound Test\n")
	// Reply with failures to every message
	fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage {
		pmes := new(Message)
		err := proto.Unmarshal(mes.Data(), pmes)
		if err != nil {
			t.Fatal(err)
		}

		resp := &Message{
			Type: pmes.Type,
		}
		m, err := msg.FromObject(mes.Peer(), resp)
		return m
	})

	// This one should fail with NotFound
	ctx2, _ := context.WithTimeout(context.Background(), time.Second)
	_, err = d.GetValue(ctx2, u.Key("test"))
	if err != nil {
		if err != u.ErrNotFound {
			t.Fatalf("Expected ErrNotFound, got: %s", err)
		}
	} else {
		t.Fatal("expected error, got none.")
	}

	fs.handlers = nil
	// Now we test this DHT's handleGetValue failure
	typ := Message_GET_VALUE
	str := "hello"
	req := Message{
		Type:  &typ,
		Key:   &str,
		Value: []byte{0},
	}

	// u.POut("handleGetValue Test\n")
	mes, err := msg.FromObject(other, &req)
	if err != nil {
		t.Error(err)
	}

	mes = d.HandleMessage(ctx, mes)

	pmes := new(Message)
	err = proto.Unmarshal(mes.Data(), pmes)
	if err != nil {
		t.Fatal(err)
	}
	if pmes.GetValue() != nil {
		t.Fatal("shouldnt have value")
	}
	if pmes.GetCloserPeers() != nil {
		t.Fatal("shouldnt have closer peers")
	}
	if pmes.GetProviderPeers() != nil {
		t.Fatal("shouldnt have provider peers")
	}

}

// TODO: Maybe put these in some sort of "ipfs_testutil" package
func _randPeer() *peer.Peer {
	p := new(peer.Peer)
	p.ID = make(peer.ID, 16)
	p.Addresses = []ma.Multiaddr{nil}
	crand.Read(p.ID)
	return p
}

func TestNotFound(t *testing.T) {
	// t.Skip("skipping test because it makes a lot of output")

	fn := &fauxNet{}
	fs := &fauxSender{}

	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")
	peerstore := peer.NewPeerstore()

	d := NewDHT(local, peerstore, fn, fs, ds.NewMapDatastore())

	var ps []*peer.Peer
	for i := 0; i < 5; i++ {
		ps = append(ps, _randPeer())
		d.Update(ps[i])
	}

	// Reply with random peers to every message
	fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage {
		pmes := new(Message)
		err := proto.Unmarshal(mes.Data(), pmes)
		if err != nil {
			t.Fatal(err)
		}

		switch pmes.GetType() {
		case Message_GET_VALUE:
			resp := &Message{Type: pmes.Type}

			peers := []*peer.Peer{}
			for i := 0; i < 7; i++ {
				peers = append(peers, _randPeer())
			}
			resp.CloserPeers = peersToPBPeers(peers)
			mes, err := msg.FromObject(mes.Peer(), resp)
			if err != nil {
				t.Error(err)
			}
			return mes
		default:
			panic("Shouldnt recieve this.")
		}

	})

	ctx, _ := context.WithTimeout(context.Background(), time.Second*5)
	v, err := d.GetValue(ctx, u.Key("hello"))
	log.Debug("get value got %v", v)
	if err != nil {
		switch err {
		case u.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}

// If less than K nodes are in the entire network, it should fail when we make
// a GET rpc and nobody has the value
func TestLessThanKResponses(t *testing.T) {
	// t.Skip("skipping test because it makes a lot of output")

	u.Debug = false
	fn := &fauxNet{}
	fs := &fauxSender{}
	peerstore := peer.NewPeerstore()
	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")

	d := NewDHT(local, peerstore, fn, fs, ds.NewMapDatastore())

	var ps []*peer.Peer
	for i := 0; i < 5; i++ {
		ps = append(ps, _randPeer())
		d.Update(ps[i])
	}
	other := _randPeer()

	// Reply with random peers to every message
	fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage {
		pmes := new(Message)
		err := proto.Unmarshal(mes.Data(), pmes)
		if err != nil {
			t.Fatal(err)
		}

		switch pmes.GetType() {
		case Message_GET_VALUE:
			resp := &Message{
				Type:        pmes.Type,
				CloserPeers: peersToPBPeers([]*peer.Peer{other}),
			}

			mes, err := msg.FromObject(mes.Peer(), resp)
			if err != nil {
				t.Error(err)
			}
			return mes
		default:
			panic("Shouldnt recieve this.")
		}

	})

	ctx, _ := context.WithTimeout(context.Background(), time.Second*30)
	_, err := d.GetValue(ctx, u.Key("hello"))
	if err != nil {
		switch err {
		case u.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}
<commit_msg>update dht tests to new network interface<commit_after>package dht

import (
	"testing"

	crand "crypto/rand"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	msg "github.com/jbenet/go-ipfs/net/message"
	mux "github.com/jbenet/go-ipfs/net/mux"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	"time"
)

// mesHandleFunc is a function that takes in outgoing messages
// and can respond to them, simulating other peers on the network.
// returning nil will chose not to respond and pass the message onto the
// next registered handler
type mesHandleFunc func(msg.NetMessage) msg.NetMessage

// fauxNet is a standin for a swarm.Network in order to more easily recreate
// different testing scenarios
type fauxSender struct {
	handlers []mesHandleFunc
}

func (f *fauxSender) AddHandler(fn func(msg.NetMessage) msg.NetMessage) {
	f.handlers = append(f.handlers, fn)
}

func (f *fauxSender) SendRequest(ctx context.Context, m msg.NetMessage) (msg.NetMessage, error) {
	for _, h := range f.handlers {
		reply := h(m)
		if reply != nil {
			return reply, nil
		}
	}

	// no reply? ok force a timeout
	select {
	case <-ctx.Done():
	}

	return nil, ctx.Err()
}

func (f *fauxSender) SendMessage(ctx context.Context, m msg.NetMessage) error {
	for _, h := range f.handlers {
		reply := h(m)
		if reply != nil {
			return nil
		}
	}
	return nil
}

// fauxNet is a standin for a swarm.Network in order to more easily recreate
// different testing scenarios
type fauxNet struct {
}

// DialPeer attempts to establish a connection to a given peer
func (f *fauxNet) DialPeer(*peer.Peer) error {
	return nil
}

// ClosePeer connection to peer
func (f *fauxNet) ClosePeer(*peer.Peer) error {
	return nil
}

// IsConnected returns whether a connection to given peer exists.
func (f *fauxNet) IsConnected(*peer.Peer) (bool, error) {
	return true, nil
}

// GetProtocols returns the protocols registered in the network.
func (f *fauxNet) GetProtocols() *mux.ProtocolMap { return nil }

// SendMessage sends given Message out
func (f *fauxNet) SendMessage(msg.NetMessage) error {
	return nil
}

func (f *fauxNet) GetPeerList() []*peer.Peer {
	return nil
}

// Close terminates all network operation
func (f *fauxNet) Close() error { return nil }

func TestGetFailures(t *testing.T) {
	// t.Skip("skipping test because it makes a lot of output")

	ctx := context.Background()
	fn := &fauxNet{}
	fs := &fauxSender{}

	peerstore := peer.NewPeerstore()
	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")

	d := NewDHT(local, peerstore, fn, fs, ds.NewMapDatastore())
	other := &peer.Peer{ID: peer.ID("other_peer")}
	d.Update(other)

	// This one should time out
	// u.POut("Timout Test\n")
	ctx1, _ := context.WithTimeout(context.Background(), time.Second)
	_, err := d.GetValue(ctx1, u.Key("test"))
	if err != nil {
		if err != context.DeadlineExceeded {
			t.Fatal("Got different error than we expected", err)
		}
	} else {
		t.Fatal("Did not get expected error!")
	}

	// u.POut("NotFound Test\n")
	// Reply with failures to every message
	fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage {
		pmes := new(Message)
		err := proto.Unmarshal(mes.Data(), pmes)
		if err != nil {
			t.Fatal(err)
		}

		resp := &Message{
			Type: pmes.Type,
		}
		m, err := msg.FromObject(mes.Peer(), resp)
		return m
	})

	// This one should fail with NotFound
	ctx2, _ := context.WithTimeout(context.Background(), time.Second)
	_, err = d.GetValue(ctx2, u.Key("test"))
	if err != nil {
		if err != u.ErrNotFound {
			t.Fatalf("Expected ErrNotFound, got: %s", err)
		}
	} else {
		t.Fatal("expected error, got none.")
	}

	fs.handlers = nil
	// Now we test this DHT's handleGetValue failure
	typ := Message_GET_VALUE
	str := "hello"
	req := Message{
		Type:  &typ,
		Key:   &str,
		Value: []byte{0},
	}

	// u.POut("handleGetValue Test\n")
	mes, err := msg.FromObject(other, &req)
	if err != nil {
		t.Error(err)
	}

	mes = d.HandleMessage(ctx, mes)

	pmes := new(Message)
	err = proto.Unmarshal(mes.Data(), pmes)
	if err != nil {
		t.Fatal(err)
	}
	if pmes.GetValue() != nil {
		t.Fatal("shouldnt have value")
	}
	if pmes.GetCloserPeers() != nil {
		t.Fatal("shouldnt have closer peers")
	}
	if pmes.GetProviderPeers() != nil {
		t.Fatal("shouldnt have provider peers")
	}

}

// TODO: Maybe put these in some sort of "ipfs_testutil" package
func _randPeer() *peer.Peer {
	p := new(peer.Peer)
	p.ID = make(peer.ID, 16)
	p.Addresses = []ma.Multiaddr{nil}
	crand.Read(p.ID)
	return p
}

func TestNotFound(t *testing.T) {
	// t.Skip("skipping test because it makes a lot of output")

	fn := &fauxNet{}
	fs := &fauxSender{}

	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")
	peerstore := peer.NewPeerstore()

	d := NewDHT(local, peerstore, fn, fs, ds.NewMapDatastore())

	var ps []*peer.Peer
	for i := 0; i < 5; i++ {
		ps = append(ps, _randPeer())
		d.Update(ps[i])
	}

	// Reply with random peers to every message
	fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage {
		pmes := new(Message)
		err := proto.Unmarshal(mes.Data(), pmes)
		if err != nil {
			t.Fatal(err)
		}

		switch pmes.GetType() {
		case Message_GET_VALUE:
			resp := &Message{Type: pmes.Type}

			peers := []*peer.Peer{}
			for i := 0; i < 7; i++ {
				peers = append(peers, _randPeer())
			}
			resp.CloserPeers = peersToPBPeers(peers)
			mes, err := msg.FromObject(mes.Peer(), resp)
			if err != nil {
				t.Error(err)
			}
			return mes
		default:
			panic("Shouldnt recieve this.")
		}

	})

	ctx, _ := context.WithTimeout(context.Background(), time.Second*5)
	v, err := d.GetValue(ctx, u.Key("hello"))
	log.Debug("get value got %v", v)
	if err != nil {
		switch err {
		case u.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}

// If less than K nodes are in the entire network, it should fail when we make
// a GET rpc and nobody has the value
func TestLessThanKResponses(t *testing.T) {
	// t.Skip("skipping test because it makes a lot of output")

	u.Debug = false
	fn := &fauxNet{}
	fs := &fauxSender{}
	peerstore := peer.NewPeerstore()
	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")

	d := NewDHT(local, peerstore, fn, fs, ds.NewMapDatastore())

	var ps []*peer.Peer
	for i := 0; i < 5; i++ {
		ps = append(ps, _randPeer())
		d.Update(ps[i])
	}
	other := _randPeer()

	// Reply with random peers to every message
	fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage {
		pmes := new(Message)
		err := proto.Unmarshal(mes.Data(), pmes)
		if err != nil {
			t.Fatal(err)
		}

		switch pmes.GetType() {
		case Message_GET_VALUE:
			resp := &Message{
				Type:        pmes.Type,
				CloserPeers: peersToPBPeers([]*peer.Peer{other}),
			}

			mes, err := msg.FromObject(mes.Peer(), resp)
			if err != nil {
				t.Error(err)
			}
			return mes
		default:
			panic("Shouldnt recieve this.")
		}

	})

	ctx, _ := context.WithTimeout(context.Background(), time.Second*30)
	_, err := d.GetValue(ctx, u.Key("hello"))
	if err != nil {
		switch err {
		case u.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}
<|endoftext|>


<commit_before>package router

import (
	"fmt"
	"runtime"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/distribution/health"

	"encoding/json"

	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/auth"
	"github.com/smancke/guble/server/cluster"
	"github.com/smancke/guble/server/kvstore"
	"github.com/smancke/guble/server/store"
	"net/http"
)

const (
	overloadedHandleChannelRatio = 0.9
	handleChannelCapacity        = 500
	subscribeChannelCapacity     = 10
	unsubscribeChannelCapacity   = 10
	routerPrefix                 = "/debug/router"
)

// Router interface provides a mechanism for PubSub messaging
type Router interface {
	Subscribe(r *Route) (*Route, error)
	Unsubscribe(r *Route)
	HandleMessage(message *protocol.Message) error
	Fetch(*store.FetchRequest) error
	GetSubscribers(topic string) ([]byte, error)

	AccessManager() (auth.AccessManager, error)
	MessageStore() (store.MessageStore, error)
	KVStore() (kvstore.KVStore, error)
	Cluster() *cluster.Cluster

	Done() <-chan bool
}

// Helper struct to pass `Route` to subscription channel and provide a notification channel.
type subRequest struct {
	route *Route
	doneC chan bool
}

type router struct {
	routes       map[protocol.Path][]*Route // mapping the path to the route slice
	handleC      chan *protocol.Message
	subscribeC   chan subRequest
	unsubscribeC chan subRequest
	stopC        chan bool      // Channel that signals stop of the router
	stopping     bool           // Flag: the router is in stopping process and no incoming messages are accepted
	wg           sync.WaitGroup // Add any operation that we need to wait upon here

	accessManager auth.AccessManager
	messageStore  store.MessageStore
	kvStore       kvstore.KVStore
	cluster       *cluster.Cluster

	sync.RWMutex
}

// New returns a pointer to Router
func New(accessManager auth.AccessManager, messageStore store.MessageStore, kvStore kvstore.KVStore, cluster *cluster.Cluster) Router {
	return &router{
		routes: make(map[protocol.Path][]*Route),

		handleC:      make(chan *protocol.Message, handleChannelCapacity),
		subscribeC:   make(chan subRequest, subscribeChannelCapacity),
		unsubscribeC: make(chan subRequest, unsubscribeChannelCapacity),
		stopC:        make(chan bool, 1),

		accessManager: accessManager,
		messageStore:  messageStore,
		kvStore:       kvStore,
		cluster:       cluster,
	}
}

func (router *router) Start() error {
	router.panicIfInternalDependenciesAreNil()
	logger.Info("Starting router")
	resetRouterMetrics()

	router.wg.Add(1)
	router.setStopping(false)

	go func() {
		for {
			if router.stopping && router.channelsAreEmpty() {
				router.closeRoutes()
				router.wg.Done()
				return
			}

			func() {
				defer protocol.PanicLogger()

				select {
				case message := <-router.handleC:
					router.handleMessage(message)
					runtime.Gosched()
				case subscriber := <-router.subscribeC:
					router.subscribe(subscriber.route)
					subscriber.doneC <- true
				case unsubscriber := <-router.unsubscribeC:
					router.unsubscribe(unsubscriber.route)
					unsubscriber.doneC <- true
				case <-router.Done():
					router.setStopping(true)
				}
			}()
		}
	}()

	return nil
}

// Stop stops the router by closing the stop channel, and waiting on the WaitGroup
func (router *router) Stop() error {
	logger.Info("Stopping router")

	router.stopC <- true
	router.wg.Wait()
	return nil
}

func (router *router) Check() error {
	if router.accessManager == nil || router.messageStore == nil || router.kvStore == nil {
		logger.WithError(ErrServiceNotProvided).Error("Some mandatory services are not provided")
		return ErrServiceNotProvided
	}
	if checkable, ok := router.messageStore.(health.Checker); ok {
		err := checkable.Check()
		if err != nil {
			logger.WithField("error", err.Error()).Error("MessageStore check failed")
			return err
		}
	}
	if checkable, ok := router.kvStore.(health.Checker); ok {
		err := checkable.Check()
		if err != nil {
			logger.WithField("error", err.Error()).Error("KVStore check failed")
			return err
		}
	}
	return nil
}

// HandleMessage stores the message in the MessageStore(and gets a new ID for it if the message was created locally)
// and then passes it to the internal channel, and asynchronously to the cluster (if available).
func (router *router) HandleMessage(message *protocol.Message) error {
	logger.WithFields(log.Fields{
		"userID": message.UserID,
		"path":   message.Path}).Debug("HandleMessage")

	mTotalMessagesIncoming.Add(1)
	if err := router.isStopping(); err != nil {
		logger.WithField("error", err.Error()).Error("Router is stopping")
		return err
	}

	if !router.accessManager.IsAllowed(auth.WRITE, message.UserID, message.Path) {
		return &PermissionDeniedError{UserID: message.UserID, AccessType: auth.WRITE, Path: message.Path}
	}

	var nodeID uint8
	if router.cluster != nil {
		nodeID = router.cluster.Config.ID
	}

	mTotalMessagesIncomingBytes.Add(int64(len(message.Bytes())))
	size, err := router.messageStore.StoreMessage(message, nodeID)
	if err != nil {
		logger.WithField("error", err.Error()).Error("Error storing message")
		mTotalMessageStoreErrors.Add(1)
		return err
	}
	mTotalMessagesStoredBytes.Add(int64(size))

	router.handleOverloadedChannel()

	router.handleC <- message

	if router.cluster != nil && message.NodeID == router.cluster.Config.ID {
		go router.cluster.BroadcastMessage(message)
	}

	return nil
}

func (router *router) Subscribe(r *Route) (*Route, error) {
	logger.WithFields(log.Fields{
		"accessManager": router.accessManager,
		"route":         r,
	}).Debug("Subscribe")

	if err := router.isStopping(); err != nil {
		return nil, err
	}

	userID := r.Get("user_id")
	routePath := r.Path

	accessAllowed := router.accessManager.IsAllowed(auth.READ, userID, routePath)
	if !accessAllowed {
		return r, &PermissionDeniedError{UserID: userID, AccessType: auth.READ, Path: routePath}
	}
	req := subRequest{
		route: r,
		doneC: make(chan bool),
	}

	router.subscribeC <- req
	<-req.doneC
	return r, nil
}

// Subscribe adds a route to the subscribers. If there is already a route with same Application Id and Path, it will be replaced.
func (router *router) Unsubscribe(r *Route) {
	logger.WithFields(log.Fields{
		"accessManager": router.accessManager,
		"route":         r,
	}).Debug("Unsubscribe")

	req := subRequest{
		route: r,
		doneC: make(chan bool),
	}
	router.unsubscribeC <- req
	<-req.doneC
}

func (router *router) GetSubscribers(topicPath string) ([]byte, error) {
	subscribers := make([]RouteParams, 0)
	routes, present := router.routes[protocol.Path(topicPath)]
	if present {
		for index, currRoute := range routes {
			logger.WithFields(log.Fields{
				"index":       index,
				"routeParams": currRoute.RouteParams,
			}).Debug("Added route to slice")
			subscribers = append(subscribers, currRoute.RouteParams)
		}
	}
	return json.Marshal(subscribers)
}

func (router *router) subscribe(r *Route) {
	logger.WithField("route", r).Debug("Internal subscribe")
	mTotalSubscriptionAttempts.Add(1)

	routePath := r.Path
	slice, present := router.routes[routePath]
	var removed bool
	if present {
		// Try to remove, to avoid double subscriptions of the same app
		slice, removed = removeIfMatching(slice, r)
	} else {
		// Path not present yet. Initialize the slice
		slice = make([]*Route, 0, 1)
		router.routes[routePath] = slice
		mCurrentRoutes.Add(1)
	}
	router.routes[routePath] = append(slice, r)
	if removed {
		mTotalDuplicateSubscriptionsAttempts.Add(1)
	} else {
		mTotalSubscriptions.Add(1)
		mCurrentSubscriptions.Add(1)
	}
}

func (router *router) unsubscribe(r *Route) {
	logger.WithField("route", r).Debug("Internal unsubscribe")
	mTotalUnsubscriptionAttempts.Add(1)

	routePath := r.Path
	slice, present := router.routes[routePath]
	if !present {
		mTotalInvalidTopicOnUnsubscriptionAttempts.Add(1)
		return
	}
	var removed bool
	router.routes[routePath], removed = removeIfMatching(slice, r)
	if removed {
		mTotalUnsubscriptions.Add(1)
		mCurrentSubscriptions.Add(-1)
	} else {
		mTotalInvalidUnsubscriptionAttempts.Add(1)
	}
	if len(router.routes[routePath]) == 0 {
		delete(router.routes, routePath)
		mCurrentRoutes.Add(-1)
	}
}

func (router *router) panicIfInternalDependenciesAreNil() {
	if router.accessManager == nil || router.kvStore == nil || router.messageStore == nil {
		panic(fmt.Sprintf("router: the internal dependencies marked with `true` are not set: AccessManager=%v, KVStore=%v, MessageStore=%v",
			router.accessManager == nil, router.kvStore == nil, router.messageStore == nil))
	}
}

func (router *router) channelsAreEmpty() bool {
	return len(router.handleC) == 0 && len(router.subscribeC) == 0 && len(router.unsubscribeC) == 0
}

func (router *router) setStopping(v bool) {
	router.Lock()
	defer router.Unlock()

	router.stopping = v
}

func (router *router) Done() <-chan bool {
	return router.stopC
}

func (router *router) isStopping() error {
	router.RLock()
	defer router.RUnlock()

	if router.stopping {
		return &ModuleStoppingError{"Router"}
	}

	return nil
}

func (router *router) handleMessage(message *protocol.Message) {
	flog := logger.WithFields(log.Fields{
		"topic":    message.Path,
		"metadata": message.Metadata(),
		"filters":  message.Filters,
	})
	flog.Debug("Called routeMessage for data")
	mTotalMessagesRouted.Add(1)

	matched := false
	for path, pathRoutes := range router.routes {
		if matchesTopic(message.Path, path) {
			matched = true
			for _, route := range pathRoutes {
				if err := route.Deliver(message); err == ErrInvalidRoute {
					// Unsubscribe invalid routes
					router.unsubscribe(route)
				}
			}
		}
	}

	if !matched {
		flog.Debug("No route matched.")
		mTotalMessagesNotMatchingTopic.Add(1)
	}
}

func (router *router) closeRoutes() {
	logger.Debug("closeRoutes")

	for _, currentRouteList := range router.routes {
		for _, route := range currentRouteList {
			router.unsubscribe(route)
			log.WithFields(log.Fields{"module": "router", "route": route.String()}).Debug("Closing route")
			route.Close()
		}
	}
}

func (router *router) handleOverloadedChannel() {
	if float32(len(router.handleC))/float32(cap(router.handleC)) > overloadedHandleChannelRatio {
		logger.WithFields(log.Fields{
			"currentLength": len(router.handleC),
			"maxCapacity":   cap(router.handleC),
		}).Warn("handleC channel is almost full")
		mTotalOverloadedHandleChannel.Add(1)
	}
}

// matchesTopic checks whether the supplied routePath matches the message topic
func matchesTopic(messagePath, routePath protocol.Path) bool {
	messagePathLen := len(string(messagePath))
	routePathLen := len(string(routePath))
	return strings.HasPrefix(string(messagePath), string(routePath)) &&
		(messagePathLen == routePathLen ||
			(messagePathLen > routePathLen && string(messagePath)[routePathLen] == '/'))
}

// removeIfMatching removes a route from the supplied list, based on same ApplicationID id and same path (if existing)
// returns: the (possibly updated) slide, and a boolean value (true if route was removed, false otherwise)
func removeIfMatching(slice []*Route, route *Route) ([]*Route, bool) {
	position := -1
	for p, r := range slice {
		if r.Equal(route) {
			position = p
			break
		}
	}
	if position == -1 {
		return slice, false
	}
	return append(slice[:position], slice[position+1:]...), true
}

func (router *router) Fetch(req *store.FetchRequest) error {
	logger.Debug("Fetch")
	if err := router.isStopping(); err != nil {
		return err
	}
	router.messageStore.Fetch(req)
	return nil
}

// AccessManager returns the `accessManager` provided for the router
func (router *router) AccessManager() (auth.AccessManager, error) {
	if router.accessManager == nil {
		return nil, ErrServiceNotProvided
	}
	return router.accessManager, nil
}

// MessageStore returns the `messageStore` provided for the router
func (router *router) MessageStore() (store.MessageStore, error) {
	if router.messageStore == nil {
		return nil, ErrServiceNotProvided
	}
	return router.messageStore, nil
}

// KVStore returns the `kvStore` provided for the router
func (router *router) KVStore() (kvstore.KVStore, error) {
	if router.kvStore == nil {
		return nil, ErrServiceNotProvided
	}
	return router.kvStore, nil
}

// Cluster returns the `cluster` provided for the router, or nil if no cluster was set-up
func (router *router) Cluster() *cluster.Cluster {
	return router.cluster
}

func (router *router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	if req.Method != http.MethodGet {
		http.Error(w, `{"error": Error method not allowed.Only HTTP GET is accepted}`, http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	err := encoder.Encode(router.routes)
	if err != nil {
		http.Error(w, `{"error":Error encoding data.}`, http.StatusInternalServerError)
		logger.WithField("error", err.Error()).Error("Error encoding data.")
		return
	}
}

func (router *router) GetPrefix() string {
	return routerPrefix
}
<commit_msg>pr refactor.<commit_after>package router

import (
	"fmt"
	"runtime"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/distribution/health"

	"encoding/json"

	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/auth"
	"github.com/smancke/guble/server/cluster"
	"github.com/smancke/guble/server/kvstore"
	"github.com/smancke/guble/server/store"
	"net/http"
)

const (
	overloadedHandleChannelRatio = 0.9
	handleChannelCapacity        = 500
	subscribeChannelCapacity     = 10
	unsubscribeChannelCapacity   = 10
	routerPrefix                 = "/admin/router"
)

// Router interface provides a mechanism for PubSub messaging
type Router interface {
	Subscribe(r *Route) (*Route, error)
	Unsubscribe(r *Route)
	HandleMessage(message *protocol.Message) error
	Fetch(*store.FetchRequest) error
	GetSubscribers(topic string) ([]byte, error)

	AccessManager() (auth.AccessManager, error)
	MessageStore() (store.MessageStore, error)
	KVStore() (kvstore.KVStore, error)
	Cluster() *cluster.Cluster

	Done() <-chan bool
}

// Helper struct to pass `Route` to subscription channel and provide a notification channel.
type subRequest struct {
	route *Route
	doneC chan bool
}

type router struct {
	routes       map[protocol.Path][]*Route // mapping the path to the route slice
	handleC      chan *protocol.Message
	subscribeC   chan subRequest
	unsubscribeC chan subRequest
	stopC        chan bool      // Channel that signals stop of the router
	stopping     bool           // Flag: the router is in stopping process and no incoming messages are accepted
	wg           sync.WaitGroup // Add any operation that we need to wait upon here

	accessManager auth.AccessManager
	messageStore  store.MessageStore
	kvStore       kvstore.KVStore
	cluster       *cluster.Cluster

	sync.RWMutex
}

// New returns a pointer to Router
func New(accessManager auth.AccessManager, messageStore store.MessageStore, kvStore kvstore.KVStore, cluster *cluster.Cluster) Router {
	return &router{
		routes: make(map[protocol.Path][]*Route),

		handleC:      make(chan *protocol.Message, handleChannelCapacity),
		subscribeC:   make(chan subRequest, subscribeChannelCapacity),
		unsubscribeC: make(chan subRequest, unsubscribeChannelCapacity),
		stopC:        make(chan bool, 1),

		accessManager: accessManager,
		messageStore:  messageStore,
		kvStore:       kvStore,
		cluster:       cluster,
	}
}

func (router *router) Start() error {
	router.panicIfInternalDependenciesAreNil()
	logger.Info("Starting router")
	resetRouterMetrics()

	router.wg.Add(1)
	router.setStopping(false)

	go func() {
		for {
			if router.stopping && router.channelsAreEmpty() {
				router.closeRoutes()
				router.wg.Done()
				return
			}

			func() {
				defer protocol.PanicLogger()

				select {
				case message := <-router.handleC:
					router.handleMessage(message)
					runtime.Gosched()
				case subscriber := <-router.subscribeC:
					router.subscribe(subscriber.route)
					subscriber.doneC <- true
				case unsubscriber := <-router.unsubscribeC:
					router.unsubscribe(unsubscriber.route)
					unsubscriber.doneC <- true
				case <-router.Done():
					router.setStopping(true)
				}
			}()
		}
	}()

	return nil
}

// Stop stops the router by closing the stop channel, and waiting on the WaitGroup
func (router *router) Stop() error {
	logger.Info("Stopping router")

	router.stopC <- true
	router.wg.Wait()
	return nil
}

func (router *router) Check() error {
	if router.accessManager == nil || router.messageStore == nil || router.kvStore == nil {
		logger.WithError(ErrServiceNotProvided).Error("Some mandatory services are not provided")
		return ErrServiceNotProvided
	}
	if checkable, ok := router.messageStore.(health.Checker); ok {
		err := checkable.Check()
		if err != nil {
			logger.WithField("error", err.Error()).Error("MessageStore check failed")
			return err
		}
	}
	if checkable, ok := router.kvStore.(health.Checker); ok {
		err := checkable.Check()
		if err != nil {
			logger.WithField("error", err.Error()).Error("KVStore check failed")
			return err
		}
	}
	return nil
}

// HandleMessage stores the message in the MessageStore(and gets a new ID for it if the message was created locally)
// and then passes it to the internal channel, and asynchronously to the cluster (if available).
func (router *router) HandleMessage(message *protocol.Message) error {
	logger.WithFields(log.Fields{
		"userID": message.UserID,
		"path":   message.Path}).Debug("HandleMessage")

	mTotalMessagesIncoming.Add(1)
	if err := router.isStopping(); err != nil {
		logger.WithField("error", err.Error()).Error("Router is stopping")
		return err
	}

	if !router.accessManager.IsAllowed(auth.WRITE, message.UserID, message.Path) {
		return &PermissionDeniedError{UserID: message.UserID, AccessType: auth.WRITE, Path: message.Path}
	}

	var nodeID uint8
	if router.cluster != nil {
		nodeID = router.cluster.Config.ID
	}

	mTotalMessagesIncomingBytes.Add(int64(len(message.Bytes())))
	size, err := router.messageStore.StoreMessage(message, nodeID)
	if err != nil {
		logger.WithField("error", err.Error()).Error("Error storing message")
		mTotalMessageStoreErrors.Add(1)
		return err
	}
	mTotalMessagesStoredBytes.Add(int64(size))

	router.handleOverloadedChannel()

	router.handleC <- message

	if router.cluster != nil && message.NodeID == router.cluster.Config.ID {
		go router.cluster.BroadcastMessage(message)
	}

	return nil
}

func (router *router) Subscribe(r *Route) (*Route, error) {
	logger.WithFields(log.Fields{
		"accessManager": router.accessManager,
		"route":         r,
	}).Debug("Subscribe")

	if err := router.isStopping(); err != nil {
		return nil, err
	}

	userID := r.Get("user_id")
	routePath := r.Path

	accessAllowed := router.accessManager.IsAllowed(auth.READ, userID, routePath)
	if !accessAllowed {
		return r, &PermissionDeniedError{UserID: userID, AccessType: auth.READ, Path: routePath}
	}
	req := subRequest{
		route: r,
		doneC: make(chan bool),
	}

	router.subscribeC <- req
	<-req.doneC
	return r, nil
}

// Subscribe adds a route to the subscribers. If there is already a route with same Application Id and Path, it will be replaced.
func (router *router) Unsubscribe(r *Route) {
	logger.WithFields(log.Fields{
		"accessManager": router.accessManager,
		"route":         r,
	}).Debug("Unsubscribe")

	req := subRequest{
		route: r,
		doneC: make(chan bool),
	}
	router.unsubscribeC <- req
	<-req.doneC
}

func (router *router) GetSubscribers(topicPath string) ([]byte, error) {
	subscribers := make([]RouteParams, 0)
	routes, present := router.routes[protocol.Path(topicPath)]
	if present {
		for index, currRoute := range routes {
			logger.WithFields(log.Fields{
				"index":       index,
				"routeParams": currRoute.RouteParams,
			}).Debug("Added route to slice")
			subscribers = append(subscribers, currRoute.RouteParams)
		}
	}
	return json.Marshal(subscribers)
}

func (router *router) subscribe(r *Route) {
	logger.WithField("route", r).Debug("Internal subscribe")
	mTotalSubscriptionAttempts.Add(1)

	routePath := r.Path
	slice, present := router.routes[routePath]
	var removed bool
	if present {
		// Try to remove, to avoid double subscriptions of the same app
		slice, removed = removeIfMatching(slice, r)
	} else {
		// Path not present yet. Initialize the slice
		slice = make([]*Route, 0, 1)
		router.routes[routePath] = slice
		mCurrentRoutes.Add(1)
	}
	router.routes[routePath] = append(slice, r)
	if removed {
		mTotalDuplicateSubscriptionsAttempts.Add(1)
	} else {
		mTotalSubscriptions.Add(1)
		mCurrentSubscriptions.Add(1)
	}
}

func (router *router) unsubscribe(r *Route) {
	logger.WithField("route", r).Debug("Internal unsubscribe")
	mTotalUnsubscriptionAttempts.Add(1)

	routePath := r.Path
	slice, present := router.routes[routePath]
	if !present {
		mTotalInvalidTopicOnUnsubscriptionAttempts.Add(1)
		return
	}
	var removed bool
	router.routes[routePath], removed = removeIfMatching(slice, r)
	if removed {
		mTotalUnsubscriptions.Add(1)
		mCurrentSubscriptions.Add(-1)
	} else {
		mTotalInvalidUnsubscriptionAttempts.Add(1)
	}
	if len(router.routes[routePath]) == 0 {
		delete(router.routes, routePath)
		mCurrentRoutes.Add(-1)
	}
}

func (router *router) panicIfInternalDependenciesAreNil() {
	if router.accessManager == nil || router.kvStore == nil || router.messageStore == nil {
		panic(fmt.Sprintf("router: the internal dependencies marked with `true` are not set: AccessManager=%v, KVStore=%v, MessageStore=%v",
			router.accessManager == nil, router.kvStore == nil, router.messageStore == nil))
	}
}

func (router *router) channelsAreEmpty() bool {
	return len(router.handleC) == 0 && len(router.subscribeC) == 0 && len(router.unsubscribeC) == 0
}

func (router *router) setStopping(v bool) {
	router.Lock()
	defer router.Unlock()

	router.stopping = v
}

func (router *router) Done() <-chan bool {
	return router.stopC
}

func (router *router) isStopping() error {
	router.RLock()
	defer router.RUnlock()

	if router.stopping {
		return &ModuleStoppingError{"Router"}
	}

	return nil
}

func (router *router) handleMessage(message *protocol.Message) {
	flog := logger.WithFields(log.Fields{
		"topic":    message.Path,
		"metadata": message.Metadata(),
		"filters":  message.Filters,
	})
	flog.Debug("Called routeMessage for data")
	mTotalMessagesRouted.Add(1)

	matched := false
	for path, pathRoutes := range router.routes {
		if matchesTopic(message.Path, path) {
			matched = true
			for _, route := range pathRoutes {
				if err := route.Deliver(message); err == ErrInvalidRoute {
					// Unsubscribe invalid routes
					router.unsubscribe(route)
				}
			}
		}
	}

	if !matched {
		flog.Debug("No route matched.")
		mTotalMessagesNotMatchingTopic.Add(1)
	}
}

func (router *router) closeRoutes() {
	logger.Debug("closeRoutes")

	for _, currentRouteList := range router.routes {
		for _, route := range currentRouteList {
			router.unsubscribe(route)
			log.WithFields(log.Fields{"module": "router", "route": route.String()}).Debug("Closing route")
			route.Close()
		}
	}
}

func (router *router) handleOverloadedChannel() {
	if float32(len(router.handleC))/float32(cap(router.handleC)) > overloadedHandleChannelRatio {
		logger.WithFields(log.Fields{
			"currentLength": len(router.handleC),
			"maxCapacity":   cap(router.handleC),
		}).Warn("handleC channel is almost full")
		mTotalOverloadedHandleChannel.Add(1)
	}
}

// matchesTopic checks whether the supplied routePath matches the message topic
func matchesTopic(messagePath, routePath protocol.Path) bool {
	messagePathLen := len(string(messagePath))
	routePathLen := len(string(routePath))
	return strings.HasPrefix(string(messagePath), string(routePath)) &&
		(messagePathLen == routePathLen ||
			(messagePathLen > routePathLen && string(messagePath)[routePathLen] == '/'))
}

// removeIfMatching removes a route from the supplied list, based on same ApplicationID id and same path (if existing)
// returns: the (possibly updated) slide, and a boolean value (true if route was removed, false otherwise)
func removeIfMatching(slice []*Route, route *Route) ([]*Route, bool) {
	position := -1
	for p, r := range slice {
		if r.Equal(route) {
			position = p
			break
		}
	}
	if position == -1 {
		return slice, false
	}
	return append(slice[:position], slice[position+1:]...), true
}

func (router *router) Fetch(req *store.FetchRequest) error {
	logger.Debug("Fetch")
	if err := router.isStopping(); err != nil {
		return err
	}
	router.messageStore.Fetch(req)
	return nil
}

// AccessManager returns the `accessManager` provided for the router
func (router *router) AccessManager() (auth.AccessManager, error) {
	if router.accessManager == nil {
		return nil, ErrServiceNotProvided
	}
	return router.accessManager, nil
}

// MessageStore returns the `messageStore` provided for the router
func (router *router) MessageStore() (store.MessageStore, error) {
	if router.messageStore == nil {
		return nil, ErrServiceNotProvided
	}
	return router.messageStore, nil
}

// KVStore returns the `kvStore` provided for the router
func (router *router) KVStore() (kvstore.KVStore, error) {
	if router.kvStore == nil {
		return nil, ErrServiceNotProvided
	}
	return router.kvStore, nil
}

// Cluster returns the `cluster` provided for the router, or nil if no cluster was set-up
func (router *router) Cluster() *cluster.Cluster {
	return router.cluster
}

func (router *router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	if req.Method != http.MethodGet {
		http.Error(w, `{"error": Error method not allowed.Only HTTP GET is accepted}`, http.StatusMethodNotAllowed)
		return
	}

	err := json.NewEncoder(w).Encode(router.routes)
	if err != nil {
		http.Error(w, `{"error":Error encoding data.}`, http.StatusInternalServerError)
		logger.WithField("error", err.Error()).Error("Error encoding data.")
		return
	}
}

func (router *router) GetPrefix() string {
	return routerPrefix
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"runtime"
	"runtime/pprof"
	"strconv"
	"testing"

	"veyron.io/apps/rps"

	mtlib "veyron.io/veyron/veyron/services/mounttable/lib"

	"veyron.io/veyron/veyron2"
	"veyron.io/veyron/veyron2/ipc"
	"veyron.io/veyron/veyron2/naming"
	"veyron.io/veyron/veyron2/options"
	"veyron.io/veyron/veyron2/rt"
	"veyron.io/veyron/veyron2/vlog"
)

var spec = ipc.ListenSpec{Protocol: "tcp", Address: "127.0.0.1:0"}

func startMountTable(t *testing.T, runtime veyron2.Runtime) (string, func()) {
	server, err := runtime.NewServer(options.ServesMountTable(true))
	if err != nil {
		t.Fatalf("NewServer() failed: %v", err)
	}
	dispatcher, err := mtlib.NewMountTable("")

	endpoint, err := server.Listen(spec)
	if err != nil {
		t.Fatalf("Listen(%v) failed: %v", spec, err)
	}
	if err := server.ServeDispatcher("", dispatcher); err != nil {
		t.Fatalf("ServeDispatcher(%v) failed: %v", dispatcher, err)
	}
	address := naming.JoinAddressName(endpoint.String(), "")
	vlog.VI(1).Infof("Mount table running at endpoint: %s", address)
	return address, func() {
		if err := server.Stop(); err != nil {
			t.Fatalf("Stop() failed: %v", err)
		}
	}
}

func startRockPaperScissors(t *testing.T, rt veyron2.Runtime, mtAddress string) (*RPS, func()) {
	ns := rt.Namespace()
	ns.SetRoots(mtAddress)
	server, err := rt.NewServer()
	if err != nil {
		t.Fatalf("NewServer failed: %v", err)
	}
	rpsService := NewRPS()

	if _, err = server.Listen(spec); err != nil {
		t.Fatalf("Listen failed: %v", err)
	}
	names := []string{"rps/judge/test", "rps/player/test", "rps/scorekeeper/test"}
	if err := server.Serve(names[0], rps.RockPaperScissorsServer(rpsService), nil); err != nil {
		t.Fatalf("Serve(%v) failed: %v", names[0], err)
	}
	for _, n := range names[1:] {
		server.AddName(n)
	}
	return rpsService, func() {
		if err := server.Stop(); err != nil {
			t.Fatalf("Stop() failed: %v", err)
		}
	}
}

// TestRockPaperScissorsImpl runs one rock-paper-scissors game and verifies
// that all the counters are consistent.
func TestRockPaperScissorsImpl(t *testing.T) {
	r, err := rt.New()
	if err != nil {
		t.Fatalf("Could not initialize runtime: %v", err)
	}
	defer r.Cleanup()

	ctx := r.NewContext()

	mtAddress, mtStop := startMountTable(t, r)
	defer mtStop()
	rpsService, rpsStop := startRockPaperScissors(t, r, mtAddress)
	defer rpsStop()

	const numGames = 100
	for x := 0; x < numGames; x++ {
		if err := rpsService.Player().InitiateGame(ctx); err != nil {
			t.Errorf("Failed to initiate game: %v", err)
		}
	}
	rpsService.Player().WaitUntilIdle()

	// For each game, the player plays twice. So, we expect the player to
	// show that it played 2×numGames, and won numGames.
	played, won := rpsService.Player().Stats()
	if want, got := int64(2*numGames), played; want != got {
		t.Errorf("Unexpected number of played games. Got %d, want %d", got, want)
	}
	if want, got := int64(numGames), won; want != got {
		t.Errorf("Unexpected number of won games. Got %d, want %d", got, want)
	}

	// The Judge ran every game.
	if want, got := int64(numGames), rpsService.Judge().Stats(); want != got {
		t.Errorf("Unexpected number of games run. Got %d, want %d", got, want)
	}

	// The Score Keeper received one score card per game.
	if want, got := int64(numGames), rpsService.ScoreKeeper().Stats(); want != got {
		t.Errorf("Unexpected number of score cards. Got %d, want %d", got, want)
	}

	// Check for leaked goroutines.
	if n := runtime.NumGoroutine(); n > 100 {
		t.Logf("Num goroutines: %d", n)
		if leak := reportLeakedGoroutines(t, n/3); len(leak) != 0 {
			t.Errorf("Potentially leaked goroutines:\n%s", leak)
		}
	}
}

// reportLeakedGoroutines reads the goroutine pprof profile and returns the
// goroutines that have more than 'threshold' instances.
func reportLeakedGoroutines(t *testing.T, threshold int) string {
	f, err := ioutil.TempFile("", "test-profile-")
	if err != nil {
		t.Fatalf("Failed to create temp file: %v", err)
	}
	defer f.Close()
	defer os.Remove(f.Name())
	p := pprof.Lookup("goroutine")
	if p == nil {
		t.Fatalf("Failed to lookup the goroutine profile")
	}
	if err := p.WriteTo(f, 1); err != nil {
		t.Fatalf("Failed to write profile: %v", err)
	}

	re, err := regexp.Compile(`^([0-9]+) @`)
	if err != nil {
		t.Fatalf("Failed to compile regexp")
	}
	f.Seek(0, 0)
	r := bufio.NewReader(f)

	var buf bytes.Buffer
	var printing bool
	for {
		line, err := r.ReadBytes('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatalf("Failed to read bytes: %v", err)
		}
		if len(line) <= 1 {
			printing = false
			continue
		}
		if !printing {
			if m := re.FindSubmatch(line); len(m) == 2 {
				count, _ := strconv.Atoi(string(m[1]))
				if count > threshold {
					printing = true
				}
			}
		}
		if printing {
			buf.Write(line)
		}
	}
	return buf.String()
}
<commit_msg>rpsbot: fix needed for CL 4708.<commit_after>package main

import (
	"bufio"
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"runtime"
	"runtime/pprof"
	"strconv"
	"testing"

	"veyron.io/apps/rps"

	mtlib "veyron.io/veyron/veyron/services/mounttable/lib"

	"veyron.io/veyron/veyron2"
	"veyron.io/veyron/veyron2/ipc"
	"veyron.io/veyron/veyron2/naming"
	"veyron.io/veyron/veyron2/options"
	"veyron.io/veyron/veyron2/rt"
	"veyron.io/veyron/veyron2/vlog"
)

var spec = ipc.ListenSpec{Addrs: ipc.ListenAddrs{{"tcp", "127.0.0.1:0"}}}

func startMountTable(t *testing.T, runtime veyron2.Runtime) (string, func()) {
	server, err := runtime.NewServer(options.ServesMountTable(true))
	if err != nil {
		t.Fatalf("NewServer() failed: %v", err)
	}
	dispatcher, err := mtlib.NewMountTable("")

	endpoint, err := server.Listen(spec)
	if err != nil {
		t.Fatalf("Listen(%v) failed: %v", spec, err)
	}
	if err := server.ServeDispatcher("", dispatcher); err != nil {
		t.Fatalf("ServeDispatcher(%v) failed: %v", dispatcher, err)
	}
	address := naming.JoinAddressName(endpoint.String(), "")
	vlog.VI(1).Infof("Mount table running at endpoint: %s", address)
	return address, func() {
		if err := server.Stop(); err != nil {
			t.Fatalf("Stop() failed: %v", err)
		}
	}
}

func startRockPaperScissors(t *testing.T, rt veyron2.Runtime, mtAddress string) (*RPS, func()) {
	ns := rt.Namespace()
	ns.SetRoots(mtAddress)
	server, err := rt.NewServer()
	if err != nil {
		t.Fatalf("NewServer failed: %v", err)
	}
	rpsService := NewRPS()

	if _, err = server.Listen(spec); err != nil {
		t.Fatalf("Listen failed: %v", err)
	}
	names := []string{"rps/judge/test", "rps/player/test", "rps/scorekeeper/test"}
	if err := server.Serve(names[0], rps.RockPaperScissorsServer(rpsService), nil); err != nil {
		t.Fatalf("Serve(%v) failed: %v", names[0], err)
	}
	for _, n := range names[1:] {
		server.AddName(n)
	}
	return rpsService, func() {
		if err := server.Stop(); err != nil {
			t.Fatalf("Stop() failed: %v", err)
		}
	}
}

// TestRockPaperScissorsImpl runs one rock-paper-scissors game and verifies
// that all the counters are consistent.
func TestRockPaperScissorsImpl(t *testing.T) {
	r, err := rt.New()
	if err != nil {
		t.Fatalf("Could not initialize runtime: %v", err)
	}
	defer r.Cleanup()

	ctx := r.NewContext()

	mtAddress, mtStop := startMountTable(t, r)
	defer mtStop()
	rpsService, rpsStop := startRockPaperScissors(t, r, mtAddress)
	defer rpsStop()

	const numGames = 100
	for x := 0; x < numGames; x++ {
		if err := rpsService.Player().InitiateGame(ctx); err != nil {
			t.Errorf("Failed to initiate game: %v", err)
		}
	}
	rpsService.Player().WaitUntilIdle()

	// For each game, the player plays twice. So, we expect the player to
	// show that it played 2×numGames, and won numGames.
	played, won := rpsService.Player().Stats()
	if want, got := int64(2*numGames), played; want != got {
		t.Errorf("Unexpected number of played games. Got %d, want %d", got, want)
	}
	if want, got := int64(numGames), won; want != got {
		t.Errorf("Unexpected number of won games. Got %d, want %d", got, want)
	}

	// The Judge ran every game.
	if want, got := int64(numGames), rpsService.Judge().Stats(); want != got {
		t.Errorf("Unexpected number of games run. Got %d, want %d", got, want)
	}

	// The Score Keeper received one score card per game.
	if want, got := int64(numGames), rpsService.ScoreKeeper().Stats(); want != got {
		t.Errorf("Unexpected number of score cards. Got %d, want %d", got, want)
	}

	// Check for leaked goroutines.
	if n := runtime.NumGoroutine(); n > 100 {
		t.Logf("Num goroutines: %d", n)
		if leak := reportLeakedGoroutines(t, n/3); len(leak) != 0 {
			t.Errorf("Potentially leaked goroutines:\n%s", leak)
		}
	}
}

// reportLeakedGoroutines reads the goroutine pprof profile and returns the
// goroutines that have more than 'threshold' instances.
func reportLeakedGoroutines(t *testing.T, threshold int) string {
	f, err := ioutil.TempFile("", "test-profile-")
	if err != nil {
		t.Fatalf("Failed to create temp file: %v", err)
	}
	defer f.Close()
	defer os.Remove(f.Name())
	p := pprof.Lookup("goroutine")
	if p == nil {
		t.Fatalf("Failed to lookup the goroutine profile")
	}
	if err := p.WriteTo(f, 1); err != nil {
		t.Fatalf("Failed to write profile: %v", err)
	}

	re, err := regexp.Compile(`^([0-9]+) @`)
	if err != nil {
		t.Fatalf("Failed to compile regexp")
	}
	f.Seek(0, 0)
	r := bufio.NewReader(f)

	var buf bytes.Buffer
	var printing bool
	for {
		line, err := r.ReadBytes('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatalf("Failed to read bytes: %v", err)
		}
		if len(line) <= 1 {
			printing = false
			continue
		}
		if !printing {
			if m := re.FindSubmatch(line); len(m) == 2 {
				count, _ := strconv.Atoi(string(m[1]))
				if count > threshold {
					printing = true
				}
			}
		}
		if printing {
			buf.Write(line)
		}
	}
	return buf.String()
}
<|endoftext|>


<commit_before>// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris
package main

// Users that are known on a Unix system to try using
var knownUsers = []string{"", "root"}
<commit_msg>Add debug "in unix" message<commit_after>// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris
package main

import "fmt"

// Users that are known on a Unix system to try using
var knownUsers = []string{"", "root"}

func init() {
	fmt.Printf("in unix\n")
}
<|endoftext|>


<commit_before>package seccomp

import (
	"fmt"
	"os"
	"syscall"

	"github.com/op/go-logging"
	"github.com/subgraph/go-seccomp"
	"github.com/subgraph/oz"
)

func createLogger() *logging.Logger {
	l := logging.MustGetLogger("seccomp-wrapper")
	be := logging.NewLogBackend(os.Stderr, "", 0)
	f := logging.MustStringFormatter("%{level:.1s} %{message}")
	fbe := logging.NewBackendFormatter(be, f)
	logging.SetBackend(fbe)
	return l
}

func Main() {
	log := createLogger()

	if len(os.Args) < 3 {
		log.Error("seccomp-wrapper: Not enough arguments.")
		os.Exit(1)
	}

	if os.Getppid() != 1 {
		log.Error("oz-seccomp wrapper must be called from oz-init!")
		os.Exit(1)
	}

	var getvar = func(name string) string {
		val := os.Getenv(name)
		if val == "" {
			log.Error("Error: missing required '%s' argument", name)
			os.Exit(1)
		}
		os.Setenv(name, "")
		return val
	}

	cmd := os.Args[2]
	cmdArgs := os.Args[2:]
	env := os.Environ()
	pname := getvar("_OZ_PROFILE")

	config, err := oz.LoadConfig(oz.DefaultConfigPath)
	if err != nil {
		if os.IsNotExist(err) {
			log.Info("Configuration file (%s) is missing, using defaults.", oz.DefaultConfigPath)
			config = oz.NewDefaultConfig()
		} else {
			log.Error("Could not load configuration: %s", oz.DefaultConfigPath, err)
			os.Exit(1)
		}
	}

	p, err := loadProfile(config.ProfileDir, pname)

	if err != nil {
		log.Error("Could not load profile %s: %v", pname, err)
		os.Exit(1)
	}

	switch os.Args[1] {
	case "-w":
		if p.Seccomp.Seccomp_Whitelist == "" {
			log.Error("No seccomp policy file.")
			os.Exit(1)
		}
		filter, err := seccomp.Compile(p.Seccomp.Seccomp_Whitelist)
		if err != nil {
			log.Error("Seccomp filter compile failed: %v", err)
			os.Exit(1)
		}
		err = seccomp.Install(filter)
		if err != nil {
			log.Error("Error (seccomp): %v", err)
			os.Exit(1)
		}
		err = syscall.Exec(cmd, cmdArgs, env)
		if err != nil {
			log.Error("Error (exec): %v", err)
			os.Exit(1)
		}
	case "-b":
		if p.Seccomp.Seccomp_Blacklist == "" {
			log.Error("No seccomp blacklist policy file.")
			os.Exit(1)
		}
		filter, err := seccomp.CompileBlacklist(p.Seccomp.Seccomp_Blacklist)
		if err != nil {
			log.Error("Seccomp blacklist filter compile failed: %v", err)
			os.Exit(1)
		}
		err = seccomp.InstallBlacklist(filter)
		if err != nil {
			log.Error("Error (seccomp): %v", err)
			os.Exit(1)
		}
		err = syscall.Exec(cmd, cmdArgs, env)
		if err != nil {
			log.Error("Error (exec): %v", err)
			os.Exit(1)
		}
	default:
		fmt.Println("Bad switch.")
		os.Exit(1)
	}
}

func loadProfile(dir, name string) (*oz.Profile, error) {
	ps, err := oz.LoadProfiles(dir)
	if err != nil {
		return nil, err
	}
	for _, p := range ps {
		if name == p.Name {
			return p, nil
		}
	}
	return nil, fmt.Errorf("no profile named '%s'", name)
}
<commit_msg>Fixed environ passing in seccomp wrapper<commit_after>package seccomp

import (
	"fmt"
	"os"
	"syscall"

	"github.com/op/go-logging"
	"github.com/subgraph/go-seccomp"
	"github.com/subgraph/oz"
)

func createLogger() *logging.Logger {
	l := logging.MustGetLogger("seccomp-wrapper")
	be := logging.NewLogBackend(os.Stderr, "", 0)
	f := logging.MustStringFormatter("%{level:.1s} %{message}")
	fbe := logging.NewBackendFormatter(be, f)
	logging.SetBackend(fbe)
	return l
}

func Main() {
	log := createLogger()

	if len(os.Args) < 3 {
		log.Error("seccomp-wrapper: Not enough arguments.")
		os.Exit(1)
	}

	if os.Getppid() != 1 {
		log.Error("oz-seccomp wrapper must be called from oz-init!")
		os.Exit(1)
	}

	var getvar = func(name string) string {
		val := os.Getenv(name)
		if val == "" {
			log.Error("Error: missing required '%s' argument", name)
			os.Exit(1)
		}
		os.Setenv(name, "")
		return val
	}

	cmd := os.Args[2]
	cmdArgs := os.Args[2:]
	pname := getvar("_OZ_PROFILE")

	config, err := oz.LoadConfig(oz.DefaultConfigPath)
	if err != nil {
		if os.IsNotExist(err) {
			log.Info("Configuration file (%s) is missing, using defaults.", oz.DefaultConfigPath)
			config = oz.NewDefaultConfig()
		} else {
			log.Error("Could not load configuration: %s", oz.DefaultConfigPath, err)
			os.Exit(1)
		}
	}

	p, err := loadProfile(config.ProfileDir, pname)

	if err != nil {
		log.Error("Could not load profile %s: %v", pname, err)
		os.Exit(1)
	}

	switch os.Args[1] {
	case "-w":
		if p.Seccomp.Seccomp_Whitelist == "" {
			log.Error("No seccomp policy file.")
			os.Exit(1)
		}
		filter, err := seccomp.Compile(p.Seccomp.Seccomp_Whitelist)
		if err != nil {
			log.Error("Seccomp filter compile failed: %v", err)
			os.Exit(1)
		}
		err = seccomp.Install(filter)
		if err != nil {
			log.Error("Error (seccomp): %v", err)
			os.Exit(1)
		}
		err = syscall.Exec(cmd, cmdArgs, oz.Environ())
		if err != nil {
			log.Error("Error (exec): %v", err)
			os.Exit(1)
		}
	case "-b":
		if p.Seccomp.Seccomp_Blacklist == "" {
			log.Error("No seccomp blacklist policy file.")
			os.Exit(1)
		}
		filter, err := seccomp.CompileBlacklist(p.Seccomp.Seccomp_Blacklist)
		if err != nil {
			log.Error("Seccomp blacklist filter compile failed: %v", err)
			os.Exit(1)
		}
		err = seccomp.InstallBlacklist(filter)
		if err != nil {
			log.Error("Error (seccomp): %v", err)
			os.Exit(1)
		}
		err = syscall.Exec(cmd, cmdArgs, os.Environ())
		if err != nil {
			log.Error("Error (exec): %v", err)
			os.Exit(1)
		}
	default:
		fmt.Println("Bad switch.")
		os.Exit(1)
	}
}

func loadProfile(dir, name string) (*oz.Profile, error) {
	ps, err := oz.LoadProfiles(dir)
	if err != nil {
		return nil, err
	}
	for _, p := range ps {
		if name == p.Name {
			return p, nil
		}
	}
	return nil, fmt.Errorf("no profile named '%s'", name)
}
<|endoftext|>


<commit_before>// Copyright 2016 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package packages

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/limetext/text"
)

func TestLoadUnLoadJSON(t *testing.T) {
	set := text.NewSettings()
	j := NewJSON("testdata/Preferences.sublime-settings", &set)

	j.Load()
	if j.err != nil {
		t.Fatalf("Error on loading json %s: %s", j.Name(), j.err)
	}
	if got, exp := set.Get("font_face").(string), "Monospace"; got != exp {
		t.Errorf("Expected font_face %s, but got %s", exp, got)
	}

	j.UnLoad()
	if j.err != nil {
		t.Fatalf("Error on unloading json %s: %s", j.Name(), j.err)
	}
	if set.Has("font_face") {
		t.Error("Expected setting to be empty but has font_face")
	}
}

func TestWatch(t *testing.T) {
	testFile := "testdata/Preferences.sublime-settings"
	testData := []byte(`{"font_face": "test"}`)

	set := text.NewSettings()
	if err := LoadJSON(testFile, &set); err != nil {
		t.Fatalf("Error LoadJSON: %s", err)
	}

	data, err := ioutil.ReadFile(testFile)
	if err != nil {
		t.Fatalf("Error reading %s: %s", testFile, err)
	}

	// FileChanged
	if err := ioutil.WriteFile(testFile, testData, 0644); err != nil {
		t.Fatalf("Error writing to file %s: %s", testFile, err)
	}
	time.Sleep(100 * time.Millisecond)
	if got, exp := set.Get("font_face").(string), "test"; got != exp {
		t.Errorf("Expected font_face %s, but got %s", exp, got)
	}

	// FileRemoved
	if err := os.Remove(testFile); err != nil {
		t.Fatalf("Couldn't remove %s: %s", testFile, err)
	}
	time.Sleep(100 * time.Millisecond)
	if set.Has("font_face") {
		t.Error("Expected setting to be empty but has font_face")
	}

	// FileCreated
	if err := ioutil.WriteFile(testFile, data, 0644); err != nil {
		t.Fatalf("Error writing to file %s: %s", testFile, err)
	}
	time.Sleep(100 * time.Millisecond)
	if got, exp := set.Get("font_face").(string), "Monospace"; got != exp {
		t.Errorf("Expected font_face %s, but got %s", exp, got)
	}
}
<commit_msg>Use safer settings functions<commit_after>// Copyright 2016 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package packages

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/limetext/text"
)

func TestLoadUnLoadJSON(t *testing.T) {
	set := text.NewSettings()
	j := NewJSON("testdata/Preferences.sublime-settings", &set)

	j.Load()
	if j.err != nil {
		t.Fatalf("Error on loading json %s: %s", j.Name(), j.err)
	}
	if got, exp := set.String("font_face"), "Monospace"; got != exp {
		t.Errorf("Expected font_face %s, but got %s", exp, got)
	}

	j.UnLoad()
	if j.err != nil {
		t.Fatalf("Error on unloading json %s: %s", j.Name(), j.err)
	}
	if set.Has("font_face") {
		t.Error("Expected setting to be empty but has font_face")
	}
}

func TestWatch(t *testing.T) {
	testFile := "testdata/Preferences.sublime-settings"
	testData := []byte(`{"font_face": "test"}`)

	set := text.NewSettings()
	if err := LoadJSON(testFile, &set); err != nil {
		t.Fatalf("Error LoadJSON: %s", err)
	}

	data, err := ioutil.ReadFile(testFile)
	if err != nil {
		t.Fatalf("Error reading %s: %s", testFile, err)
	}

	// FileChanged
	if err := ioutil.WriteFile(testFile, testData, 0644); err != nil {
		t.Fatalf("Error writing to file %s: %s", testFile, err)
	}
	time.Sleep(100 * time.Millisecond)
	if got, exp := set.String("font_face"), "test"; got != exp {
		t.Errorf("Expected font_face %s, but got %s", exp, got)
	}

	// FileRemoved
	if err := os.Remove(testFile); err != nil {
		t.Fatalf("Couldn't remove %s: %s", testFile, err)
	}
	time.Sleep(100 * time.Millisecond)
	if set.Has("font_face") {
		t.Error("Expected setting to be empty but has font_face")
	}

	// FileCreated
	if err := ioutil.WriteFile(testFile, data, 0644); err != nil {
		t.Fatalf("Error writing to file %s: %s", testFile, err)
	}
	time.Sleep(100 * time.Millisecond)
	if got, exp := set.String("font_face"), "Monospace"; got != exp {
		t.Errorf("Expected font_face %s, but got %s", exp, got)
	}
}
<|endoftext|>


<commit_before>package ali

import (
	"bytes"
	"crypto"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
)

// misc constants
const (
	RSA  = "RSA"
	RSA2 = "RSA2"
)

// PayParam is the interface of all AliPay APIs.
type PayParam interface {
	URI() string
	ExtraParams() map[string]string
	BizContent() string
}

// Config contains all configuration info.
type Config struct {
	APIGateway    string
	AppID         string
	NotifyURL     string
	SandBox       bool
	SignType      string
	AliPublicKey  []byte
	AppPublicKey  []byte
	AppPrivateKey []byte
}

// Client handles all transactions.
type Client struct {
	config    Config
	tlsClient http.Client
}

// NewClient returns a *Client for Alipay.
func NewClient(cfg Config) *Client {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: false},
	}
	client := http.Client{Transport: tr}
	return &Client{
		config:    cfg,
		tlsClient: client,
	}
}

// CreateTrade creates order from Alipay.
func (c *Client) CreateTrade(p CreateTradeParam) (*CreateTradeRsp, error) {
	p.NotifyURL = c.config.NotifyURL
	data, err := c.doHTTPRequest(p)
	if err != nil {
		return nil, err
	}
	rsp := &CreateTradeRsp{}
	if err = json.NewDecoder(bytes.NewReader(data)).Decode(rsp); err != nil {
		return nil, err
	}

	if rsp.TradeCreateResponse.Code != "10000" {
		return nil, fmt.Errorf("code %s msg %s err %s err msg %s",
			rsp.TradeCreateResponse.Code, rsp.TradeCreateResponse.Msg,
			rsp.TradeCreateResponse.SubCode, rsp.TradeCreateResponse.SubMsg)
	}

	responseStr := marshalJSON(rsp.TradeCreateResponse)
	var ok bool
	if c.config.SignType == RSA {
		ok = verifyPKCS1v15([]byte(responseStr), []byte(rsp.Sign), c.config.AliPublicKey, crypto.SHA1)
	} else if c.config.SignType == RSA2 {
		ok = verifyPKCS1v15([]byte(responseStr), []byte(rsp.Sign), c.config.AliPublicKey, crypto.SHA256)
	}

	if !ok {
		return nil, errors.New("verify signature failed")
	}

	return rsp, nil
}

// QueryTrade queries order from Alipay.
func (c *Client) QueryTrade(p PayParam) (*QueryTradeRsp, error) {
	data, err := c.doHTTPRequest(p)
	if err != nil {
		return nil, err
	}
	rsp := &QueryTradeRsp{}
	if err = json.NewDecoder(bytes.NewReader(data)).Decode(rsp); err != nil {
		return nil, err
	}

	if rsp.TradeQueryResponse.Code != "10000" {
		return nil, fmt.Errorf("code %s msg %s err %s err msg %s",
			rsp.TradeQueryResponse.Code, rsp.TradeQueryResponse.Msg,
			rsp.TradeQueryResponse.SubCode, rsp.TradeQueryResponse.SubMsg)
	}

	responseStr := marshalJSON(rsp.TradeQueryResponse)
	var ok bool
	if c.config.SignType == RSA {
		ok = verifyPKCS1v15([]byte(responseStr), []byte(rsp.Sign), c.config.AliPublicKey, crypto.SHA1)
	} else if c.config.SignType == RSA2 {
		ok = verifyPKCS1v15([]byte(responseStr), []byte(rsp.Sign), c.config.AliPublicKey, crypto.SHA256)
	}

	if !ok {
		return nil, errors.New("verify signature failed")
	}

	return rsp, nil
}

func (c *Client) doHTTPRequest(param PayParam) ([]byte, error) {
	reader := strings.NewReader(urlValues(c, param).Encode())
	req, err := http.NewRequest(http.MethodPost, c.config.APIGateway, reader)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")

	rsp, err := c.tlsClient.Do(req)
	if err != nil {
		return nil, err
	}

	defer rsp.Body.Close()

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return nil, err
	}

	return data, nil
}

// AsyncNotificationResult is the result return from Alipay.
type AsyncNotificationResult struct {
	NotifyTime        string `json:"notify_time"`         // 通知时间
	NotifyType        string `json:"notify_type"`         // 通知类型
	NotifyID          string `json:"notify_id"`           // 通知校验ID
	AppID             string `json:"app_id"`              // 开发者的app_id
	Charset           string `json:"charset"`             // 编码格式
	Version           string `json:"version"`             // 接口版本
	SignType          string `json:"sign_type"`           // 签名类型
	Sign              string `json:"sign"`                // 签名
	TradeNo           string `json:"trade_no"`            // 支付宝交易号
	OutTradeNo        string `json:"out_trade_no"`        // 商户订单号
	OutBizNo          string `json:"out_biz_no"`          // 商户业务号
	BuyerID           string `json:"buyer_id"`            // 买家支付宝用户号
	BuyerLogonID      string `json:"buyer_logon_id"`      // 买家支付宝账号
	SellerID          string `json:"seller_id"`           // 卖家支付宝用户号
	SellerEmail       string `json:"seller_email"`        // 卖家支付宝账号
	TradeStatus       string `json:"trade_status"`        // 交易状态
	TotalAmount       string `json:"total_amount"`        // 订单金额
	ReceiptAmount     string `json:"receipt_amount"`      // 实收金额
	InvoiceAmount     string `json:"invoice_amount"`      // 开票金额
	BuyerPayAmount    string `json:"buyer_pay_amount"`    // 付款金额
	PointAmount       string `json:"point_amount"`        // 集分宝金额
	RefundFee         string `json:"refund_fee"`          // 总退款金额
	Subject           string `json:"subject"`             // 总退款金额
	Body              string `json:"body"`                // 商品描述
	GmtCreate         string `json:"gmt_create"`          // 交易创建时间
	GmtPayment        string `json:"gmt_payment"`         // 交易付款时间
	GmtRefund         string `json:"gmt_refund"`          // 交易退款时间
	GmtClose          string `json:"gmt_close"`           // 交易结束时间
	FundBillList      string `json:"fund_bill_list"`      // 支付金额信息
	PassbackParams    string `json:"passback_params"`     // 回传参数
	VoucherDetailList string `json:"voucher_detail_list"` // 优惠券信息
}

// AsyncNotification retrieves the asynchronous notification from Weixin.
func (c *Client) AsyncNotification(req *http.Request) (*AsyncNotificationResult, error) {
	if req == nil {
		return nil, errors.New("http request nil")
	}
	req.ParseForm()

	result := &AsyncNotificationResult{}
	result.NotifyTime = req.PostFormValue("notify_time")
	result.NotifyType = req.PostFormValue("notify_type")
	result.NotifyID = req.PostFormValue("notify_id")
	result.AppID = req.PostFormValue("app_id")
	result.Charset = req.PostFormValue("charset")
	result.Version = req.PostFormValue("version")
	result.SignType = req.PostFormValue("sign_type")
	result.Sign = req.PostFormValue("sign")
	result.TradeNo = req.PostFormValue("trade_no")
	result.OutTradeNo = req.PostFormValue("out_trade_no")
	result.OutBizNo = req.PostFormValue("out_biz_no")
	result.BuyerID = req.PostFormValue("buyer_id")
	result.BuyerLogonID = req.PostFormValue("buyer_logon_id")
	result.SellerID = req.PostFormValue("seller_id")
	result.SellerEmail = req.PostFormValue("seller_email")
	result.TradeStatus = req.PostFormValue("trade_status")
	result.TotalAmount = req.PostFormValue("total_amount")
	result.ReceiptAmount = req.PostFormValue("receipt_amount")
	result.InvoiceAmount = req.PostFormValue("invoice_amount")
	result.BuyerPayAmount = req.PostFormValue("buyer_pay_amount")
	result.PointAmount = req.PostFormValue("point_amount")
	result.RefundFee = req.PostFormValue("refund_fee")
	result.Subject = req.PostFormValue("subject")
	result.Body = req.PostFormValue("body")
	result.GmtCreate = req.PostFormValue("gmt_create")
	result.GmtPayment = req.PostFormValue("gmt_payment")
	result.GmtRefund = req.PostFormValue("gmt_refund")
	result.GmtClose = req.PostFormValue("gmt_close")
	result.FundBillList = req.PostFormValue("fund_bill_list")
	result.PassbackParams = req.PostFormValue("passback_params")
	result.VoucherDetailList = req.PostFormValue("voucher_detail_list")

	if result.NotifyID == "" {
		return nil, errors.New("invalid notify ID")
	}

	fmt.Println("ASYNC RESULT", result, result.Sign)

	// ok := verify(req.PostForm, c.config.AliPublicKey, c.config.SignType)

	var excluded []string
	for k := range req.PostForm {
		if k == "sign" || k == "sign_type" {
			continue
		}
		if len(req.PostFormValue(k)) > 0 {
			excluded = append(excluded, k)
		}
	}
	sort.Strings(excluded)

	sign := signature(excluded, req.PostForm, c.config.AppPrivateKey, c.config.SignType)
	ok := sign == result.Sign
	if ok {
		return result, nil
	}
	return nil, errors.New("verify signature failed")
}
<commit_msg>remove<commit_after>package ali

import (
	"bytes"
	"crypto"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

// misc constants
const (
	RSA  = "RSA"
	RSA2 = "RSA2"
)

// PayParam is the interface of all AliPay APIs.
type PayParam interface {
	URI() string
	ExtraParams() map[string]string
	BizContent() string
}

// Config contains all configuration info.
type Config struct {
	APIGateway    string
	AppID         string
	NotifyURL     string
	SandBox       bool
	SignType      string
	AliPublicKey  []byte
	AppPublicKey  []byte
	AppPrivateKey []byte
}

// Client handles all transactions.
type Client struct {
	config    Config
	tlsClient http.Client
}

// NewClient returns a *Client for Alipay.
func NewClient(cfg Config) *Client {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: false},
	}
	client := http.Client{Transport: tr}
	return &Client{
		config:    cfg,
		tlsClient: client,
	}
}

// CreateTrade creates order from Alipay.
func (c *Client) CreateTrade(p CreateTradeParam) (*CreateTradeRsp, error) {
	p.NotifyURL = c.config.NotifyURL
	data, err := c.doHTTPRequest(p)
	if err != nil {
		return nil, err
	}
	rsp := &CreateTradeRsp{}
	if err = json.NewDecoder(bytes.NewReader(data)).Decode(rsp); err != nil {
		return nil, err
	}

	if rsp.TradeCreateResponse.Code != "10000" {
		return nil, fmt.Errorf("code %s msg %s err %s err msg %s",
			rsp.TradeCreateResponse.Code, rsp.TradeCreateResponse.Msg,
			rsp.TradeCreateResponse.SubCode, rsp.TradeCreateResponse.SubMsg)
	}

	responseStr := marshalJSON(rsp.TradeCreateResponse)
	var ok bool
	if c.config.SignType == RSA {
		ok = verifyPKCS1v15([]byte(responseStr), []byte(rsp.Sign), c.config.AliPublicKey, crypto.SHA1)
	} else if c.config.SignType == RSA2 {
		ok = verifyPKCS1v15([]byte(responseStr), []byte(rsp.Sign), c.config.AliPublicKey, crypto.SHA256)
	}

	if !ok {
		return nil, errors.New("verify signature failed")
	}

	return rsp, nil
}

// QueryTrade queries order from Alipay.
func (c *Client) QueryTrade(p PayParam) (*QueryTradeRsp, error) {
	data, err := c.doHTTPRequest(p)
	if err != nil {
		return nil, err
	}
	rsp := &QueryTradeRsp{}
	if err = json.NewDecoder(bytes.NewReader(data)).Decode(rsp); err != nil {
		return nil, err
	}

	if rsp.TradeQueryResponse.Code != "10000" {
		return nil, fmt.Errorf("code %s msg %s err %s err msg %s",
			rsp.TradeQueryResponse.Code, rsp.TradeQueryResponse.Msg,
			rsp.TradeQueryResponse.SubCode, rsp.TradeQueryResponse.SubMsg)
	}

	responseStr := marshalJSON(rsp.TradeQueryResponse)
	var ok bool
	if c.config.SignType == RSA {
		ok = verifyPKCS1v15([]byte(responseStr), []byte(rsp.Sign), c.config.AliPublicKey, crypto.SHA1)
	} else if c.config.SignType == RSA2 {
		ok = verifyPKCS1v15([]byte(responseStr), []byte(rsp.Sign), c.config.AliPublicKey, crypto.SHA256)
	}

	if !ok {
		return nil, errors.New("verify signature failed")
	}

	return rsp, nil
}

func (c *Client) doHTTPRequest(param PayParam) ([]byte, error) {
	reader := strings.NewReader(urlValues(c, param).Encode())
	req, err := http.NewRequest(http.MethodPost, c.config.APIGateway, reader)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")

	rsp, err := c.tlsClient.Do(req)
	if err != nil {
		return nil, err
	}

	defer rsp.Body.Close()

	data, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		return nil, err
	}

	return data, nil
}

// AsyncNotificationResult is the result return from Alipay.
type AsyncNotificationResult struct {
	NotifyTime        string `json:"notify_time"`         // 通知时间
	NotifyType        string `json:"notify_type"`         // 通知类型
	NotifyID          string `json:"notify_id"`           // 通知校验ID
	AppID             string `json:"app_id"`              // 开发者的app_id
	Charset           string `json:"charset"`             // 编码格式
	Version           string `json:"version"`             // 接口版本
	SignType          string `json:"sign_type"`           // 签名类型
	Sign              string `json:"sign"`                // 签名
	TradeNo           string `json:"trade_no"`            // 支付宝交易号
	OutTradeNo        string `json:"out_trade_no"`        // 商户订单号
	OutBizNo          string `json:"out_biz_no"`          // 商户业务号
	BuyerID           string `json:"buyer_id"`            // 买家支付宝用户号
	BuyerLogonID      string `json:"buyer_logon_id"`      // 买家支付宝账号
	SellerID          string `json:"seller_id"`           // 卖家支付宝用户号
	SellerEmail       string `json:"seller_email"`        // 卖家支付宝账号
	TradeStatus       string `json:"trade_status"`        // 交易状态
	TotalAmount       string `json:"total_amount"`        // 订单金额
	ReceiptAmount     string `json:"receipt_amount"`      // 实收金额
	InvoiceAmount     string `json:"invoice_amount"`      // 开票金额
	BuyerPayAmount    string `json:"buyer_pay_amount"`    // 付款金额
	PointAmount       string `json:"point_amount"`        // 集分宝金额
	RefundFee         string `json:"refund_fee"`          // 总退款金额
	Subject           string `json:"subject"`             // 总退款金额
	Body              string `json:"body"`                // 商品描述
	GmtCreate         string `json:"gmt_create"`          // 交易创建时间
	GmtPayment        string `json:"gmt_payment"`         // 交易付款时间
	GmtRefund         string `json:"gmt_refund"`          // 交易退款时间
	GmtClose          string `json:"gmt_close"`           // 交易结束时间
	FundBillList      string `json:"fund_bill_list"`      // 支付金额信息
	PassbackParams    string `json:"passback_params"`     // 回传参数
	VoucherDetailList string `json:"voucher_detail_list"` // 优惠券信息
}

// AsyncNotification retrieves the asynchronous notification from Weixin.
func (c *Client) AsyncNotification(req *http.Request) (*AsyncNotificationResult, error) {
	if req == nil {
		return nil, errors.New("http request nil")
	}
	req.ParseForm()

	result := &AsyncNotificationResult{}
	result.NotifyTime = req.PostFormValue("notify_time")
	result.NotifyType = req.PostFormValue("notify_type")
	result.NotifyID = req.PostFormValue("notify_id")
	result.AppID = req.PostFormValue("app_id")
	result.Charset = req.PostFormValue("charset")
	result.Version = req.PostFormValue("version")
	result.SignType = req.PostFormValue("sign_type")
	result.Sign = req.PostFormValue("sign")
	result.TradeNo = req.PostFormValue("trade_no")
	result.OutTradeNo = req.PostFormValue("out_trade_no")
	result.OutBizNo = req.PostFormValue("out_biz_no")
	result.BuyerID = req.PostFormValue("buyer_id")
	result.BuyerLogonID = req.PostFormValue("buyer_logon_id")
	result.SellerID = req.PostFormValue("seller_id")
	result.SellerEmail = req.PostFormValue("seller_email")
	result.TradeStatus = req.PostFormValue("trade_status")
	result.TotalAmount = req.PostFormValue("total_amount")
	result.ReceiptAmount = req.PostFormValue("receipt_amount")
	result.InvoiceAmount = req.PostFormValue("invoice_amount")
	result.BuyerPayAmount = req.PostFormValue("buyer_pay_amount")
	result.PointAmount = req.PostFormValue("point_amount")
	result.RefundFee = req.PostFormValue("refund_fee")
	result.Subject = req.PostFormValue("subject")
	result.Body = req.PostFormValue("body")
	result.GmtCreate = req.PostFormValue("gmt_create")
	result.GmtPayment = req.PostFormValue("gmt_payment")
	result.GmtRefund = req.PostFormValue("gmt_refund")
	result.GmtClose = req.PostFormValue("gmt_close")
	result.FundBillList = req.PostFormValue("fund_bill_list")
	result.PassbackParams = req.PostFormValue("passback_params")
	result.VoucherDetailList = req.PostFormValue("voucher_detail_list")

	if result.NotifyID == "" {
		return nil, errors.New("invalid notify ID")
	}

	fmt.Println("ASYNC RESULT", result, result.Sign)

	ok := verify(req.PostForm, c.config.AliPublicKey, c.config.SignType)

	if ok {
		return result, nil
	}
	return nil, errors.New("verify signature failed")
}
<|endoftext|>


<commit_before>// Copyright 2015 Kazuhisa TAKEI<xtakei@me.com>. All rights reserved.
// Use of this source code is governed by MPL-2.0 license tha can be
// found in the LICENSE file
//
// Package buffer_list implements a double linked list with sequencial buffer data.
//
// implement AList(alias list)
// example
//   bl :=  buffer_list.New(Hoge{}, 10000)
//   alist := Alist{parent: bl}
//   ae := alist.NewElem()
//   alist.Front().Insert(ae)
//   v := ae.Value().(*Hoge)
//   v.Commit()
//
// iteration
//      for e := range  alist.Generator() {
//			v := e.Value().(*Hoge)
//
//	  }
//

package buffer_list

import (
	"sync"
)

type AList struct {
	m      sync.Mutex
	parent *List
	root   *AElement
	Len    int
	e2ae   map[*Element]*AElement
}

// AElement is an element of alias linked list.
type AElement struct {
	list   *AList
	prev   *AElement
	next   *AElement
	parent *Element
}

// create new alias element with element

func NewAList(l *List) AList {
	return AList{parent: l}
}

func (al *AList) NewElem() (ae *AElement) {
	ae = &AElement{list: al, parent: al.parent.InsertLast()}
	return ae
}

// add element to last of list
func (al *AList) Push(e *AElement) bool {
	al.m.Lock()
	defer al.m.Unlock()
	e.prev = e
	e.next = e
	if al.root == nil {
		al.root = e
	}
	at := al.root.prev
	n := at.next
	at.next = e
	e.prev = at
	e.next = n
	n.prev = e
	e.list = al
	al.Len++
	return true
}

// first element of list
func (al *AList) Front() *AElement {
	return al.root
}

// last element of list
func (al *AList) Back() *AElement {
	if al.root == nil {
		return nil
	}
	return al.root.prev
}

// add elemnet after at_element
func (at *AElement) Insert(e *AElement) *AElement {
	l := at.list

	l.m.Lock()
	defer l.m.Unlock()

	n := at.next
	at.next = e
	e.prev = at
	e.next = n
	n.prev = e
	e.list = at.list

	l.Len++
	return e
}

// return value of reel element object
func (ae *AElement) Value() interface{} {
	if ae.list.e2ae == nil {
		ae.list.e2ae = make(map[*Element]*AElement)
	}
	ae.list.e2ae[ae.parent] = ae

	return ae.parent.Value()
}
func (ae *AElement) InitValue() {
	ae.parent.InitValue()
}

// register pointer protect GC
func (ae *AElement) Commit() {
	ae.parent.Commit()
}

// remove element from alias list
func (e *AElement) Remove() bool {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	delete(e.list.e2ae, e.parent)

	e.prev.next = e.next
	e.next.prev = e.prev
	if e.list.root == e {
		e.list.root = e.next
	}
	e.next = nil
	e.prev = nil
	e.list.Len -= 1
	e.list = nil
	return true
}

// Next return next list element or nil
func (e *AElement) Next() *AElement {
	return e.next
}

// Prev return previous list element or nil
func (e *AElement) Prev() *AElement {
	return e.prev
}

// remove  from alias list and free real list
func (e *AElement) Free() {
	e.Remove()
	e.parent.Free()
	e.parent = nil
}

func (e *AElement) Base() *Element {
	return e.parent
}
func (e *AElement) List() *AList {
	return e.list
}

func (l *AList) ElemByValue(v interface{}) *AElement {

	e := l.parent.ElemByValue(v)
	if e == nil {
		return nil
	}
	if l.e2ae == nil {
		return nil
	}
	return l.e2ae[e]
}

func (l *AList) Generator() chan *AElement {
	ch_size := l.Len

	if ch_size == 0 {
		ch_size = 1
	}

	ch := make(chan *AElement, ch_size)

	go func() {
		if l == nil {
			close(ch)
			return
		}
		cnt := 0
		e := l.Back()
		for {
			if e == nil || cnt > ch_size-1 {
				break
			}
			ch <- e
			e = e.Prev()
			cnt++
		}
		close(ch)
	}()

	return ch
}
<commit_msg>- logging on race condition - fix: dont all removed on removing last one.<commit_after>// Copyright 2015 Kazuhisa TAKEI<xtakei@me.com>. All rights reserved.
// Use of this source code is governed by MPL-2.0 license tha can be
// found in the LICENSE file
//
// Package buffer_list implements a double linked list with sequencial buffer data.
//
// implement AList(alias list)
// example
//   bl :=  buffer_list.New(Hoge{}, 10000)
//   alist := Alist{parent: bl}
//   ae := alist.NewElem()
//   alist.Front().Insert(ae)
//   v := ae.Value().(*Hoge)
//   v.Commit()
//
// iteration
//      for e := range  alist.Generator() {
//			v := e.Value().(*Hoge)
//
//	  }
//

package buffer_list

import (
	"fmt"
	"sync"
)

type AList struct {
	m      sync.Mutex
	parent *List
	root   *AElement
	Len    int
	e2ae   map[*Element]*AElement
}

// AElement is an element of alias linked list.
type AElement struct {
	list   *AList
	prev   *AElement
	next   *AElement
	parent *Element
}

// create new alias element with element

func NewAList(l *List) AList {
	return AList{parent: l}
}

func (al *AList) NewElem() (ae *AElement) {
	ae = &AElement{list: al, parent: al.parent.InsertLast()}
	return ae
}

// add element to last of list
func (al *AList) Push(e *AElement) bool {

	al.detect_empty_record("before_Push()")

	al.m.Lock()
	defer al.m.Unlock()
	e.prev = e
	e.next = e
	if al.root == nil {
		al.root = e
	}
	at := al.root.prev
	n := at.next
	at.next = e
	e.prev = at
	e.next = n
	n.prev = e
	e.list = al
	al.Len++
	return true
}

// first element of list
func (al *AList) Front() *AElement {
	return al.root
}

// last element of list
func (al *AList) Back() *AElement {
	if al.root == nil {
		return nil
	}
	return al.root.prev
}

// add elemnet after at_element
func (at *AElement) Insert(e *AElement) *AElement {
	l := at.list

	l.detect_empty_record("before_Insert()")

	l.m.Lock()
	defer l.m.Unlock()

	n := at.next
	at.next = e
	e.prev = at
	e.next = n
	n.prev = e
	e.list = at.list

	l.Len++
	return e
}

// return value of reel element object
func (ae *AElement) Value() interface{} {
	if ae.list.e2ae == nil {
		ae.list.e2ae = make(map[*Element]*AElement)
	}
	ae.list.e2ae[ae.parent] = ae

	return ae.parent.Value()
}
func (ae *AElement) InitValue() {
	ae.parent.InitValue()
}

// register pointer protect GC
func (ae *AElement) Commit() {
	ae.parent.Commit()
}
func (l *AList) detect_empty_record(s string) {
	if l.root == nil {
		return
	}

	if l.root.list == nil || l.root.parent == nil || l.root.prev == nil || l.root.next == nil {
		fmt.Printf("detect invalid on %s al=%+v AList.root=%+v\n", s, l, l.root)
	}

}

// remove element from alias list
func (e *AElement) Remove() bool {
	e.list.detect_empty_record("before_Remove()")
	e.list.m.Lock()
	defer e.list.m.Unlock()

	delete(e.list.e2ae, e.parent)

	e.prev.next = e.next
	e.next.prev = e.prev
	e.list.Len -= 1
	if e.list.root == e {
		if e.list.Len == 0 {
			e.list.root = nil
		} else {
			e.list.root = e.next
		}
	}
	e.next = nil
	e.prev = nil

	e.list.detect_empty_record("After_Remove()")
	e.list = nil

	return true
}

// Next return next list element or nil
func (e *AElement) Next() *AElement {
	return e.next
}

// Prev return previous list element or nil
func (e *AElement) Prev() *AElement {
	return e.prev
}

// remove  from alias list and free real list
func (e *AElement) Free() {
	e.Remove()
	e.parent.Free()
	e.parent = nil
}

func (e *AElement) Base() *Element {
	return e.parent
}
func (e *AElement) List() *AList {
	return e.list
}

func (l *AList) ElemByValue(v interface{}) *AElement {

	e := l.parent.ElemByValue(v)
	if e == nil {
		return nil
	}
	if l.e2ae == nil {
		return nil
	}
	return l.e2ae[e]
}

func (l *AList) Generator() chan *AElement {
	ch_size := l.Len

	if ch_size == 0 {
		ch_size = 1
	}

	ch := make(chan *AElement, ch_size)

	go func() {
		if l == nil {
			close(ch)
			return
		}
		cnt := 0
		e := l.Back()
		for {
			if e == nil || cnt > ch_size-1 {
				break
			}
			ch <- e
			e = e.Prev()
			cnt++
		}
		close(ch)
	}()

	return ch
}
<|endoftext|>