<commit_before>package firebase_test

import (
	"testing"
	"time"
)

func TestReference(t *testing.T) {
	app := initApp()
	db := app.Database()

	r1 := db.Ref("test/path")
	n := time.Now()
	rr, err := r1.Push(n)
	if err != nil {
		t.Fatal(err)
	}
	if rr == nil {
		t.Fatalf("expected push result not nil; got nil")
	}
	if rr.Key() == r1.Key() {
		t.Fatalf("expected push result key changed")
	}
	err = r1.Set(n)
	if err != nil {
		t.Fatal(err)
	}

	snapshot, err := r1.OnceValue()
	if err != nil {
		t.Fatal(err)
	}
	if !snapshot.Exists() {
		t.Fatal("expected snapshot exists")
	}
	var ts time.Time
	err = snapshot.Val(&ts)
	if err != nil {
		t.Fatal(err)
	}
	if !n.Equal(ts) {
		t.Fatalf("expected data to be %v; got %v", n, ts)
	}

	err = r1.Remove()
	if err != nil {
		t.Fatal(err)
	}
	snapshot, err = r1.OnceValue()
	if err != nil {
		t.Fatal(err)
	}
	if snapshot.Exists() {
		t.Fatal("expected snapshot not exists")
	}
}

type dinosaurs struct {
	Appeared int64   `json:"appeared"`
	Height   float32 `json:"height"`
	Length   float32 `json:"length"`
	Order    string  `json:"order"`
	Vanished int64   `json:"vanished"`
	Weight   int     `json:"weight"`
}

func TestOrderBy(t *testing.T) {
	app := initApp()
	db := app.Database()
	var err error

	// feed data
	r := db.Ref("test/path")
	defer r.Remove()
	err = r.Child("bruhathkayosaurus").Set(&dinosaurs{-70000000, 25, 44, "saurischia", -70000000, 135000})
	if err != nil {
		t.Fatal(err)
	}
	err = r.Child("lambeosaurus").Set(&dinosaurs{-76000000, 2.1, 12.5, "ornithischia", -75000000, 5000})
	if err != nil {
		t.Fatal(err)
	}
	err = r.Child("linhenykus").Set(&dinosaurs{-85000000, 0.6, 1, "theropoda", -75000000, 3})
	if err != nil {
		t.Fatal(err)
	}
	err = r.Child("pterodactyl").Set(&dinosaurs{-150000000, 0.6, 0.8, "pterosauria", -148500000, 2})
	if err != nil {
		t.Fatal(err)
	}
	err = r.Child("stegosaurus").Set(&dinosaurs{-155000000, 4, 9, "ornithischia", -150000000, 2500})
	if err != nil {
		t.Fatal(err)
	}
	err = r.Child("triceratops").Set(&dinosaurs{-68000000, 3, 8, "ornithischia", -66000000, 11000})
	if err != nil {
		t.Fatal(err)
	}

	snapshot, err := r.OrderByKey().OnceValue()
	if err != nil {
		t.Fatal(err)
	}
	var d map[string]*dinosaurs
	snapshot.Val(&d)
	if len(d) != 6 {
		t.Fatalf("expected dinosours have 6 len; got %d", len(d))
	}

	snapshot, err = r.OrderByChild("height").EqualTo(0.6).OnceValue()
	if err != nil {
		t.Fatal(err)
	}
	d = nil
	snapshot.Val(&d)
	if len(d) != 2 {
		t.Fatalf("expected dinosours have 2 len; got %d", len(d))
	}
}
<commit_msg>fix test, refactor test<commit_after>package firebase_test

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestReference(t *testing.T) {
	app := initApp()
	db := app.Database()

	err := db.Ref(".settings/rules").Set(map[string]interface{}{
		"rules": map[string]interface{}{
			".read":  false,
			".write": false,
			"test": map[string]interface{}{
				"path": map[string]interface{}{
					".indexOn": []string{"height"},
				},
			},
		},
	})
	assert.NoError(t, err)

	r1 := db.Ref("test/path")
	n := time.Now()
	rr, err := r1.Push(n)
	assert.NoError(t, err)
	assert.NotNil(t, rr, "expected push result not nil")
	assert.NotEqual(t, rr.Key(), r1.Key(), "expected push result key changed")

	err = r1.Set(n)
	assert.NoError(t, err)

	snapshot, err := r1.OnceValue()
	assert.NoError(t, err)
	assert.True(t, snapshot.Exists())

	var ts time.Time
	err = snapshot.Val(&ts)
	assert.NoError(t, err)
	assert.Equal(t, n, ts, "expected data to be %v; got %v", n, ts)

	err = r1.Remove()
	assert.NoError(t, err)

	snapshot, err = r1.OnceValue()
	assert.NoError(t, err)
	assert.False(t, snapshot.Exists(), "expected snapshot not exists")
}

type dinosaurs struct {
	Appeared int64   `json:"appeared"`
	Height   float32 `json:"height"`
	Length   float32 `json:"length"`
	Order    string  `json:"order"`
	Vanished int64   `json:"vanished"`
	Weight   int     `json:"weight"`
}

func TestOrderBy(t *testing.T) {
	app := initApp()
	db := app.Database()
	var err error

	// feed data
	r := db.Ref("test/path")
	defer r.Remove()
	err = r.Child("bruhathkayosaurus").Set(&dinosaurs{-70000000, 25, 44, "saurischia", -70000000, 135000})
	assert.NoError(t, err)

	err = r.Child("lambeosaurus").Set(&dinosaurs{-76000000, 2.1, 12.5, "ornithischia", -75000000, 5000})
	assert.NoError(t, err)

	err = r.Child("linhenykus").Set(&dinosaurs{-85000000, 0.6, 1, "theropoda", -75000000, 3})
	assert.NoError(t, err)

	err = r.Child("pterodactyl").Set(&dinosaurs{-150000000, 0.6, 0.8, "pterosauria", -148500000, 2})
	assert.NoError(t, err)

	err = r.Child("stegosaurus").Set(&dinosaurs{-155000000, 4, 9, "ornithischia", -150000000, 2500})
	assert.NoError(t, err)

	err = r.Child("triceratops").Set(&dinosaurs{-68000000, 3, 8, "ornithischia", -66000000, 11000})
	assert.NoError(t, err)

	snapshot, err := r.OrderByKey().OnceValue()
	assert.NoError(t, err)

	var d map[string]*dinosaurs
	snapshot.Val(&d)
	assert.Len(t, d, 6, "expected dinosours have 6 len; got %d", len(d))

	snapshot, err = r.OrderByChild("height").EqualTo(0.6).OnceValue()
	assert.NoError(t, err)

	d = nil
	snapshot.Val(&d)
	assert.Len(t, d, 2, "expected dinosours have 2 len; got %d", len(d))
}
<|endoftext|>


<commit_before>// Copyright 2017, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package opencensus contains Go support for OpenCensus.
package opencensus // import "go.opencensus.io"

// Version is the current release version of OpenCensus in use.
func Version() string {
	return "0.15.0"
}
<commit_msg>Bump version string to 0.16.0 (#863)<commit_after>// Copyright 2017, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package opencensus contains Go support for OpenCensus.
package opencensus // import "go.opencensus.io"

// Version is the current release version of OpenCensus in use.
func Version() string {
	return "0.16.0"
}
<|endoftext|>


<commit_before>package opts // import "github.com/docker/docker/opts"

import (
	"fmt"
	"net"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/docker/docker/pkg/homedir"
	"github.com/pkg/errors"
)

const (
	// DefaultHTTPPort Default HTTP Port used if only the protocol is provided to -H flag e.g. dockerd -H tcp://
	// These are the IANA registered port numbers for use with Docker
	// see http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=docker
	DefaultHTTPPort = 2375 // Default HTTP Port
	// DefaultTLSHTTPPort Default HTTP Port used when TLS enabled
	DefaultTLSHTTPPort = 2376 // Default TLS encrypted HTTP Port
	// DefaultUnixSocket Path for the unix socket.
	// Docker daemon by default always listens on the default unix socket
	DefaultUnixSocket = "/var/run/docker.sock"
	// DefaultTCPHost constant defines the default host string used by docker on Windows
	DefaultTCPHost = "tcp://" + DefaultHTTPHost + ":2375"
	// DefaultTLSHost constant defines the default host string used by docker for TLS sockets
	DefaultTLSHost = "tcp://" + DefaultHTTPHost + ":2376"
	// DefaultNamedPipe defines the default named pipe used by docker on Windows
	DefaultNamedPipe = `//./pipe/docker_engine`
	// HostGatewayName is the string value that can be passed
	// to the IPAddr section in --add-host that is replaced by
	// the value of HostGatewayIP daemon config value
	HostGatewayName = "host-gateway"
)

// ValidateHost validates that the specified string is a valid host and returns it.
func ValidateHost(val string) (string, error) {
	host := strings.TrimSpace(val)
	// The empty string means default and is not handled by parseDaemonHost
	if host != "" {
		_, err := parseDaemonHost(host)
		if err != nil {
			return val, err
		}
	}
	// Note: unlike most flag validators, we don't return the mutated value here
	//       we need to know what the user entered later (using ParseHost) to adjust for TLS
	return val, nil
}

// ParseHost and set defaults for a Daemon host string.
// defaultToTLS is preferred over defaultToUnixXDG.
func ParseHost(defaultToTLS, defaultToUnixXDG bool, val string) (string, error) {
	host := strings.TrimSpace(val)
	if host == "" {
		if defaultToTLS {
			host = DefaultTLSHost
		} else if defaultToUnixXDG {
			runtimeDir, err := homedir.GetRuntimeDir()
			if err != nil {
				return "", err
			}
			socket := filepath.Join(runtimeDir, "docker.sock")
			host = "unix://" + socket
		} else {
			host = DefaultHost
		}
	} else {
		var err error
		host, err = parseDaemonHost(host)
		if err != nil {
			return val, err
		}
	}
	return host, nil
}

// parseDaemonHost parses the specified address and returns an address that will be used as the host.
// Depending of the address specified, this may return one of the global Default* strings defined in hosts.go.
func parseDaemonHost(addr string) (string, error) {
	addrParts := strings.SplitN(addr, "://", 2)
	if len(addrParts) == 1 && addrParts[0] != "" {
		addrParts = []string{"tcp", addrParts[0]}
	}

	switch addrParts[0] {
	case "tcp":
		return ParseTCPAddr(addrParts[1], DefaultTCPHost)
	case "unix":
		return parseSimpleProtoAddr("unix", addrParts[1], DefaultUnixSocket)
	case "npipe":
		return parseSimpleProtoAddr("npipe", addrParts[1], DefaultNamedPipe)
	case "fd":
		return addr, nil
	default:
		return "", fmt.Errorf("Invalid bind address format: %s", addr)
	}
}

// parseSimpleProtoAddr parses and validates that the specified address is a valid
// socket address for simple protocols like unix and npipe. It returns a formatted
// socket address, either using the address parsed from addr, or the contents of
// defaultAddr if addr is a blank string.
func parseSimpleProtoAddr(proto, addr, defaultAddr string) (string, error) {
	addr = strings.TrimPrefix(addr, proto+"://")
	if strings.Contains(addr, "://") {
		return "", fmt.Errorf("Invalid proto, expected %s: %s", proto, addr)
	}
	if addr == "" {
		addr = defaultAddr
	}
	return fmt.Sprintf("%s://%s", proto, addr), nil
}

// ParseTCPAddr parses and validates that the specified address is a valid TCP
// address. It returns a formatted TCP address, either using the address parsed
// from tryAddr, or the contents of defaultAddr if tryAddr is a blank string.
// tryAddr is expected to have already been Trim()'d
// defaultAddr must be in the full `tcp://host:port` form
func ParseTCPAddr(tryAddr string, defaultAddr string) (string, error) {
	if tryAddr == "" || tryAddr == "tcp://" {
		return defaultAddr, nil
	}
	addr := strings.TrimPrefix(tryAddr, "tcp://")
	if strings.Contains(addr, "://") || addr == "" {
		return "", fmt.Errorf("Invalid proto, expected tcp: %s", tryAddr)
	}

	defaultAddr = strings.TrimPrefix(defaultAddr, "tcp://")
	defaultHost, defaultPort, err := net.SplitHostPort(defaultAddr)
	if err != nil {
		return "", err
	}
	// url.Parse fails for trailing colon on IPv6 brackets on Go 1.5, but
	// not 1.4. See https://github.com/golang/go/issues/12200 and
	// https://github.com/golang/go/issues/6530.
	if strings.HasSuffix(addr, "]:") {
		addr += defaultPort
	}

	u, err := url.Parse("tcp://" + addr)
	if err != nil {
		return "", err
	}
	host, port, err := net.SplitHostPort(u.Host)
	if err != nil {
		// try port addition once
		host, port, err = net.SplitHostPort(net.JoinHostPort(u.Host, defaultPort))
	}
	if err != nil {
		return "", fmt.Errorf("Invalid bind address format: %s", tryAddr)
	}

	if host == "" {
		host = defaultHost
	}
	if port == "" {
		port = defaultPort
	}
	p, err := strconv.Atoi(port)
	if err != nil && p == 0 {
		return "", fmt.Errorf("Invalid bind address format: %s", tryAddr)
	}

	if u.Path != "" {
		return "", errors.Errorf("invalid bind address (%s): should not contain a path element", tryAddr)
	}

	return "tcp://" + net.JoinHostPort(host, port), nil
}

// ValidateExtraHost validates that the specified string is a valid extrahost and returns it.
// ExtraHost is in the form of name:ip where the ip has to be a valid ip (IPv4 or IPv6).
func ValidateExtraHost(val string) (string, error) {
	// allow for IPv6 addresses in extra hosts by only splitting on first ":"
	arr := strings.SplitN(val, ":", 2)
	if len(arr) != 2 || len(arr[0]) == 0 {
		return "", fmt.Errorf("bad format for add-host: %q", val)
	}
	// Skip IPaddr validation for special "host-gateway" string
	if arr[1] != HostGatewayName {
		if _, err := ValidateIPAddress(arr[1]); err != nil {
			return "", fmt.Errorf("invalid IP address in add-host: %q", arr[1])
		}
	}
	return val, nil
}
<commit_msg>opts: ParseTCPAddr(): remove workaround for go1.5<commit_after>package opts // import "github.com/docker/docker/opts"

import (
	"fmt"
	"net"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/docker/docker/pkg/homedir"
	"github.com/pkg/errors"
)

const (
	// DefaultHTTPPort Default HTTP Port used if only the protocol is provided to -H flag e.g. dockerd -H tcp://
	// These are the IANA registered port numbers for use with Docker
	// see http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=docker
	DefaultHTTPPort = 2375 // Default HTTP Port
	// DefaultTLSHTTPPort Default HTTP Port used when TLS enabled
	DefaultTLSHTTPPort = 2376 // Default TLS encrypted HTTP Port
	// DefaultUnixSocket Path for the unix socket.
	// Docker daemon by default always listens on the default unix socket
	DefaultUnixSocket = "/var/run/docker.sock"
	// DefaultTCPHost constant defines the default host string used by docker on Windows
	DefaultTCPHost = "tcp://" + DefaultHTTPHost + ":2375"
	// DefaultTLSHost constant defines the default host string used by docker for TLS sockets
	DefaultTLSHost = "tcp://" + DefaultHTTPHost + ":2376"
	// DefaultNamedPipe defines the default named pipe used by docker on Windows
	DefaultNamedPipe = `//./pipe/docker_engine`
	// HostGatewayName is the string value that can be passed
	// to the IPAddr section in --add-host that is replaced by
	// the value of HostGatewayIP daemon config value
	HostGatewayName = "host-gateway"
)

// ValidateHost validates that the specified string is a valid host and returns it.
func ValidateHost(val string) (string, error) {
	host := strings.TrimSpace(val)
	// The empty string means default and is not handled by parseDaemonHost
	if host != "" {
		_, err := parseDaemonHost(host)
		if err != nil {
			return val, err
		}
	}
	// Note: unlike most flag validators, we don't return the mutated value here
	//       we need to know what the user entered later (using ParseHost) to adjust for TLS
	return val, nil
}

// ParseHost and set defaults for a Daemon host string.
// defaultToTLS is preferred over defaultToUnixXDG.
func ParseHost(defaultToTLS, defaultToUnixXDG bool, val string) (string, error) {
	host := strings.TrimSpace(val)
	if host == "" {
		if defaultToTLS {
			host = DefaultTLSHost
		} else if defaultToUnixXDG {
			runtimeDir, err := homedir.GetRuntimeDir()
			if err != nil {
				return "", err
			}
			socket := filepath.Join(runtimeDir, "docker.sock")
			host = "unix://" + socket
		} else {
			host = DefaultHost
		}
	} else {
		var err error
		host, err = parseDaemonHost(host)
		if err != nil {
			return val, err
		}
	}
	return host, nil
}

// parseDaemonHost parses the specified address and returns an address that will be used as the host.
// Depending of the address specified, this may return one of the global Default* strings defined in hosts.go.
func parseDaemonHost(addr string) (string, error) {
	addrParts := strings.SplitN(addr, "://", 2)
	if len(addrParts) == 1 && addrParts[0] != "" {
		addrParts = []string{"tcp", addrParts[0]}
	}

	switch addrParts[0] {
	case "tcp":
		return ParseTCPAddr(addrParts[1], DefaultTCPHost)
	case "unix":
		return parseSimpleProtoAddr("unix", addrParts[1], DefaultUnixSocket)
	case "npipe":
		return parseSimpleProtoAddr("npipe", addrParts[1], DefaultNamedPipe)
	case "fd":
		return addr, nil
	default:
		return "", fmt.Errorf("Invalid bind address format: %s", addr)
	}
}

// parseSimpleProtoAddr parses and validates that the specified address is a valid
// socket address for simple protocols like unix and npipe. It returns a formatted
// socket address, either using the address parsed from addr, or the contents of
// defaultAddr if addr is a blank string.
func parseSimpleProtoAddr(proto, addr, defaultAddr string) (string, error) {
	addr = strings.TrimPrefix(addr, proto+"://")
	if strings.Contains(addr, "://") {
		return "", fmt.Errorf("Invalid proto, expected %s: %s", proto, addr)
	}
	if addr == "" {
		addr = defaultAddr
	}
	return fmt.Sprintf("%s://%s", proto, addr), nil
}

// ParseTCPAddr parses and validates that the specified address is a valid TCP
// address. It returns a formatted TCP address, either using the address parsed
// from tryAddr, or the contents of defaultAddr if tryAddr is a blank string.
// tryAddr is expected to have already been Trim()'d
// defaultAddr must be in the full `tcp://host:port` form
func ParseTCPAddr(tryAddr string, defaultAddr string) (string, error) {
	if tryAddr == "" || tryAddr == "tcp://" {
		return defaultAddr, nil
	}
	addr := strings.TrimPrefix(tryAddr, "tcp://")
	if strings.Contains(addr, "://") || addr == "" {
		return "", fmt.Errorf("Invalid proto, expected tcp: %s", tryAddr)
	}

	defaultAddr = strings.TrimPrefix(defaultAddr, "tcp://")
	defaultHost, defaultPort, err := net.SplitHostPort(defaultAddr)
	if err != nil {
		return "", err
	}
	u, err := url.Parse("tcp://" + addr)
	if err != nil {
		return "", err
	}
	host, port, err := net.SplitHostPort(u.Host)
	if err != nil {
		// try port addition once
		host, port, err = net.SplitHostPort(net.JoinHostPort(u.Host, defaultPort))
	}
	if err != nil {
		return "", fmt.Errorf("Invalid bind address format: %s", tryAddr)
	}

	if host == "" {
		host = defaultHost
	}
	if port == "" {
		port = defaultPort
	}
	p, err := strconv.Atoi(port)
	if err != nil && p == 0 {
		return "", fmt.Errorf("Invalid bind address format: %s", tryAddr)
	}

	if u.Path != "" {
		return "", errors.Errorf("invalid bind address (%s): should not contain a path element", tryAddr)
	}

	return "tcp://" + net.JoinHostPort(host, port), nil
}

// ValidateExtraHost validates that the specified string is a valid extrahost and returns it.
// ExtraHost is in the form of name:ip where the ip has to be a valid ip (IPv4 or IPv6).
func ValidateExtraHost(val string) (string, error) {
	// allow for IPv6 addresses in extra hosts by only splitting on first ":"
	arr := strings.SplitN(val, ":", 2)
	if len(arr) != 2 || len(arr[0]) == 0 {
		return "", fmt.Errorf("bad format for add-host: %q", val)
	}
	// Skip IPaddr validation for special "host-gateway" string
	if arr[1] != HostGatewayName {
		if _, err := ValidateIPAddress(arr[1]); err != nil {
			return "", fmt.Errorf("invalid IP address in add-host: %q", arr[1])
		}
	}
	return val, nil
}
<|endoftext|>


<commit_before>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetDAG(context.Context, *Node) <-chan *Node
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

// MakeLink creates a link to the given node
func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

// Remove a link on this node by the given name
func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	k, _ := nd.Key()
	log.Debugf("DagService Add [%s]", k)
	if n == nil {
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph asynchronously fetches all nodes that are children of the given
// node, and returns a channel that may be waited upon for the fetch to complete
func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{} {
	log.Warning("Untested.")
	var wg sync.WaitGroup
	done := make(chan struct{})

	for _, l := range root.Links {
		wg.Add(1)
		go func(lnk *Link) {

			// Signal child is done on way out
			defer wg.Done()
			select {
			case <-ctx.Done():
				return
			}

			nd, err := lnk.GetNode(serv)
			if err != nil {
				log.Error(err)
				return
			}

			// Wait for children to finish
			<-FetchGraph(ctx, nd, serv)
		}(l)
	}

	go func() {
		wg.Wait()
		done <- struct{}{}
	}()

	return done
}

// FindLinks searches this nodes links for the given key,
// returns the indexes of any links pointing to it
func FindLinks(n *Node, k u.Key) []int {
	var out []int
	keybytes := []byte(k)
	for i, lnk := range n.Links {
		if bytes.Equal([]byte(lnk.Hash), keybytes) {
			out = append(out, i)
		}
	}
	return out
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
	sig := make(chan *Node)
	go func() {
		defer close(sig)

		var keys []u.Key
		for _, lnk := range root.Links {
			keys = append(keys, u.Key(lnk.Hash))
		}
		blkchan := ds.Blocks.GetBlocks(ctx, keys)

		nodes := make([]*Node, len(root.Links))
		next := 0
		for blk := range blkchan {
			nd, err := Decoded(blk.Data)
			if err != nil {
				// NB: can occur in normal situations, with improperly formatted
				// input data
				log.Error("Got back bad block!")
				break
			}
			is := FindLinks(root, blk.Key())
			for _, i := range is {
				nodes[i] = nd
			}

			for ; next < len(nodes) && nodes[next] != nil; next++ {
				sig <- nodes[next]
			}
		}
		if next < len(nodes) {
			// TODO: bubble errors back up.
			log.Errorf("Did not receive correct number of nodes!")
		}
	}()

	return sig
}
<commit_msg>remote low SnR debug statement<commit_after>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetDAG(context.Context, *Node) <-chan *Node
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

// MakeLink creates a link to the given node
func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

// Remove a link on this node by the given name
func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	if n == nil { // FIXME remove this assertion. protect with constructor invariant
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph asynchronously fetches all nodes that are children of the given
// node, and returns a channel that may be waited upon for the fetch to complete
func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{} {
	log.Warning("Untested.")
	var wg sync.WaitGroup
	done := make(chan struct{})

	for _, l := range root.Links {
		wg.Add(1)
		go func(lnk *Link) {

			// Signal child is done on way out
			defer wg.Done()
			select {
			case <-ctx.Done():
				return
			}

			nd, err := lnk.GetNode(serv)
			if err != nil {
				log.Error(err)
				return
			}

			// Wait for children to finish
			<-FetchGraph(ctx, nd, serv)
		}(l)
	}

	go func() {
		wg.Wait()
		done <- struct{}{}
	}()

	return done
}

// FindLinks searches this nodes links for the given key,
// returns the indexes of any links pointing to it
func FindLinks(n *Node, k u.Key) []int {
	var out []int
	keybytes := []byte(k)
	for i, lnk := range n.Links {
		if bytes.Equal([]byte(lnk.Hash), keybytes) {
			out = append(out, i)
		}
	}
	return out
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
	sig := make(chan *Node)
	go func() {
		defer close(sig)

		var keys []u.Key
		for _, lnk := range root.Links {
			keys = append(keys, u.Key(lnk.Hash))
		}
		blkchan := ds.Blocks.GetBlocks(ctx, keys)

		nodes := make([]*Node, len(root.Links))
		next := 0
		for blk := range blkchan {
			nd, err := Decoded(blk.Data)
			if err != nil {
				// NB: can occur in normal situations, with improperly formatted
				// input data
				log.Error("Got back bad block!")
				break
			}
			is := FindLinks(root, blk.Key())
			for _, i := range is {
				nodes[i] = nd
			}

			for ; next < len(nodes) && nodes[next] != nil; next++ {
				sig <- nodes[next]
			}
		}
		if next < len(nodes) {
			// TODO: bubble errors back up.
			log.Errorf("Did not receive correct number of nodes!")
		}
	}()

	return sig
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/coreos/etcd-operator/pkg/spec"
	"github.com/coreos/etcd-operator/pkg/util/constants"
	"github.com/coreos/etcd-operator/test/e2e/framework"

	"golang.org/x/net/context"
)

func TestClusterRestore(t *testing.T) {
	t.Run("restore cluster from backup", func(t *testing.T) {
		t.Run("restore from the same name cluster", testClusterRestoreSameName)
		t.Run("restore from a different name cluster", testClusterRestoreDifferentName)
	})
}

func TestClusterRestoreS3(t *testing.T) {
	if os.Getenv("AWS_TEST_ENABLED") != "true" {
		t.Skip("skipping test since AWS_TEST_ENABLED is not set.")
	}

	t.Run("restore cluster from backup on S3", func(t *testing.T) {
		t.Run("restore from the same name cluster", testClusterRestoreS3SameName)
		t.Run("restore from a different name cluster", testClusterRestoreS3DifferentName)
	})
}

func testClusterRestoreSameName(t *testing.T) {
	if os.Getenv(envParallelTest) == envParallelTestTrue {
		t.Parallel()
	}
	testClusterRestore(t, false)
}

func testClusterRestoreDifferentName(t *testing.T) {
	if os.Getenv(envParallelTest) == envParallelTestTrue {
		t.Parallel()
	}
	testClusterRestore(t, true)
}

func testClusterRestore(t *testing.T, needDataClone bool) {
	testClusterRestoreWithBackupPolicy(t, needDataClone, newBackupPolicyPV())
}

func testClusterRestoreWithBackupPolicy(t *testing.T, needDataClone bool, backupPolicy *spec.BackupPolicy) {
	f := framework.Global

	origEtcd := newClusterSpec("test-etcd-", 3)
	testEtcd, err := createCluster(t, f, etcdClusterWithBackup(origEtcd, backupPolicy))
	if err != nil {
		t.Fatal(err)
	}

	names, err := waitUntilSizeReached(t, f, testEtcd.Name, 3, 60*time.Second)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}

	pod, err := f.KubeClient.Core().Pods(f.Namespace).Get(names[0])
	if err != nil {
		t.Fatal(err)
	}
	err = putDataToEtcd(fmt.Sprintf("http://%s:2379", pod.Status.PodIP))
	if err != nil {
		t.Fatal(err)
	}

	if err := waitBackupPodUp(f, testEtcd.Name, 60*time.Second); err != nil {
		t.Fatalf("failed to create backup pod: %v", err)
	}
	if err := makeBackup(f, testEtcd.Name); err != nil {
		t.Fatalf("fail to make a backup: %v", err)
	}
	if err := deleteEtcdCluster(t, f, testEtcd); err != nil {
		t.Fatal(err)
	}
	// waits a bit to make sure resources are finally deleted on APIServer.
	time.Sleep(5 * time.Second)

	if !needDataClone {
		// Restore the etcd cluster of the same name:
		// - use the name already generated. We don't need to regenerate again.
		// - set BackupClusterName to the same name in RestorePolicy.
		// Then operator will use the existing backup in the same storage and
		// restore cluster with the same data.
		origEtcd.GenerateName = ""
		origEtcd.Name = testEtcd.Name
	}
	waitRestoreTimeout := calculateRestoreWaitTime(needDataClone)

	origEtcd = etcdClusterWithRestore(origEtcd, &spec.RestorePolicy{
		BackupClusterName: testEtcd.Name,
		StorageType:       backupPolicy.StorageType,
	})

	backupPolicy.CleanupBackupsOnClusterDelete = true
	testEtcd, err = createCluster(t, f, etcdClusterWithBackup(origEtcd, backupPolicy))
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := deleteEtcdCluster(t, f, testEtcd); err != nil {
			t.Fatal(err)
		}
	}()

	names, err = waitUntilSizeReached(t, f, testEtcd.Name, 3, waitRestoreTimeout)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}

	pod, err = f.KubeClient.Core().Pods(f.Namespace).Get(names[0])
	if err != nil {
		t.Fatal(err)
	}
	checkEtcdData(t, fmt.Sprintf("http://%s:2379", pod.Status.PodIP))
}

func putDataToEtcd(addr string) error {
	etcdcli, err := createEtcdClient(addr)
	if err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultRequestTimeout)
	_, err = etcdcli.Put(ctx, etcdKeyFoo, etcdValBar)
	cancel()
	etcdcli.Close()
	return err
}

func checkEtcdData(t *testing.T, addr string) {
	etcdcli, err := createEtcdClient(addr)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultRequestTimeout)
	resp, err := etcdcli.Get(ctx, etcdKeyFoo)
	cancel()
	etcdcli.Close()
	if err != nil {
		t.Fatal(err)
	}
	if len(resp.Kvs) != 1 {
		t.Errorf("want only 1 key result, get %d", len(resp.Kvs))
	} else {
		val := string(resp.Kvs[0].Value)
		if val != etcdValBar {
			t.Errorf("value want = '%s', get = '%s'", etcdValBar, val)
		}
	}
}

func calculateRestoreWaitTime(needDataClone bool) time.Duration {
	waitTime := 180 * time.Second
	if needDataClone {
		// Take additional time to clone the data.
		waitTime += 60 * time.Second
	}
	return waitTime
}

func testClusterRestoreS3SameName(t *testing.T) {
	if os.Getenv(envParallelTest) == envParallelTestTrue {
		t.Parallel()
	}

	testClusterRestoreWithBackupPolicy(t, false, newBackupPolicyS3())
}

func testClusterRestoreS3DifferentName(t *testing.T) {
	if os.Getenv(envParallelTest) == envParallelTestTrue {
		t.Parallel()
	}

	testClusterRestoreWithBackupPolicy(t, true, newBackupPolicyS3())
}
<commit_msg>e2e: bump timeout for wait restore<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/coreos/etcd-operator/pkg/spec"
	"github.com/coreos/etcd-operator/pkg/util/constants"
	"github.com/coreos/etcd-operator/test/e2e/framework"

	"golang.org/x/net/context"
)

func TestClusterRestore(t *testing.T) {
	t.Run("restore cluster from backup", func(t *testing.T) {
		t.Run("restore from the same name cluster", testClusterRestoreSameName)
		t.Run("restore from a different name cluster", testClusterRestoreDifferentName)
	})
}

func TestClusterRestoreS3(t *testing.T) {
	if os.Getenv("AWS_TEST_ENABLED") != "true" {
		t.Skip("skipping test since AWS_TEST_ENABLED is not set.")
	}

	t.Run("restore cluster from backup on S3", func(t *testing.T) {
		t.Run("restore from the same name cluster", testClusterRestoreS3SameName)
		t.Run("restore from a different name cluster", testClusterRestoreS3DifferentName)
	})
}

func testClusterRestoreSameName(t *testing.T) {
	if os.Getenv(envParallelTest) == envParallelTestTrue {
		t.Parallel()
	}
	testClusterRestore(t, false)
}

func testClusterRestoreDifferentName(t *testing.T) {
	if os.Getenv(envParallelTest) == envParallelTestTrue {
		t.Parallel()
	}
	testClusterRestore(t, true)
}

func testClusterRestore(t *testing.T, needDataClone bool) {
	testClusterRestoreWithBackupPolicy(t, needDataClone, newBackupPolicyPV())
}

func testClusterRestoreWithBackupPolicy(t *testing.T, needDataClone bool, backupPolicy *spec.BackupPolicy) {
	f := framework.Global

	origEtcd := newClusterSpec("test-etcd-", 3)
	testEtcd, err := createCluster(t, f, etcdClusterWithBackup(origEtcd, backupPolicy))
	if err != nil {
		t.Fatal(err)
	}

	names, err := waitUntilSizeReached(t, f, testEtcd.Name, 3, 60*time.Second)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}

	pod, err := f.KubeClient.Core().Pods(f.Namespace).Get(names[0])
	if err != nil {
		t.Fatal(err)
	}
	err = putDataToEtcd(fmt.Sprintf("http://%s:2379", pod.Status.PodIP))
	if err != nil {
		t.Fatal(err)
	}

	if err := waitBackupPodUp(f, testEtcd.Name, 60*time.Second); err != nil {
		t.Fatalf("failed to create backup pod: %v", err)
	}
	if err := makeBackup(f, testEtcd.Name); err != nil {
		t.Fatalf("fail to make a backup: %v", err)
	}
	if err := deleteEtcdCluster(t, f, testEtcd); err != nil {
		t.Fatal(err)
	}
	// waits a bit to make sure resources are finally deleted on APIServer.
	time.Sleep(5 * time.Second)

	if !needDataClone {
		// Restore the etcd cluster of the same name:
		// - use the name already generated. We don't need to regenerate again.
		// - set BackupClusterName to the same name in RestorePolicy.
		// Then operator will use the existing backup in the same storage and
		// restore cluster with the same data.
		origEtcd.GenerateName = ""
		origEtcd.Name = testEtcd.Name
	}
	waitRestoreTimeout := calculateRestoreWaitTime(needDataClone)

	origEtcd = etcdClusterWithRestore(origEtcd, &spec.RestorePolicy{
		BackupClusterName: testEtcd.Name,
		StorageType:       backupPolicy.StorageType,
	})

	backupPolicy.CleanupBackupsOnClusterDelete = true
	testEtcd, err = createCluster(t, f, etcdClusterWithBackup(origEtcd, backupPolicy))
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := deleteEtcdCluster(t, f, testEtcd); err != nil {
			t.Fatal(err)
		}
	}()

	names, err = waitUntilSizeReached(t, f, testEtcd.Name, 3, waitRestoreTimeout)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}

	pod, err = f.KubeClient.Core().Pods(f.Namespace).Get(names[0])
	if err != nil {
		t.Fatal(err)
	}
	checkEtcdData(t, fmt.Sprintf("http://%s:2379", pod.Status.PodIP))
}

func putDataToEtcd(addr string) error {
	etcdcli, err := createEtcdClient(addr)
	if err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultRequestTimeout)
	_, err = etcdcli.Put(ctx, etcdKeyFoo, etcdValBar)
	cancel()
	etcdcli.Close()
	return err
}

func checkEtcdData(t *testing.T, addr string) {
	etcdcli, err := createEtcdClient(addr)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultRequestTimeout)
	resp, err := etcdcli.Get(ctx, etcdKeyFoo)
	cancel()
	etcdcli.Close()
	if err != nil {
		t.Fatal(err)
	}
	if len(resp.Kvs) != 1 {
		t.Errorf("want only 1 key result, get %d", len(resp.Kvs))
	} else {
		val := string(resp.Kvs[0].Value)
		if val != etcdValBar {
			t.Errorf("value want = '%s', get = '%s'", etcdValBar, val)
		}
	}
}

func calculateRestoreWaitTime(needDataClone bool) time.Duration {
	waitTime := 240 * time.Second
	if needDataClone {
		// Take additional time to clone the data.
		waitTime += 60 * time.Second
	}
	return waitTime
}

func testClusterRestoreS3SameName(t *testing.T) {
	if os.Getenv(envParallelTest) == envParallelTestTrue {
		t.Parallel()
	}

	testClusterRestoreWithBackupPolicy(t, false, newBackupPolicyS3())
}

func testClusterRestoreS3DifferentName(t *testing.T) {
	if os.Getenv(envParallelTest) == envParallelTestTrue {
		t.Parallel()
	}

	testClusterRestoreWithBackupPolicy(t, true, newBackupPolicyS3())
}
<|endoftext|>


<commit_before>// $G -N -o slow.$A $D/bug369.dir/pkg.go &&
// $G -o fast.$A $D/bug369.dir/pkg.go &&
// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that compiling with optimization turned on produces faster code.

package main

import (
	"flag"
	"os"
	"runtime"
	"testing"

	fast "./fast"
	slow "./slow"
)

var buf = make([]byte, 1048576)

func BenchmarkFastNonASCII(b *testing.B) {
	for i := 0; i < b.N; i++ {
		fast.NonASCII(buf, 0)
	}
}

func BenchmarkSlowNonASCII(b *testing.B) {
	for i := 0; i < b.N; i++ {
		slow.NonASCII(buf, 0)
	}
}

func main() {
	os.Args = []string{os.Args[0], "-test.benchtime=0.1"}
	flag.Parse()

	rslow := testing.Benchmark(BenchmarkSlowNonASCII)
	rfast := testing.Benchmark(BenchmarkFastNonASCII)
	tslow := rslow.NsPerOp()
	tfast := rfast.NsPerOp()

	// Optimization should be good for at least 2x, but be forgiving.
	// On the ARM simulator we see closer to 1.5x.
	speedup := float64(tslow)/float64(tfast)
	want := 1.8
	if runtime.GOARCH == "arm" {
		want = 1.3
	}
	if speedup < want {
		println("fast:", tfast, "slow:", tslow, "speedup:", speedup, "want:", want)
		println("not fast enough")
	}
}
<commit_msg>Fix build, disabling flaky registerization test.<commit_after>// $G -N -o slow.$A $D/bug369.dir/pkg.go &&
// $G -o fast.$A $D/bug369.dir/pkg.go &&
// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that compiling with optimization turned on produces faster code.

package main

import (
	"flag"
	"os"
	"runtime"
	"testing"

	fast "./fast"
	slow "./slow"
)

var buf = make([]byte, 1048576)

func BenchmarkFastNonASCII(b *testing.B) {
	for i := 0; i < b.N; i++ {
		fast.NonASCII(buf, 0)
	}
}

func BenchmarkSlowNonASCII(b *testing.B) {
	for i := 0; i < b.N; i++ {
		slow.NonASCII(buf, 0)
	}
}

func main() {
	os.Args = []string{os.Args[0], "-test.benchtime=0.1"}
	flag.Parse()

	rslow := testing.Benchmark(BenchmarkSlowNonASCII)
	rfast := testing.Benchmark(BenchmarkFastNonASCII)
	tslow := rslow.NsPerOp()
	tfast := rfast.NsPerOp()

	// Optimization should be good for at least 2x, but be forgiving.
	// On the ARM simulator we see closer to 1.5x.
	speedup := float64(tslow)/float64(tfast)
	want := 1.8
	if runtime.GOARCH == "arm" {
		want = 1.3
	}
	if speedup < want {
		// TODO(rsc): doesn't work on linux-amd64 or darwin-amd64 builders, nor on
		// a Lenovo x200 (linux-amd64) laptop.
		//println("fast:", tfast, "slow:", tslow, "speedup:", speedup, "want:", want)
		//println("not fast enough")
	}
}
<|endoftext|>


<commit_before>package orderer

import (
	"fmt"
	"net"
	"os"
	"strings"

	context "golang.org/x/net/context"

	"time"

	"github.com/ellcrys/util"
	"github.com/ncodes/cocoon/core/common"
	"github.com/ncodes/cocoon/core/orderer/proto"
	"github.com/ncodes/cocoon/core/scheduler"
	"github.com/ncodes/cocoon/core/types"
	"github.com/ncodes/cstructs"
	logging "github.com/op/go-logging"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("orderer")

// SetLogLevel sets the log level of the logger
func SetLogLevel(l logging.Level) {
	logging.SetLevel(l, log.Module)
}

// DiscoverOrderers fetches a list of orderer service addresses
// via consul service discovery API. For development purpose,
// If DEV_ORDERER_ADDR is set, it will fetch the orderer
// address from the env variable.
func DiscoverOrderers() ([]string, error) {

	if len(os.Getenv("DEV_ORDERER_ADDR")) > 0 {
		return []string{os.Getenv("DEV_ORDERER_ADDR")}, nil
	}

	ds := scheduler.NomadServiceDiscovery{
		ConsulAddr: util.Env("CONSUL_ADDR", "localhost:8500"),
		Protocol:   "http",
	}

	_orderers, err := ds.GetByID("orderers", nil)
	if err != nil {
		return []string{}, nil
	}

	var orderers []string
	for _, orderer := range _orderers {
		orderers = append(orderers, fmt.Sprintf("%s:%d", orderer.IP, int(orderer.Port)))
	}

	return orderers, nil
}

// DialOrderer returns a connection to a orderer from a list of addresses. It randomly
// picks an orderer address from the list for orderers.
func DialOrderer(ordererAddrs []string) (*grpc.ClientConn, error) {
	var ordererAddr string

	if len(ordererAddrs) == 0 {
		return nil, fmt.Errorf("no known orderer address")
	} else if len(ordererAddrs) == 1 {
		ordererAddr = ordererAddrs[0]
	} else {
		ordererAddr = ordererAddrs[util.RandNum(0, len(ordererAddrs))]
	}

	client, err := grpc.Dial(ordererAddr, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}

	return client, nil
}

// Orderer defines a transaction ordering, block creation
// and inclusion module
type Orderer struct {
	server     *grpc.Server
	store      types.Store
	blockchain types.Blockchain
	endedCh    chan bool
}

// NewOrderer creates a new Orderer object
func NewOrderer() *Orderer {
	return new(Orderer)
}

// Start starts the order service
func (od *Orderer) Start(addr, storeConStr string, endedCh chan bool) {

	od.endedCh = endedCh

	lis, err := net.Listen("tcp", fmt.Sprintf("%s", addr))
	if err != nil {
		log.Fatalf("failed to listen on port=%s. Err: %s", strings.Split(addr, ":")[1], err)
	}

	time.AfterFunc(2*time.Second, func() {

		log.Infof("Started orderer GRPC server on port %s", strings.Split(addr, ":")[1])

		// establish connection to store backend
		_, err := od.store.Connect(storeConStr)
		if err != nil {
			log.Info(err)
			od.Stop(1)
			return
		}

		// initialize store
		if od.store == nil {
			log.Error("Store implementation not set")
			od.Stop(1)
			return
		}

		err = od.store.Init(od.store.MakeLedgerName("", types.GetGlobalLedgerName()))
		if err != nil {
			log.Info(err)
			od.Stop(1)
			return
		}

		if od.blockchain == nil {
			log.Error("Blockchain implementation not set")
			od.Stop(1)
			return
		}

		// establish connection to blockchain backend
		if od.blockchain != nil {
			_, err = od.blockchain.Connect(storeConStr)
			if err != nil {
				log.Info(err)
				od.Stop(1)
				return
			}
		}

		// initialize the blockchain
		err = od.blockchain.Init(od.blockchain.MakeChainName("", types.GetGlobalChainName()))
		if err != nil {
			log.Info(err)
			od.Stop(1)
			return
		}

		log.Info("Backend successfully connnected")
	})

	od.server = grpc.NewServer()
	proto.RegisterOrdererServer(od.server, od)
	od.server.Serve(lis)
}

// Stop stops the orderer and returns an exit code.
func (od *Orderer) Stop(exitCode int) int {
	od.server.Stop()
	od.store.Close()
	close(od.endedCh)
	return exitCode
}

// SetStore sets the store implementation to use.
func (od *Orderer) SetStore(ch types.Store) {
	log.Infof("Setting store implementation named %s", ch.GetImplmentationName())
	od.store = ch
}

// SetBlockchain sets the blockchain implementation
func (od *Orderer) SetBlockchain(b types.Blockchain) {
	log.Infof("Setting blockchain implementation named %s", b.GetImplmentationName())
	od.blockchain = b
}

// CreateLedger creates a new ledger
func (od *Orderer) CreateLedger(ctx context.Context, params *proto.CreateLedgerParams) (*proto.Ledger, error) {

	name := od.store.MakeLedgerName(params.GetCocoonID(), params.GetName())

	var createChainFunc func() error
	if params.Chained {
		createChainFunc = func() error {
			_, err := od.blockchain.CreateChain(name, params.Public)
			return err
		}
	}

	ledger, err := od.store.CreateLedgerThen(name, params.GetChained(), params.GetPublic(), createChainFunc)
	if err != nil {
		return nil, err
	}

	// replace hashed name to user readable name
	ledger.Name = params.GetName()

	var protoLedger proto.Ledger
	cstructs.Copy(ledger, &protoLedger)

	return &protoLedger, nil
}

// GetLedger returns a ledger
func (od *Orderer) GetLedger(ctx context.Context, params *proto.GetLedgerParams) (*proto.Ledger, error) {

	name := od.store.MakeLedgerName(params.GetCocoonID(), params.GetName())
	ledger, err := od.store.GetLedger(name)
	if err != nil {
		return nil, err
	} else if ledger == nil && err == nil {
		return nil, types.ErrLedgerNotFound
	}

	// replace hashed name to user readable name
	ledger.Name = params.GetName()

	var protoLedger proto.Ledger
	cstructs.Copy(ledger, &protoLedger)

	return &protoLedger, nil
}

// Put creates a new transaction
func (od *Orderer) Put(ctx context.Context, params *proto.PutTransactionParams) (*proto.PutResult, error) {

	start := time.Now()

	// check if ledger exists
	ledgerName := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedgerName())
	ledger, err := od.store.GetLedger(ledgerName)
	if err != nil {
		return nil, err
	} else if err == nil && ledger == nil {
		return nil, types.ErrLedgerNotFound
	}

	// copy individual tx from []proto.Transaction to []types.Transaction
	// and set transactions key and block id
	blockID := util.Sha256(util.UUID4())
	var transactions = make([]*types.Transaction, len(params.GetTransactions()))
	for i, protoTx := range params.GetTransactions() {
		var tx = types.Transaction{}
		cstructs.Copy(protoTx, &tx)
		tx.Key = od.store.MakeTxKey(params.GetCocoonID(), tx.Key)
		tx.BlockID = blockID
		transactions[i] = &tx
	}

	var block *proto.Block
	var createBlockFunc func() error
	if ledger.Chained {
		block = &proto.Block{}

		createBlockFunc = func() error {
			var err error
			retryDelay := time.Duration(2) * time.Second
			common.ReRunOnError(func() error {
				b, _err := od.blockchain.CreateBlock(blockID, ledgerName, transactions)
				if b != nil {
					block.Id = b.ID
					block.ChainName = b.ChainName
					block.Hash = b.Hash
					block.Number = int64(b.Number)
					block.PrevBlockHash = b.PrevBlockHash
					block.Transactions = b.Transactions
					block.CreatedAt = b.CreatedAt
				}

				err = _err

				// If error is not a duplicate previous block hash error, don't re-run.
				// return nil to end the re-run routine
				if _err != nil && !types.IsDuplicatePrevBlockHashError(_err) {
					return nil
				}

				return _err
			}, 5, &retryDelay)
			return err
		}
	}

	err = od.store.PutThen(ledgerName, transactions, createBlockFunc)
	if err != nil {
		return nil, err
	}

	log.Debug("Put(): Time taken: ", time.Since(start))

	return &proto.PutResult{
		Added: int32(len(transactions)),
		Block: block,
	}, nil
}

// Get returns a transaction with a matching key
func (od *Orderer) Get(ctx context.Context, params *proto.GetParams) (*proto.Transaction, error) {

	start := time.Now()
	log.Error("Get it")

	ledger, err := od.GetLedger(ctx, &proto.GetLedgerParams{
		CocoonID: params.GetCocoonID(),
		Name:     params.GetLedger(),
	})
	if err != nil {
		return nil, err
	}

	ledgerName := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedger())
	key := od.store.MakeTxKey(params.GetCocoonID(), params.GetKey())
	tx, err := od.store.Get(ledgerName, key)
	if err != nil {
		return nil, err
	} else if tx == nil && err == nil {
		return nil, types.ErrTxNotFound
	}

	if ledger.Chained {
		block, err := od.blockchain.GetBlock(ledgerName, tx.BlockID)
		if err != nil {
			log.Error(err)
			return nil, fmt.Errorf("failed to populate block to transaction")
		} else if block == nil && err == nil {
			return nil, fmt.Errorf("orphaned transaction")
		}

		tx.Block = block
		tx.BlockID = ""
	}

	var protoTx proto.Transaction
	cstructs.Copy(tx, &protoTx)

	log.Debug("Get(): Time taken: ", time.Since(start))

	return &protoTx, nil
}

// GetByID finds and returns a transaction with a matching id
func (od *Orderer) GetByID(ctx context.Context, params *proto.GetParams) (*proto.Transaction, error) {

	ledger, err := od.GetLedger(ctx, &proto.GetLedgerParams{
		CocoonID: params.GetCocoonID(),
		Name:     params.GetLedger(),
	})
	if err != nil {
		return nil, err
	}

	ledgerName := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedger())
	tx, err := od.store.GetByID(ledgerName, params.GetId())
	if err != nil {
		return nil, err
	} else if tx == nil && err == nil {
		return nil, types.ErrTxNotFound
	}

	if ledger.Chained {
		block, err := od.blockchain.GetBlock(ledgerName, tx.BlockID)
		if err != nil {
			log.Error(err)
			return nil, fmt.Errorf("failed to populate block to transaction")
		} else if block == nil && err == nil {
			return nil, fmt.Errorf("orphaned transaction")
		}

		tx.Block = block
		tx.BlockID = ""
	}

	var protoTx proto.Transaction
	cstructs.Copy(tx, &protoTx)

	return &protoTx, nil
}

// GetBlockByID returns a block by its id and chain/ledger name
func (od *Orderer) GetBlockByID(ctx context.Context, params *proto.GetBlockParams) (*proto.Block, error) {

	name := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedger())
	ledger, err := od.store.GetLedger(name)
	if err != nil {
		return nil, err
	} else if ledger == nil && err == nil {
		return nil, types.ErrLedgerNotFound
	}

	blk, err := od.blockchain.GetBlock(name, params.GetId())
	if err != nil {
		return nil, err
	} else if blk == nil && err == nil {
		return nil, types.ErrBlockNotFound
	}

	var protoBlk proto.Block
	cstructs.Copy(blk, &protoBlk)

	return &protoBlk, nil
}

// GetRange fetches transactions between a range of keys
func (od *Orderer) GetRange(ctx context.Context, params *proto.GetRangeParams) (*proto.Transactions, error) {

	name := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedger())
	ledger, err := od.store.GetLedger(name)
	if err != nil {
		return nil, err
	} else if ledger == nil && err == nil {
		return nil, types.ErrLedgerNotFound
	}

	if len(params.GetStartKey()) > 0 {
		params.StartKey = od.store.MakeTxKey(params.GetCocoonID(), params.GetStartKey())
	}

	if len(params.GetEndKey()) > 0 {
		if len(params.GetStartKey()) > 0 {
			params.EndKey = od.store.MakeTxKey(params.GetCocoonID(), params.GetEndKey())
		} else {
			params.EndKey = od.store.MakeTxKey(params.GetCocoonID(), "%"+params.GetEndKey())
		}
	}

	txs, err := od.store.GetRange(name, params.GetStartKey(), params.GetEndKey(), params.GetInclusive(), int(params.GetLimit()), int(params.GetOffset()))
	if err != nil {
		return nil, err
	}

	// copy individual tx from []types.Transaction to []proto.Transaction
	var protoTxs = make([]*proto.Transaction, len(txs))
	for i, tx := range txs {
		var protoTx = proto.Transaction{}
		cstructs.Copy(tx, &protoTx)
		protoTxs[i] = &protoTx
	}

	return &proto.Transactions{
		Transactions: protoTxs,
	}, nil
}
<commit_msg>Only set block id to transaction if destination ledger is chained<commit_after>package orderer

import (
	"fmt"
	"net"
	"os"
	"strings"

	context "golang.org/x/net/context"

	"time"

	"github.com/ellcrys/util"
	"github.com/ncodes/cocoon/core/common"
	"github.com/ncodes/cocoon/core/orderer/proto"
	"github.com/ncodes/cocoon/core/scheduler"
	"github.com/ncodes/cocoon/core/types"
	"github.com/ncodes/cstructs"
	logging "github.com/op/go-logging"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("orderer")

// SetLogLevel sets the log level of the logger
func SetLogLevel(l logging.Level) {
	logging.SetLevel(l, log.Module)
}

// DiscoverOrderers fetches a list of orderer service addresses
// via consul service discovery API. For development purpose,
// If DEV_ORDERER_ADDR is set, it will fetch the orderer
// address from the env variable.
func DiscoverOrderers() ([]string, error) {

	if len(os.Getenv("DEV_ORDERER_ADDR")) > 0 {
		return []string{os.Getenv("DEV_ORDERER_ADDR")}, nil
	}

	ds := scheduler.NomadServiceDiscovery{
		ConsulAddr: util.Env("CONSUL_ADDR", "localhost:8500"),
		Protocol:   "http",
	}

	_orderers, err := ds.GetByID("orderers", nil)
	if err != nil {
		return []string{}, nil
	}

	var orderers []string
	for _, orderer := range _orderers {
		orderers = append(orderers, fmt.Sprintf("%s:%d", orderer.IP, int(orderer.Port)))
	}

	return orderers, nil
}

// DialOrderer returns a connection to a orderer from a list of addresses. It randomly
// picks an orderer address from the list for orderers.
func DialOrderer(ordererAddrs []string) (*grpc.ClientConn, error) {
	var ordererAddr string

	if len(ordererAddrs) == 0 {
		return nil, fmt.Errorf("no known orderer address")
	} else if len(ordererAddrs) == 1 {
		ordererAddr = ordererAddrs[0]
	} else {
		ordererAddr = ordererAddrs[util.RandNum(0, len(ordererAddrs))]
	}

	client, err := grpc.Dial(ordererAddr, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}

	return client, nil
}

// Orderer defines a transaction ordering, block creation
// and inclusion module
type Orderer struct {
	server     *grpc.Server
	store      types.Store
	blockchain types.Blockchain
	endedCh    chan bool
}

// NewOrderer creates a new Orderer object
func NewOrderer() *Orderer {
	return new(Orderer)
}

// Start starts the order service
func (od *Orderer) Start(addr, storeConStr string, endedCh chan bool) {

	od.endedCh = endedCh

	lis, err := net.Listen("tcp", fmt.Sprintf("%s", addr))
	if err != nil {
		log.Fatalf("failed to listen on port=%s. Err: %s", strings.Split(addr, ":")[1], err)
	}

	time.AfterFunc(2*time.Second, func() {

		log.Infof("Started orderer GRPC server on port %s", strings.Split(addr, ":")[1])

		// establish connection to store backend
		_, err := od.store.Connect(storeConStr)
		if err != nil {
			log.Info(err)
			od.Stop(1)
			return
		}

		// initialize store
		if od.store == nil {
			log.Error("Store implementation not set")
			od.Stop(1)
			return
		}

		err = od.store.Init(od.store.MakeLedgerName("", types.GetGlobalLedgerName()))
		if err != nil {
			log.Info(err)
			od.Stop(1)
			return
		}

		if od.blockchain == nil {
			log.Error("Blockchain implementation not set")
			od.Stop(1)
			return
		}

		// establish connection to blockchain backend
		if od.blockchain != nil {
			_, err = od.blockchain.Connect(storeConStr)
			if err != nil {
				log.Info(err)
				od.Stop(1)
				return
			}
		}

		// initialize the blockchain
		err = od.blockchain.Init(od.blockchain.MakeChainName("", types.GetGlobalChainName()))
		if err != nil {
			log.Info(err)
			od.Stop(1)
			return
		}

		log.Info("Backend successfully connnected")
	})

	od.server = grpc.NewServer()
	proto.RegisterOrdererServer(od.server, od)
	od.server.Serve(lis)
}

// Stop stops the orderer and returns an exit code.
func (od *Orderer) Stop(exitCode int) int {
	od.server.Stop()
	od.store.Close()
	close(od.endedCh)
	return exitCode
}

// SetStore sets the store implementation to use.
func (od *Orderer) SetStore(ch types.Store) {
	log.Infof("Setting store implementation named %s", ch.GetImplmentationName())
	od.store = ch
}

// SetBlockchain sets the blockchain implementation
func (od *Orderer) SetBlockchain(b types.Blockchain) {
	log.Infof("Setting blockchain implementation named %s", b.GetImplmentationName())
	od.blockchain = b
}

// CreateLedger creates a new ledger
func (od *Orderer) CreateLedger(ctx context.Context, params *proto.CreateLedgerParams) (*proto.Ledger, error) {

	name := od.store.MakeLedgerName(params.GetCocoonID(), params.GetName())

	var createChainFunc func() error
	if params.Chained {
		createChainFunc = func() error {
			_, err := od.blockchain.CreateChain(name, params.Public)
			return err
		}
	}

	ledger, err := od.store.CreateLedgerThen(name, params.GetChained(), params.GetPublic(), createChainFunc)
	if err != nil {
		return nil, err
	}

	// replace hashed name to user readable name
	ledger.Name = params.GetName()

	var protoLedger proto.Ledger
	cstructs.Copy(ledger, &protoLedger)

	return &protoLedger, nil
}

// GetLedger returns a ledger
func (od *Orderer) GetLedger(ctx context.Context, params *proto.GetLedgerParams) (*proto.Ledger, error) {

	name := od.store.MakeLedgerName(params.GetCocoonID(), params.GetName())
	ledger, err := od.store.GetLedger(name)
	if err != nil {
		return nil, err
	} else if ledger == nil && err == nil {
		return nil, types.ErrLedgerNotFound
	}

	// replace hashed name to user readable name
	ledger.Name = params.GetName()

	var protoLedger proto.Ledger
	cstructs.Copy(ledger, &protoLedger)

	return &protoLedger, nil
}

// Put creates a new transaction
func (od *Orderer) Put(ctx context.Context, params *proto.PutTransactionParams) (*proto.PutResult, error) {

	start := time.Now()

	// check if ledger exists
	ledgerName := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedgerName())
	ledger, err := od.store.GetLedger(ledgerName)
	if err != nil {
		return nil, err
	} else if err == nil && ledger == nil {
		return nil, types.ErrLedgerNotFound
	}

	// copy individual tx from []proto.Transaction to []types.Transaction
	// and set transactions key and block id
	blockID := util.Sha256(util.UUID4())
	var transactions = make([]*types.Transaction, len(params.GetTransactions()))
	for i, protoTx := range params.GetTransactions() {
		var tx = types.Transaction{}
		cstructs.Copy(protoTx, &tx)
		tx.Key = od.store.MakeTxKey(params.GetCocoonID(), tx.Key)
		transactions[i] = &tx
		if ledger.Chained {
			tx.BlockID = blockID
		}
	}

	var block *proto.Block
	var createBlockFunc func() error
	if ledger.Chained {
		block = &proto.Block{}

		createBlockFunc = func() error {
			var err error
			retryDelay := time.Duration(2) * time.Second
			common.ReRunOnError(func() error {
				b, _err := od.blockchain.CreateBlock(blockID, ledgerName, transactions)
				if b != nil {
					block.Id = b.ID
					block.ChainName = b.ChainName
					block.Hash = b.Hash
					block.Number = int64(b.Number)
					block.PrevBlockHash = b.PrevBlockHash
					block.Transactions = b.Transactions
					block.CreatedAt = b.CreatedAt
				}

				err = _err

				// If error is not a duplicate previous block hash error, don't re-run.
				// return nil to end the re-run routine
				if _err != nil && !types.IsDuplicatePrevBlockHashError(_err) {
					return nil
				}

				return _err
			}, 5, &retryDelay)
			return err
		}
	}

	err = od.store.PutThen(ledgerName, transactions, createBlockFunc)
	if err != nil {
		return nil, err
	}

	log.Debug("Put(): Time taken: ", time.Since(start))

	return &proto.PutResult{
		Added: int32(len(transactions)),
		Block: block,
	}, nil
}

// Get returns a transaction with a matching key
func (od *Orderer) Get(ctx context.Context, params *proto.GetParams) (*proto.Transaction, error) {

	start := time.Now()
	log.Error("Get it")

	ledger, err := od.GetLedger(ctx, &proto.GetLedgerParams{
		CocoonID: params.GetCocoonID(),
		Name:     params.GetLedger(),
	})
	if err != nil {
		return nil, err
	}

	ledgerName := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedger())
	key := od.store.MakeTxKey(params.GetCocoonID(), params.GetKey())
	tx, err := od.store.Get(ledgerName, key)
	if err != nil {
		return nil, err
	} else if tx == nil && err == nil {
		return nil, types.ErrTxNotFound
	}

	if ledger.Chained {
		block, err := od.blockchain.GetBlock(ledgerName, tx.BlockID)
		if err != nil {
			log.Error(err)
			return nil, fmt.Errorf("failed to populate block to transaction")
		} else if block == nil && err == nil {
			return nil, fmt.Errorf("orphaned transaction")
		}

		tx.Block = block
		tx.BlockID = ""
	}

	var protoTx proto.Transaction
	cstructs.Copy(tx, &protoTx)

	log.Debug("Get(): Time taken: ", time.Since(start))

	return &protoTx, nil
}

// GetByID finds and returns a transaction with a matching id
func (od *Orderer) GetByID(ctx context.Context, params *proto.GetParams) (*proto.Transaction, error) {

	ledger, err := od.GetLedger(ctx, &proto.GetLedgerParams{
		CocoonID: params.GetCocoonID(),
		Name:     params.GetLedger(),
	})
	if err != nil {
		return nil, err
	}

	ledgerName := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedger())
	tx, err := od.store.GetByID(ledgerName, params.GetId())
	if err != nil {
		return nil, err
	} else if tx == nil && err == nil {
		return nil, types.ErrTxNotFound
	}

	if ledger.Chained {
		block, err := od.blockchain.GetBlock(ledgerName, tx.BlockID)
		if err != nil {
			log.Error(err)
			return nil, fmt.Errorf("failed to populate block to transaction")
		} else if block == nil && err == nil {
			return nil, fmt.Errorf("orphaned transaction")
		}

		tx.Block = block
		tx.BlockID = ""
	}

	var protoTx proto.Transaction
	cstructs.Copy(tx, &protoTx)

	return &protoTx, nil
}

// GetBlockByID returns a block by its id and chain/ledger name
func (od *Orderer) GetBlockByID(ctx context.Context, params *proto.GetBlockParams) (*proto.Block, error) {

	name := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedger())
	ledger, err := od.store.GetLedger(name)
	if err != nil {
		return nil, err
	} else if ledger == nil && err == nil {
		return nil, types.ErrLedgerNotFound
	}

	blk, err := od.blockchain.GetBlock(name, params.GetId())
	if err != nil {
		return nil, err
	} else if blk == nil && err == nil {
		return nil, types.ErrBlockNotFound
	}

	var protoBlk proto.Block
	cstructs.Copy(blk, &protoBlk)

	return &protoBlk, nil
}

// GetRange fetches transactions between a range of keys
func (od *Orderer) GetRange(ctx context.Context, params *proto.GetRangeParams) (*proto.Transactions, error) {

	name := od.store.MakeLedgerName(params.GetCocoonID(), params.GetLedger())
	ledger, err := od.store.GetLedger(name)
	if err != nil {
		return nil, err
	} else if ledger == nil && err == nil {
		return nil, types.ErrLedgerNotFound
	}

	if len(params.GetStartKey()) > 0 {
		params.StartKey = od.store.MakeTxKey(params.GetCocoonID(), params.GetStartKey())
	}

	if len(params.GetEndKey()) > 0 {
		if len(params.GetStartKey()) > 0 {
			params.EndKey = od.store.MakeTxKey(params.GetCocoonID(), params.GetEndKey())
		} else {
			params.EndKey = od.store.MakeTxKey(params.GetCocoonID(), "%"+params.GetEndKey())
		}
	}

	txs, err := od.store.GetRange(name, params.GetStartKey(), params.GetEndKey(), params.GetInclusive(), int(params.GetLimit()), int(params.GetOffset()))
	if err != nil {
		return nil, err
	}

	// copy individual tx from []types.Transaction to []proto.Transaction
	var protoTxs = make([]*proto.Transaction, len(txs))
	for i, tx := range txs {
		var protoTx = proto.Transaction{}
		cstructs.Copy(tx, &protoTx)
		protoTxs[i] = &protoTx
	}

	return &proto.Transactions{
		Transactions: protoTxs,
	}, nil
}
<|endoftext|>


<commit_before>// Package provider contains implementations for the built-in usrv transports.
package provider

import (
	"fmt"
	"sync"

	"github.com/achilleasa/usrv/transport"
)

// InMemory implements the in-memory transport. It uses channels and go-routines
// to facilitate the exchange of messages making it very easy to use when
// writing tests.
type InMemory struct {
	mutex  sync.Mutex
	dialed bool

	bindings map[string]transport.Handler
}

// NewInMemory creates a new in-memory transport instance.
func NewInMemory() *InMemory {
	return &InMemory{
		bindings: make(map[string]transport.Handler, 0),
	}
}

// Dial connects the transport and starts relaying messages.
func (t *InMemory) Dial() error {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	t.dialed = true
	return nil
}

// Close shuts down the transport.
func (t *InMemory) Close() error {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	if !t.dialed {
		return transport.ErrTransportClosed
	}

	t.dialed = false

	return nil
}

// Bind listens for messages send to a particular service and
// endpoint combination and invokes the supplied handler to process them.
//
// Bindings can only be established on a closed transport. Calls to Bind
// after a call to Dial will result in an error.
func (t *InMemory) Bind(service, endpoint string, handler transport.Handler) error {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	if t.dialed {
		return transport.ErrTransportAlreadyDialed
	}

	key := fmt.Sprintf("%s/%s", service, endpoint)
	if t.bindings[key] != nil {
		return fmt.Errorf("binding %q already defined", key)
	}
	t.bindings[key] = handler

	return nil
}

// Request performs an RPC and returns back a read-only channel for
// receiving the result.
func (t *InMemory) Request(msg transport.Message) <-chan transport.ImmutableMessage {
	resChan := make(chan transport.ImmutableMessage, 1)

	// Build destination key for looking up the binding
	key := fmt.Sprintf("%s/%s", msg.Receiver(), msg.ReceiverEndpoint())

	go func(handler transport.Handler, req transport.Message, resChan chan transport.ImmutableMessage) {
		res := transport.MakeGenericMessage()
		res.SenderField = msg.Receiver()
		res.SenderEndpointField = msg.ReceiverEndpoint()
		res.ReceiverField = msg.Sender()
		res.ReceiverEndpointField = msg.SenderEndpoint()

		// Unknown target; return back an error
		if handler == nil {
			res.SetPayload(nil, transport.ErrNotFound)
		} else {
			handler.Process(req, res)
		}

		resChan <- res
		close(resChan)
	}(t.bindings[key], msg, resChan)

	return resChan
}

// InMemoryTransportFactory is a factory for creating usrv transport instances
// whose concrete implementation is the InMemory transport. This function behaves
// exactly the same as NewInMemory() but returns back a Transport interface allowing
// it to be used as usrv.DefaultTransportFactory.
func InMemoryTransportFactory() transport.Transport {
	return NewInMemory()
}
<commit_msg>Add mutex to in_mem provider so go -race doesn't produce false-positives<commit_after>// Package provider contains implementations for the built-in usrv transports.
package provider

import (
	"fmt"
	"sync"

	"github.com/achilleasa/usrv/transport"
)

// InMemory implements the in-memory transport. It uses channels and go-routines
// to facilitate the exchange of messages making it very easy to use when
// writing tests.
type InMemory struct {
	mutex  sync.Mutex
	dialed bool

	bindings map[string]transport.Handler
}

// NewInMemory creates a new in-memory transport instance.
func NewInMemory() *InMemory {
	return &InMemory{
		bindings: make(map[string]transport.Handler, 0),
	}
}

// Dial connects the transport and starts relaying messages.
func (t *InMemory) Dial() error {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	t.dialed = true
	return nil
}

// Close shuts down the transport.
func (t *InMemory) Close() error {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	if !t.dialed {
		return transport.ErrTransportClosed
	}

	t.dialed = false

	return nil
}

// Bind listens for messages send to a particular service and
// endpoint combination and invokes the supplied handler to process them.
//
// Bindings can only be established on a closed transport. Calls to Bind
// after a call to Dial will result in an error.
func (t *InMemory) Bind(service, endpoint string, handler transport.Handler) error {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	if t.dialed {
		return transport.ErrTransportAlreadyDialed
	}

	key := fmt.Sprintf("%s/%s", service, endpoint)
	if t.bindings[key] != nil {
		return fmt.Errorf("binding %q already defined", key)
	}
	t.bindings[key] = handler

	return nil
}

// Request performs an RPC and returns back a read-only channel for
// receiving the result.
func (t *InMemory) Request(msg transport.Message) <-chan transport.ImmutableMessage {
	resChan := make(chan transport.ImmutableMessage, 1)

	// Build destination key for looking up the binding
	key := fmt.Sprintf("%s/%s", msg.Receiver(), msg.ReceiverEndpoint())

	// This is required to prevent go test -race from flagging this as a false-positive data race.
	t.mutex.Lock()
	defer t.mutex.Unlock()

	go func(handler transport.Handler, req transport.Message, resChan chan transport.ImmutableMessage) {
		res := transport.MakeGenericMessage()
		res.SenderField = msg.Receiver()
		res.SenderEndpointField = msg.ReceiverEndpoint()
		res.ReceiverField = msg.Sender()
		res.ReceiverEndpointField = msg.SenderEndpoint()

		// Unknown target; return back an error
		if handler == nil {
			res.SetPayload(nil, transport.ErrNotFound)
		} else {
			handler.Process(req, res)
		}

		resChan <- res
		close(resChan)
	}(t.bindings[key], msg, resChan)

	return resChan
}

// InMemoryTransportFactory is a factory for creating usrv transport instances
// whose concrete implementation is the InMemory transport. This function behaves
// exactly the same as NewInMemory() but returns back a Transport interface allowing
// it to be used as usrv.DefaultTransportFactory.
func InMemoryTransportFactory() transport.Transport {
	return NewInMemory()
}
<|endoftext|>


<commit_before>// Package journal provides write bindings to the systemd journal
package journal

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
	"encoding/binary"
)

// Priority of a journal message
type Priority int

const (
	PriEmerg Priority = iota
	PriAlert
	PriCrit
	PriErr
	PriWarning
	PriNotice
	PriInfo
	PriDebug
)

var conn net.Conn

func init() {
	conn, _ = net.Dial("unixgram", "/run/systemd/journal/socket")
}

// Enabled returns true iff the systemd journal is available for logging
func Enabled() bool {
	return conn != nil
}

// Send a message to the systemd journal. vars is a map of journald fields to
// values.  Fields must be composed of uppercase letters, numbers, and
// underscores, but must not start with an underscore. Within these
// restrictions, any arbitrary field name may be used.  Some names have special
// significance: see the journalctl documentation
// (http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html)
// for more details.  vars may be nil.
func Send(message string, priority Priority, vars map[string]string) error {
	if conn == nil {
		return journalError("could not connect to journald socket")
	}

	data := new(bytes.Buffer)
	appendVariable(data, "PRIORITY", strconv.Itoa(int(priority)))
	appendVariable(data, "MESSAGE", message)
	for k, v := range vars {
		appendVariable(data, k, v)
	}

	_, err := io.Copy(conn, data)
	if err != nil && isSocketSpaceError(err) {
		file, err := tempFd()
		if err != nil {
			return journalError(err.Error())
		}
		_, err = io.Copy(file, data)
		if err != nil {
			return journalError(err.Error())
		}

		rights := syscall.UnixRights(int(file.Fd()))

		/* this connection should always be a UnixConn, but better safe than sorry */
		unixConn, ok := conn.(*net.UnixConn)
		if !ok {
			return journalError("can't send file through non-Unix connection")
		}
		unixConn.WriteMsgUnix([]byte{}, rights, nil)
	} else if err != nil {
		return journalError(err.Error())
	}
	return nil
}

func appendVariable(w io.Writer, name, value string) {
	if !validVarName(name) {
		journalError("variable name contains invalid character, ignoring")
	}
	if strings.ContainsRune(value, '\n') {
		/* When the value contains a newline, we write:
		 * - the variable name, followed by a newline
		 * - the size (in 64bit little endian format)
		 * - the data, followed by a newline
		 */
		fmt.Fprintln(w, name)
		binary.Write(w, binary.LittleEndian, uint64(len(value)))
		fmt.Fprintln(w, value)
	} else {
		/* just write the variable and value all on one line */
		fmt.Fprintf(w, "%s=%s\n", name, value)
	}
}

func validVarName(name string) bool {
	/* The variable name must be in uppercase and consist only of characters,
	 * numbers and underscores, and may not begin with an underscore. (from the docs)
	 */
	valid := true
	valid = valid && name[0] != '_'
	for _, c := range name {
		valid = valid && ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '_'
	}
	return valid
}

func isSocketSpaceError(err error) bool {
	opErr, ok := err.(*net.OpError)
	if !ok {
		return false
	}

	sysErr, ok := opErr.Err.(syscall.Errno)
	if !ok {
		return false
	}

	return sysErr == syscall.EMSGSIZE || sysErr == syscall.ENOBUFS
}

func tempFd() (*os.File, error) {
	file, err := ioutil.TempFile("/dev/shm/", "journal.XXXXX")
	if err != nil {
		return nil, err
	}
	syscall.Unlink(file.Name())
	if err != nil {
		return nil, err
	}
	return file, nil
}

func journalError(s string) error {
	s = "journal error: " + s
	fmt.Fprintln(os.Stderr, s)
	return errors.New(s)
}
<commit_msg>fix(journal): check whether net.Dial succeeds<commit_after>// Package journal provides write bindings to the systemd journal
package journal

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
)

// Priority of a journal message
type Priority int

const (
	PriEmerg Priority = iota
	PriAlert
	PriCrit
	PriErr
	PriWarning
	PriNotice
	PriInfo
	PriDebug
)

var conn net.Conn

func init() {
	var err error
	conn, err = net.Dial("unixgram", "/run/systemd/journal/socket")
	if err != nil {
		conn = nil
	}
}

// Enabled returns true iff the systemd journal is available for logging
func Enabled() bool {
	return conn != nil
}

// Send a message to the systemd journal. vars is a map of journald fields to
// values.  Fields must be composed of uppercase letters, numbers, and
// underscores, but must not start with an underscore. Within these
// restrictions, any arbitrary field name may be used.  Some names have special
// significance: see the journalctl documentation
// (http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html)
// for more details.  vars may be nil.
func Send(message string, priority Priority, vars map[string]string) error {
	if conn == nil {
		return journalError("could not connect to journald socket")
	}

	data := new(bytes.Buffer)
	appendVariable(data, "PRIORITY", strconv.Itoa(int(priority)))
	appendVariable(data, "MESSAGE", message)
	for k, v := range vars {
		appendVariable(data, k, v)
	}

	_, err := io.Copy(conn, data)
	if err != nil && isSocketSpaceError(err) {
		file, err := tempFd()
		if err != nil {
			return journalError(err.Error())
		}
		_, err = io.Copy(file, data)
		if err != nil {
			return journalError(err.Error())
		}

		rights := syscall.UnixRights(int(file.Fd()))

		/* this connection should always be a UnixConn, but better safe than sorry */
		unixConn, ok := conn.(*net.UnixConn)
		if !ok {
			return journalError("can't send file through non-Unix connection")
		}
		unixConn.WriteMsgUnix([]byte{}, rights, nil)
	} else if err != nil {
		return journalError(err.Error())
	}
	return nil
}

func appendVariable(w io.Writer, name, value string) {
	if !validVarName(name) {
		journalError("variable name contains invalid character, ignoring")
	}
	if strings.ContainsRune(value, '\n') {
		/* When the value contains a newline, we write:
		 * - the variable name, followed by a newline
		 * - the size (in 64bit little endian format)
		 * - the data, followed by a newline
		 */
		fmt.Fprintln(w, name)
		binary.Write(w, binary.LittleEndian, uint64(len(value)))
		fmt.Fprintln(w, value)
	} else {
		/* just write the variable and value all on one line */
		fmt.Fprintf(w, "%s=%s\n", name, value)
	}
}

func validVarName(name string) bool {
	/* The variable name must be in uppercase and consist only of characters,
	 * numbers and underscores, and may not begin with an underscore. (from the docs)
	 */
	valid := true
	valid = valid && name[0] != '_'
	for _, c := range name {
		valid = valid && ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '_'
	}
	return valid
}

func isSocketSpaceError(err error) bool {
	opErr, ok := err.(*net.OpError)
	if !ok {
		return false
	}

	sysErr, ok := opErr.Err.(syscall.Errno)
	if !ok {
		return false
	}

	return sysErr == syscall.EMSGSIZE || sysErr == syscall.ENOBUFS
}

func tempFd() (*os.File, error) {
	file, err := ioutil.TempFile("/dev/shm/", "journal.XXXXX")
	if err != nil {
		return nil, err
	}
	syscall.Unlink(file.Name())
	if err != nil {
		return nil, err
	}
	return file, nil
}

func journalError(s string) error {
	s = "journal error: " + s
	fmt.Fprintln(os.Stderr, s)
	return errors.New(s)
}
<|endoftext|>