<commit_before>package v1

import (
	"k8s.io/kubernetes/pkg/api/unversioned"
	kapi "k8s.io/kubernetes/pkg/api/v1"
)

const (
	ClusterNetworkDefault = "default"
)

// +genclient=true

// ClusterNetwork describes a cluster network
type ClusterNetwork struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	kapi.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// Network is a CIDR string to specify the global overlay network's L3 space
	Network string `json:"network" protobuf:"bytes,2,opt,name=network"`
	// HostSubnetLength is the number of bits to allocate to each host's subnet e.g. 8 would mean a /24 network on the host
	HostSubnetLength uint32 `json:"hostsubnetlength" protobuf:"varint,3,opt,name=hostsubnetlength"`
	// ServiceNetwork is the CIDR string to specify the service network
	ServiceNetwork string `json:"serviceNetwork" protobuf:"bytes,4,opt,name=serviceNetwork"`
	// PluginName is the name of the network plugin
	PluginName string `json:"pluginName,omitempty" protobuf:"bytes,5,opt,name=pluginName"`
}

// ClusterNetworkList is a collection of ClusterNetworks
type ClusterNetworkList struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
	// Items is the list of cluster networks
	Items []ClusterNetwork `json:"items" protobuf:"bytes,2,rep,name=items"`
}

// HostSubnet encapsulates the inputs needed to define the container subnet network on a node
type HostSubnet struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	kapi.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// Host is the name of the host that is registered at the master. May just be an IP address, resolvable hostname or a complete DNS.
	// A lease will be sought after this name.
	Host string `json:"host" protobuf:"bytes,2,opt,name=host"`
	// HostIP is the IP address to be used as vtep by other hosts in the overlay network
	HostIP string `json:"hostIP" protobuf:"bytes,3,opt,name=hostIP"`
	// Subnet is the actual subnet CIDR lease assigned to the host
	Subnet string `json:"subnet" protobuf:"bytes,4,opt,name=subnet"`
}

// HostSubnetList is a collection of HostSubnets
type HostSubnetList struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
	// Items is the list of host subnets
	Items []HostSubnet `json:"items" protobuf:"bytes,2,rep,name=items"`
}

// NetNamespace encapsulates the inputs needed to define a unique network namespace on the cluster
type NetNamespace struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	kapi.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// NetName is the name of the network namespace
	NetName string `json:"netname" protobuf:"bytes,2,opt,name=netname"`
	// NetID is the network identifier of the network namespace assigned to each overlay network packet
	NetID uint32 `json:"netid" protobuf:"varint,3,opt,name=netid"`
}

// NetNamespaceList is a collection of NetNamespaces
type NetNamespaceList struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
	// Items is the list of net namespaces
	Items []NetNamespace `json:"items" protobuf:"bytes,2,rep,name=items"`
}

// EgressNetworkPolicyRuleType indicates whether an EgressNetworkPolicyRule allows or denies traffic
type EgressNetworkPolicyRuleType string

const (
	EgressNetworkPolicyRuleAllow EgressNetworkPolicyRuleType = "Allow"
	EgressNetworkPolicyRuleDeny  EgressNetworkPolicyRuleType = "Deny"
)

// EgressNetworkPolicyPeer specifies a target to apply egress network policy to
type EgressNetworkPolicyPeer struct {
	// cidrSelector is the CIDR range to allow/deny traffic to
	CIDRSelector string `json:"cidrSelector" protobuf:"bytes,1,rep,name=cidrSelector"`
}

// EgressNetworkPolicyRule contains a single egress network policy rule
type EgressNetworkPolicyRule struct {
	// type marks this as an "Allow" or "Deny" rule
	Type EgressNetworkPolicyRuleType `json:"type" protobuf:"bytes,1,rep,name=type"`
	// to is the target that traffic is allowed/denied to
	To EgressNetworkPolicyPeer `json:"to" protobuf:"bytes,2,rep,name=to"`
}

// EgressNetworkPolicySpec provides a list of policies on outgoing network traffic
type EgressNetworkPolicySpec struct {
	// egress contains the list of egress policy rules
	Egress []EgressNetworkPolicyRule `json:"egress" protobuf:"bytes,1,rep,name=egress"`
}

// EgressNetworkPolicy describes the current egress network policy for a Namespace. When using
// the 'redhat/openshift-ovs-multitenant' network plugin, traffic from a pod to an IP address
// outside the cluster will be checked against each EgressNetworkPolicyRule in the pod's
// namespace's EgressNetworkPolicy, in order. If no rule matches (or no EgressNetworkPolicy
// is present) then the traffic will be allowed by default.
type EgressNetworkPolicy struct {
	unversioned.TypeMeta `json:",inline"`
	// metadata for EgressNetworkPolicy
	kapi.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// spec is the specification of the current egress network policy
	Spec EgressNetworkPolicySpec `json:"spec" protobuf:"bytes,2,opt,name=spec"`
}

// EgressNetworkPolicyList is a collection of EgressNetworkPolicy
type EgressNetworkPolicyList struct {
	unversioned.TypeMeta `json:",inline"`
	// metadata for EgressNetworkPolicyList
	unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
	// items is the list of policies
	Items []EgressNetworkPolicy `json:"items" protobuf:"bytes,2,rep,name=items"`
}
<commit_msg>Improve the SDN API docs a little bit<commit_after>package v1

import (
	"k8s.io/kubernetes/pkg/api/unversioned"
	kapi "k8s.io/kubernetes/pkg/api/v1"
)

const (
	ClusterNetworkDefault = "default"
)

// +genclient=true

// ClusterNetwork describes the cluster network. There is normally only one object of this type,
// named "default", which is created by the SDN network plugin based on the master configuration
// when the cluster is brought up for the first time.
type ClusterNetwork struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	kapi.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// Network is a CIDR string specifying the global overlay network's L3 space
	Network string `json:"network" protobuf:"bytes,2,opt,name=network"`
	// HostSubnetLength is the number of bits of network to allocate to each node. eg, 8 would mean that each node would have a /24 slice of the overlay network for its pods
	HostSubnetLength uint32 `json:"hostsubnetlength" protobuf:"varint,3,opt,name=hostsubnetlength"`
	// ServiceNetwork is the CIDR range that Service IP addresses are allocated from
	ServiceNetwork string `json:"serviceNetwork" protobuf:"bytes,4,opt,name=serviceNetwork"`
	// PluginName is the name of the network plugin being used
	PluginName string `json:"pluginName,omitempty" protobuf:"bytes,5,opt,name=pluginName"`
}

// ClusterNetworkList is a collection of ClusterNetworks
type ClusterNetworkList struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
	// Items is the list of cluster networks
	Items []ClusterNetwork `json:"items" protobuf:"bytes,2,rep,name=items"`
}

// HostSubnet describes the container subnet network on a node. The HostSubnet object must have the
// same name as the Node object it corresponds to.
type HostSubnet struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	kapi.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// Host is the name of the node. (This is redundant with the object's name, and this
	// field is not actually used any more.)
	Host string `json:"host" protobuf:"bytes,2,opt,name=host"`
	// HostIP is the IP address to be used as a VTEP by other nodes in the overlay network
	HostIP string `json:"hostIP" protobuf:"bytes,3,opt,name=hostIP"`
	// Subnet is the CIDR range of the overlay network assigned to the node for its pods
	Subnet string `json:"subnet" protobuf:"bytes,4,opt,name=subnet"`
}

// HostSubnetList is a collection of HostSubnets
type HostSubnetList struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
	// Items is the list of host subnets
	Items []HostSubnet `json:"items" protobuf:"bytes,2,rep,name=items"`
}

// NetNamespace describes a single isolated network. When using the redhat/openshift-ovs-multitenant
// plugin, every Namespace will have a corresponding NetNamespace object with the same name.
// (When using redhat/openshift-ovs-subnet, NetNamespaces are not used.)
type NetNamespace struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	kapi.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// NetName is the name of the network namespace. (This is the same as the object's name, but both fields must be set.)
	NetName string `json:"netname" protobuf:"bytes,2,opt,name=netname"`
	// NetID is the network identifier of the network namespace assigned to each overlay network packet. This can be manipulated with the "oadm pod-network" commands.
	NetID uint32 `json:"netid" protobuf:"varint,3,opt,name=netid"`
}

// NetNamespaceList is a collection of NetNamespaces
type NetNamespaceList struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
	// Items is the list of net namespaces
	Items []NetNamespace `json:"items" protobuf:"bytes,2,rep,name=items"`
}

// EgressNetworkPolicyRuleType indicates whether an EgressNetworkPolicyRule allows or denies traffic
type EgressNetworkPolicyRuleType string

const (
	EgressNetworkPolicyRuleAllow EgressNetworkPolicyRuleType = "Allow"
	EgressNetworkPolicyRuleDeny  EgressNetworkPolicyRuleType = "Deny"
)

// EgressNetworkPolicyPeer specifies a target to apply egress network policy to
type EgressNetworkPolicyPeer struct {
	// cidrSelector is the CIDR range to allow/deny traffic to
	CIDRSelector string `json:"cidrSelector" protobuf:"bytes,1,rep,name=cidrSelector"`
}

// EgressNetworkPolicyRule contains a single egress network policy rule
type EgressNetworkPolicyRule struct {
	// type marks this as an "Allow" or "Deny" rule
	Type EgressNetworkPolicyRuleType `json:"type" protobuf:"bytes,1,rep,name=type"`
	// to is the target that traffic is allowed/denied to
	To EgressNetworkPolicyPeer `json:"to" protobuf:"bytes,2,rep,name=to"`
}

// EgressNetworkPolicySpec provides a list of policies on outgoing network traffic
type EgressNetworkPolicySpec struct {
	// egress contains the list of egress policy rules
	Egress []EgressNetworkPolicyRule `json:"egress" protobuf:"bytes,1,rep,name=egress"`
}

// EgressNetworkPolicy describes the current egress network policy for a Namespace. When using
// the 'redhat/openshift-ovs-multitenant' network plugin, traffic from a pod to an IP address
// outside the cluster will be checked against each EgressNetworkPolicyRule in the pod's
// namespace's EgressNetworkPolicy, in order. If no rule matches (or no EgressNetworkPolicy
// is present) then the traffic will be allowed by default.
type EgressNetworkPolicy struct {
	unversioned.TypeMeta `json:",inline"`
	// metadata for EgressNetworkPolicy
	kapi.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// spec is the specification of the current egress network policy
	Spec EgressNetworkPolicySpec `json:"spec" protobuf:"bytes,2,opt,name=spec"`
}

// EgressNetworkPolicyList is a collection of EgressNetworkPolicy
type EgressNetworkPolicyList struct {
	unversioned.TypeMeta `json:",inline"`
	// metadata for EgressNetworkPolicyList
	unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
	// items is the list of policies
	Items []EgressNetworkPolicy `json:"items" protobuf:"bytes,2,rep,name=items"`
}
<|endoftext|>


<commit_before>package particle

import (
	"time"

	"github.com/Bredgren/gogame/geo"
)

// Particle is a generatic particle that has a position and velocity. It also allows
// forces to be applied to it.
type Particle struct {
	// Pos is the position of the particle. No particular units are assumed.
	Pos geo.Vec
	// Vel is the velocity of the particle. It is the number units per second Pos changes
	// on a call to Update.
	Vel geo.Vec
	// A Mass of 0 will ignore applied forces.
	Mass  float64
	accel geo.Vec
}

// Update causes all applied forces since the last call to Update to take effect on the
// particle and updates the position based on the, possibly new, velocity. dt is the
// amount of time to advance.
func (p *Particle) Update(dt time.Duration) {
	p.Vel.Add(p.accel.Times(dt.Seconds()))
	p.Pos.Add(p.Vel.Times(dt.Seconds()))
	p.accel.Mul(0)
}

// ApplyForce applies the given force to the particle taking its Mass into account. It will
// not affect the particle until Update is called. If multiple forces are applied then they
// will accumulate. Forces will not persist between calls to Update.
func (p *Particle) ApplyForce(force geo.Vec) {
	if p.Mass > 0 {
		p.accel.Add(force.DividedBy(p.Mass))
	}
}
<commit_msg>Accept negative mass.<commit_after>package particle

import (
	"time"

	"github.com/Bredgren/gogame/geo"
)

// Particle is a generatic particle that has a position and velocity. It also allows
// forces to be applied to it.
type Particle struct {
	// Pos is the position of the particle. No particular units are assumed.
	Pos geo.Vec
	// Vel is the velocity of the particle. It is the number units per second Pos changes
	// on a call to Update.
	Vel geo.Vec
	// A Mass of 0 will ignore applied forces.
	Mass  float64
	accel geo.Vec
}

// Update causes all applied forces since the last call to Update to take effect on the
// particle and updates the position based on the, possibly new, velocity. dt is the
// amount of time to advance.
func (p *Particle) Update(dt time.Duration) {
	p.Vel.Add(p.accel.Times(dt.Seconds()))
	p.Pos.Add(p.Vel.Times(dt.Seconds()))
	p.accel.Mul(0)
}

// ApplyForce applies the given force to the particle taking its Mass into account. It will
// not affect the particle until Update is called. If multiple forces are applied then they
// will accumulate. Forces will not persist between calls to Update.
func (p *Particle) ApplyForce(force geo.Vec) {
	if p.Mass != 0 {
		p.accel.Add(force.DividedBy(p.Mass))
	}
}
<|endoftext|>


<commit_before>package goleg

import (
	"io/ioutil"
	"os"
	"strconv"
	"testing"
	"fmt"
	"bytes"
)

func openRandomDB(features int) (Database, string, error) {
	name, err := ioutil.TempDir("/tmp", "goleg")
	if err != nil {
		return Database{}, "", err
	}

	//F_APPENDONLY|F_AOL_FFLUSH|F_LZ4|F_SPLAYTREE
	database, err := Open(name, "test", features)
	if err != nil {
		return Database{}, "", err
	}

	return database, name, nil
}

func cleanTemp(dir string) {
	os.RemoveAll(dir)
}

func TestOpen(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping in short mode")
	}

	database, dir, err := openRandomDB(F_APPENDONLY)
	if err != nil {
		t.Fatalf("Can't open database: %s", err.Error())
	}

	database.Close()
	cleanTemp(dir)
}

const JARN = 1000

func TestJar(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping in short mode")
	}

	database, dir, err := openRandomDB(F_LZ4 | F_SPLAYTREE)
	if err != nil {
		t.Fatalf("Can't open database: %s", err.Error())
	}

	for i := 0; i < JARN; i++ {
		if database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) != 0 {
			t.Fatalf("Can't jar value #%d", i)
		}
	}

	database.Close()
	cleanTemp(dir)
}

func TestUnjar(t *testing.T) {
	database, dir, err := openRandomDB(F_LZ4 | F_SPLAYTREE)
	if err != nil {
		t.Fatalf("Can't open database: %s", err.Error())
	}

	for i := 0; i < JARN; i++ {
		if database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) != 0 {
			t.Fatalf("Can't jar value #%d", i)
		}
	}

	for i := 0; i < JARN; i++ {
		val := database.Unjar("record" + strconv.Itoa(i))
		fmt.Printf("%#v\n", val)
		if bytes.Equal(val, []byte("value"+strconv.Itoa(i))) {
			t.Errorf("Value #%d doesn't match", i)
		}
	}

	database.Close()
	cleanTemp(dir)
}
<commit_msg>oops, this passes now<commit_after>package goleg

import (
	"io/ioutil"
	"os"
	"strconv"
	"testing"
	"bytes"
)

func openRandomDB(features int) (Database, string, error) {
	name, err := ioutil.TempDir("/tmp", "goleg")
	if err != nil {
		return Database{}, "", err
	}

	//F_APPENDONLY|F_AOL_FFLUSH|F_LZ4|F_SPLAYTREE
	database, err := Open(name, "test", features)
	if err != nil {
		return Database{}, "", err
	}

	return database, name, nil
}

func cleanTemp(dir string) {
	os.RemoveAll(dir)
}

func TestOpen(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping in short mode")
	}

	database, dir, err := openRandomDB(F_APPENDONLY)
	if err != nil {
		t.Fatalf("Can't open database: %s", err.Error())
	}

	database.Close()
	cleanTemp(dir)
}

const JARN = 1000

func TestJar(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping in short mode")
	}

	database, dir, err := openRandomDB(F_LZ4 | F_SPLAYTREE)
	if err != nil {
		t.Fatalf("Can't open database: %s", err.Error())
	}

	for i := 0; i < JARN; i++ {
		if database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) != 0 {
			t.Fatalf("Can't jar value #%d", i)
		}
	}

	database.Close()
	cleanTemp(dir)
}

func TestUnjar(t *testing.T) {
	database, dir, err := openRandomDB(F_LZ4 | F_SPLAYTREE)
	if err != nil {
		t.Fatalf("Can't open database: %s", err.Error())
	}

	for i := 0; i < JARN; i++ {
		if database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) != 0 {
			t.Fatalf("Can't jar value #%d", i)
		}
	}

	for i := 0; i < JARN; i++ {
		val := database.Unjar("record" + strconv.Itoa(i))
		if !bytes.Equal(val, []byte("value"+strconv.Itoa(i))) {
			t.Errorf("Value #%d doesn't match", i)
		}
	}

	database.Close()
	cleanTemp(dir)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The go-patricia AUTHORS
//
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package patricia

import (
	"fmt"
	"io"
	"sort"
)

type childList interface {
	length() int
	head() *Trie
	add(child *Trie) childList
	remove(b byte)
	replace(b byte, child *Trie)
	next(b byte) *Trie
	walk(prefix *Prefix, visitor VisitorFunc) error
	print(w io.Writer, indent int)
	total() int
}

type tries []*Trie

func (t tries) Len() int {
	return len(t)
}

func (t tries) Less(i, j int) bool {
	strings := sort.StringSlice{string(t[i].prefix), string(t[j].prefix)}
	return strings.Less(0, 1)
}

func (t tries) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}

type sparseChildList struct {
	children tries
}

func newSparseChildList(maxChildrenPerSparseNode int) childList {
	return &sparseChildList{
		children: make(tries, 0, maxChildrenPerSparseNode),
	}
}

func (list *sparseChildList) length() int {
	return len(list.children)
}

func (list *sparseChildList) head() *Trie {
	return list.children[0]
}

func (list *sparseChildList) add(child *Trie) childList {
	// Search for an empty spot and insert the child if possible.
	if len(list.children) != cap(list.children) {
		list.children = append(list.children, child)
		return list
	}

	// Otherwise we have to transform to the dense list type.
	return newDenseChildList(list, child)
}

func (list *sparseChildList) remove(b byte) {
	for i, node := range list.children {
		if node.prefix[0] == b {
			copy(list.children[i:], list.children[i+1:])
			list.children = list.children[:len(list.children)-1]
			return
		}
	}

	// This is not supposed to be reached.
	panic("removing non-existent child")
}

func (list *sparseChildList) replace(b byte, child *Trie) {
	// Make a consistency check.
	if p0 := child.prefix[0]; p0 != b {
		panic(fmt.Errorf("child prefix mismatch: %v != %v", p0, b))
	}

	// Seek the child and replace it.
	for i, node := range list.children {
		if node.prefix[0] == b {
			list.children[i] = child
			return
		}
	}
}

func (list *sparseChildList) next(b byte) *Trie {
	for _, child := range list.children {
		if child.prefix[0] == b {
			return child
		}
	}
	return nil
}

func (list *sparseChildList) walk(prefix *Prefix, visitor VisitorFunc) error {

	sort.Sort(list.children)

	for _, child := range list.children {
		*prefix = append(*prefix, child.prefix...)
		if child.item != nil {
			err := visitor(*prefix, child.item)
			if err != nil {
				if err == SkipSubtree {
					*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
					continue
				}
				*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
				return err
			}
		}

		err := child.children.walk(prefix, visitor)
		*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
		if err != nil {
			return err
		}
	}

	return nil
}

func (list *sparseChildList) total() int {
	tot := 0
	for _, child := range list.children {
		if child != nil {
			tot = tot + child.total()
		}
	}
	return tot
}

func (list *sparseChildList) print(w io.Writer, indent int) {
	for _, child := range list.children {
		if child != nil {
			child.print(w, indent)
		}
	}
}

type denseChildList struct {
	min         int
	max         int
	numChildren int
	headIndex   int
	children    []*Trie
}

func newDenseChildList(list *sparseChildList, child *Trie) childList {
	var (
		min int = 255
		max int = 0
	)
	for _, child := range list.children {
		b := int(child.prefix[0])
		if b < min {
			min = b
		}
		if b > max {
			max = b
		}
	}

	b := int(child.prefix[0])
	if b < min {
		min = b
	}
	if b > max {
		max = b
	}

	children := make([]*Trie, max-min+1)
	for _, child := range list.children {
		children[int(child.prefix[0])-min] = child
	}
	children[int(child.prefix[0])-min] = child

	return &denseChildList{
		min:         min,
		max:         max,
		numChildren: list.length() + 1,
		headIndex:   0,
		children:    children,
	}
}

func (list *denseChildList) length() int {
	return list.numChildren
}

func (list *denseChildList) head() *Trie {
	return list.children[list.headIndex]
}

func (list *denseChildList) add(child *Trie) childList {
	b := int(child.prefix[0])
	var i int

	switch {
	case list.min <= b && b <= list.max:
		if list.children[b-list.min] != nil {
			panic("dense child list collision detected")
		}
		i = b - list.min
		list.children[i] = child

	case b < list.min:
		children := make([]*Trie, list.max-b+1)
		i = 0
		children[i] = child
		copy(children[list.min-b:], list.children)
		list.children = children
		list.min = b

	default: // b > list.max
		children := make([]*Trie, b-list.min+1)
		i = b - list.min
		children[i] = child
		copy(children, list.children)
		list.children = children
		list.max = b
	}

	list.numChildren++
	if i < list.headIndex {
		list.headIndex = i
	}
	return list
}

func (list *denseChildList) remove(b byte) {
	i := int(b) - list.min
	if list.children[i] == nil {
		// This is not supposed to be reached.
		panic("removing non-existent child")
	}
	list.numChildren--
	list.children[i] = nil

	// Update head index.
	if i == list.headIndex {
		for ; i < len(list.children); i++ {
			if list.children[i] != nil {
				list.headIndex = i
				return
			}
		}
	}
}

func (list *denseChildList) replace(b byte, child *Trie) {
	// Make a consistency check.
	if p0 := child.prefix[0]; p0 != b {
		panic(fmt.Errorf("child prefix mismatch: %v != %v", p0, b))
	}

	// Replace the child.
	list.children[int(b)-list.min] = child
}

func (list *denseChildList) next(b byte) *Trie {
	i := int(b)
	if i < list.min || list.max < i {
		return nil
	}
	return list.children[i-list.min]
}

func (list *denseChildList) walk(prefix *Prefix, visitor VisitorFunc) error {
	for _, child := range list.children {
		if child == nil {
			continue
		}
		*prefix = append(*prefix, child.prefix...)
		if child.item != nil {
			if err := visitor(*prefix, child.item); err != nil {
				if err == SkipSubtree {
					*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
					continue
				}
				*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
				return err
			}
		}

		err := child.children.walk(prefix, visitor)
		*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
		if err != nil {
			return err
		}
	}

	return nil
}

func (list *denseChildList) print(w io.Writer, indent int) {
	for _, child := range list.children {
		if child != nil {
			child.print(w, indent)
		}
	}
}

func (list *denseChildList) total() int {
	tot := 0
	for _, child := range list.children {
		if child != nil {
			tot = tot + child.total()
		}
	}
	return tot
}
<commit_msg>Improve remove() in the sparse mode<commit_after>// Copyright (c) 2014 The go-patricia AUTHORS
//
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package patricia

import (
	"fmt"
	"io"
	"sort"
)

type childList interface {
	length() int
	head() *Trie
	add(child *Trie) childList
	remove(b byte)
	replace(b byte, child *Trie)
	next(b byte) *Trie
	walk(prefix *Prefix, visitor VisitorFunc) error
	print(w io.Writer, indent int)
	total() int
}

type tries []*Trie

func (t tries) Len() int {
	return len(t)
}

func (t tries) Less(i, j int) bool {
	strings := sort.StringSlice{string(t[i].prefix), string(t[j].prefix)}
	return strings.Less(0, 1)
}

func (t tries) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}

type sparseChildList struct {
	children tries
}

func newSparseChildList(maxChildrenPerSparseNode int) childList {
	return &sparseChildList{
		children: make(tries, 0, maxChildrenPerSparseNode),
	}
}

func (list *sparseChildList) length() int {
	return len(list.children)
}

func (list *sparseChildList) head() *Trie {
	return list.children[0]
}

func (list *sparseChildList) add(child *Trie) childList {
	// Search for an empty spot and insert the child if possible.
	if len(list.children) != cap(list.children) {
		list.children = append(list.children, child)
		return list
	}

	// Otherwise we have to transform to the dense list type.
	return newDenseChildList(list, child)
}

func (list *sparseChildList) remove(b byte) {
	for i, node := range list.children {
		if node.prefix[0] == b {
			list.children[i] = list.children[len(list.children)-1]
			list.children[len(list.children)-1] = nil
			list.children = list.children[:len(list.children)-1]
			return
		}
	}

	// This is not supposed to be reached.
	panic("removing non-existent child")
}

func (list *sparseChildList) replace(b byte, child *Trie) {
	// Make a consistency check.
	if p0 := child.prefix[0]; p0 != b {
		panic(fmt.Errorf("child prefix mismatch: %v != %v", p0, b))
	}

	// Seek the child and replace it.
	for i, node := range list.children {
		if node.prefix[0] == b {
			list.children[i] = child
			return
		}
	}
}

func (list *sparseChildList) next(b byte) *Trie {
	for _, child := range list.children {
		if child.prefix[0] == b {
			return child
		}
	}
	return nil
}

func (list *sparseChildList) walk(prefix *Prefix, visitor VisitorFunc) error {

	sort.Sort(list.children)

	for _, child := range list.children {
		*prefix = append(*prefix, child.prefix...)
		if child.item != nil {
			err := visitor(*prefix, child.item)
			if err != nil {
				if err == SkipSubtree {
					*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
					continue
				}
				*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
				return err
			}
		}

		err := child.children.walk(prefix, visitor)
		*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
		if err != nil {
			return err
		}
	}

	return nil
}

func (list *sparseChildList) total() int {
	tot := 0
	for _, child := range list.children {
		if child != nil {
			tot = tot + child.total()
		}
	}
	return tot
}

func (list *sparseChildList) print(w io.Writer, indent int) {
	for _, child := range list.children {
		if child != nil {
			child.print(w, indent)
		}
	}
}

type denseChildList struct {
	min         int
	max         int
	numChildren int
	headIndex   int
	children    []*Trie
}

func newDenseChildList(list *sparseChildList, child *Trie) childList {
	var (
		min int = 255
		max int = 0
	)
	for _, child := range list.children {
		b := int(child.prefix[0])
		if b < min {
			min = b
		}
		if b > max {
			max = b
		}
	}

	b := int(child.prefix[0])
	if b < min {
		min = b
	}
	if b > max {
		max = b
	}

	children := make([]*Trie, max-min+1)
	for _, child := range list.children {
		children[int(child.prefix[0])-min] = child
	}
	children[int(child.prefix[0])-min] = child

	return &denseChildList{
		min:         min,
		max:         max,
		numChildren: list.length() + 1,
		headIndex:   0,
		children:    children,
	}
}

func (list *denseChildList) length() int {
	return list.numChildren
}

func (list *denseChildList) head() *Trie {
	return list.children[list.headIndex]
}

func (list *denseChildList) add(child *Trie) childList {
	b := int(child.prefix[0])
	var i int

	switch {
	case list.min <= b && b <= list.max:
		if list.children[b-list.min] != nil {
			panic("dense child list collision detected")
		}
		i = b - list.min
		list.children[i] = child

	case b < list.min:
		children := make([]*Trie, list.max-b+1)
		i = 0
		children[i] = child
		copy(children[list.min-b:], list.children)
		list.children = children
		list.min = b

	default: // b > list.max
		children := make([]*Trie, b-list.min+1)
		i = b - list.min
		children[i] = child
		copy(children, list.children)
		list.children = children
		list.max = b
	}

	list.numChildren++
	if i < list.headIndex {
		list.headIndex = i
	}
	return list
}

func (list *denseChildList) remove(b byte) {
	i := int(b) - list.min
	if list.children[i] == nil {
		// This is not supposed to be reached.
		panic("removing non-existent child")
	}
	list.numChildren--
	list.children[i] = nil

	// Update head index.
	if i == list.headIndex {
		for ; i < len(list.children); i++ {
			if list.children[i] != nil {
				list.headIndex = i
				return
			}
		}
	}
}

func (list *denseChildList) replace(b byte, child *Trie) {
	// Make a consistency check.
	if p0 := child.prefix[0]; p0 != b {
		panic(fmt.Errorf("child prefix mismatch: %v != %v", p0, b))
	}

	// Replace the child.
	list.children[int(b)-list.min] = child
}

func (list *denseChildList) next(b byte) *Trie {
	i := int(b)
	if i < list.min || list.max < i {
		return nil
	}
	return list.children[i-list.min]
}

func (list *denseChildList) walk(prefix *Prefix, visitor VisitorFunc) error {
	for _, child := range list.children {
		if child == nil {
			continue
		}
		*prefix = append(*prefix, child.prefix...)
		if child.item != nil {
			if err := visitor(*prefix, child.item); err != nil {
				if err == SkipSubtree {
					*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
					continue
				}
				*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
				return err
			}
		}

		err := child.children.walk(prefix, visitor)
		*prefix = (*prefix)[:len(*prefix)-len(child.prefix)]
		if err != nil {
			return err
		}
	}

	return nil
}

func (list *denseChildList) print(w io.Writer, indent int) {
	for _, child := range list.children {
		if child != nil {
			child.print(w, indent)
		}
	}
}

func (list *denseChildList) total() int {
	tot := 0
	for _, child := range list.children {
		if child != nil {
			tot = tot + child.total()
		}
	}
	return tot
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Manish R Jain <manishrjain@gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 		http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package query

import (
	"fmt"

	"github.com/google/flatbuffers/go"
	"github.com/dgraph-io/dgraph/posting"
	"github.com/dgraph-io/dgraph/task"
	"github.com/dgraph-io/dgraph/uid"
	"github.com/dgraph-io/dgraph/x"
)

/*
 * QUERY:
 * Let's take this query from GraphQL as example:
 * {
 *   me {
 *     id
 *     firstName
 *     lastName
 *     birthday {
 *       month
 *       day
 *     }
 *     friends {
 *       name
 *     }
 *   }
 * }
 *
 * REPRESENTATION:
 * This would be represented in SubGraph format internally, as such:
 * SubGraph [result uid = me]
 *    |
 *  Children
 *    |
 *    --> SubGraph [Attr = "xid"]
 *    --> SubGraph [Attr = "firstName"]
 *    --> SubGraph [Attr = "lastName"]
 *    --> SubGraph [Attr = "birthday"]
 *           |
 *         Children
 *           |
 *           --> SubGraph [Attr = "month"]
 *           --> SubGraph [Attr = "day"]
 *    --> SubGraph [Attr = "friends"]
 *           |
 *         Children
 *           |
 *           --> SubGraph [Attr = "name"]
 *
 * ALGORITHM:
 * This is a rough and simple algorithm of how to process this SubGraph query
 * and populate the results:
 *
 * For a given entity, a new SubGraph can be started off with NewGraph(id).
 * Given a SubGraph, is the Query field empty? [Step a]
 *   - If no, run (or send it to server serving the attribute) query
 *     and populate result.
 * Iterate over children and copy Result Uids to child Query Uids.
 *     Set Attr. Then for each child, use goroutine to run Step:a.
 * Wait for goroutines to finish.
 * Return errors, if any.
 */

var log = x.Log("query")

// SubGraph is the way to represent data internally. It contains both the
// query and the response. Once generated, this can then be encoded to other
// client convenient formats, like GraphQL / JSON.
type SubGraph struct {
	Attr     string
	Children []*SubGraph

	query  []byte
	result []byte
}

func NewGraph(euid uint64, exid string) (*SubGraph, error) {
	// This would set the Result field in SubGraph,
	// and populate the children for attributes.
	if len(exid) > 0 {
		u, err := uid.GetOrAssign(exid)
		if err != nil {
			x.Err(log, err).WithField("xid", exid).Error(
				"While GetOrAssign uid from external id")
			return nil, err
		}
		log.WithField("xid", exid).WithField("uid", u).Debug("GetOrAssign")
		euid = u
	}

	if euid == 0 {
		err := fmt.Errorf("Query internal id is zero")
		x.Err(log, err).Error("Invalid query")
		return nil, err
	}

	// Encode uid into result flatbuffer.
	b := flatbuffers.NewBuilder(0)
	task.ResultStartUidsVector(b, 1)
	b.PrependUint64(euid)
	vend := b.EndVector(1)
	task.ResultStart(b)
	task.ResultAddUids(b, vend)
	rend := task.ResultEnd(b)
	b.Finish(rend)

	sg := new(SubGraph)
	sg.result = b.Bytes[b.Head():]
	return sg, nil
}

func createTaskQuery(attr string, r *task.Result) []byte {
	b := flatbuffers.NewBuilder(0)
	ao := b.CreateString(attr)

	task.QueryStartUidsVector(b, r.UidsLength())
	for i := r.UidsLength() - 1; i >= 0; i-- {
		uid := r.Uids(i)
		b.PrependUint64(uid)
	}
	vend := b.EndVector(r.UidsLength())

	task.QueryStart(b)
	task.QueryAddAttr(b, ao)
	task.QueryAddUids(b, vend)
	qend := task.QueryEnd(b)
	b.Finish(qend)
	return b.Bytes[b.Head():]
}

func ProcessGraph(sg *SubGraph, rch chan error) {
	var err error
	if len(sg.query) > 0 {
		// This task execution would go over the wire in later versions.
		sg.result, err = posting.ProcessTask(sg.query)
		if err != nil {
			rch <- err
			return
		}
	}

	uo := flatbuffers.GetUOffsetT(sg.result)
	r := new(task.Result)
	r.Init(sg.result, uo)
	if r.UidsLength() == 0 {
		// Looks like we're done here.
		if len(sg.Children) > 0 {
			log.Debug("Have some children but no results. Life got cut short early.")
		}
		rch <- nil
		return
	}

	// Let's execute it in a tree fashion. Each SubGraph would break off
	// as many goroutines as it's children; which would then recursively
	// do the same thing.
	// Buffered channel to ensure no-blockage.
	childchan := make(chan error, len(sg.Children))
	for i := 0; i < len(sg.Children); i++ {
		child := sg.Children[i]
		child.query = createTaskQuery(child.Attr, r)
		go ProcessGraph(child, childchan)
	}

	// Now get all the results back.
	for i := 0; i < len(sg.Children); i++ {
		err = <-childchan
		if err != nil {
			rch <- err
			return
		}
	}
	rch <- nil
}
<commit_msg>BREAKING change: convert to JSON. Given we're not tracking uids for each query uid, this will cause problems for the result. So, have to re-structure stuff here.<commit_after>/*
 * Copyright 2015 Manish R Jain <manishrjain@gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 		http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package query

import (
	"fmt"

	"github.com/dgraph-io/dgraph/posting"
	"github.com/dgraph-io/dgraph/task"
	"github.com/dgraph-io/dgraph/uid"
	"github.com/dgraph-io/dgraph/x"
	"github.com/google/flatbuffers/go"
)

/*
 * QUERY:
 * Let's take this query from GraphQL as example:
 * {
 *   me {
 *     id
 *     firstName
 *     lastName
 *     birthday {
 *       month
 *       day
 *     }
 *     friends {
 *       name
 *     }
 *   }
 * }
 *
 * REPRESENTATION:
 * This would be represented in SubGraph format internally, as such:
 * SubGraph [result uid = me]
 *    |
 *  Children
 *    |
 *    --> SubGraph [Attr = "xid"]
 *    --> SubGraph [Attr = "firstName"]
 *    --> SubGraph [Attr = "lastName"]
 *    --> SubGraph [Attr = "birthday"]
 *           |
 *         Children
 *           |
 *           --> SubGraph [Attr = "month"]
 *           --> SubGraph [Attr = "day"]
 *    --> SubGraph [Attr = "friends"]
 *           |
 *         Children
 *           |
 *           --> SubGraph [Attr = "name"]
 *
 * ALGORITHM:
 * This is a rough and simple algorithm of how to process this SubGraph query
 * and populate the results:
 *
 * For a given entity, a new SubGraph can be started off with NewGraph(id).
 * Given a SubGraph, is the Query field empty? [Step a]
 *   - If no, run (or send it to server serving the attribute) query
 *     and populate result.
 * Iterate over children and copy Result Uids to child Query Uids.
 *     Set Attr. Then for each child, use goroutine to run Step:a.
 * Wait for goroutines to finish.
 * Return errors, if any.
 */

var log = x.Log("query")

// SubGraph is the way to represent data internally. It contains both the
// query and the response. Once generated, this can then be encoded to other
// client convenient formats, like GraphQL / JSON.
type SubGraph struct {
	Attr     string
	Children []*SubGraph

	query  []byte
	result []byte
}

func getChildren(r *task.Result, sg *SubGraph) (result interface{}, rerr error) {
	var l []interface{}
	for i := 0; i < r.UidsLength(); i++ {
		m := make(map[string]interface{})
		uid := r.Uids(i)
		m["uid"] = uid
		if len(sg.Children) > 0 {
			for _, cg := range sg.Children {
			}

			// do something.
		}

		var v task.Value
		if ok := r.Values(&v, i); !ok {
			return nil, fmt.Errorf("While reading value at index: %v", i)
		}
		var i interface{}
		if err := posting.ParseValue(i, v.ValBytes()); err != nil {
			return nil, err
		}

		if r.UidsLength() == 0 {
		}
	}
}

func processChild(result *[]map[string]interface{}, g *SubGraph) error {
	ro := flatbuffers.GetUOffsetT(g.result)
	r := new(task.Result)
	r.Init(g.result, ro)
	if r.ValuesLength() > 0 {
		var v task.Value
		for i := 0; i < r.ValuesLength(); i++ {
			if ok := r.Values(&v, i); !ok {
				glog.WithField("idx", i).Error("While loading value")
				return fmt.Errorf("While parsing value at index: %v", i)
			}
			var i interface{}
			if err := posting.ParseValue(i, v.ValBytes()); err != nil {
				x.Log(glog, err).Error("While parsing value")
				return err
			}
			result[i][g.Attr] = i
		}
	}

	if r.UidsLength() > 0 {
		rlist := make([]map[string]interface{}, r.UidsLength())
		for i := 0; i < r.UidsLength(); i++ {
			rlist[i]["uid"] = r.Uids(i)
			for _, cg := range g.Children {
				if err := processChild(&rlist, cg); err != nil {
					x.Log(glog, err).Error("While processing child with attr: %v", cg.Attr)
					return err
				}
			}
		}
	}
}

func (sg SubGraph) ToJson() (result []byte, rerr error) {
	ro := flatbuffers.GetUOffsetT(sg.result)
	r := new(task.Result)
	r.Init(sg.result, ro)
	rlist := make([]map[string]interface{}, r.UidsLength())
	for i := 0; i < r.UidsLength(); i++ {
		rlist[i]["uid"] = r.Uids(i)
	}
}

func NewGraph(euid uint64, exid string) (*SubGraph, error) {
	// This would set the Result field in SubGraph,
	// and populate the children for attributes.
	if len(exid) > 0 {
		u, err := uid.GetOrAssign(exid)
		if err != nil {
			x.Err(log, err).WithField("xid", exid).Error(
				"While GetOrAssign uid from external id")
			return nil, err
		}
		log.WithField("xid", exid).WithField("uid", u).Debug("GetOrAssign")
		euid = u
	}

	if euid == 0 {
		err := fmt.Errorf("Query internal id is zero")
		x.Err(log, err).Error("Invalid query")
		return nil, err
	}

	// Encode uid into result flatbuffer.
	b := flatbuffers.NewBuilder(0)
	task.ResultStartUidsVector(b, 1)
	b.PrependUint64(euid)
	vend := b.EndVector(1)
	task.ResultStart(b)
	task.ResultAddUids(b, vend)
	rend := task.ResultEnd(b)
	b.Finish(rend)

	sg := new(SubGraph)
	sg.result = b.Bytes[b.Head():]
	return sg, nil
}

func createTaskQuery(attr string, r *task.Result) []byte {
	b := flatbuffers.NewBuilder(0)
	ao := b.CreateString(attr)

	task.QueryStartUidsVector(b, r.UidsLength())
	for i := r.UidsLength() - 1; i >= 0; i-- {
		uid := r.Uids(i)
		b.PrependUint64(uid)
	}
	vend := b.EndVector(r.UidsLength())

	task.QueryStart(b)
	task.QueryAddAttr(b, ao)
	task.QueryAddUids(b, vend)
	qend := task.QueryEnd(b)
	b.Finish(qend)
	return b.Bytes[b.Head():]
}

func ProcessGraph(sg *SubGraph, rch chan error) {
	var err error
	if len(sg.query) > 0 {
		// This task execution would go over the wire in later versions.
		sg.result, err = posting.ProcessTask(sg.query)
		if err != nil {
			rch <- err
			return
		}
	}

	uo := flatbuffers.GetUOffsetT(sg.result)
	r := new(task.Result)
	r.Init(sg.result, uo)
	if r.UidsLength() == 0 {
		// Looks like we're done here.
		if len(sg.Children) > 0 {
			log.Debug("Have some children but no results. Life got cut short early.")
		}
		rch <- nil
		return
	}

	// Let's execute it in a tree fashion. Each SubGraph would break off
	// as many goroutines as it's children; which would then recursively
	// do the same thing.
	// Buffered channel to ensure no-blockage.
	childchan := make(chan error, len(sg.Children))
	for i := 0; i < len(sg.Children); i++ {
		child := sg.Children[i]
		child.query = createTaskQuery(child.Attr, r)
		go ProcessGraph(child, childchan)
	}

	// Now get all the results back.
	for i := 0; i < len(sg.Children); i++ {
		err = <-childchan
		if err != nil {
			rch <- err
			return
		}
	}
	rch <- nil
}
<|endoftext|>


<commit_before>// +build windows

package main

import "github.com/tischda/timer/registry"

func init() {
	shell = "cmd"
	shellCmdFlag = "/c"
	timer = theTimer{registry: registry.RealRegistry{}}
}
<commit_msg>methods have pointer receiver<commit_after>// +build windows

package main

import "github.com/tischda/timer/registry"

func init() {
	shell = "cmd"
	shellCmdFlag = "/c"
	timer = &theTimer{registry: registry.RealRegistry{}}
}
<|endoftext|>


<commit_before>package git

/*
#cgo pkg-config: libgit2
#include <git2.h>
#include <git2/errors.h>
*/
import "C"

import (
	"errors"
	"strings"
	"unsafe"
)

var ErrEUser = errors.New("Error in user callback function")

type ListFlags uint

type BranchType uint

const (
	BRANCH_LOCAL  BranchType = C.GIT_BRANCH_LOCAL
	BRANCH_REMOTE            = C.GIT_BRANCH_REMOTE
)

const (
	REFS_DIR         = "refs/"
	REFS_HEADS_DIR   = REFS_DIR + "heads/"
	REFS_TAGS_DIR    = REFS_DIR + "tags/"
	REFS_REMOTES_DIR = REFS_DIR + "remotes/"
)

type Branch struct {
	Reference
}

func (repo *Repository) CreateBranch(branchName string, target *Commit, force bool) (*Reference, error) {
	ref := new(Reference)
	cBranchName := C.CString(branchName)
	cForce := cbool(force)
	err := C.git_branch_create(&ref.ptr, repo.ptr, cBranchName, target.ptr, cForce)
	if err < 0 {
		return nil, LastError()
	}
	return ref, nil
}

func (b *Branch) BranchDelete() error {
	if err := C.git_branch_delete(b.ptr); err < 0 {
		return LastError()
	}
	return nil
}

type BranchForeachCB func(name string, flags ListFlags, payload interface{}) error

func (repo *Repository) BranchForeach(flags ListFlags, callback BranchForeachCB, payload interface{}) error {
	iter, err := repo.NewReferenceIterator()
	if err != nil {
		return err
	}

	for {
		ref, err := iter.Next()
		if err == ErrIterOver {
			break
		}

		if (flags == ListFlags(BRANCH_LOCAL)) && strings.HasPrefix(ref.Name(), REFS_HEADS_DIR) {
			name := strings.TrimPrefix(ref.Name(), REFS_HEADS_DIR)
			err = callback(name, ListFlags(BRANCH_LOCAL), payload)
			if err != nil {
				return err
			}
		}

		if (flags == ListFlags(BRANCH_REMOTE)) && strings.HasPrefix(ref.Name(), REFS_REMOTES_DIR) {
			name := strings.TrimPrefix(ref.Name(), REFS_REMOTES_DIR)
			err = callback(name, ListFlags(BRANCH_REMOTE), payload)
			if err != nil {
				return err
			}
		}
	}

	if err == ErrIterOver {
		err = nil
	}
	return err
}

func (b *Branch) Move(newBranchName string, force bool) (*Branch, error) {
	newBranch := new(Branch)
	cNewBranchName := C.CString(newBranchName)
	cForce := cbool(force)

	err := C.git_branch_move(&newBranch.ptr, b.ptr, cNewBranchName, cForce)
	if err < 0 {
		return nil, LastError()
	}
	return newBranch, nil
}

func (b *Branch) IsHead() (bool, error) {
	isHead := C.git_branch_is_head(b.ptr)
	switch isHead {
	case 1:
		return true, nil
	case 0:
		return false, nil
	default:
		return false, LastError()
	}

}

func (repo *Repository) BranchLookup(branchName string, bt BranchType) (*Branch, error) {
	branch := new(Branch)
	cName := C.CString(branchName)

	err := C.git_branch_lookup(&branch.ptr, repo.ptr, cName, C.git_branch_t(bt))
	if err < 0 {
		return nil, LastError()
	}
	return branch, nil
}

func (b *Branch) Name() (string, error) {
	var cName *C.char
	defer C.free(unsafe.Pointer(cName))

	err := C.git_branch_name(&cName, b.ptr)
	if err < 0 {
		return "", LastError()
	}

	return C.GoString(cName), nil
}

func (repo *Repository) RemoteName(canonicalBranchName string) (string, error) {
	cName := C.CString(canonicalBranchName)

	// Obtain the length of the name
	ret := C.git_branch_remote_name(nil, 0, repo.ptr, cName)
	if ret < 0 {
		return "", LastError()
	}

	cBuf := (*C.char)(C.malloc(C.size_t(ret)))
	defer C.free(unsafe.Pointer(cBuf))

	// Actually obtain the name
	ret = C.git_branch_remote_name(cBuf, C.size_t(ret), repo.ptr, cName)
	if ret < 0 {
		return "", LastError()
	}

	return C.GoString(cBuf), nil
}

func (b *Branch) SetUpstream(upstreamName string) error {
	cName := C.CString(upstreamName)

	err := C.git_branch_set_upstream(b.ptr, cName)
	if err < 0 {
		return LastError()
	}
	return nil
}

func (b *Branch) Upstream() (*Branch, error) {
	upstream := new(Branch)
	err := C.git_branch_upstream(&upstream.ptr, b.ptr)
	if err < 0 {
		return nil, LastError()
	}
	return upstream, nil
}

func (repo *Repository) UpstreamName(canonicalBranchName string) (string, error) {
	cName := C.CString(canonicalBranchName)

	// Obtain the length of the name
	ret := C.git_branch_upstream_name(nil, 0, repo.ptr, cName)
	if ret < 0 {
		return "", LastError()
	}

	cBuf := (*C.char)(C.malloc(C.size_t(ret)))
	defer C.free(unsafe.Pointer(cBuf))

	// Actually obtain the name
	ret = C.git_branch_upstream_name(cBuf, C.size_t(ret), repo.ptr, cName)
	if ret < 0 {
		return "", LastError()
	}
	return C.GoString(cBuf), nil
}
<commit_msg>branch: Renamed BranchLookup to LookupBrnach<commit_after>package git

/*
#cgo pkg-config: libgit2
#include <git2.h>
#include <git2/errors.h>
*/
import "C"

import (
	"errors"
	"strings"
	"unsafe"
)

var ErrEUser = errors.New("Error in user callback function")

type ListFlags uint

type BranchType uint

const (
	BRANCH_LOCAL  BranchType = C.GIT_BRANCH_LOCAL
	BRANCH_REMOTE            = C.GIT_BRANCH_REMOTE
)

const (
	REFS_DIR         = "refs/"
	REFS_HEADS_DIR   = REFS_DIR + "heads/"
	REFS_TAGS_DIR    = REFS_DIR + "tags/"
	REFS_REMOTES_DIR = REFS_DIR + "remotes/"
)

type Branch struct {
	Reference
}

func (repo *Repository) CreateBranch(branchName string, target *Commit, force bool) (*Reference, error) {
	ref := new(Reference)
	cBranchName := C.CString(branchName)
	cForce := cbool(force)
	err := C.git_branch_create(&ref.ptr, repo.ptr, cBranchName, target.ptr, cForce)
	if err < 0 {
		return nil, LastError()
	}
	return ref, nil
}

func (b *Branch) BranchDelete() error {
	if err := C.git_branch_delete(b.ptr); err < 0 {
		return LastError()
	}
	return nil
}

type BranchForeachCB func(name string, flags ListFlags, payload interface{}) error

func (repo *Repository) BranchForeach(flags ListFlags, callback BranchForeachCB, payload interface{}) error {
	iter, err := repo.NewReferenceIterator()
	if err != nil {
		return err
	}

	for {
		ref, err := iter.Next()
		if err == ErrIterOver {
			break
		}

		if (flags == ListFlags(BRANCH_LOCAL)) && strings.HasPrefix(ref.Name(), REFS_HEADS_DIR) {
			name := strings.TrimPrefix(ref.Name(), REFS_HEADS_DIR)
			err = callback(name, ListFlags(BRANCH_LOCAL), payload)
			if err != nil {
				return err
			}
		}

		if (flags == ListFlags(BRANCH_REMOTE)) && strings.HasPrefix(ref.Name(), REFS_REMOTES_DIR) {
			name := strings.TrimPrefix(ref.Name(), REFS_REMOTES_DIR)
			err = callback(name, ListFlags(BRANCH_REMOTE), payload)
			if err != nil {
				return err
			}
		}
	}

	if err == ErrIterOver {
		err = nil
	}
	return err
}

func (b *Branch) Move(newBranchName string, force bool) (*Branch, error) {
	newBranch := new(Branch)
	cNewBranchName := C.CString(newBranchName)
	cForce := cbool(force)

	err := C.git_branch_move(&newBranch.ptr, b.ptr, cNewBranchName, cForce)
	if err < 0 {
		return nil, LastError()
	}
	return newBranch, nil
}

func (b *Branch) IsHead() (bool, error) {
	isHead := C.git_branch_is_head(b.ptr)
	switch isHead {
	case 1:
		return true, nil
	case 0:
		return false, nil
	default:
		return false, LastError()
	}

}

func (repo *Repository) LookupBranch(branchName string, bt BranchType) (*Branch, error) {
	branch := new(Branch)
	cName := C.CString(branchName)

	err := C.git_branch_lookup(&branch.ptr, repo.ptr, cName, C.git_branch_t(bt))
	if err < 0 {
		return nil, LastError()
	}
	return branch, nil
}

func (b *Branch) Name() (string, error) {
	var cName *C.char
	defer C.free(unsafe.Pointer(cName))

	err := C.git_branch_name(&cName, b.ptr)
	if err < 0 {
		return "", LastError()
	}

	return C.GoString(cName), nil
}

func (repo *Repository) RemoteName(canonicalBranchName string) (string, error) {
	cName := C.CString(canonicalBranchName)

	// Obtain the length of the name
	ret := C.git_branch_remote_name(nil, 0, repo.ptr, cName)
	if ret < 0 {
		return "", LastError()
	}

	cBuf := (*C.char)(C.malloc(C.size_t(ret)))
	defer C.free(unsafe.Pointer(cBuf))

	// Actually obtain the name
	ret = C.git_branch_remote_name(cBuf, C.size_t(ret), repo.ptr, cName)
	if ret < 0 {
		return "", LastError()
	}

	return C.GoString(cBuf), nil
}

func (b *Branch) SetUpstream(upstreamName string) error {
	cName := C.CString(upstreamName)

	err := C.git_branch_set_upstream(b.ptr, cName)
	if err < 0 {
		return LastError()
	}
	return nil
}

func (b *Branch) Upstream() (*Branch, error) {
	upstream := new(Branch)
	err := C.git_branch_upstream(&upstream.ptr, b.ptr)
	if err < 0 {
		return nil, LastError()
	}
	return upstream, nil
}

func (repo *Repository) UpstreamName(canonicalBranchName string) (string, error) {
	cName := C.CString(canonicalBranchName)

	// Obtain the length of the name
	ret := C.git_branch_upstream_name(nil, 0, repo.ptr, cName)
	if ret < 0 {
		return "", LastError()
	}

	cBuf := (*C.char)(C.malloc(C.size_t(ret)))
	defer C.free(unsafe.Pointer(cBuf))

	// Actually obtain the name
	ret = C.git_branch_upstream_name(cBuf, C.size_t(ret), repo.ptr, cName)
	if ret < 0 {
		return "", LastError()
	}
	return C.GoString(cBuf), nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"

	"github.com/mozilla-services/heka/message"
	"github.com/prometheus/client_golang/prometheus"
)

type Config struct {
	Metrics []MetricConfig `json:"metrics"`
}

type MetricConfig struct {
	Name        string            `json:"name"`
	Help        string            `json:"help"`
	Type        string            `json:"type"`
	Value       string            `json:"value"`
	Matcher     string            `json:"matcher"`
	Labels      map[string]string `json:"labels"`
	ConstLabels map[string]string `json:"const_labels"`
	Buckets     []float64         `json:buckets`
}

func (m *MetricConfig) LabelKeysValues() ([]string, []string) {
	keys := make([]string, len(m.Labels))
	values := make([]string, len(m.Labels))
	i := 0
	for k, v := range m.Labels {
		keys[i] = k
		values[i] = v
		i++
	}
	return keys, values
}

type metric struct {
	counter      prometheus.Counter
	counterVec   *prometheus.CounterVec
	gauge        prometheus.Gauge
	gaugeVec     *prometheus.GaugeVec
	histogram    prometheus.Histogram
	histogramVec *prometheus.HistogramVec
	summary      prometheus.Summary
	summaryVec   *prometheus.SummaryVec
	matcher      *message.MatcherSpecification
	LabelFields  []string
	MetricConfig
}

func (m *metric) Process(msg *message.Message) {
	if m.matcher != nil && !m.matcher.Match(msg) {
		return
	}
	value := 0.0
	v := getFieldValue(m.MetricConfig.Value, msg)
	if v == nil {
		log.Println("Couldn't find field", m.MetricConfig.Value)
		return
	}
	switch v := v.(type) {
	case float64:
		value = v
	case int:
		value = float64(v)
	default:
		log.Printf("Invalid type %T for field %s", v, m.MetricConfig.Value)
		return
	}

	switch m.MetricConfig.Type {
	case "counter":
		if len(m.LabelFields) > 0 {
			if m.MetricConfig.Value == "" {
				m.counterVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Inc()
			} else {
				m.counterVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Set(value)
			}
		} else {
			m.counter.Inc()
		}
	case "gauge":
		if len(m.LabelFields) > 0 {
			m.gaugeVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Set(value)
		} else {
			m.gauge.Set(value)
		}
	case "histogram":
		if len(m.LabelFields) > 0 {
			m.histogramVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Observe(value)
		} else {
			m.histogram.Observe(value)
		}
	case "summary":
		if len(m.LabelFields) > 0 {
			m.summaryVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Observe(value)
		} else {
			m.summary.Observe(value)
		}
	default:
	}
}

func getFieldValue(field string, msg *message.Message) (value interface{}) {
	var funcMap = map[string]func() string{
		"hostname":  msg.GetHostname,
		"logger":    msg.GetLogger,
		"pid":       func() string { return strconv.Itoa(int(msg.GetPid())) },
		"severity":  func() string { return strconv.Itoa(int(msg.GetSeverity())) },
		"timestamp": func() string { return strconv.Itoa(int(msg.GetTimestamp())) },
	}
	if field[0] == '@' {
		if f, ok := funcMap[field[1:]]; ok {
			return f()
		}
	}
	f, _ := msg.GetFieldValue(field)
	return f
}

func extractLabels(labels []string, msg *message.Message) []string {
	lvs := make([]string, len(labels))
	for i, l := range labels {
		v := getFieldValue(l, msg)
		lvs[i] = fmt.Sprintf("%v", v)
	}
	return lvs
}
func fieldToFloat(field *message.Field) (float64, error) {
	switch *field.ValueType {
	case message.Field_INTEGER:
		return float64(field.ValueInteger[0]), nil
	case message.Field_DOUBLE:
		return field.ValueDouble[0], nil
	case message.Field_BOOL:
		if field.ValueBool[0] {
			return 1.0, nil
		}
		return 0.0, fmt.Errorf("Can't convert %s to number", field.ValueType)
	default:
		return 0.0, fmt.Errorf("Value type %s not supported", *field.ValueType)
	}
}

type Bridge struct {
	metrics []metric
}

func newBridge(filename string) (*Bridge, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	config := &Config{}
	if err := json.Unmarshal(data, config); err != nil {
		return nil, err
	}
	bridge := &Bridge{}
	bridge.metrics = make([]metric, len(config.Metrics))
	for i, metric := range config.Metrics {
		if metric.Value == "" && metric.Type != "counter" {
			return nil, fmt.Errorf("Type %s requires a value field name", metric.Type)
		}
		bridge.metrics[i].MetricConfig = metric
		labels, fields := metric.LabelKeysValues()
		bridge.metrics[i].LabelFields = fields
		if len(labels) > 0 {
			switch metric.Type {
			case "counter":
				bridge.metrics[i].counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels}, labels)
				prometheus.MustRegister(bridge.metrics[i].counterVec)
			case "gauge":
				bridge.metrics[i].gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels}, labels)
				prometheus.MustRegister(bridge.metrics[i].gaugeVec)
			case "histogram":
				bridge.metrics[i].histogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: metric.Name, Help: metric.Help, Buckets: metric.Buckets, ConstLabels: metric.ConstLabels}, labels)
				prometheus.MustRegister(bridge.metrics[i].histogramVec)
			case "summary":
				bridge.metrics[i].summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels}, labels)
				prometheus.MustRegister(bridge.metrics[i].summaryVec)
			default:
				return nil, fmt.Errorf("Metric type %s is invalid", metric.Type)
			}
		} else {
			switch metric.Type {
			case "counter":
				bridge.metrics[i].counter = prometheus.NewCounter(prometheus.CounterOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels})
				prometheus.MustRegister(bridge.metrics[i].counter)
			case "gauge":
				bridge.metrics[i].gauge = prometheus.NewGauge(prometheus.GaugeOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels})
				prometheus.MustRegister(bridge.metrics[i].gauge)
			case "histogram":
				bridge.metrics[i].histogram = prometheus.NewHistogram(prometheus.HistogramOpts{Name: metric.Name, Help: metric.Help, Buckets: metric.Buckets, ConstLabels: metric.ConstLabels})
				prometheus.MustRegister(bridge.metrics[i].histogram)
			case "summary":
				bridge.metrics[i].summary = prometheus.NewSummary(prometheus.SummaryOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels})
				prometheus.MustRegister(bridge.metrics[i].summary)
			default:
				return nil, fmt.Errorf("Metric type %s is invalid", metric.Type)
			}
		}

		if metric.Matcher != "" {
			matcher, err := message.CreateMatcherSpecification(metric.Matcher)
			if err != nil {
				return nil, err
			}
			bridge.metrics[i].matcher = matcher
		}
	}
	http.Handle("/metrics", prometheus.Handler())
	return bridge, nil
}

func (b *Bridge) Process(msg *message.Message) error {
	for _, metric := range b.metrics {
		metric.Process(msg)
	}
	return nil
}
<commit_msg>Fix handling of empty value field<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"

	"github.com/mozilla-services/heka/message"
	"github.com/prometheus/client_golang/prometheus"
)

type Config struct {
	Metrics []MetricConfig `json:"metrics"`
}

type MetricConfig struct {
	Name        string            `json:"name"`
	Help        string            `json:"help"`
	Type        string            `json:"type"`
	Value       string            `json:"value"`
	Matcher     string            `json:"matcher"`
	Labels      map[string]string `json:"labels"`
	ConstLabels map[string]string `json:"const_labels"`
	Buckets     []float64         `json:buckets`
}

func (m *MetricConfig) LabelKeysValues() ([]string, []string) {
	keys := make([]string, len(m.Labels))
	values := make([]string, len(m.Labels))
	i := 0
	for k, v := range m.Labels {
		keys[i] = k
		values[i] = v
		i++
	}
	return keys, values
}

type metric struct {
	counter      prometheus.Counter
	counterVec   *prometheus.CounterVec
	gauge        prometheus.Gauge
	gaugeVec     *prometheus.GaugeVec
	histogram    prometheus.Histogram
	histogramVec *prometheus.HistogramVec
	summary      prometheus.Summary
	summaryVec   *prometheus.SummaryVec
	matcher      *message.MatcherSpecification
	LabelFields  []string
	MetricConfig
}

func (m *metric) Process(msg *message.Message) {
	if m.matcher != nil && !m.matcher.Match(msg) {
		return
	}
	value := 0.0
	if m.MetricConfig.Value != "" {
		v := getFieldValue(m.MetricConfig.Value, msg)
		if v == nil {
			log.Println("Couldn't find field", m.MetricConfig.Value)
			return
		}
		switch v := v.(type) {
		case float64:
			value = v
		case int:
			value = float64(v)
		default:
			log.Printf("Invalid type %T for field %s", v, m.MetricConfig.Value)
			return
		}
	}

	switch m.MetricConfig.Type {
	case "counter":
		if len(m.LabelFields) > 0 {
			if m.MetricConfig.Value == "" {
				m.counterVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Inc()
			} else {
				m.counterVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Set(value)
			}
		} else {
			m.counter.Inc()
		}
	case "gauge":
		if len(m.LabelFields) > 0 {
			m.gaugeVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Set(value)
		} else {
			m.gauge.Set(value)
		}
	case "histogram":
		if len(m.LabelFields) > 0 {
			m.histogramVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Observe(value)
		} else {
			m.histogram.Observe(value)
		}
	case "summary":
		if len(m.LabelFields) > 0 {
			m.summaryVec.WithLabelValues(extractLabels(m.LabelFields, msg)...).Observe(value)
		} else {
			m.summary.Observe(value)
		}
	default:
	}
}

func getFieldValue(field string, msg *message.Message) (value interface{}) {
	var funcMap = map[string]func() string{
		"hostname":  msg.GetHostname,
		"logger":    msg.GetLogger,
		"pid":       func() string { return strconv.Itoa(int(msg.GetPid())) },
		"severity":  func() string { return strconv.Itoa(int(msg.GetSeverity())) },
		"timestamp": func() string { return strconv.Itoa(int(msg.GetTimestamp())) },
	}
	if field[0] == '@' {
		if f, ok := funcMap[field[1:]]; ok {
			return f()
		}
	}
	f, _ := msg.GetFieldValue(field)
	return f
}

func extractLabels(labels []string, msg *message.Message) []string {
	lvs := make([]string, len(labels))
	for i, l := range labels {
		v := getFieldValue(l, msg)
		lvs[i] = fmt.Sprintf("%v", v)
	}
	return lvs
}
func fieldToFloat(field *message.Field) (float64, error) {
	switch *field.ValueType {
	case message.Field_INTEGER:
		return float64(field.ValueInteger[0]), nil
	case message.Field_DOUBLE:
		return field.ValueDouble[0], nil
	case message.Field_BOOL:
		if field.ValueBool[0] {
			return 1.0, nil
		}
		return 0.0, fmt.Errorf("Can't convert %s to number", field.ValueType)
	default:
		return 0.0, fmt.Errorf("Value type %s not supported", *field.ValueType)
	}
}

type Bridge struct {
	metrics []metric
}

func newBridge(filename string) (*Bridge, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	config := &Config{}
	if err := json.Unmarshal(data, config); err != nil {
		return nil, err
	}
	bridge := &Bridge{}
	bridge.metrics = make([]metric, len(config.Metrics))
	for i, metric := range config.Metrics {
		if metric.Value == "" && metric.Type != "counter" {
			return nil, fmt.Errorf("Type %s requires a value field name", metric.Type)
		}
		bridge.metrics[i].MetricConfig = metric
		labels, fields := metric.LabelKeysValues()
		bridge.metrics[i].LabelFields = fields
		if len(labels) > 0 {
			switch metric.Type {
			case "counter":
				bridge.metrics[i].counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels}, labels)
				prometheus.MustRegister(bridge.metrics[i].counterVec)
			case "gauge":
				bridge.metrics[i].gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels}, labels)
				prometheus.MustRegister(bridge.metrics[i].gaugeVec)
			case "histogram":
				bridge.metrics[i].histogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: metric.Name, Help: metric.Help, Buckets: metric.Buckets, ConstLabels: metric.ConstLabels}, labels)
				prometheus.MustRegister(bridge.metrics[i].histogramVec)
			case "summary":
				bridge.metrics[i].summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels}, labels)
				prometheus.MustRegister(bridge.metrics[i].summaryVec)
			default:
				return nil, fmt.Errorf("Metric type %s is invalid", metric.Type)
			}
		} else {
			switch metric.Type {
			case "counter":
				bridge.metrics[i].counter = prometheus.NewCounter(prometheus.CounterOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels})
				prometheus.MustRegister(bridge.metrics[i].counter)
			case "gauge":
				bridge.metrics[i].gauge = prometheus.NewGauge(prometheus.GaugeOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels})
				prometheus.MustRegister(bridge.metrics[i].gauge)
			case "histogram":
				bridge.metrics[i].histogram = prometheus.NewHistogram(prometheus.HistogramOpts{Name: metric.Name, Help: metric.Help, Buckets: metric.Buckets, ConstLabels: metric.ConstLabels})
				prometheus.MustRegister(bridge.metrics[i].histogram)
			case "summary":
				bridge.metrics[i].summary = prometheus.NewSummary(prometheus.SummaryOpts{Name: metric.Name, Help: metric.Help, ConstLabels: metric.ConstLabels})
				prometheus.MustRegister(bridge.metrics[i].summary)
			default:
				return nil, fmt.Errorf("Metric type %s is invalid", metric.Type)
			}
		}

		if metric.Matcher != "" {
			matcher, err := message.CreateMatcherSpecification(metric.Matcher)
			if err != nil {
				return nil, err
			}
			bridge.metrics[i].matcher = matcher
		}
	}
	http.Handle("/metrics", prometheus.Handler())
	return bridge, nil
}

func (b *Bridge) Process(msg *message.Message) error {
	for _, metric := range b.metrics {
		metric.Process(msg)
	}
	return nil
}
<|endoftext|>


<commit_before>package MQTTg

import (
	"net"
	"strconv"
	"time"
)

type Broker struct {
	Bt *Transport
	// TODO: check whether not good to use addr as key
	Clients   map[*net.UDPAddr]*ClientInfo // map[addr]*ClientInfo
	ClientIDs map[string]*ClientInfo       // map[clientID]*ClientInfo
	TopicRoot *TopicNode
}

func (self *Broker) DisconnectFromBroker(client *ClientInfo) {
	client.Will = nil
	client.IsConnecting = false
	client.KeepAliveTimer.Stop()
	if client.CleanSession {
		delete(self.Clients, client.RemoteAddr)
		delete(self.ClientIDs, client.ID)
	}

}

func (self *Broker) RunClientTimer(client *ClientInfo) {
	<-client.KeepAliveTimer.C
	self.DisconnectFromBroker(client)
	// TODO: logging?
}

type ClientInfo struct {
	*Client
	KeepAliveTimer *time.Timer
	Duration       time.Duration
}

func NewClientInfo(c *Client, duration time.Duration) *ClientInfo {
	return &ClientInfo{
		Client:         c,
		KeepAliveTimer: time.NewTimer(duration),
		Duration:       duration,
	}
}

func (self *ClientInfo) ResetTimer() {
	self.KeepAliveTimer.Reset(self.Duration)
}

func (self *Broker) ApplyDummyClientID() string {
	return "DummyClientID:" + strconv.Itoa(len(self.ClientIDs)+1)
}

func (self *Broker) recvConnectMessage(m *ConnectMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if ok && client.IsConnecting {
		self.DisconnectFromBroker(client)
		return PROTOCOL_VIOLATION
	}

	if m.Protocol.Name != MQTT_3_1_1.Name {
		// server MAY disconnect
		return INVALID_PROTOCOL_NAME
	}

	if m.Protocol.Level != MQTT_3_1_1.Level {
		// CHECK: Is false correct?
		err = self.Bt.SendMessage(NewConnackMessage(false, UnacceptableProtocolVersion), addr)
		return INVALID_PROTOCOL_LEVEL
	}

	_, ok = self.ClientIDs[m.ClientID]
	if ok {
		// TODO: this might cause problem
		err = self.Bt.SendMessage(NewConnackMessage(false, IdentifierRejected), addr)
		return CLIENT_ID_IS_USED_ALREADY
	}
	if len(m.ClientID) == 0 {
		m.ClientID = self.ApplyDummyClientID()
	}

	if m.Flags&Reserved_Flag == Reserved_Flag {
		// TODO: disconnect the connection
		return MALFORMED_CONNECT_FLAG_BIT
	}

	if m.Flags&UserName_Flag != UserName_Flag && m.Flags&Password_Flag == Password_Flag {
		return USERNAME_DOES_NOT_EXIST_WITH_PASSWORD
	}
	// TODO: authorization

	client, ok = self.Clients[addr]
	sessionPresent := ok
	cleanSession := m.Flags&CleanSession_Flag == CleanSession_Flag
	if cleanSession || !ok {
		// TODO: need to manage QoS base processing
		duration := time.Duration(float32(m.KeepAlive) * 100000000 * 1.5)
		client = NewClientInfo(NewClient(self.Bt, addr, m.ClientID,
			m.User, m.KeepAlive, m.Will, cleanSession), duration)
		self.Clients[addr] = client
		self.ClientIDs[m.ClientID] = client
		sessionPresent = false
	}

	if m.Flags&Will_Flag == Will_Flag {
		client.Will = m.Will
		// TODO: consider QoS and Retain as broker need
	} else {

	}

	if m.KeepAlive != 0 {
		go self.RunClientTimer(client)
	}
	err = self.Bt.SendMessage(NewConnackMessage(sessionPresent, Accepted), addr)
	client.IsConnecting = true
	client.Redelivery()
	return err
}

func (self *Broker) recvConnackMessage(m *ConnackMessage, addr *net.UDPAddr) (err error) {
	return INVALID_MESSAGE_CAME
}

func (self *Broker) recvPublishMessage(m *PublishMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		self.DisconnectFromBroker(client)
		return CLIENT_NOT_EXIST
	}

	if m.Dup {
		// re-delivered
	} else {
		// first time delivery
	}

	if m.Retain {
		// store tehe application message to designated topic
		data := string(m.Payload)
		if m.QoS == 0 && len(data) > 0 {
			// TODO: warnning, in this case data cannot be stored.
			// discard retained message
			data = ""
		}
		self.TopicRoot.ApplyRetain(m.TopicName, m.QoS, data)
	}

	switch m.QoS {
	// in any case, Dub must be 0
	case 0:
	case 1:
		err = client.SendMessage(NewPubackMessage(m.PacketID))
	case 2:
		err = client.SendMessage(NewPubrecMessage(m.PacketID))
	}
	return err
}

func (self *Broker) recvPubackMessage(m *PubackMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	// acknowledge the sent Publish packet
	if m.PacketID > 0 {
		err = client.AckMessage(m.PacketID)
	}
	return err
}

func (self *Broker) recvPubrecMessage(m *PubrecMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	// acknowledge the sent Publish packet
	err = client.AckMessage(m.PacketID)
	err = client.SendMessage(NewPubrelMessage(m.PacketID))
	return err
}

func (self *Broker) recvPubrelMessage(m *PubrelMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	// acknowledge the sent Pubrel packet
	err = client.AckMessage(m.PacketID)
	err = client.SendMessage(NewPubcompMessage(m.PacketID))
	return err
}

func (self *Broker) recvPubcompMessage(m *PubcompMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	// acknowledge the sent Pubrel packet
	err = client.AckMessage(m.PacketID)
	return err
}

func (self *Broker) recvSubscribeMessage(m *SubscribeMessage, addr *net.UDPAddr) (err error) {
	// TODO: check The wild card is permitted
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	returnCodes := make([]SubscribeReturnCode, 0)
	for _, subTopic := range m.SubscribeTopics {
		// TODO: need to validate wheter there are same topics or not
		edges, err := self.TopicRoot.GetTopicNodes(subTopic.Topic)
		codes := make([]SubscribeReturnCode, len(edges))
		if err != nil || !ok {
			for i, _ := range codes {
				codes[i] = SubscribeFailure
			}
		} else {
			for i, edge := range edges {
				edge.Subscribers[client.ID] = subTopic.QoS
				codes[i] = SubscribeReturnCode(subTopic.QoS)
				client.SubTopics = append(client.SubTopics,
					SubscribeTopic{SubscribeAck,
						edge.FullPath,
						uint8(subTopic.QoS),
					})
				if len(edge.RetainMessage) > 0 {
					// publish retain
					// TODO: check all arguments
					err = client.SendMessage(NewPublishMessage(false, edge.RetainQoS, true,
						edge.FullPath, m.PacketID, []uint8(edge.RetainMessage)))
					// TODO: error validation
				}
			}
		}
		returnCodes = append(returnCodes, codes...)
	}
	// TODO: check whether the number of return codes are correct?
	err = client.SendMessage(NewSubackMessage(m.PacketID, returnCodes))
	return err
}

func (self *Broker) recvSubackMessage(m *SubackMessage, addr *net.UDPAddr) (err error) {
	return INVALID_MESSAGE_CAME
}
func (self *Broker) recvUnsubscribeMessage(m *UnsubscribeMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	if len(m.TopicNames) == 0 {
		// protocol violation
	}
	// TODO: optimize here
	result := []SubscribeTopic{}
	for _, t := range client.SubTopics {
		del := false
		for _, name := range m.TopicNames {
			if string(t.Topic) == string(name) {
				del = true
			}
		}
		if !del {
			result = append(result, t)
		}
	}
	client.SubTopics = result
	err = client.SendMessage(NewUnsubackMessage(m.PacketID))
	return err
}
func (self *Broker) recvUnsubackMessage(m *UnsubackMessage, addr *net.UDPAddr) (err error) {
	return INVALID_MESSAGE_CAME
}

func (self *Broker) recvPingreqMessage(m *PingreqMessage, addr *net.UDPAddr) (err error) {
	// Pingresp
	// TODO: calc elapsed time from previous pingreq.
	//       and store the time to duration of Transport
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	err = client.SendMessage(NewPingrespMessage())
	if client.KeepAlive != 0 {
		client.ResetTimer()
		go self.RunClientTimer(client)
	}
	return err
}

func (self *Broker) recvPingrespMessage(m *PingrespMessage, addr *net.UDPAddr) (err error) {
	return INVALID_MESSAGE_CAME
}

func (self *Broker) recvDisconnectMessage(m *DisconnectMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	self.DisconnectFromBroker(client)
	// close the client
	return err
}

func (self *Broker) ReadMessageFrom() (Message, *net.UDPAddr, error) {
	return self.Bt.ReadMessageFrom()

}
<commit_msg>added Publish message propagation<commit_after>package MQTTg

import (
	"net"
	"strconv"
	"time"
)

type Broker struct {
	Bt *Transport
	// TODO: check whether not good to use addr as key
	Clients   map[*net.UDPAddr]*ClientInfo // map[addr]*ClientInfo
	ClientIDs map[string]*ClientInfo       // map[clientID]*ClientInfo
	TopicRoot *TopicNode
}

func (self *Broker) DisconnectFromBroker(client *ClientInfo) {
	client.Will = nil
	client.IsConnecting = false
	client.KeepAliveTimer.Stop()
	if client.CleanSession {
		delete(self.Clients, client.RemoteAddr)
		delete(self.ClientIDs, client.ID)
	}

}

func (self *Broker) RunClientTimer(client *ClientInfo) {
	<-client.KeepAliveTimer.C
	self.DisconnectFromBroker(client)
	// TODO: logging?
}

type ClientInfo struct {
	*Client
	KeepAliveTimer *time.Timer
	Duration       time.Duration
}

func NewClientInfo(c *Client, duration time.Duration) *ClientInfo {
	return &ClientInfo{
		Client:         c,
		KeepAliveTimer: time.NewTimer(duration),
		Duration:       duration,
	}
}

func (self *ClientInfo) ResetTimer() {
	self.KeepAliveTimer.Reset(self.Duration)
}

func (self *Broker) ApplyDummyClientID() string {
	return "DummyClientID:" + strconv.Itoa(len(self.ClientIDs)+1)
}

func (self *Broker) recvConnectMessage(m *ConnectMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if ok && client.IsConnecting {
		self.DisconnectFromBroker(client)
		return PROTOCOL_VIOLATION
	}

	if m.Protocol.Name != MQTT_3_1_1.Name {
		// server MAY disconnect
		return INVALID_PROTOCOL_NAME
	}

	if m.Protocol.Level != MQTT_3_1_1.Level {
		// CHECK: Is false correct?
		err = self.Bt.SendMessage(NewConnackMessage(false, UnacceptableProtocolVersion), addr)
		return INVALID_PROTOCOL_LEVEL
	}

	_, ok = self.ClientIDs[m.ClientID]
	if ok {
		// TODO: this might cause problem
		err = self.Bt.SendMessage(NewConnackMessage(false, IdentifierRejected), addr)
		return CLIENT_ID_IS_USED_ALREADY
	}
	if len(m.ClientID) == 0 {
		m.ClientID = self.ApplyDummyClientID()
	}

	if m.Flags&Reserved_Flag == Reserved_Flag {
		// TODO: disconnect the connection
		return MALFORMED_CONNECT_FLAG_BIT
	}

	if m.Flags&UserName_Flag != UserName_Flag && m.Flags&Password_Flag == Password_Flag {
		return USERNAME_DOES_NOT_EXIST_WITH_PASSWORD
	}
	// TODO: authorization

	client, ok = self.Clients[addr]
	sessionPresent := ok
	cleanSession := m.Flags&CleanSession_Flag == CleanSession_Flag
	if cleanSession || !ok {
		// TODO: need to manage QoS base processing
		duration := time.Duration(float32(m.KeepAlive) * 100000000 * 1.5)
		client = NewClientInfo(NewClient(self.Bt, addr, m.ClientID,
			m.User, m.KeepAlive, m.Will, cleanSession), duration)
		self.Clients[addr] = client
		self.ClientIDs[m.ClientID] = client
		sessionPresent = false
	}

	if m.Flags&Will_Flag == Will_Flag {
		client.Will = m.Will
		// TODO: consider QoS and Retain as broker need
	} else {

	}

	if m.KeepAlive != 0 {
		go self.RunClientTimer(client)
	}
	err = self.Bt.SendMessage(NewConnackMessage(sessionPresent, Accepted), addr)
	client.IsConnecting = true
	client.Redelivery()
	return err
}

func (self *Broker) recvConnackMessage(m *ConnackMessage, addr *net.UDPAddr) (err error) {
	return INVALID_MESSAGE_CAME
}

func (self *Broker) recvPublishMessage(m *PublishMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		self.DisconnectFromBroker(client)
		return CLIENT_NOT_EXIST
	}

	if m.Dup {
		// re-delivered
	} else {
		// first time delivery
	}

	if m.Retain {
		// store tehe application message to designated topic
		data := string(m.Payload)
		if m.QoS == 0 && len(data) > 0 {
			// TODO: warnning, in this case data cannot be stored.
			// discard retained message
			data = ""
		}
		self.TopicRoot.ApplyRetain(m.TopicName, m.QoS, data)
	}

	nodes, err := self.TopicRoot.GetTopicNodes(m.TopicName)
	if err != nil {
		return err
	}
	for subscriberID, qos := range nodes[0].Subscribers {
		subscriber, _ := self.ClientIDs[subscriberID]
		subscriber.Publish(m.TopicName, string(m.Payload), qos, false)
	}

	switch m.QoS {
	// in any case, Dub must be 0
	case 0:
	case 1:
		err = client.SendMessage(NewPubackMessage(m.PacketID))
	case 2:
		err = client.SendMessage(NewPubrecMessage(m.PacketID))
	}
	return err
}

func (self *Broker) recvPubackMessage(m *PubackMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	// acknowledge the sent Publish packet
	if m.PacketID > 0 {
		err = client.AckMessage(m.PacketID)
	}
	return err
}

func (self *Broker) recvPubrecMessage(m *PubrecMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	// acknowledge the sent Publish packet
	err = client.AckMessage(m.PacketID)
	err = client.SendMessage(NewPubrelMessage(m.PacketID))
	return err
}

func (self *Broker) recvPubrelMessage(m *PubrelMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	// acknowledge the sent Pubrel packet
	err = client.AckMessage(m.PacketID)
	err = client.SendMessage(NewPubcompMessage(m.PacketID))
	return err
}

func (self *Broker) recvPubcompMessage(m *PubcompMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	// acknowledge the sent Pubrel packet
	err = client.AckMessage(m.PacketID)
	return err
}

func (self *Broker) recvSubscribeMessage(m *SubscribeMessage, addr *net.UDPAddr) (err error) {
	// TODO: check The wild card is permitted
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	returnCodes := make([]SubscribeReturnCode, 0)
	for _, subTopic := range m.SubscribeTopics {
		// TODO: need to validate wheter there are same topics or not
		edges, err := self.TopicRoot.GetTopicNodes(subTopic.Topic)
		codes := make([]SubscribeReturnCode, len(edges))
		if err != nil || !ok {
			for i, _ := range codes {
				codes[i] = SubscribeFailure
			}
		} else {
			for i, edge := range edges {
				edge.Subscribers[client.ID] = subTopic.QoS
				codes[i] = SubscribeReturnCode(subTopic.QoS)
				client.SubTopics = append(client.SubTopics,
					SubscribeTopic{SubscribeAck,
						edge.FullPath,
						uint8(subTopic.QoS),
					})
				if len(edge.RetainMessage) > 0 {
					// publish retain
					// TODO: check all arguments
					err = client.SendMessage(NewPublishMessage(false, edge.RetainQoS, true,
						edge.FullPath, m.PacketID, []uint8(edge.RetainMessage)))
					// TODO: error validation
				}
			}
		}
		returnCodes = append(returnCodes, codes...)
	}
	// TODO: check whether the number of return codes are correct?
	err = client.SendMessage(NewSubackMessage(m.PacketID, returnCodes))
	return err
}

func (self *Broker) recvSubackMessage(m *SubackMessage, addr *net.UDPAddr) (err error) {
	return INVALID_MESSAGE_CAME
}
func (self *Broker) recvUnsubscribeMessage(m *UnsubscribeMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	if len(m.TopicNames) == 0 {
		// protocol violation
	}
	// TODO: optimize here
	result := []SubscribeTopic{}
	for _, t := range client.SubTopics {
		del := false
		for _, name := range m.TopicNames {
			if string(t.Topic) == string(name) {
				del = true
			}
		}
		if !del {
			result = append(result, t)
		}
	}
	client.SubTopics = result
	err = client.SendMessage(NewUnsubackMessage(m.PacketID))
	return err
}
func (self *Broker) recvUnsubackMessage(m *UnsubackMessage, addr *net.UDPAddr) (err error) {
	return INVALID_MESSAGE_CAME
}

func (self *Broker) recvPingreqMessage(m *PingreqMessage, addr *net.UDPAddr) (err error) {
	// Pingresp
	// TODO: calc elapsed time from previous pingreq.
	//       and store the time to duration of Transport
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	err = client.SendMessage(NewPingrespMessage())
	if client.KeepAlive != 0 {
		client.ResetTimer()
		go self.RunClientTimer(client)
	}
	return err
}

func (self *Broker) recvPingrespMessage(m *PingrespMessage, addr *net.UDPAddr) (err error) {
	return INVALID_MESSAGE_CAME
}

func (self *Broker) recvDisconnectMessage(m *DisconnectMessage, addr *net.UDPAddr) (err error) {
	client, ok := self.Clients[addr]
	if !ok {
		return CLIENT_NOT_EXIST
	}
	self.DisconnectFromBroker(client)
	// close the client
	return err
}

func (self *Broker) ReadMessageFrom() (Message, *net.UDPAddr, error) {
	return self.Bt.ReadMessageFrom()

}
<|endoftext|>


<commit_before>package queue

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"github.com/FoundationDB/fdb-go/fdb"
	"github.com/FoundationDB/fdb-go/fdb/tuple"
	"github.com/happypancake/go-layers/subspace"
	"time"
)

type Queue struct {
	Subspace       subspace.Subspace
	HighContention bool
	conflictedPop  subspace.Subspace // stores int64 index, randId []byte
	conflictedItem subspace.Subspace
	queueItem      subspace.Subspace
}

func New(sub subspace.Subspace, highContention bool) Queue {

	conflict := sub.Item(tuple.Tuple{"conflict"})
	pop := sub.Item(tuple.Tuple{"pop"})
	item := sub.Item(tuple.Tuple{"item"})

	return Queue{sub, highContention, pop, conflict, item}
}

func (queue *Queue) Clear(tr fdb.Transaction) {

	tr.ClearRange(queue.Subspace.FullRange())
}

func (queue *Queue) Peek(tr fdb.Transaction) (value []byte, ok bool) {
	if val, ok := queue.getFirstItem(tr); ok {
		return decodeValue(val.Value), true
	}
	return

}

func decodeValue(val []byte) []byte {
	if t, err := tuple.Unpack(val); err != nil {
		panic(err)
	} else {
		return t[0].([]byte)
	}

}
func encodeValue(value []byte) []byte {
	return tuple.Tuple{value}.Pack()
}

type KeyReader interface {
	GetKey(key fdb.Selectable) fdb.FutureKey
}

// to make private
func (queue *Queue) GetNextIndex(tr KeyReader, sub subspace.Subspace) int64 {

	r := sub.Range(tuple.Tuple{})

	key := tr.GetKey(fdb.LastLessThan(r.End)).GetOrPanic()

	if i := bytes.Compare(key, []byte(r.BeginKey())); i < 0 {
		return 0
	}

	if t, err := sub.Unpack(key); err != nil {
		panic("Failed to unpack key")
	} else {
		return t[0].(int64) + 1
	}
}

func (queue *Queue) GetNextQueueIndex(tr fdb.Transaction) int64 {
	return queue.GetNextIndex(tr.Snapshot(), queue.queueItem)
}
func (queue *Queue) Push(tr fdb.Transaction, value []byte) {
	snap := tr.Snapshot()
	index := queue.GetNextIndex(snap, queue.queueItem)
	queue.pushAt(tr, value, index)
}

// Pop the next item from the queue. Cannot be composed with other functions in a single transaction.
func (queue *Queue) Pop(db fdb.Database) (value []byte, ok bool) {

	if queue.HighContention {
		if result, ok := queue.popHighContention(db); ok {
			return decodeValue(result), true
		}
	} else {

		val, _ := db.Transact(func(tr fdb.Transaction) (interface{}, error) {
			if result, ok := queue.popSimple(tr); ok {
				return decodeValue(result), nil
			}
			return nil, nil

		})
		if val != nil {
			return val.([]byte), true
		}

	}
	return
}

func (queue *Queue) pushAt(tr fdb.Transaction, value []byte, index int64) {
	key := queue.queueItem.Pack(tuple.Tuple{index, nextRandom()})
	val := encodeValue(value)

	tr.Set(fdb.Key(key), val)
}

// popSimple does not attempt to avoid conflicts
// if many clients are trying to pop simultaneously, only one will be able to succeed at a time.
func (queue *Queue) popSimple(tr fdb.Transaction) (value []byte, ok bool) {
	if kv, ok := queue.getFirstItem(tr); ok {
		tr.Clear(kv.Key)
		return kv.Value, true
	}

	return
}

func (queue *Queue) addConflictedPop(tr fdb.Transaction, forced bool) (val []byte) {
	index := queue.GetNextIndex(tr.Snapshot(), queue.conflictedPop)

	if (index == 0) && (!forced) {
		return nil
	}
	key := queue.conflictedPop.Pack(tuple.Tuple{index, nextRandom()})
	// why do we read no
	_ = tr.Get(fdb.Key(key))
	tr.Set(fdb.Key(key), []byte(""))
	return key
}

// popHighContention attempts to avoid collisions by registering
// itself in a semi-ordered set of poppers if it doesn't initially succeed.
// It then enters a polling loop where it attempts to fulfill outstanding pops
// and then checks to see if it has been fulfilled.
func (queue *Queue) popHighContention(db fdb.Database) (value []byte, ok bool) {
	//panic("Not implemented")
	backoff := 0.01

	tr, err := db.CreateTransaction()

	if err != nil {
		panic(err)
	}

	// Check if there are other people waiting to be popped. If so, we
	// cannot pop before them.

	waitKey := queue.addConflictedPop(tr, false)
	if waitKey == nil {
		value, ok := queue.popSimple(tr)

		// if we managed to commit without collisions
		if err := tr.Commit().GetWithError(); err == nil {

			return value, ok
		}
	}

	if err := tr.Commit().GetWithError(); err != nil {
		fmt.Println("Panic in #", err)
	}

	if waitKey == nil {
		waitKey = queue.addConflictedPop(tr, true)
	}

	randId := queue.conflictedPop.MustUnpack(waitKey)[1].([]byte)
	// The result of the pop will be stored at this key once it has been fulfilled
	resultKey := queue.conflictedItemKey(randId)

	tr.Reset()

	for {
		for done := queue.fulfilConflictedPops(db); !done; {

		}

		tr.Reset()
		value := tr.Get(fdb.Key(waitKey))
		result := tr.Get(fdb.Key(resultKey))

		// If waitKey is present, then we have not been fulfilled
		if value.IsReady() {
			time.Sleep(time.Duration(backoff) * time.Second)
			backoff = backoff * 2
			if backoff > 1 {
				backoff = 1
			}
			continue
		}
		if !result.IsReady() {
			return nil, false
		}
		tr.Clear(fdb.Key(resultKey))
		tr.Commit().BlockUntilReady()

		return result.GetOrPanic(), true

	}

	return nil, false
}

func (queue *Queue) getWaitingPops(tr fdb.Transaction, numPops int) fdb.RangeResult {
	r := queue.conflictedPop.FullRange()
	return tr.GetRange(r, fdb.RangeOptions{Limit: numPops})
}

func (queue *Queue) getItems(tr fdb.Transaction, numPops int) fdb.RangeResult {
	r := queue.queueItem.FullRange()
	return tr.GetRange(r, fdb.RangeOptions{Limit: numPops})
}

func minLength(a, b []fdb.KeyValue) int {
	if al, bl := len(a), len(b); al < bl {
		return al
	} else {
		return bl
	}

}

func (queue *Queue) conflictedItemKey(subkey []byte) []byte {
	return queue.conflictedItem.Pack(tuple.Tuple{subkey})
}

func (queue *Queue) fulfilConflictedPops(db fdb.Database) bool {
	numPops := 100

	v, err := db.Transact(func(tr fdb.Transaction) (interface{}, error) {
		pops := queue.getWaitingPops(tr, numPops).GetSliceOrPanic()
		items := queue.getItems(tr, numPops).GetSliceOrPanic()

		min := minLength(pops, items)

		for i := 0; i < min; i++ {
			pop, k, v := pops[i], items[i].Key, items[i].Value

			key := queue.conflictedPop.MustUnpack(pop.Key)
			storageKey := queue.conflictedItemKey(key[1].([]byte))
			tr.Set(fdb.Key(storageKey), v)
			_ = tr.Get(k)
			_ = tr.Get(pop.Key)
			tr.Clear(pop.Key)
			tr.Clear(k)
		}

		for _, pop := range pops[min:] {
			_ = tr.Get(pop.Key)
			tr.Clear(pop.Key)
		}

		return len(pops) < numPops, nil

	})

	if err != nil {
		panic(err)
	}

	return v.(bool)
}

func nextRandom() []byte {
	b := make([]byte, 20)
	if _, err := rand.Read(b); err == nil {
		return b
	} else {

		panic(err)
	}
}

func (queue *Queue) Empty(tr fdb.Transaction) bool {
	_, ok := queue.getFirstItem(tr)
	return ok == false
}

func (queue *Queue) getFirstItem(tr fdb.Transaction) (kv fdb.KeyValue, ok bool) {
	r := queue.queueItem.FullRange()
	opt := fdb.RangeOptions{Limit: 1}

	if kvs := tr.GetRange(r, opt).GetSliceOrPanic(); len(kvs) == 1 {
		return kvs[0], true
	}
	return
}
<commit_msg>Comments to the queue class<commit_after>/*
Package queue provides a high-contention queue class. It is a part of
FoundationDb layer.

Queue has two operating modes. The high contention mode (default) is
designed for environments where many clients will be popping the queue
simultaneously. Pop operations in this mode are slower when performed in
isolation, but their performance scales much better with the number of
popping clients.

If high contention mode is off, then no attempt will be made to avoid
transaction conflicts in pop operations. This mode performs well with
only one popping client, but will not scale well to many popping clients.

This code is a port from official python layer
*/

package queue

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"github.com/FoundationDB/fdb-go/fdb"
	"github.com/FoundationDB/fdb-go/fdb/tuple"
	"github.com/happypancake/go-layers/subspace"
	"time"
)

type Queue struct {
	Subspace       subspace.Subspace
	HighContention bool
	conflictedPop  subspace.Subspace // stores int64 index, randId []byte
	conflictedItem subspace.Subspace
	queueItem      subspace.Subspace
}

// New queue is created within a given subspace
func New(sub subspace.Subspace, highContention bool) Queue {

	conflict := sub.Item(tuple.Tuple{"conflict"})
	pop := sub.Item(tuple.Tuple{"pop"})
	item := sub.Item(tuple.Tuple{"item"})

	return Queue{sub, highContention, pop, conflict, item}
}

// Clear all items from the queue
func (queue *Queue) Clear(tr fdb.Transaction) {
	tr.ClearRange(queue.Subspace.FullRange())
}

// Peek at value of the next item without popping it
func (queue *Queue) Peek(tr fdb.Transaction) (value []byte, ok bool) {
	if val, ok := queue.getFirstItem(tr); ok {
		return decodeValue(val.Value), true
	}
	return

}

func decodeValue(val []byte) []byte {
	if t, err := tuple.Unpack(val); err != nil {
		panic(err)
	} else {
		return t[0].([]byte)
	}

}
func encodeValue(value []byte) []byte {
	return tuple.Tuple{value}.Pack()
}

type KeyReader interface {
	GetKey(key fdb.Selectable) fdb.FutureKey
}

// to make private
func (queue *Queue) GetNextIndex(tr KeyReader, sub subspace.Subspace) int64 {

	r := sub.Range(tuple.Tuple{})

	key := tr.GetKey(fdb.LastLessThan(r.End)).GetOrPanic()

	if i := bytes.Compare(key, []byte(r.BeginKey())); i < 0 {
		return 0
	}

	if t, err := sub.Unpack(key); err != nil {
		panic("Failed to unpack key")
	} else {
		return t[0].(int64) + 1
	}
}

func (queue *Queue) GetNextQueueIndex(tr fdb.Transaction) int64 {
	return queue.GetNextIndex(tr.Snapshot(), queue.queueItem)
}

// Push a single item onto the queue
func (queue *Queue) Push(tr fdb.Transaction, value []byte) {
	snap := tr.Snapshot()
	index := queue.GetNextIndex(snap, queue.queueItem)
	queue.pushAt(tr, value, index)
}

// Pop the next item from the queue. Cannot be composed with other functions
// in a single transaction.
func (queue *Queue) Pop(db fdb.Database) (value []byte, ok bool) {

	if queue.HighContention {
		if result, ok := queue.popHighContention(db); ok {
			return decodeValue(result), true
		}
	} else {

		val, _ := db.Transact(func(tr fdb.Transaction) (interface{}, error) {
			if result, ok := queue.popSimple(tr); ok {
				return decodeValue(result), nil
			}
			return nil, nil

		})
		if val != nil {
			return val.([]byte), true
		}

	}
	return
}

// pushAt inserts item in the queue at (index, randomId) position. Items
// pushed at the same time will have the same index, and so their ordering
// will be random
// This makes pushes fast and usually conflict free (unless the queue becomes)
// empty during the push
func (queue *Queue) pushAt(tr fdb.Transaction, value []byte, index int64) {
	key := queue.queueItem.Pack(tuple.Tuple{index, nextRandom()})
	val := encodeValue(value)

	tr.Set(fdb.Key(key), val)
}

// popSimple gets the message without trying to avoid conflicts
// if many clients are trying to pop simultaneously, only one will be able to
// succeed at a time.
func (queue *Queue) popSimple(tr fdb.Transaction) (value []byte, ok bool) {
	if kv, ok := queue.getFirstItem(tr); ok {
		tr.Clear(kv.Key)
		return kv.Value, true
	}

	return
}

func (queue *Queue) addConflictedPop(tr fdb.Transaction, forced bool) (val []byte) {
	index := queue.GetNextIndex(tr.Snapshot(), queue.conflictedPop)

	if (index == 0) && (!forced) {
		return nil
	}
	key := queue.conflictedPop.Pack(tuple.Tuple{index, nextRandom()})
	// why do we read no
	_ = tr.Get(fdb.Key(key))
	tr.Set(fdb.Key(key), []byte(""))
	return key
}

func errIsCommitFailure(e error) bool {
	if nil != e {
		if f, ok := e.(fdb.Error); ok {
			if f == fdb.Error(1020) {
				return true
			}
		}
	}
	return false
}

// popHighContention attempts to avoid collisions by registering
// itself in a semi-ordered set of poppers if it doesn't initially succeed.
// It then enters a polling loop where it attempts to fulfill outstanding pops
// and then checks to see if it has been fulfilled.
func (queue *Queue) popHighContention(db fdb.Database) (value []byte, ok bool) {
	//panic("Not implemented")
	backoff := 0.01

	tr, err := db.CreateTransaction()

	if err != nil {
		panic(err)
	}

	// Check if there are other people waiting to be popped. If so, we
	// cannot pop before them.

	waitKey := queue.addConflictedPop(tr, false)
	if waitKey == nil {
		value, ok := queue.popSimple(tr)

		// if we managed to commit without collisions

		if err := tr.Commit().GetWithError(); err == nil {
			return value, ok
		} else {
			if !errIsCommitFailure(err) {
				panic(err)
			}
		}

	}

	if err := tr.Commit().GetWithError(); err != nil {
		fmt.Println("Panic in #", err)
	}

	if waitKey == nil {
		waitKey = queue.addConflictedPop(tr, true)
	}

	randId := queue.conflictedPop.MustUnpack(waitKey)[1].([]byte)
	// The result of the pop will be stored at this key once it has been fulfilled
	resultKey := queue.conflictedItemKey(randId)

	tr.Reset()

	for {
		for done := queue.fulfilConflictedPops(db); !done; {

		}

		tr.Reset()
		value := tr.Get(fdb.Key(waitKey))
		result := tr.Get(fdb.Key(resultKey))

		// If waitKey is present, then we have not been fulfilled
		if value.IsReady() {
			time.Sleep(time.Duration(backoff) * time.Second)
			backoff = backoff * 2
			if backoff > 1 {
				backoff = 1
			}
			continue
		}
		if !result.IsReady() {
			return nil, false
		}
		tr.Clear(fdb.Key(resultKey))
		tr.Commit().BlockUntilReady()

		return result.GetOrPanic(), true

	}

	return nil, false
}

func (queue *Queue) getWaitingPops(tr fdb.Transaction, numPops int) fdb.RangeResult {
	r := queue.conflictedPop.FullRange()
	return tr.GetRange(r, fdb.RangeOptions{Limit: numPops})
}

func (queue *Queue) getItems(tr fdb.Transaction, numPops int) fdb.RangeResult {
	r := queue.queueItem.FullRange()
	return tr.GetRange(r, fdb.RangeOptions{Limit: numPops})
}

func minLength(a, b []fdb.KeyValue) int {
	if al, bl := len(a), len(b); al < bl {
		return al
	} else {
		return bl
	}

}

func (queue *Queue) conflictedItemKey(subkey []byte) []byte {
	return queue.conflictedItem.Pack(tuple.Tuple{subkey})
}

func (queue *Queue) fulfilConflictedPops(db fdb.Database) bool {
	numPops := 100

	v, err := db.Transact(func(tr fdb.Transaction) (interface{}, error) {
		pops := queue.getWaitingPops(tr, numPops).GetSliceOrPanic()
		items := queue.getItems(tr, numPops).GetSliceOrPanic()

		min := minLength(pops, items)

		for i := 0; i < min; i++ {
			pop, k, v := pops[i], items[i].Key, items[i].Value

			key := queue.conflictedPop.MustUnpack(pop.Key)
			storageKey := queue.conflictedItemKey(key[1].([]byte))
			tr.Set(fdb.Key(storageKey), v)
			_ = tr.Get(k)
			_ = tr.Get(pop.Key)
			tr.Clear(pop.Key)
			tr.Clear(k)
		}

		for _, pop := range pops[min:] {
			_ = tr.Get(pop.Key)
			tr.Clear(pop.Key)
		}

		return len(pops) < numPops, nil

	})

	if err != nil {
		panic(err)
	}

	return v.(bool)
}

func nextRandom() []byte {
	b := make([]byte, 20)
	if _, err := rand.Read(b); err == nil {
		return b
	} else {

		panic(err)
	}
}

// Empty returns true is queue does not have any messages
func (queue *Queue) Empty(tr fdb.Transaction) bool {
	_, ok := queue.getFirstItem(tr)
	return ok == false
}

func (queue *Queue) getFirstItem(tr fdb.Transaction) (kv fdb.KeyValue, ok bool) {
	r := queue.queueItem.FullRange()
	opt := fdb.RangeOptions{Limit: 1}

	if kvs := tr.GetRange(r, opt).GetSliceOrPanic(); len(kvs) == 1 {
		return kvs[0], true
	}
	return
}
<|endoftext|>