<commit_before>package core_test

import (
	"fmt"
	"os"
	"reflect"
	"sort"
	"strconv"
	"testing"
	"time"

	"veyron.io/veyron/veyron2/naming"
	"veyron.io/veyron/veyron2/rt"

	"veyron.io/veyron/veyron/lib/expect"
	"veyron.io/veyron/veyron/lib/modules"
	"veyron.io/veyron/veyron/lib/modules/core"
	_ "veyron.io/veyron/veyron/lib/testutil"
)

func TestCommands(t *testing.T) {
	shell := core.NewShell()
	defer shell.Cleanup(os.Stderr)
	for _, c := range []string{core.RootMTCommand, core.MTCommand} {
		if len(shell.Help(c)) == 0 {
			t.Fatalf("missing command %q", c)
		}
	}
}

func init() {
	rt.Init()
}

func TestRoot(t *testing.T) {
	shell := core.NewShell()
	defer shell.Cleanup(os.Stderr)

	root, err := shell.Start(core.RootMTCommand)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	s := expect.NewSession(t, root.Stdout(), time.Second)
	s.ExpectVar("MT_NAME")
	s.ExpectVar("PID")
	root.CloseStdin()
	s.Expect("PASS")
}

func startMountTables(t *testing.T, sh *modules.Shell, mountPoints ...string) (map[string]string, error) {
	// Start root mount table
	root, err := sh.Start(core.RootMTCommand)
	if err != nil {
		t.Fatalf("unexpected error for root mt: %s", err)
	}
	rootSession := expect.NewSession(t, root.Stdout(), time.Minute)
	rootName := rootSession.ExpectVar("MT_NAME")
	if t.Failed() {
		return nil, rootSession.Error()
	}
	sh.SetVar("NAMESPACE_ROOT", rootName)
	mountAddrs := make(map[string]string)
	mountAddrs["root"] = rootName

	// Start the mount tables
	for _, mp := range mountPoints {
		h, err := sh.Start(core.MTCommand, mp)
		if err != nil {
			return nil, fmt.Errorf("unexpected error for mt %q: %s", mp, err)
		}
		s := expect.NewSession(t, h.Stdout(), time.Minute)
		// Wait until each mount table has at least called Serve to
		// mount itself.
		mountAddrs[mp] = s.ExpectVar("MT_NAME")
		if s.Failed() {
			return nil, s.Error()
		}
	}
	return mountAddrs, nil

}

func getMatchingMountpoint(r [][]string) string {
	if len(r) != 1 {
		return ""
	}
	shortest := ""
	for _, p := range r[0][1:] {
		if len(p) > 0 {
			if len(shortest) == 0 {
				shortest = p
			}
			if len(shortest) > 0 && len(p) < len(shortest) {
				shortest = p
			}
		}
	}
	return shortest
}

func TestMountTableAndGlob(t *testing.T) {
	shell := core.NewShell()
	if testing.Verbose() {
		defer shell.Cleanup(os.Stderr)
	} else {
		defer shell.Cleanup(nil)
	}

	mountPoints := []string{"a", "b", "c", "d", "e"}
	mountAddrs, err := startMountTables(t, shell, mountPoints...)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	rootName := mountAddrs["root"]
	ls, err := shell.Start(core.LSCommand, rootName+"/...")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	lsSession := expect.NewSession(t, ls.Stdout(), time.Minute)
	lsSession.SetVerbosity(testing.Verbose())

	if got, want := lsSession.ExpectVar("RN"), strconv.Itoa(len(mountPoints)+1); got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	lsSession.Expect("R0=" + rootName)

	// Look for names that correspond to the mountpoints above (i.e, a, b or c)
	pattern := ""
	for _, n := range mountPoints {
		pattern = pattern + "^R[\\d]+=(" + rootName + "/(" + n + ")$)|"
	}
	pattern = pattern[:len(pattern)-1]

	found := []string{}
	for i := 0; i < len(mountPoints); i++ {
		found = append(found, getMatchingMountpoint(lsSession.ExpectRE(pattern, 1)))
	}
	sort.Strings(found)
	sort.Strings(mountPoints)
	if got, want := found, mountPoints; !reflect.DeepEqual(got, want) {
		t.Errorf("got %v, want %v", got, want)
	}

	// Run the ls command in a subprocess, with NAMESPACE_ROOT as set above.
	lse, err := shell.Start(core.LSExternalCommand, "...")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	lseSession := expect.NewSession(t, lse.Stdout(), time.Minute)
	lseSession.SetVerbosity(testing.Verbose())

	if got, want := lseSession.ExpectVar("RN"), strconv.Itoa(len(mountPoints)); got != want {
		t.Fatalf("got %v, want %v", got, want)
	}

	pattern = ""
	for _, n := range mountPoints {
		// Since the LSExternalCommand runs in a subprocess with NAMESPACE_ROOT
		// set to the name of the root mount table it sees to the relative name
		// format of the mounted mount tables.
		pattern = pattern + "^R[\\d]+=(" + n + "$)|"
	}
	pattern = pattern[:len(pattern)-1]
	found = []string{}
	for i := 0; i < len(mountPoints); i++ {
		found = append(found, getMatchingMountpoint(lseSession.ExpectRE(pattern, 1)))
	}
	sort.Strings(found)
	sort.Strings(mountPoints)
	if got, want := found, mountPoints; !reflect.DeepEqual(got, want) {
		t.Errorf("got %v, want %v", got, want)
	}
}

func TestEcho(t *testing.T) {
	shell := core.NewShell()
	if testing.Verbose() {
		defer shell.Cleanup(os.Stderr)
	} else {
		defer shell.Cleanup(nil)
	}
	srv, err := shell.Start(core.EchoServerCommand, "test", "")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	srvSession := expect.NewSession(t, srv.Stdout(), time.Minute)
	name := srvSession.ExpectVar("NAME")
	if len(name) == 0 {
		t.Fatalf("failed to get name")
	}

	clt, err := shell.Start(core.EchoClientCommand, name, "a message")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	cltSession := expect.NewSession(t, clt.Stdout(), time.Minute)
	cltSession.Expect("test: a message")
}

func TestResolve(t *testing.T) {
	shell := core.NewShell()
	if testing.Verbose() {
		defer shell.Cleanup(os.Stderr)
	} else {
		defer shell.Cleanup(nil)
	}
	mountPoints := []string{"a", "b"}
	mountAddrs, err := startMountTables(t, shell, mountPoints...)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	rootName := mountAddrs["root"]
	mtName := "b"
	echoName := naming.Join(mtName, "echo")
	srv, err := shell.Start(core.EchoServerCommand, "test", echoName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	srvSession := expect.NewSession(t, srv.Stdout(), time.Minute)
	srvSession.ExpectVar("NAME")
	addr := srvSession.ExpectVar("ADDR")
	addr = naming.JoinAddressName(addr, "//")

	// Resolve an object
	resolver, err := shell.Start(core.ResolveCommand, rootName+"/"+echoName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	resolverSession := expect.NewSession(t, resolver.Stdout(), time.Minute)
	if got, want := resolverSession.ExpectVar("RN"), "1"; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if got, want := resolverSession.ExpectVar("R0"), addr; got != want {
		t.Errorf("got %v, want %v", got, want)
	}
	if err = resolver.Shutdown(nil); err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	// Resolve to a mount table using a rooted name.
	addr = naming.JoinAddressName(mountAddrs[mtName], "//echo")
	resolver, err = shell.Start(core.ResolveMTCommand, rootName+"/"+echoName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	resolverSession = expect.NewSession(t, resolver.Stdout(), time.Minute)
	if got, want := resolverSession.ExpectVar("RN"), "1"; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if got, want := resolverSession.ExpectVar("R0"), addr; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if err := resolver.Shutdown(nil); err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	// Resolve to a mount table, but using a relative name.
	nsroots, err := shell.Start(core.SetNamespaceRootsCommand, rootName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	if err := nsroots.Shutdown(nil); err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	resolver, err = shell.Start(core.ResolveMTCommand, echoName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	resolverSession = expect.NewSession(t, resolver.Stdout(), time.Minute)
	if got, want := resolverSession.ExpectVar("RN"), "1"; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if got, want := resolverSession.ExpectVar("R0"), addr; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if err := resolver.Shutdown(nil); err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
}

func TestHelperProcess(t *testing.T) {
	if !modules.IsTestHelperProcess() {
		return
	}
	if err := modules.Dispatch(); err != nil {
		t.Fatalf("failed: %v", err)
	}
}
<commit_msg>veyron/lib/modules/core: generate VEYRON_IDENTITY file.<commit_after>package core_test

import (
	"fmt"
	"os"
	"reflect"
	"sort"
	"strconv"
	"testing"
	"time"

	"veyron.io/veyron/veyron2/naming"
	"veyron.io/veyron/veyron2/rt"

	"veyron.io/veyron/veyron/lib/expect"
	"veyron.io/veyron/veyron/lib/modules"
	"veyron.io/veyron/veyron/lib/modules/core"
	_ "veyron.io/veyron/veyron/lib/testutil"
	"veyron.io/veyron/veyron/lib/testutil/security"
)

func TestCommands(t *testing.T) {
	shell := core.NewShell()
	defer shell.Cleanup(os.Stderr)
	for _, c := range []string{core.RootMTCommand, core.MTCommand} {
		if len(shell.Help(c)) == 0 {
			t.Fatalf("missing command %q", c)
		}
	}
}

func init() {
	rt.Init()
}

func newShell() (*modules.Shell, func()) {
	shell := core.NewShell()
	idpath := security.SaveIdentityToFile(security.NewBlessedIdentity(rt.R().Identity(), "test"))
	shell.SetVar("VEYRON_IDENTITY", idpath)
	return shell, func() {
		os.Remove(idpath)
		if testing.Verbose() {
			shell.Cleanup(os.Stderr)
		} else {
			shell.Cleanup(nil)
		}
	}
}

func TestRoot(t *testing.T) {
	shell, fn := newShell()
	defer fn()
	root, err := shell.Start(core.RootMTCommand)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	s := expect.NewSession(t, root.Stdout(), time.Second)
	s.ExpectVar("MT_NAME")
	s.ExpectVar("PID")
	root.CloseStdin()
	s.Expect("PASS")
}

func startMountTables(t *testing.T, sh *modules.Shell, mountPoints ...string) (map[string]string, error) {
	// Start root mount table
	root, err := sh.Start(core.RootMTCommand)
	if err != nil {
		t.Fatalf("unexpected error for root mt: %s", err)
	}
	rootSession := expect.NewSession(t, root.Stdout(), time.Minute)
	rootName := rootSession.ExpectVar("MT_NAME")
	if t.Failed() {
		return nil, rootSession.Error()
	}
	sh.SetVar("NAMESPACE_ROOT", rootName)
	mountAddrs := make(map[string]string)
	mountAddrs["root"] = rootName

	// Start the mount tables
	for _, mp := range mountPoints {
		h, err := sh.Start(core.MTCommand, mp)
		if err != nil {
			return nil, fmt.Errorf("unexpected error for mt %q: %s", mp, err)
		}
		s := expect.NewSession(t, h.Stdout(), time.Minute)
		// Wait until each mount table has at least called Serve to
		// mount itself.
		mountAddrs[mp] = s.ExpectVar("MT_NAME")
		if s.Failed() {
			return nil, s.Error()
		}
	}
	return mountAddrs, nil

}

func getMatchingMountpoint(r [][]string) string {
	if len(r) != 1 {
		return ""
	}
	shortest := ""
	for _, p := range r[0][1:] {
		if len(p) > 0 {
			if len(shortest) == 0 {
				shortest = p
			}
			if len(shortest) > 0 && len(p) < len(shortest) {
				shortest = p
			}
		}
	}
	return shortest
}

func TestMountTableAndGlob(t *testing.T) {
	shell, fn := newShell()
	defer fn()

	mountPoints := []string{"a", "b", "c", "d", "e"}
	mountAddrs, err := startMountTables(t, shell, mountPoints...)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	rootName := mountAddrs["root"]
	ls, err := shell.Start(core.LSCommand, rootName+"/...")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	lsSession := expect.NewSession(t, ls.Stdout(), time.Minute)
	lsSession.SetVerbosity(testing.Verbose())

	if got, want := lsSession.ExpectVar("RN"), strconv.Itoa(len(mountPoints)+1); got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	lsSession.Expect("R0=" + rootName)

	// Look for names that correspond to the mountpoints above (i.e, a, b or c)
	pattern := ""
	for _, n := range mountPoints {
		pattern = pattern + "^R[\\d]+=(" + rootName + "/(" + n + ")$)|"
	}
	pattern = pattern[:len(pattern)-1]

	found := []string{}
	for i := 0; i < len(mountPoints); i++ {
		found = append(found, getMatchingMountpoint(lsSession.ExpectRE(pattern, 1)))
	}
	sort.Strings(found)
	sort.Strings(mountPoints)
	if got, want := found, mountPoints; !reflect.DeepEqual(got, want) {
		t.Errorf("got %v, want %v", got, want)
	}

	// Run the ls command in a subprocess, with NAMESPACE_ROOT as set above.
	lse, err := shell.Start(core.LSExternalCommand, "...")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	lseSession := expect.NewSession(t, lse.Stdout(), time.Minute)
	lseSession.SetVerbosity(testing.Verbose())

	if got, want := lseSession.ExpectVar("RN"), strconv.Itoa(len(mountPoints)); got != want {
		t.Fatalf("got %v, want %v", got, want)
	}

	pattern = ""
	for _, n := range mountPoints {
		// Since the LSExternalCommand runs in a subprocess with NAMESPACE_ROOT
		// set to the name of the root mount table it sees to the relative name
		// format of the mounted mount tables.
		pattern = pattern + "^R[\\d]+=(" + n + "$)|"
	}
	pattern = pattern[:len(pattern)-1]
	found = []string{}
	for i := 0; i < len(mountPoints); i++ {
		found = append(found, getMatchingMountpoint(lseSession.ExpectRE(pattern, 1)))
	}
	sort.Strings(found)
	sort.Strings(mountPoints)
	if got, want := found, mountPoints; !reflect.DeepEqual(got, want) {
		t.Errorf("got %v, want %v", got, want)
	}
}

func TestEcho(t *testing.T) {
	shell, fn := newShell()
	defer fn()

	srv, err := shell.Start(core.EchoServerCommand, "test", "")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	srvSession := expect.NewSession(t, srv.Stdout(), time.Minute)
	name := srvSession.ExpectVar("NAME")
	if len(name) == 0 {
		t.Fatalf("failed to get name")
	}

	clt, err := shell.Start(core.EchoClientCommand, name, "a message")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	cltSession := expect.NewSession(t, clt.Stdout(), time.Minute)
	cltSession.Expect("test: a message")
}

func TestResolve(t *testing.T) {
	shell, fn := newShell()
	defer fn()

	mountPoints := []string{"a", "b"}
	mountAddrs, err := startMountTables(t, shell, mountPoints...)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	rootName := mountAddrs["root"]
	mtName := "b"
	echoName := naming.Join(mtName, "echo")
	srv, err := shell.Start(core.EchoServerCommand, "test", echoName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	srvSession := expect.NewSession(t, srv.Stdout(), time.Minute)
	srvSession.ExpectVar("NAME")
	addr := srvSession.ExpectVar("ADDR")
	addr = naming.JoinAddressName(addr, "//")

	// Resolve an object
	resolver, err := shell.Start(core.ResolveCommand, rootName+"/"+echoName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	resolverSession := expect.NewSession(t, resolver.Stdout(), time.Minute)
	if got, want := resolverSession.ExpectVar("RN"), "1"; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if got, want := resolverSession.ExpectVar("R0"), addr; got != want {
		t.Errorf("got %v, want %v", got, want)
	}
	if err = resolver.Shutdown(nil); err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	// Resolve to a mount table using a rooted name.
	addr = naming.JoinAddressName(mountAddrs[mtName], "//echo")
	resolver, err = shell.Start(core.ResolveMTCommand, rootName+"/"+echoName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	resolverSession = expect.NewSession(t, resolver.Stdout(), time.Minute)
	if got, want := resolverSession.ExpectVar("RN"), "1"; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if got, want := resolverSession.ExpectVar("R0"), addr; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if err := resolver.Shutdown(nil); err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	// Resolve to a mount table, but using a relative name.
	nsroots, err := shell.Start(core.SetNamespaceRootsCommand, rootName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	if err := nsroots.Shutdown(nil); err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	resolver, err = shell.Start(core.ResolveMTCommand, echoName)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
	resolverSession = expect.NewSession(t, resolver.Stdout(), time.Minute)
	if got, want := resolverSession.ExpectVar("RN"), "1"; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if got, want := resolverSession.ExpectVar("R0"), addr; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	if err := resolver.Shutdown(nil); err != nil {
		t.Fatalf("unexpected error: %s", err)
	}
}

func TestHelperProcess(t *testing.T) {
	if !modules.IsTestHelperProcess() {
		return
	}
	if err := modules.Dispatch(); err != nil {
		t.Fatalf("failed: %v", err)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
	"bytes"
	"net"
	"strings"

	namespacemodel "github.com/contiv/vpp/plugins/ksr/model/namespace"
	podmodel "github.com/contiv/vpp/plugins/ksr/model/pod"
	policymodel "github.com/contiv/vpp/plugins/ksr/model/policy"
)

// RemoveDuplicates removes duplicates entries for a slice of strings
func RemoveDuplicates(el []string) []string {
	found := map[string]bool{}

	// Create a map of all unique elements.
	for v := range el {
		found[el[v]] = true
	}

	// Place all keys from the map into a slice.
	result := []string{}
	for key := range found {
		result = append(result, key)
	}
	return result
}

// Intersect returns the common elements of or more slices
// Intersect needs at least two lists
func Intersect(a []string, b []string, s ...[]string) []string {
	if len(a) == 0 || len(b) == 0 {
		return []string{}
	}
	set := make([]string, 0)
	hash := make(map[string]bool)
	for _, el := range a {
		hash[el] = true
	}
	for _, el := range b {
		if _, found := hash[el]; found {
			set = append(set, el)
		}
	}
	if len(s) == 0 {
		return set
	}
	return Intersect(set, s[0], s[1:]...)
}

// Difference returns the difference of two slices
func Difference(a []string, b []string) []string {
	diffStr := []string{}
	m := map[string]int{}

	for _, aVal := range a {
		m[aVal] = 1
	}
	for _, bVal := range b {
		m[bVal] = m[bVal] + 1
	}

	for mKey, mVal := range m {
		if mVal == 1 {
			diffStr = append(diffStr, mKey)
		}
	}

	return diffStr
}

// UnstringPodID converts string podIDs to podmodelIDs
func UnstringPodID(pods []string) []podmodel.ID {
	podIDs := []podmodel.ID{}
	for _, pod := range pods {
		parts := strings.Split(pod, "/")
		podID := podmodel.ID{
			Name:      parts[1],
			Namespace: parts[0],
		}
		podIDs = append(podIDs, podID)
	}
	return podIDs
}

// StringPodID converts  podmodelIDs to string podIDs
func StringPodID(pods []podmodel.ID) []string {
	podIDs := []string{}
	for _, pod := range pods {
		podID := pod.Namespace + "/" + pod.Name
		podIDs = append(podIDs, podID)
	}
	return podIDs
}

// UnstringPolicyID converts string policyIDs to policymodelIDs
func UnstringPolicyID(policies []string) []policymodel.ID {
	policyIDs := []policymodel.ID{}
	for _, policy := range policies {
		parts := strings.Split(policy, "/")
		policyID := policymodel.ID{
			Name:      parts[1],
			Namespace: parts[0],
		}
		policyIDs = append(policyIDs, policyID)
	}
	return policyIDs
}

// StringPolicyID converts policymodelIDs to string policyIDs
func StringPolicyID(policies []policymodel.ID) []string {
	policyIDs := []string{}
	for _, policy := range policies {
		policyID := policy.Namespace + "/" + policy.Name
		policyIDs = append(policyIDs, policyID)
	}
	return policyIDs
}

// UnstringNamespaceID converts string namespaceIDs to namespacemodelIDs
func UnstringNamespaceID(namespaces []string) []namespacemodel.ID {
	namespaceIDs := []namespacemodel.ID{}
	for _, namespace := range namespaces {
		namespaceID := namespacemodel.ID(namespace)
		namespaceIDs = append(namespaceIDs, namespaceID)
	}
	return namespaceIDs
}

// ConstructLabels returns a key-value pair as a label given an expression
func ConstructLabels(key string, values []string) []*policymodel.Policy_Label {
	policyLabel := []*policymodel.Policy_Label{}
	for _, label := range values {
		policyLabel = append(policyLabel,
			&policymodel.Policy_Label{
				Key:   key,
				Value: label,
			})
	}
	return policyLabel
}

// CompareInts is a comparison function for two integers.
func CompareInts(a, b int) int {
	if a < b {
		return -1
	}
	if a > b {
		return 1
	}
	return 0
}

// CompareIPNets returns an integer comparing two IP network addresses
// lexicographically.
func CompareIPNets(a, b *net.IPNet) int {
	ipOrder := bytes.Compare(a.IP, b.IP)
	if ipOrder == 0 {
		return bytes.Compare(a.Mask, b.Mask)
	}
	return ipOrder
}

// CompareIPNetsBytes returns an integer comparing two IP network addresses
// represented as raw bytes lexicographically.
func CompareIPNetsBytes(aPrefixLen uint8, aIP [16]byte, bPrefixLen uint8, bIP [16]byte) int {
	prefixOrder := CompareInts(int(aPrefixLen), int(bPrefixLen))
	if prefixOrder != 0 {
		return prefixOrder
	}
	return bytes.Compare(aIP[:], bIP[:])
}

// GetOneHostSubnet returns the IP subnet that contains only the given host
// (i.e. /32 for IPv4, /128 for IPv6).
func GetOneHostSubnet(hostAddr string) *net.IPNet {
	ip := net.ParseIP(hostAddr)
	if ip == nil {
		return nil
	}
	ipNet := &net.IPNet{IP: ip}
	if ip.To4() != nil {
		ipNet.Mask = net.CIDRMask(net.IPv4len*8, net.IPv4len*8)
	} else {
		ipNet.Mask = net.CIDRMask(net.IPv6len*8, net.IPv6len*8)
	}
	return ipNet
}
<commit_msg>Fixed typo on utils.go godoc<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
	"bytes"
	"net"
	"strings"

	namespacemodel "github.com/contiv/vpp/plugins/ksr/model/namespace"
	podmodel "github.com/contiv/vpp/plugins/ksr/model/pod"
	policymodel "github.com/contiv/vpp/plugins/ksr/model/policy"
)

// RemoveDuplicates removes duplicates entries for a slice of strings
func RemoveDuplicates(el []string) []string {
	found := map[string]bool{}

	// Create a map of all unique elements.
	for v := range el {
		found[el[v]] = true
	}

	// Place all keys from the map into a slice.
	result := []string{}
	for key := range found {
		result = append(result, key)
	}
	return result
}

// Intersect returns the common elements of two or more slices
func Intersect(a []string, b []string, s ...[]string) []string {
	if len(a) == 0 || len(b) == 0 {
		return []string{}
	}
	set := make([]string, 0)
	hash := make(map[string]bool)
	for _, el := range a {
		hash[el] = true
	}
	for _, el := range b {
		if _, found := hash[el]; found {
			set = append(set, el)
		}
	}
	if len(s) == 0 {
		return set
	}
	return Intersect(set, s[0], s[1:]...)
}

// Difference returns the difference of two slices
func Difference(a []string, b []string) []string {
	diffStr := []string{}
	m := map[string]int{}

	for _, aVal := range a {
		m[aVal] = 1
	}
	for _, bVal := range b {
		m[bVal] = m[bVal] + 1
	}

	for mKey, mVal := range m {
		if mVal == 1 {
			diffStr = append(diffStr, mKey)
		}
	}

	return diffStr
}

// UnstringPodID converts string podIDs to podmodelIDs
func UnstringPodID(pods []string) []podmodel.ID {
	podIDs := []podmodel.ID{}
	for _, pod := range pods {
		parts := strings.Split(pod, "/")
		podID := podmodel.ID{
			Name:      parts[1],
			Namespace: parts[0],
		}
		podIDs = append(podIDs, podID)
	}
	return podIDs
}

// StringPodID converts  podmodelIDs to string podIDs
func StringPodID(pods []podmodel.ID) []string {
	podIDs := []string{}
	for _, pod := range pods {
		podID := pod.Namespace + "/" + pod.Name
		podIDs = append(podIDs, podID)
	}
	return podIDs
}

// UnstringPolicyID converts string policyIDs to policymodelIDs
func UnstringPolicyID(policies []string) []policymodel.ID {
	policyIDs := []policymodel.ID{}
	for _, policy := range policies {
		parts := strings.Split(policy, "/")
		policyID := policymodel.ID{
			Name:      parts[1],
			Namespace: parts[0],
		}
		policyIDs = append(policyIDs, policyID)
	}
	return policyIDs
}

// StringPolicyID converts policymodelIDs to string policyIDs
func StringPolicyID(policies []policymodel.ID) []string {
	policyIDs := []string{}
	for _, policy := range policies {
		policyID := policy.Namespace + "/" + policy.Name
		policyIDs = append(policyIDs, policyID)
	}
	return policyIDs
}

// UnstringNamespaceID converts string namespaceIDs to namespacemodelIDs
func UnstringNamespaceID(namespaces []string) []namespacemodel.ID {
	namespaceIDs := []namespacemodel.ID{}
	for _, namespace := range namespaces {
		namespaceID := namespacemodel.ID(namespace)
		namespaceIDs = append(namespaceIDs, namespaceID)
	}
	return namespaceIDs
}

// ConstructLabels returns a key-value pair as a label given an expression
func ConstructLabels(key string, values []string) []*policymodel.Policy_Label {
	policyLabel := []*policymodel.Policy_Label{}
	for _, label := range values {
		policyLabel = append(policyLabel,
			&policymodel.Policy_Label{
				Key:   key,
				Value: label,
			})
	}
	return policyLabel
}

// CompareInts is a comparison function for two integers.
func CompareInts(a, b int) int {
	if a < b {
		return -1
	}
	if a > b {
		return 1
	}
	return 0
}

// CompareIPNets returns an integer comparing two IP network addresses
// lexicographically.
func CompareIPNets(a, b *net.IPNet) int {
	ipOrder := bytes.Compare(a.IP, b.IP)
	if ipOrder == 0 {
		return bytes.Compare(a.Mask, b.Mask)
	}
	return ipOrder
}

// CompareIPNetsBytes returns an integer comparing two IP network addresses
// represented as raw bytes lexicographically.
func CompareIPNetsBytes(aPrefixLen uint8, aIP [16]byte, bPrefixLen uint8, bIP [16]byte) int {
	prefixOrder := CompareInts(int(aPrefixLen), int(bPrefixLen))
	if prefixOrder != 0 {
		return prefixOrder
	}
	return bytes.Compare(aIP[:], bIP[:])
}

// GetOneHostSubnet returns the IP subnet that contains only the given host
// (i.e. /32 for IPv4, /128 for IPv6).
func GetOneHostSubnet(hostAddr string) *net.IPNet {
	ip := net.ParseIP(hostAddr)
	if ip == nil {
		return nil
	}
	ipNet := &net.IPNet{IP: ip}
	if ip.To4() != nil {
		ipNet.Mask = net.CIDRMask(net.IPv4len*8, net.IPv4len*8)
	} else {
		ipNet.Mask = net.CIDRMask(net.IPv6len*8, net.IPv6len*8)
	}
	return ipNet
}
<|endoftext|>


<commit_before>package objectclient

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/objectserver"
	"io"
)

func (objClient *ObjectClient) addObject(reader io.Reader, length uint64,
	expectedHash *hash.Hash) (hash.Hash, bool, error) {
	var request objectserver.AddObjectRequest
	var reply objectserver.AddObjectResponse
	if length < 1 {
		return reply.Hash, false, errors.New(
			"zero length object cannot be added")
	}
	srpcClient, err := srpc.DialHTTP("tcp", objClient.address, 0)
	if err != nil {
		return reply.Hash, false, errors.New(
			fmt.Sprintf("Error dialing\t%s\n", err.Error()))
	}
	defer srpcClient.Close()
	conn, err := srpcClient.Call("ObjectServer.AddObjects")
	if err != nil {
		return reply.Hash, false, err
	}
	defer conn.Close()
	request.Length = length
	request.ExpectedHash = expectedHash
	encoder := gob.NewEncoder(conn)
	encoder.Encode(request)
	nCopied, err := io.Copy(conn, reader)
	if err != nil {
		return reply.Hash, false, err
	}
	if uint64(nCopied) != length {
		return reply.Hash, false, errors.New(fmt.Sprintf(
			"failed to copy, wanted: %d, got: %d bytes", length, nCopied))
	}
	conn.Flush()
	decoder := gob.NewDecoder(conn)
	if err := decoder.Decode(&reply); err != nil {
		return reply.Hash, false, err
	}
	if reply.Error != nil {
		return reply.Hash, false, err
	}
	if expectedHash != nil && *expectedHash != reply.Hash {
		return reply.Hash, false, errors.New(fmt.Sprintf(
			"received hash: %x != expected: %x",
			reply.Hash, *expectedHash))
	}
	return reply.Hash, reply.Added, nil
}
<commit_msg>Fix bug in objectclient.AddObject(): end-of-stream marker was not sent.<commit_after>package objectclient

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/objectserver"
	"io"
)

func (objClient *ObjectClient) addObject(reader io.Reader, length uint64,
	expectedHash *hash.Hash) (hash.Hash, bool, error) {
	var request objectserver.AddObjectRequest
	var reply objectserver.AddObjectResponse
	if length < 1 {
		return reply.Hash, false, errors.New(
			"zero length object cannot be added")
	}
	srpcClient, err := srpc.DialHTTP("tcp", objClient.address, 0)
	if err != nil {
		return reply.Hash, false, errors.New(
			fmt.Sprintf("Error dialing\t%s\n", err.Error()))
	}
	defer srpcClient.Close()
	conn, err := srpcClient.Call("ObjectServer.AddObjects")
	if err != nil {
		return reply.Hash, false, err
	}
	defer conn.Close()
	request.Length = length
	request.ExpectedHash = expectedHash
	encoder := gob.NewEncoder(conn)
	encoder.Encode(request)
	nCopied, err := io.Copy(conn, reader)
	if err != nil {
		return reply.Hash, false, err
	}
	if uint64(nCopied) != length {
		return reply.Hash, false, errors.New(fmt.Sprintf(
			"failed to copy, wanted: %d, got: %d bytes", length, nCopied))
	}
	// Send end-of-stream marker.
	request = objectserver.AddObjectRequest{}
	encoder.Encode(request)
	conn.Flush()
	decoder := gob.NewDecoder(conn)
	if err := decoder.Decode(&reply); err != nil {
		return reply.Hash, false, err
	}
	if reply.Error != nil {
		return reply.Hash, false, err
	}
	if expectedHash != nil && *expectedHash != reply.Hash {
		return reply.Hash, false, errors.New(fmt.Sprintf(
			"received hash: %x != expected: %x",
			reply.Hash, *expectedHash))
	}
	return reply.Hash, reply.Added, nil
}
<|endoftext|>


<commit_before><commit_msg>Change the base test match kind to be qualifying<commit_after><|endoftext|>


<commit_before>package transmit

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/binary"
	"errors"
	"hash/adler32"
	"io"
	"io/ioutil"
	"log"
	"net"
	"sync"
	"sync/atomic"

	"github.com/midbel/uuid"
)

var ErrClosed = errors.New("closed")

var (
	ErrEmpty     = errors.New("empty")
	ErrCorrupted = errors.New("packet corrupted")
	ErrUnknownId = errors.New("unknown packet id")
)

const (
	Size    = 26
	Padding = 512
)

const (
	Bind uint16 = iota
	Accept
	Reject
	Copy
	Abort
	Done
)

type Route struct {
	Id   string `json:"id"`
	Addr string `json:"addr"`
	Eth  string `json:"ifi"`
}

func Subscribe(a, n string) (net.Conn, error) {
	addr, err := net.ResolveUDPAddr("udp", a)
	if err != nil {
		return nil, err
	}
	i, err := net.InterfaceByName(n)
	if err != nil && len(n) > 0 {
		return nil, err
	}
	c, err := net.ListenMulticastUDP("udp", i, addr)
	if err != nil {
		return nil, err
	}
	return &subscriber{c}, nil
}

func Dispatch(a string) (net.Conn, error) {
	c, err := net.Dial("udp", a)
	if err != nil {
		return nil, err
	}
	return &subscriber{c}, nil
}

func Forward(a, s string) (net.Conn, error) {
	c, err := net.Dial("tcp", a)
	if err != nil {
		return nil, err
	}
	id, _ := uuid.UUID5(uuid.URL, []byte(s))

	f := &forwarder{
		Conn:    c,
		id:      id.Bytes(),
		padding: Padding,
		reader:  bufio.NewReaderSize(rand.Reader, 4096),
	}
	if _, err := f.Write(f.id); err != nil {
		f.Close()
		return nil, err
	}
	return f, nil
}

type Router struct {
	net.Listener

	routes map[string]*pool
}

func (r *Router) Accept() (net.Conn, net.Conn, error) {
	c, err := r.Listener.Accept()
	if err != nil {
		return nil, nil, err
	}
	id := make([]byte, Size+uuid.Size)
	if _, err := io.ReadFull(c, id); err != nil {
		c.Close()
		return nil, nil, err
	} else {
		id = id[:uuid.Size]
	}
	p, ok := r.routes[string(id)]
	if !ok {
		c.Close()
		return nil, nil, ErrUnknownId
	}
	io.CopyN(ioutil.Discard, c, Padding)
	w, err := p.Acquire()
	if err != nil {
		return nil, nil, err
	}
	return &forwarder{Conn: c, id: id}, w, nil
}

func (r *Router) Close() error {
	err := r.Listener.Close()
	for _, p := range r.routes {
		if e := p.Close(); err == nil && e != nil {
			err = e
		}
	}
	return err
}

func NewRouter(a string, rs []Route) (*Router, error) {
	l, err := net.Listen("tcp", a)
	if err != nil {
		return nil, err
	}
	gs := make(map[string]*pool)
	for _, r := range rs {
		id, _ := uuid.UUID5(uuid.URL, []byte(r.Id))
		p := &pool{
			c: make(chan net.Conn, 5),
			a: r.Addr,
		}
		gs[string(id.Bytes())] = p
	}
	return &Router{Listener: l, routes: gs}, nil
}

type subscriber struct {
	net.Conn
}

func (s *subscriber) Read(b []byte) (int, error) {
	d := make([]byte, len(b))
	r, err := s.Conn.Read(d)
	if err != nil && r == 0 {
		return r, err
	} else {
		d = d[:r]
		go log.Printf("%d bytes read from %s (%x)", r, s.LocalAddr(), md5.Sum(d))
	}
	sum := make([]byte, 4)
	binary.BigEndian.PutUint32(sum, adler32.Checksum(d))
	d = append(d, sum...)

	return copy(b, d), err
}

func (s *subscriber) Write(b []byte) (int, error) {
	d, sum := b[:len(b)-adler32.Size], b[len(b)-adler32.Size:]
	if a := adler32.Checksum(d); a != binary.BigEndian.Uint32(sum) {
		return len(b), ErrCorrupted
	}
	_, err := s.Conn.Write(d)
	if err == nil {
		go log.Printf("%d bytes written to %s (%x)", len(d), s.RemoteAddr(), md5.Sum(d))
	}
	return len(b), err
}

type forwarder struct {
	net.Conn

	id []byte

	sequence uint32
	padding  uint16

	reader io.Reader
}

func (f *forwarder) Read(b []byte) (int, error) {
	d, err := readFrom(f.Conn, f.id)
	if len(d) == 0 {
		return 0, ErrEmpty
	}
	go log.Printf("%d bytes received from %s", len(d), f.RemoteAddr())
	return copy(b, d), err
}

func (f *forwarder) Write(b []byte) (int, error) {
	s := atomic.AddUint32(&f.sequence, 1)
	err := writeTo(f.Conn, s, f.id, b)
	go log.Printf("%d bytes sent to %s", len(b), f.RemoteAddr())
	return len(b), err
}

type block struct {
	Id       [uuid.Size]byte
	Length   uint16
	Sequence uint32
	N        uint8
	C        uint8
	R        uint16
}

func readFrom(r io.Reader, i []byte) ([]byte, error) {
	w := new(bytes.Buffer)

	for {
		v := new(block)
		if err := binary.Read(r, binary.BigEndian, v); err != nil {
			return nil, err
		}
		if _, err := io.CopyN(w, r, int64(v.R)); err != nil {
			return nil, err
		}
		if v.R < Padding {
			io.CopyN(ioutil.Discard, r, Padding)
		}
		if !bytes.Equal(v.Id[:], i) {
			return nil, ErrUnknownId
		}
		if v.N == v.C {
			break
		}
	}
	return w.Bytes(), nil
}

func writeTo(w io.Writer, s uint32, id, b []byte) error {
	const size = 1024
	r := bytes.NewReader(b)

	n := r.Size() / size
	var j uint8
	for r.Len() > 0 {
		buf := new(bytes.Buffer)

		buf.Write(id)
		binary.Write(buf, binary.BigEndian, s)
		binary.Write(buf, binary.BigEndian, uint16(len(b)))
		binary.Write(buf, binary.BigEndian, uint8(n))
		binary.Write(buf, binary.BigEndian, uint8(j))

		count := size
		if r.Len() < size {
			count = r.Len()
		}
		binary.Write(buf, binary.BigEndian, uint16(count))

		io.CopyN(buf, r, int64(count))
		if count < Padding {
			io.CopyN(buf, rand.Reader, Padding)
		}
		if _, err := io.Copy(w, buf); err != nil {
			return err
		}
		j++
	}
	return nil
}

type pool struct {
	a string
	c chan net.Conn

	closed bool
	mu     sync.Mutex
}

func (p *pool) Acquire() (net.Conn, error) {
	select {
	case c, ok := <-p.c:
		if !ok {
			return nil, ErrClosed
		}
		return c, nil
	default:
		return Dispatch(p.a)
	}
}

func (p *pool) Release(c net.Conn) error {
	if p.closed {
		return ErrClosed
	}
	select {
	case p.c <- c:
		return nil
	default:
		return c.Close()
	}
}

func (p *pool) Close() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.closed {
		return ErrClosed
	}
	var err error
	for c := range p.c {
		if e := c.Close(); err == nil && e != nil {
			err = e
		}
	}
	close(p.c)
	return err
}
<commit_msg>protect routes registry by mutex<commit_after>package transmit

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/binary"
	"errors"
	"hash/adler32"
	"io"
	"io/ioutil"
	"log"
	"net"
	"sync"
	"sync/atomic"

	"github.com/midbel/uuid"
)

var ErrClosed = errors.New("closed")

var (
	ErrEmpty     = errors.New("empty")
	ErrCorrupted = errors.New("packet corrupted")
	ErrUnknownId = errors.New("unknown packet id")
)

const (
	Size    = 26
	Padding = 512
)

const (
	Bind uint16 = iota
	Accept
	Reject
	Copy
	Abort
	Done
)

type Route struct {
	Id   string `json:"id"`
	Addr string `json:"addr"`
	Eth  string `json:"ifi"`
}

func Subscribe(a, n string) (net.Conn, error) {
	addr, err := net.ResolveUDPAddr("udp", a)
	if err != nil {
		return nil, err
	}
	i, err := net.InterfaceByName(n)
	if err != nil && len(n) > 0 {
		return nil, err
	}
	c, err := net.ListenMulticastUDP("udp", i, addr)
	if err != nil {
		return nil, err
	}
	return &subscriber{c}, nil
}

func Dispatch(a string) (net.Conn, error) {
	c, err := net.Dial("udp", a)
	if err != nil {
		return nil, err
	}
	return &subscriber{c}, nil
}

func Forward(a, s string) (net.Conn, error) {
	c, err := net.Dial("tcp", a)
	if err != nil {
		return nil, err
	}
	id, _ := uuid.UUID5(uuid.URL, []byte(s))

	f := &forwarder{
		Conn:    c,
		id:      id.Bytes(),
		padding: Padding,
		reader:  bufio.NewReaderSize(rand.Reader, 4096),
	}
	if _, err := f.Write(f.id); err != nil {
		f.Close()
		return nil, err
	}
	return f, nil
}

type Router struct {
	net.Listener

	mu sync.Mutex
	routes map[string]*pool
}

func (r *Router) Accept() (net.Conn, net.Conn, error) {
	c, err := r.Listener.Accept()
	if err != nil {
		return nil, nil, err
	}
	id := make([]byte, Size+uuid.Size)
	if _, err := io.ReadFull(c, id); err != nil {
		c.Close()
		return nil, nil, err
	} else {
		id = id[:uuid.Size]
	}
	r.mu.Lock()
	p, ok := r.routes[string(id)]
	r.mu.Unlock()
	if !ok {
		c.Close()
		return nil, nil, ErrUnknownId
	}
	io.CopyN(ioutil.Discard, c, Padding)
	w, err := p.Acquire()
	if err != nil {
		return nil, nil, err
	}
	return &forwarder{Conn: c, id: id}, w, nil
}

func (r *Router) Close() error {
	err := r.Listener.Close()
	for _, p := range r.routes {
		if e := p.Close(); err == nil && e != nil {
			err = e
		}
	}
	return err
}

func NewRouter(a string, rs []Route) (*Router, error) {
	l, err := net.Listen("tcp", a)
	if err != nil {
		return nil, err
	}
	gs := make(map[string]*pool)
	for _, r := range rs {
		id, _ := uuid.UUID5(uuid.URL, []byte(r.Id))
		p := &pool{
			c: make(chan net.Conn, 5),
			a: r.Addr,
		}
		gs[string(id.Bytes())] = p
	}
	return &Router{Listener: l, routes: gs}, nil
}

type subscriber struct {
	net.Conn
}

func (s *subscriber) Read(b []byte) (int, error) {
	d := make([]byte, len(b))
	r, err := s.Conn.Read(d)
	if err != nil && r == 0 {
		return r, err
	} else {
		d = d[:r]
		go log.Printf("%d bytes read from %s (%x)", r, s.LocalAddr(), md5.Sum(d))
	}
	sum := make([]byte, 4)
	binary.BigEndian.PutUint32(sum, adler32.Checksum(d))
	d = append(d, sum...)

	return copy(b, d), err
}

func (s *subscriber) Write(b []byte) (int, error) {
	d, sum := b[:len(b)-adler32.Size], b[len(b)-adler32.Size:]
	if a := adler32.Checksum(d); a != binary.BigEndian.Uint32(sum) {
		return len(b), ErrCorrupted
	}
	_, err := s.Conn.Write(d)
	if err == nil {
		go log.Printf("%d bytes written to %s (%x)", len(d), s.RemoteAddr(), md5.Sum(d))
	}
	return len(b), err
}

type forwarder struct {
	net.Conn

	id []byte

	sequence uint32
	padding  uint16

	reader io.Reader
}

func (f *forwarder) Read(b []byte) (int, error) {
	d, err := readFrom(f.Conn, f.id)
	if len(d) == 0 {
		return 0, ErrEmpty
	}
	go log.Printf("%d bytes received from %s", len(d), f.RemoteAddr())
	return copy(b, d), err
}

func (f *forwarder) Write(b []byte) (int, error) {
	s := atomic.AddUint32(&f.sequence, 1)
	err := writeTo(f.Conn, s, f.id, b)
	go log.Printf("%d bytes sent to %s", len(b), f.RemoteAddr())
	return len(b), err
}

type block struct {
	Id       [uuid.Size]byte
	Length   uint16
	Sequence uint32
	N        uint8
	C        uint8
	R        uint16
}

func readFrom(r io.Reader, i []byte) ([]byte, error) {
	w := new(bytes.Buffer)

	for {
		v := new(block)
		if err := binary.Read(r, binary.BigEndian, v); err != nil {
			return nil, err
		}
		if _, err := io.CopyN(w, r, int64(v.R)); err != nil {
			return nil, err
		}
		if v.R < Padding {
			io.CopyN(ioutil.Discard, r, Padding)
		}
		if !bytes.Equal(v.Id[:], i) {
			return nil, ErrUnknownId
		}
		if v.N == v.C {
			break
		}
	}
	return w.Bytes(), nil
}

func writeTo(w io.Writer, s uint32, id, b []byte) error {
	const size = 1024
	r := bytes.NewReader(b)

	n := r.Size() / size
	var j uint8
	for r.Len() > 0 {
		buf := new(bytes.Buffer)

		buf.Write(id)
		binary.Write(buf, binary.BigEndian, s)
		binary.Write(buf, binary.BigEndian, uint16(len(b)))
		binary.Write(buf, binary.BigEndian, uint8(n))
		binary.Write(buf, binary.BigEndian, uint8(j))

		count := size
		if r.Len() < size {
			count = r.Len()
		}
		binary.Write(buf, binary.BigEndian, uint16(count))

		io.CopyN(buf, r, int64(count))
		if count < Padding {
			io.CopyN(buf, rand.Reader, Padding)
		}
		if _, err := io.Copy(w, buf); err != nil {
			return err
		}
		j++
	}
	return nil
}

type pool struct {
	a string
	c chan net.Conn

	closed bool
	mu     sync.Mutex
}

func (p *pool) Acquire() (net.Conn, error) {
	select {
	case c, ok := <-p.c:
		if !ok {
			return nil, ErrClosed
		}
		return c, nil
	default:
		return Dispatch(p.a)
	}
}

func (p *pool) Release(c net.Conn) error {
	if p.closed {
		return ErrClosed
	}
	select {
	case p.c <- c:
		return nil
	default:
		return c.Close()
	}
}

func (p *pool) Close() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.closed {
		return ErrClosed
	}
	var err error
	for c := range p.c {
		if e := c.Close(); err == nil && e != nil {
			err = e
		}
	}
	close(p.c)
	return err
}
<|endoftext|>


<commit_before>package parser

import (
	docker "github.com/docker/docker/builder/dockerfile/parser"
	"os"
	"regexp"
	"strconv"
	"strings"
)

func parceEnv(line string) string {
	var tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`)
	var str string

	line = strings.Replace(line, "ENV ", "", -1)
	if strings.Contains(line, "=") {
		str = line
	} else {
		slice := tokenWhitespace.Split(line, 2)
		str = slice[0] + "=" + `"` + strings.Join(slice[1:], " ") + `"`
	}
	str = strings.Replace(str, "\t", " ", -1)
	return "export " + str + "\n"
}

func parceCopy(line []string) string {
	if len(line) > 1 {
		return "cp -rf " + strings.Join(line[1:], " ") + "\n"
	}
	return ""
}

func parceRun(line []string) string {
	if len(line) > 1 {
		str, _ := strconv.Unquote(strings.Join(line[1:], " "))
		if !(strings.Contains(str, "ln") && (strings.Contains(str, "/dev/stdout") || strings.Contains(str, "/dev/stderr"))) {
			str = strings.Replace(str, "\t", " ", -1)
			// str = strings.Replace(str, " && ", "\n", -1)
			return str + "\n"
		}
	}
	return ""
}

func parceFrom(line []string) string {
	if len(line) > 1 {
		if strings.Contains(line[1], "debian") {
			return "debian"
		} else if strings.Contains(line[1], "ubuntu") {
			return "ubuntu"
		}
	}
	return ""
}

func parceCmd(line []string) string {
	if len(line) > 1 {
		str := strings.Join(line[1:], " ")
		str = strings.Replace(str, "\t", " ", -1)
		return str
	}
	return ""
}

func Parce(name string) (out, env, cmd, image string) {
	file, _ := os.Open(name)
	node, _ := docker.Parse(file)
	file.Close()

	for _, n := range node.Children {
		if str := strings.Fields(n.Dump()); len(str) > 0 {
			switch str[0] {
			case "env":
				// env = env + parceEnv(str)
				env = env + parceEnv(n.Original)
			case "run":
				out = out + parceRun(str)
			case "add", "copy":
				out = out + parceCopy(str)
			case "from":
				image = parceFrom(str)
			case "cmd":
				cmd = cmd + parceCmd(str)
			case "entrypoint":
				cmd = cmd + parceCmd(str) + " "
			}
		}
	}
	if len(cmd) > 1 {
		slice := strings.Split(cmd, " ")
		for i, _ := range slice {
			slice[i] = strings.Replace(slice[i], `"`, "", -1)
		}

		cmd = slice[0] + ` "` + strings.Join(slice[1:], " ") + `"`
	}
	if len(out) > 0 {
		return out, env, cmd, image
	}
	return
}
<commit_msg>d2s / changed CMD command parsing method #156 all params in one quote-pair -> all params have own quotes<commit_after>package parser

import (
	docker "github.com/docker/docker/builder/dockerfile/parser"
	"os"
	"regexp"
	"strconv"
	"strings"
)

func parceEnv(line string) string {
	var tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`)
	var str string

	line = strings.Replace(line, "ENV ", "", -1)
	if strings.Contains(line, "=") {
		str = line
	} else {
		slice := tokenWhitespace.Split(line, 2)
		str = slice[0] + "=" + `"` + strings.Join(slice[1:], " ") + `"`
	}
	str = strings.Replace(str, "\t", " ", -1)
	return "export " + str + "\n"
}

func parceCopy(line []string) string {
	if len(line) > 1 {
		return "cp -rf " + strings.Join(line[1:], " ") + "\n"
	}
	return ""
}

func parceRun(line []string) string {
	if len(line) > 1 {
		str, _ := strconv.Unquote(strings.Join(line[1:], " "))
		if !(strings.Contains(str, "ln") && (strings.Contains(str, "/dev/stdout") || strings.Contains(str, "/dev/stderr"))) {
			str = strings.Replace(str, "\t", " ", -1)
			// str = strings.Replace(str, " && ", "\n", -1)
			return str + "\n"
		}
	}
	return ""
}

func parceFrom(line []string) string {
	if len(line) > 1 {
		if strings.Contains(line[1], "debian") {
			return "debian"
		} else if strings.Contains(line[1], "ubuntu") {
			return "ubuntu"
		}
	}
	return ""
}

func parceCmd(line []string, isEntrypoint bool) string {
	if len(line) > 1 {
		str := strings.Join(line[1:], " ")
		str = strings.Replace(str, "\t", " ", -1)
		if isEntrypoint {
			str, _ = strconv.Unquote(str)
		}
		return str
	}
	return ""
}

func Parce(name string) (out, env, cmd, image string) {
	file, _ := os.Open(name)
	node, _ := docker.Parse(file)
	file.Close()

	for _, n := range node.Children {
		if str := strings.Fields(n.Dump()); len(str) > 0 {
			switch str[0] {
			case "env":
				// env = env + parceEnv(str)
				env = env + parceEnv(n.Original)
			case "run":
				out = out + parceRun(str)
			case "add", "copy":
				out = out + parceCopy(str)
			case "from":
				image = parceFrom(str)
			case "cmd":
				cmd = cmd + parceCmd(str, false)
			case "entrypoint":
				cmd = cmd + parceCmd(str, true) + " "
			}
		}
	}
	if len(out) > 0 {
		return out, env, cmd, image
	}
	return
}
<|endoftext|>


<commit_before>package ahrsweb

import (
	"encoding/json"
	"log"
	"time"
	//"math"
	"net/url"

	"github.com/gorilla/websocket"
	"github.com/westphae/goflying/ahrs"
	"fmt"
)

type KalmanListener struct {
	data 	*AHRSData
	c	*websocket.Conn
}

func NewKalmanListener() (kl *KalmanListener, err error) {
	kl = new(KalmanListener)
	kl.data = new(AHRSData)
	err = kl.connect()

	return kl, err
}

func (kl *KalmanListener) connect() (err error) {
	u := url.URL{Scheme: "ws", Host: fmt.Sprintf("localhost:%d", Port), Path: "/ahrsweb"}

	if c, _, err := websocket.DefaultDialer.Dial(u.String(), nil); err != nil {
		log.Printf("AHRSWeb dial error: %s\n", err)
	} else {
		kl.c = c
	}
	return
}

func (kl *KalmanListener) update(s *ahrs.State, m *ahrs.Measurement) {
	kl.data.T = float64(time.Now().UnixNano()/1000)/1e6

	if s != nil {
		kl.data.U1 = s.U1
		kl.data.U2 = s.U2
		kl.data.U3 = s.U3
		kl.data.Z1 = s.Z1
		kl.data.Z2 = s.Z2
		kl.data.Z3 = s.Z3
		kl.data.E0 = s.E0
		kl.data.E1 = s.E1
		kl.data.E2 = s.E2
		kl.data.E3 = s.E3
		kl.data.H1 = s.H1
		kl.data.H2 = s.H2
		kl.data.H3 = s.H3
		kl.data.N1 = s.N1
		kl.data.N2 = s.N2
		kl.data.N3 = s.N3

		if s.M != nil {
			kl.data.DU1 = s.M.Get(0, 0)
			kl.data.DU2 = s.M.Get(1, 1)
			kl.data.DU3 = s.M.Get(2, 2)
			kl.data.DZ1 = s.M.Get(3, 3)
			kl.data.DZ2 = s.M.Get(4, 4)
			kl.data.DZ3 = s.M.Get(5, 5)
			kl.data.DE0 = s.M.Get(6, 6)
			kl.data.DE1 = s.M.Get(7, 7)
			kl.data.DE2 = s.M.Get(8, 8)
			kl.data.DE3 = s.M.Get(9, 9)
			kl.data.DH1 = s.M.Get(10, 10)
			kl.data.DH2 = s.M.Get(11, 11)
			kl.data.DH3 = s.M.Get(12, 12)
			kl.data.DN1 = s.M.Get(13, 13)
			kl.data.DN2 = s.M.Get(14, 14)
			kl.data.DN3 = s.M.Get(15, 15)

			kl.data.DV1 = s.M.Get(16, 16)
			kl.data.DV2 = s.M.Get(17, 17)
			kl.data.DV3 = s.M.Get(18, 18)
			kl.data.DC1 = s.M.Get(19, 19)
			kl.data.DC2 = s.M.Get(20, 20)
			kl.data.DC3 = s.M.Get(21, 21)
			kl.data.DF0 = s.M.Get(22, 22)
			kl.data.DF1 = s.M.Get(23, 23)
			kl.data.DF2 = s.M.Get(24, 24)
			kl.data.DF3 = s.M.Get(25, 25)
			kl.data.DD1 = s.M.Get(26, 26)
			kl.data.DD2 = s.M.Get(27, 27)
			kl.data.DD3 = s.M.Get(28, 28)
			kl.data.DL1 = s.M.Get(29, 29)
			kl.data.DL2 = s.M.Get(30, 30)
			kl.data.DL3 = s.M.Get(31, 31)
		}

		kl.data.V1 = s.V1
		kl.data.V2 = s.V2
		kl.data.V3 = s.V3
		kl.data.C1 = s.C1
		kl.data.C2 = s.C2
		kl.data.C3 = s.C3
		kl.data.F0 = s.F0
		kl.data.F1 = s.F1
		kl.data.F2 = s.F2
		kl.data.F3 = s.F3
		kl.data.D1 = s.D1
		kl.data.D2 = s.D2
		kl.data.D3 = s.D3
		kl.data.L1 = s.L1
		kl.data.L2 = s.L2
		kl.data.L3 = s.L3

		roll, pitch, heading := ahrs.FromQuaternion(s.E0, s.E1, s.E2, s.E3)
		kl.data.Pitch = pitch / ahrs.Deg
		kl.data.Roll = roll / ahrs.Deg
		kl.data.Heading = heading / ahrs.Deg
	} else {
		log.Println("AHRSWeb: state is nil, not updating data")
	}

	if m != nil {
		kl.data.UValid = m.UValid
		kl.data.WValid = m.WValid
		kl.data.SValid = m.SValid
		kl.data.MValid = m.MValid

		kl.data.S1 = m.U1
		kl.data.S2 = m.U2
		kl.data.S3 = m.U3
		kl.data.W1 = m.W1
		kl.data.W2 = m.W2
		kl.data.W3 = m.W3
		kl.data.A1 = m.A1
		kl.data.A2 = m.A2
		kl.data.A3 = m.A3
		kl.data.B1 = m.B1
		kl.data.B2 = m.B2
		kl.data.B3 = m.B3
		kl.data.M1 = m.M1
		kl.data.M2 = m.M2
		kl.data.M3 = m.M3
	} else {
		log.Println("AHRSWeb: measurement is nil, not updating data")
	}
}

func (kl *KalmanListener) Send(s *ahrs.State, m *ahrs.Measurement) error {
	kl.update(s, m)

	if msg, err := json.Marshal(kl.data); err != nil {
		log.Println("AHRSWeb: Error marshalling json data:", err)
		log.Println("AHRSWeb: Data was:", kl.data)
		return err
	} else {
		if err := kl.c.WriteMessage(websocket.TextMessage, msg); err != nil {
			log.Println("AHRSWeb: Error writing to websocket:", err)
			err2 := kl.connect()
			return fmt.Errorf("AHRSWeb: %v: %v", err, err2) // Just drop this message
		}
	}
	return nil
}

func (kl *KalmanListener) Close() {
	if err := kl.c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")); err != nil {
		log.Println("AHRSWeb: Error closing websocket:", err)
		return
	}
	kl.c.Close()
}
<commit_msg>Better websocket connection error handling in ahrsweb<commit_after>package ahrsweb

import (
	"encoding/json"
	"log"
	"time"
	//"math"
	"net/url"

	"github.com/gorilla/websocket"
	"github.com/westphae/goflying/ahrs"
	"fmt"
)

type KalmanListener struct {
	data 	*AHRSData
	c	*websocket.Conn
}

func NewKalmanListener() (kl *KalmanListener, err error) {
	kl = new(KalmanListener)
	kl.data = new(AHRSData)
	err = kl.connect()

	return kl, err
}

func (kl *KalmanListener) connect() (err error) {
	u := url.URL{Scheme: "ws", Host: fmt.Sprintf("localhost:%d", Port), Path: "/ahrsweb"}
	kl.c, _, err = websocket.DefaultDialer.Dial(u.String(), nil)
	return
}

func (kl *KalmanListener) update(s *ahrs.State, m *ahrs.Measurement) {
	kl.data.T = float64(time.Now().UnixNano()/1000)/1e6

	if s != nil {
		kl.data.U1 = s.U1
		kl.data.U2 = s.U2
		kl.data.U3 = s.U3
		kl.data.Z1 = s.Z1
		kl.data.Z2 = s.Z2
		kl.data.Z3 = s.Z3
		kl.data.E0 = s.E0
		kl.data.E1 = s.E1
		kl.data.E2 = s.E2
		kl.data.E3 = s.E3
		kl.data.H1 = s.H1
		kl.data.H2 = s.H2
		kl.data.H3 = s.H3
		kl.data.N1 = s.N1
		kl.data.N2 = s.N2
		kl.data.N3 = s.N3

		if s.M != nil {
			kl.data.DU1 = s.M.Get(0, 0)
			kl.data.DU2 = s.M.Get(1, 1)
			kl.data.DU3 = s.M.Get(2, 2)
			kl.data.DZ1 = s.M.Get(3, 3)
			kl.data.DZ2 = s.M.Get(4, 4)
			kl.data.DZ3 = s.M.Get(5, 5)
			kl.data.DE0 = s.M.Get(6, 6)
			kl.data.DE1 = s.M.Get(7, 7)
			kl.data.DE2 = s.M.Get(8, 8)
			kl.data.DE3 = s.M.Get(9, 9)
			kl.data.DH1 = s.M.Get(10, 10)
			kl.data.DH2 = s.M.Get(11, 11)
			kl.data.DH3 = s.M.Get(12, 12)
			kl.data.DN1 = s.M.Get(13, 13)
			kl.data.DN2 = s.M.Get(14, 14)
			kl.data.DN3 = s.M.Get(15, 15)

			kl.data.DV1 = s.M.Get(16, 16)
			kl.data.DV2 = s.M.Get(17, 17)
			kl.data.DV3 = s.M.Get(18, 18)
			kl.data.DC1 = s.M.Get(19, 19)
			kl.data.DC2 = s.M.Get(20, 20)
			kl.data.DC3 = s.M.Get(21, 21)
			kl.data.DF0 = s.M.Get(22, 22)
			kl.data.DF1 = s.M.Get(23, 23)
			kl.data.DF2 = s.M.Get(24, 24)
			kl.data.DF3 = s.M.Get(25, 25)
			kl.data.DD1 = s.M.Get(26, 26)
			kl.data.DD2 = s.M.Get(27, 27)
			kl.data.DD3 = s.M.Get(28, 28)
			kl.data.DL1 = s.M.Get(29, 29)
			kl.data.DL2 = s.M.Get(30, 30)
			kl.data.DL3 = s.M.Get(31, 31)
		}

		kl.data.V1 = s.V1
		kl.data.V2 = s.V2
		kl.data.V3 = s.V3
		kl.data.C1 = s.C1
		kl.data.C2 = s.C2
		kl.data.C3 = s.C3
		kl.data.F0 = s.F0
		kl.data.F1 = s.F1
		kl.data.F2 = s.F2
		kl.data.F3 = s.F3
		kl.data.D1 = s.D1
		kl.data.D2 = s.D2
		kl.data.D3 = s.D3
		kl.data.L1 = s.L1
		kl.data.L2 = s.L2
		kl.data.L3 = s.L3

		roll, pitch, heading := ahrs.FromQuaternion(s.E0, s.E1, s.E2, s.E3)
		kl.data.Pitch = pitch / ahrs.Deg
		kl.data.Roll = roll / ahrs.Deg
		kl.data.Heading = heading / ahrs.Deg
	} else {
		log.Println("AHRSWeb: state is nil, not updating data")
	}

	if m != nil {
		kl.data.UValid = m.UValid
		kl.data.WValid = m.WValid
		kl.data.SValid = m.SValid
		kl.data.MValid = m.MValid

		kl.data.S1 = m.U1
		kl.data.S2 = m.U2
		kl.data.S3 = m.U3
		kl.data.W1 = m.W1
		kl.data.W2 = m.W2
		kl.data.W3 = m.W3
		kl.data.A1 = m.A1
		kl.data.A2 = m.A2
		kl.data.A3 = m.A3
		kl.data.B1 = m.B1
		kl.data.B2 = m.B2
		kl.data.B3 = m.B3
		kl.data.M1 = m.M1
		kl.data.M2 = m.M2
		kl.data.M3 = m.M3
	} else {
		log.Println("AHRSWeb: measurement is nil, not updating data")
	}
}

func (kl *KalmanListener) Send(s *ahrs.State, m *ahrs.Measurement) error {
	kl.update(s, m)

	if msg, err := json.Marshal(kl.data); err != nil {
		log.Println("AHRSWeb: Error marshalling json data:", err)
		log.Println("AHRSWeb: Data was:", kl.data)
		return err
	} else {
		if err := kl.c.WriteMessage(websocket.TextMessage, msg); err != nil {
			log.Println("AHRSWeb: Error writing to websocket:", err)
			err2 := kl.connect()
			return fmt.Errorf("AHRSWeb: %v: %v", err, err2) // Just drop this message
		}
	}
	return nil
}

func (kl *KalmanListener) Close() {
	if err := kl.c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")); err != nil {
		log.Println("AHRSWeb: Error closing websocket:", err)
		return
	}
	kl.c.Close()
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

const (
	EventTypeIndexSettings = `
{
	"settings": {
		"index.mapper.dynamic": false
	},
	"mappings": {
		"EventType": {
			"properties": {
				"eventTypeId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"name": {
					"type": "string",
					"index": "not_analyzed"
				},
				"mapping": {
					"dynamic": true
				}
			}
		}
	}
}
`
)

type EventTypeDB struct {
	*ResourceDB
	mapping string
}

func NewEventTypeDB(service *WorkflowService, esi elasticsearch.IIndex) (*EventTypeDB, error) {

	rdb, err := NewResourceDB(service, esi)
	if err != nil {
		return nil, err
	}
	esi.SetMapping(EventTypeDBMapping, EventTypeIndexSettings)
	etrdb := EventTypeDB{ResourceDB: rdb, mapping: EventTypeDBMapping}
	return &etrdb, nil
}

func (db *EventTypeDB) PostData(obj interface{}, id piazza.Ident) (piazza.Ident, error) {

	indexResult, err := db.Esi.PostData(db.mapping, id.String(), obj)
	if err != nil {
		return piazza.NoIdent, LoggedError("EventTypeDB.PostData failed: %s", err)
	}
	if !indexResult.Created {
		return piazza.NoIdent, LoggedError("EventTypeDB.PostData failed: not created")
	}

	return id, nil
}

func (db *EventTypeDB) GetAll(format *piazza.JsonPagination) (*[]EventType, int64, error) {
	var eventTypes []EventType
	var count = int64(-1)

	exists := db.Esi.TypeExists(db.mapping)
	if !exists {
		return &eventTypes, count, nil
	}

	searchResult, err := db.Esi.FilterByMatchAll(db.mapping, format)
	if err != nil {
		return nil, count, LoggedError("EventTypeDB.GetAll failed: %s", err)
	}
	if searchResult == nil {
		return nil, count, LoggedError("EventTypeDB.GetAll failed: no searchResult")
	}

	if searchResult != nil && searchResult.GetHits() != nil {
		count = searchResult.NumberMatched()
		for _, hit := range *searchResult.GetHits() {
			var eventType EventType
			err := json.Unmarshal(*hit.Source, &eventType)
			if err != nil {
				return nil, count, err
			}
			eventTypes = append(eventTypes, eventType)
		}
	}

	return &eventTypes, count, nil
}

func (db *EventTypeDB) GetOne(id piazza.Ident) (*EventType, error) {

	getResult, err := db.Esi.GetByID(db.mapping, id.String())
	if err != nil {
		return nil, LoggedError("EventTypeDB.GetOne failed: %s", err)
	}
	if getResult == nil {
		return nil, LoggedError("EventTypeDB.GetOne failed: no getResult")
	}

	if !getResult.Found {
		return nil, nil
	}

	src := getResult.Source
	var eventType EventType
	err = json.Unmarshal(*src, &eventType)
	if err != nil {
		return nil, err
	}

	return &eventType, nil
}

func (db *EventTypeDB) DeleteByID(id piazza.Ident) (bool, error) {
	deleteResult, err := db.Esi.DeleteByID(db.mapping, string(id))
	if err != nil {
		return false, LoggedError("EventTypeDB.DeleteById failed: %s", err)
	}
	if deleteResult == nil {
		return false, LoggedError("EventTypeDB.DeleteById failed: no deleteResult")
	}

	return deleteResult.Found, nil
}
<commit_msg>added CreatedBy, CreatedOn<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

const (
	EventTypeIndexSettings = `
{
	"settings": {
		"index.mapper.dynamic": false
	},
	"mappings": {
		"EventType": {
			"properties": {
				"eventTypeId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"name": {
					"type": "string",
					"index": "not_analyzed"
				},
 				"createdOn": {
 					"type": "date",
 					"index": "not_analyzed"
 				},
 				"createdBy": {
 					"type": "string",
 					"index": "not_analyzed"
 				},
				"mapping": {
					"dynamic": true
				}
			}
		}
	}
}
`
)

type EventTypeDB struct {
	*ResourceDB
	mapping string
}

func NewEventTypeDB(service *WorkflowService, esi elasticsearch.IIndex) (*EventTypeDB, error) {

	rdb, err := NewResourceDB(service, esi)
	if err != nil {
		return nil, err
	}
	esi.SetMapping(EventTypeDBMapping, EventTypeIndexSettings)
	etrdb := EventTypeDB{ResourceDB: rdb, mapping: EventTypeDBMapping}
	return &etrdb, nil
}

func (db *EventTypeDB) PostData(obj interface{}, id piazza.Ident) (piazza.Ident, error) {

	indexResult, err := db.Esi.PostData(db.mapping, id.String(), obj)
	if err != nil {
		return piazza.NoIdent, LoggedError("EventTypeDB.PostData failed: %s", err)
	}
	if !indexResult.Created {
		return piazza.NoIdent, LoggedError("EventTypeDB.PostData failed: not created")
	}

	return id, nil
}

func (db *EventTypeDB) GetAll(format *piazza.JsonPagination) (*[]EventType, int64, error) {
	var eventTypes []EventType
	var count = int64(-1)

	exists := db.Esi.TypeExists(db.mapping)
	if !exists {
		return &eventTypes, count, nil
	}

	searchResult, err := db.Esi.FilterByMatchAll(db.mapping, format)
	if err != nil {
		return nil, count, LoggedError("EventTypeDB.GetAll failed: %s", err)
	}
	if searchResult == nil {
		return nil, count, LoggedError("EventTypeDB.GetAll failed: no searchResult")
	}

	if searchResult != nil && searchResult.GetHits() != nil {
		count = searchResult.NumberMatched()
		for _, hit := range *searchResult.GetHits() {
			var eventType EventType
			err := json.Unmarshal(*hit.Source, &eventType)
			if err != nil {
				return nil, count, err
			}
			eventTypes = append(eventTypes, eventType)
		}
	}

	return &eventTypes, count, nil
}

func (db *EventTypeDB) GetOne(id piazza.Ident) (*EventType, error) {

	getResult, err := db.Esi.GetByID(db.mapping, id.String())
	if err != nil {
		return nil, LoggedError("EventTypeDB.GetOne failed: %s", err)
	}
	if getResult == nil {
		return nil, LoggedError("EventTypeDB.GetOne failed: no getResult")
	}

	if !getResult.Found {
		return nil, nil
	}

	src := getResult.Source
	var eventType EventType
	err = json.Unmarshal(*src, &eventType)
	if err != nil {
		return nil, err
	}

	return &eventType, nil
}

func (db *EventTypeDB) DeleteByID(id piazza.Ident) (bool, error) {
	deleteResult, err := db.Esi.DeleteByID(db.mapping, string(id))
	if err != nil {
		return false, LoggedError("EventTypeDB.DeleteById failed: %s", err)
	}
	if deleteResult == nil {
		return false, LoggedError("EventTypeDB.DeleteById failed: no deleteResult")
	}

	return deleteResult.Found, nil
}
<|endoftext|>


<commit_before>// #ignore

// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run runs tests in the test directory.
//
// TODO(bradfitz): docs of some sort, once we figure out how we're changing
// headers of files
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

var (
	verbose     = flag.Bool("v", false, "verbose. if set, parallelism is set to 1.")
	numParallel = flag.Int("n", 8, "number of parallel tests to run")
	summary     = flag.Bool("summary", false, "show summary of results")
)

var (
	// gc and ld are [568][gl].
	gc, ld string

	// letter is the build.ArchChar
	letter string

	// dirs are the directories to look for *.go files in.
	// TODO(bradfitz): just use all directories?
	dirs = []string{".", "ken", "chan", "interface", "syntax", "dwarf", "fixedbugs", "bugs"}

	// ratec controls the max number of tests running at a time.
	ratec chan bool

	// toRun is the channel of tests to run.
	// It is nil until the first test is started.
	toRun chan *test
)

// maxTests is an upper bound on the total number of tests.
// It is used as a channel buffer size to make sure sends don't block.
const maxTests = 5000

func main() {
	flag.Parse()
	if *verbose {
		*numParallel = 1
	}

	ratec = make(chan bool, *numParallel)
	var err error
	letter, err = build.ArchChar(build.DefaultContext.GOARCH)
	check(err)
	gc = letter + "g"
	ld = letter + "l"

	var tests []*test
	if flag.NArg() > 0 {
		for _, arg := range flag.Args() {
			if arg == "-" || arg == "--" {
				// Permit running either:
				// $ go run run.go - env.go
				// $ go run run.go -- env.go
				continue
			}
			if !strings.HasSuffix(arg, ".go") {
				log.Fatalf("can't yet deal with non-go file %q", arg)
			}
			dir, file := filepath.Split(arg)
			tests = append(tests, startTest(dir, file))
		}
	} else {
		for _, dir := range dirs {
			for _, baseGoFile := range goFiles(dir) {
				tests = append(tests, startTest(dir, baseGoFile))
			}
		}
	}

	failed := false
	resCount := map[string]int{}
	for _, test := range tests {
		<-test.donec
		_, isSkip := test.err.(skipError)
		if isSkip {
			resCount["skip"]++
			if !*verbose {
				continue
			}
		}
		errStr := "pass"
		if test.err != nil {
			errStr = test.err.Error()
			if !isSkip {
				failed = true
			}
		}
		resCount[errStr]++
		if !*verbose && test.err == nil {
			continue
	}
		fmt.Printf("%-10s %-20s: %s\n", test.action, test.goFileName(), errStr)
	}

	if *summary {
		for k, v := range resCount {
			fmt.Printf("%5d %s\n", v, k)
		}
	}

	if failed {
		os.Exit(1)
	}
}

func toolPath(name string) string {
	p := filepath.Join(os.Getenv("GOROOT"), "bin", "tool", name)
	if _, err := os.Stat(p); err != nil {
		log.Fatalf("didn't find binary at %s", p)
	}
	return p
}

func goFiles(dir string) []string {
	f, err := os.Open(dir)
	check(err)
	dirnames, err := f.Readdirnames(-1)
	check(err)
	names := []string{}
	for _, name := range dirnames {
		if strings.HasSuffix(name, ".go") {
			names = append(names, name)
		}
	}
	sort.Strings(names)
	return names
}

// skipError describes why a test was skipped.
type skipError string

func (s skipError) Error() string { return string(s) }

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// test holds the state of a test.
type test struct {
	dir, gofile string
	donec       chan bool // closed when done

	src    string
	action string // "compile", "build", "run", "errorcheck"

	tempDir string
	err     error
}

// startTest
func startTest(dir, gofile string) *test {
	t := &test{
		dir:    dir,
		gofile: gofile,
		donec:  make(chan bool, 1),
	}
	if toRun == nil {
		toRun = make(chan *test, maxTests)
		go runTests()
	}
	select {
	case toRun <- t:
	default:
		panic("toRun buffer size (maxTests) is too small")
	}
	return t
}

// runTests runs tests in parallel, but respecting the order they
// were enqueued on the toRun channel.
func runTests() {
	for {
		ratec <- true
		t := <-toRun
		go func() {
			t.run()
			<-ratec
		}()
	}
}

func (t *test) goFileName() string {
	return filepath.Join(t.dir, t.gofile)
}

// run runs a test.
func (t *test) run() {
	defer close(t.donec)

	srcBytes, err := ioutil.ReadFile(t.goFileName())
	if err != nil {
		t.err = err
		return
	}
	t.src = string(srcBytes)
	if t.src[0] == '\n' {
		t.err = skipError("starts with newline")
		return
	}
	pos := strings.Index(t.src, "\n\n")
	if pos == -1 {
		t.err = errors.New("double newline not found")
		return
	}
	action := t.src[:pos]
	if strings.HasPrefix(action, "//") {
		action = action[2:]
	}
	action = strings.TrimSpace(action)

	switch action {
	case "compile", "build", "run", "errorcheck":
		t.action = action
	default:
		t.err = skipError("skipped; unknown pattern: " + action)
		t.action = "??"
		return
	}

	t.makeTempDir()
	defer os.RemoveAll(t.tempDir)

	err = ioutil.WriteFile(filepath.Join(t.tempDir, t.gofile), srcBytes, 0644)
	check(err)

	cmd := exec.Command("go", "tool", gc, "-e", "-o", "a."+letter, t.gofile)
	var buf bytes.Buffer
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	cmd.Dir = t.tempDir
	err = cmd.Run()
	out := buf.String()

	if action == "errorcheck" {
		t.err = t.errorCheck(out)
		return
	}

	if err != nil {
		t.err = fmt.Errorf("build = %v (%q)", err, out)
		return
	}

	if action == "compile" {
		return
	}

	if action == "build" || action == "run" {
		buf.Reset()
		cmd = exec.Command("go", "tool", ld, "-o", "a.out", "a."+letter)
		cmd.Stdout = &buf
		cmd.Stderr = &buf
		cmd.Dir = t.tempDir
		err = cmd.Run()
		out = buf.String()
		if err != nil {
			t.err = fmt.Errorf("link = %v (%q)", err, out)
			return
		}
		if action == "build" {
			return
		}
	}

	if action == "run" {
		buf.Reset()
		cmd = exec.Command(filepath.Join(t.tempDir, "a.out"))
		cmd.Stdout = &buf
		cmd.Stderr = &buf
		cmd.Dir = t.tempDir
		cmd.Env = append(cmd.Env, "GOARCH="+runtime.GOARCH)
		err = cmd.Run()
		out = buf.String()
		if err != nil {
			t.err = fmt.Errorf("run = %v (%q)", err, out)
			return
		}

		if out != t.expectedOutput() {
			t.err = fmt.Errorf("output differs; got:\n%s", out)
		}
		return
	}

	t.err = fmt.Errorf("unimplemented action %q", action)
}

func (t *test) String() string {
	return filepath.Join(t.dir, t.gofile)
}

func (t *test) makeTempDir() {
	var err error
	t.tempDir, err = ioutil.TempDir("", "")
	check(err)
}

func (t *test) expectedOutput() string {
	filename := filepath.Join(t.dir, t.gofile)
	filename = filename[:len(filename)-len(".go")]
	filename += ".out"
	b, _ := ioutil.ReadFile(filename)
	return string(b)
}

func (t *test) errorCheck(outStr string) (err error) {
	defer func() {
		if *verbose && err != nil {
			log.Printf("%s gc output:\n%s", t, outStr)
		}
	}()
	var errs []error

	var out []string
	// 6g error messages continue onto additional lines with leading tabs.
	// Split the output at the beginning of each line that doesn't begin with a tab.
	for _, line := range strings.Split(outStr, "\n") {
		if strings.HasPrefix(line, "\t") {
			out[len(out)-1] += "\n" + line
		} else {
			out = append(out, line)
		}
	}

	for _, we := range t.wantedErrors() {
		var errmsgs []string
		errmsgs, out = partitionStrings(we.filterRe, out)
		if len(errmsgs) == 0 {
			errs = append(errs, fmt.Errorf("errchk: %s:%d: missing expected error: %s", we.file, we.lineNum, we.reStr))
			continue
		}
		matched := false
		for _, errmsg := range errmsgs {
			if we.re.MatchString(errmsg) {
				matched = true
			} else {
				out = append(out, errmsg)
			}
		}
		if !matched {
			errs = append(errs, fmt.Errorf("errchk: %s:%d: error(s) on line didn't match pattern: %s", we.file, we.lineNum, we.reStr))
			continue
		}
	}

	if len(errs) == 0 {
		return nil
	}
	if len(errs) == 1 {
		return errs[0]
	}
	var buf bytes.Buffer
	buf.WriteString("Multiple errors:\n")
	for _, err := range errs {
		fmt.Fprintf(&buf, "%s\n", err.Error())
	}
	return errors.New(buf.String())

}

func partitionStrings(rx *regexp.Regexp, strs []string) (matched, unmatched []string) {
	for _, s := range strs {
		if rx.MatchString(s) {
			matched = append(matched, s)
		} else {
			unmatched = append(unmatched, s)
		}
	}
	return
}

type wantedError struct {
	reStr    string
	re       *regexp.Regexp
	lineNum  int
	file     string
	filterRe *regexp.Regexp // /^file:linenum\b/m
}

var (
	errRx       = regexp.MustCompile(`// (?:GC_)?ERROR (.*)`)
	errQuotesRx = regexp.MustCompile(`"([^"]*)"`)
	lineRx      = regexp.MustCompile(`LINE(([+-])([0-9]+))?`)
)

func (t *test) wantedErrors() (errs []wantedError) {
	for i, line := range strings.Split(t.src, "\n") {
		lineNum := i + 1
		if strings.Contains(line, "////") {
			// double comment disables ERROR
			continue
		}
		m := errRx.FindStringSubmatch(line)
		if m == nil {
			continue
		}
		all := m[1]
		mm := errQuotesRx.FindAllStringSubmatch(all, -1)
		if mm == nil {
			log.Fatalf("invalid errchk line in %s: %s", t.goFileName(), line)
		}
		for _, m := range mm {
			rx := lineRx.ReplaceAllStringFunc(m[1], func(m string) string {
				n := lineNum
				if strings.HasPrefix(m, "LINE+") {
					delta, _ := strconv.Atoi(m[5:])
					n += delta
				} else if strings.HasPrefix(m, "LINE-") {
					delta, _ := strconv.Atoi(m[5:])
					n -= delta
				}
				return fmt.Sprintf("%s:%d", t.gofile, n)
			})
			filterPattern := fmt.Sprintf(`^(\w+/)?%s:%d[:[]`, t.gofile, lineNum)
			errs = append(errs, wantedError{
				reStr:    rx,
				re:       regexp.MustCompile(rx),
				filterRe: regexp.MustCompile(filterPattern),
				lineNum:  lineNum,
				file:     t.gofile,
			})
		}
	}

	return
}
<commit_msg>test: add temporary show_skips flag.<commit_after>// #ignore

// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run runs tests in the test directory.
//
// TODO(bradfitz): docs of some sort, once we figure out how we're changing
// headers of files
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

var (
	verbose     = flag.Bool("v", false, "verbose. if set, parallelism is set to 1.")
	numParallel = flag.Int("n", 8, "number of parallel tests to run")
	summary     = flag.Bool("summary", false, "show summary of results")
	showSkips   = flag.Bool("show_skips", false, "show skipped tests")
)

var (
	// gc and ld are [568][gl].
	gc, ld string

	// letter is the build.ArchChar
	letter string

	// dirs are the directories to look for *.go files in.
	// TODO(bradfitz): just use all directories?
	dirs = []string{".", "ken", "chan", "interface", "syntax", "dwarf", "fixedbugs", "bugs"}

	// ratec controls the max number of tests running at a time.
	ratec chan bool

	// toRun is the channel of tests to run.
	// It is nil until the first test is started.
	toRun chan *test
)

// maxTests is an upper bound on the total number of tests.
// It is used as a channel buffer size to make sure sends don't block.
const maxTests = 5000

func main() {
	flag.Parse()
	if *verbose {
		*numParallel = 1
	}

	ratec = make(chan bool, *numParallel)
	var err error
	letter, err = build.ArchChar(build.DefaultContext.GOARCH)
	check(err)
	gc = letter + "g"
	ld = letter + "l"

	var tests []*test
	if flag.NArg() > 0 {
		for _, arg := range flag.Args() {
			if arg == "-" || arg == "--" {
				// Permit running either:
				// $ go run run.go - env.go
				// $ go run run.go -- env.go
				continue
			}
			if !strings.HasSuffix(arg, ".go") {
				log.Fatalf("can't yet deal with non-go file %q", arg)
			}
			dir, file := filepath.Split(arg)
			tests = append(tests, startTest(dir, file))
		}
	} else {
		for _, dir := range dirs {
			for _, baseGoFile := range goFiles(dir) {
				tests = append(tests, startTest(dir, baseGoFile))
			}
		}
	}

	failed := false
	resCount := map[string]int{}
	for _, test := range tests {
		<-test.donec
		_, isSkip := test.err.(skipError)
		errStr := "pass"
		if isSkip {
			errStr = "skip"
		}
		if test.err != nil {
			errStr = test.err.Error()
			if !isSkip {
				failed = true
			}
		}
		resCount[errStr]++
		if isSkip && !*verbose && !*showSkips {
			continue
		}
		if !*verbose && test.err == nil {
			continue
		}
		fmt.Printf("%-10s %-20s: %s\n", test.action, test.goFileName(), errStr)
	}

	if *summary {
		for k, v := range resCount {
			fmt.Printf("%5d %s\n", v, k)
		}
	}

	if failed {
		os.Exit(1)
	}
}

func toolPath(name string) string {
	p := filepath.Join(os.Getenv("GOROOT"), "bin", "tool", name)
	if _, err := os.Stat(p); err != nil {
		log.Fatalf("didn't find binary at %s", p)
	}
	return p
}

func goFiles(dir string) []string {
	f, err := os.Open(dir)
	check(err)
	dirnames, err := f.Readdirnames(-1)
	check(err)
	names := []string{}
	for _, name := range dirnames {
		if strings.HasSuffix(name, ".go") {
			names = append(names, name)
		}
	}
	sort.Strings(names)
	return names
}

// skipError describes why a test was skipped.
type skipError string

func (s skipError) Error() string { return string(s) }

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// test holds the state of a test.
type test struct {
	dir, gofile string
	donec       chan bool // closed when done

	src    string
	action string // "compile", "build", "run", "errorcheck"

	tempDir string
	err     error
}

// startTest
func startTest(dir, gofile string) *test {
	t := &test{
		dir:    dir,
		gofile: gofile,
		donec:  make(chan bool, 1),
	}
	if toRun == nil {
		toRun = make(chan *test, maxTests)
		go runTests()
	}
	select {
	case toRun <- t:
	default:
		panic("toRun buffer size (maxTests) is too small")
	}
	return t
}

// runTests runs tests in parallel, but respecting the order they
// were enqueued on the toRun channel.
func runTests() {
	for {
		ratec <- true
		t := <-toRun
		go func() {
			t.run()
			<-ratec
		}()
	}
}

func (t *test) goFileName() string {
	return filepath.Join(t.dir, t.gofile)
}

// run runs a test.
func (t *test) run() {
	defer close(t.donec)

	srcBytes, err := ioutil.ReadFile(t.goFileName())
	if err != nil {
		t.err = err
		return
	}
	t.src = string(srcBytes)
	if t.src[0] == '\n' {
		t.err = skipError("starts with newline")
		return
	}
	pos := strings.Index(t.src, "\n\n")
	if pos == -1 {
		t.err = errors.New("double newline not found")
		return
	}
	action := t.src[:pos]
	if strings.HasPrefix(action, "//") {
		action = action[2:]
	}
	action = strings.TrimSpace(action)

	switch action {
	case "compile", "build", "run", "errorcheck":
		t.action = action
	default:
		t.err = skipError("skipped; unknown pattern: " + action)
		t.action = "??"
		return
	}

	t.makeTempDir()
	defer os.RemoveAll(t.tempDir)

	err = ioutil.WriteFile(filepath.Join(t.tempDir, t.gofile), srcBytes, 0644)
	check(err)

	cmd := exec.Command("go", "tool", gc, "-e", "-o", "a."+letter, t.gofile)
	var buf bytes.Buffer
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	cmd.Dir = t.tempDir
	err = cmd.Run()
	out := buf.String()

	if action == "errorcheck" {
		t.err = t.errorCheck(out)
		return
	}

	if err != nil {
		t.err = fmt.Errorf("build = %v (%q)", err, out)
		return
	}

	if action == "compile" {
		return
	}

	if action == "build" || action == "run" {
		buf.Reset()
		cmd = exec.Command("go", "tool", ld, "-o", "a.out", "a."+letter)
		cmd.Stdout = &buf
		cmd.Stderr = &buf
		cmd.Dir = t.tempDir
		err = cmd.Run()
		out = buf.String()
		if err != nil {
			t.err = fmt.Errorf("link = %v (%q)", err, out)
			return
		}
		if action == "build" {
			return
		}
	}

	if action == "run" {
		buf.Reset()
		cmd = exec.Command(filepath.Join(t.tempDir, "a.out"))
		cmd.Stdout = &buf
		cmd.Stderr = &buf
		cmd.Dir = t.tempDir
		cmd.Env = append(cmd.Env, "GOARCH="+runtime.GOARCH)
		err = cmd.Run()
		out = buf.String()
		if err != nil {
			t.err = fmt.Errorf("run = %v (%q)", err, out)
			return
		}

		if out != t.expectedOutput() {
			t.err = fmt.Errorf("output differs; got:\n%s", out)
		}
		return
	}

	t.err = fmt.Errorf("unimplemented action %q", action)
}

func (t *test) String() string {
	return filepath.Join(t.dir, t.gofile)
}

func (t *test) makeTempDir() {
	var err error
	t.tempDir, err = ioutil.TempDir("", "")
	check(err)
}

func (t *test) expectedOutput() string {
	filename := filepath.Join(t.dir, t.gofile)
	filename = filename[:len(filename)-len(".go")]
	filename += ".out"
	b, _ := ioutil.ReadFile(filename)
	return string(b)
}

func (t *test) errorCheck(outStr string) (err error) {
	defer func() {
		if *verbose && err != nil {
			log.Printf("%s gc output:\n%s", t, outStr)
		}
	}()
	var errs []error

	var out []string
	// 6g error messages continue onto additional lines with leading tabs.
	// Split the output at the beginning of each line that doesn't begin with a tab.
	for _, line := range strings.Split(outStr, "\n") {
		if strings.HasPrefix(line, "\t") {
			out[len(out)-1] += "\n" + line
		} else {
			out = append(out, line)
		}
	}

	for _, we := range t.wantedErrors() {
		var errmsgs []string
		errmsgs, out = partitionStrings(we.filterRe, out)
		if len(errmsgs) == 0 {
			errs = append(errs, fmt.Errorf("errchk: %s:%d: missing expected error: %s", we.file, we.lineNum, we.reStr))
			continue
		}
		matched := false
		for _, errmsg := range errmsgs {
			if we.re.MatchString(errmsg) {
				matched = true
			} else {
				out = append(out, errmsg)
			}
		}
		if !matched {
			errs = append(errs, fmt.Errorf("errchk: %s:%d: error(s) on line didn't match pattern: %s", we.file, we.lineNum, we.reStr))
			continue
		}
	}

	if len(errs) == 0 {
		return nil
	}
	if len(errs) == 1 {
		return errs[0]
	}
	var buf bytes.Buffer
	buf.WriteString("Multiple errors:\n")
	for _, err := range errs {
		fmt.Fprintf(&buf, "%s\n", err.Error())
	}
	return errors.New(buf.String())

}

func partitionStrings(rx *regexp.Regexp, strs []string) (matched, unmatched []string) {
	for _, s := range strs {
		if rx.MatchString(s) {
			matched = append(matched, s)
		} else {
			unmatched = append(unmatched, s)
		}
	}
	return
}

type wantedError struct {
	reStr    string
	re       *regexp.Regexp
	lineNum  int
	file     string
	filterRe *regexp.Regexp // /^file:linenum\b/m
}

var (
	errRx       = regexp.MustCompile(`// (?:GC_)?ERROR (.*)`)
	errQuotesRx = regexp.MustCompile(`"([^"]*)"`)
	lineRx      = regexp.MustCompile(`LINE(([+-])([0-9]+))?`)
)

func (t *test) wantedErrors() (errs []wantedError) {
	for i, line := range strings.Split(t.src, "\n") {
		lineNum := i + 1
		if strings.Contains(line, "////") {
			// double comment disables ERROR
			continue
		}
		m := errRx.FindStringSubmatch(line)
		if m == nil {
			continue
		}
		all := m[1]
		mm := errQuotesRx.FindAllStringSubmatch(all, -1)
		if mm == nil {
			log.Fatalf("invalid errchk line in %s: %s", t.goFileName(), line)
		}
		for _, m := range mm {
			rx := lineRx.ReplaceAllStringFunc(m[1], func(m string) string {
				n := lineNum
				if strings.HasPrefix(m, "LINE+") {
					delta, _ := strconv.Atoi(m[5:])
					n += delta
				} else if strings.HasPrefix(m, "LINE-") {
					delta, _ := strconv.Atoi(m[5:])
					n -= delta
				}
				return fmt.Sprintf("%s:%d", t.gofile, n)
			})
			filterPattern := fmt.Sprintf(`^(\w+/)?%s:%d[:[]`, t.gofile, lineNum)
			errs = append(errs, wantedError{
				reStr:    rx,
				re:       regexp.MustCompile(rx),
				filterRe: regexp.MustCompile(filterPattern),
				lineNum:  lineNum,
				file:     t.gofile,
			})
		}
	}

	return
}
<|endoftext|>


<commit_before>package functional

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/coreos/fleet/functional/platform"
)

var fleetctlBinPath string

func init() {
	fleetctlBinPath = os.Getenv("FLEETCTL_BIN")
	if fleetctlBinPath == "" {
		fmt.Println("FLEETCTL_BIN environment variable must be set")
		os.Exit(1)
	} else if _, err := os.Stat(fleetctlBinPath); err != nil {
		fmt.Printf("%v\n", err)
		os.Exit(1)
	}

	if os.Getenv("SSH_AUTH_SOCK") == "" {
		fmt.Println("SSH_AUTH_SOCK environment variable must be set")
		os.Exit(1)
	}
}

func TestCluster(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatalf(err.Error())
	}
	defer cluster.DestroyAll()

	// Start with a simple three-node cluster
	if err := cluster.Create(3); err != nil {
		t.Fatalf(err.Error())
	}
	machines, err := waitForNMachines(3)
	if err != nil {
		t.Fatalf(err.Error())
	}

	// Ensure we can SSH into each machine using fleetctl
	for _, machine := range machines {
		if _, _, err := fleetctl("--strict-host-key-checking=false", "ssh", machine, "uptime"); err != nil {
			t.Errorf("Unable to SSH into fleet machine: %v", err)
		}
	}

	// Start the 5 services
	for i := 0; i < 5; i++ {
		unitName := fmt.Sprintf("fixtures/units/conflict.%d.service", i)
		_, _, err := fleetctl("start", "--no-block", unitName)
		if err != nil {
			t.Errorf("Failed starting %s: %v", unitName, err)
		}
	}

	// All 5 services should be visible immediately and become ACTIVE
	// shortly thereafter
	stdout, _, err := fleetctl("list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units := strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 5 {
		t.Fatalf("Did not find five units in cluster: \n%s", stdout)
	}
	if err := waitForNActiveUnits(3); err != nil {
		t.Fatalf(err.Error())
	}

	// Add two more machines to the cluster and ensure the remaining
	// unscheduled services are picked up.
	if err := cluster.Create(2); err != nil {
		t.Fatalf(err.Error())
	}
	machines, err = waitForNMachines(5)
	if err != nil {
		t.Fatalf(err.Error())
	}
	if err := waitForNActiveUnits(5); err != nil {
		t.Fatalf(err.Error())
	}
}

func parseUnitStates(units []string) []string {
	states := make([]string, len(units))
	for i, unit := range units {
		cols := strings.SplitN(unit, "\t", 6)
		if len(cols) == 6 {
			states[i] = cols[2]
		}
	}
	return states
}

func activeCount(states []string) (count int) {
	for _, state := range states {
		if state == "active" {
			count++
		}
	}
	return
}

func fleetctl(args ...string) (string, string, error) {
	log.Printf("%s %s", fleetctlBinPath, strings.Join(args, " "))
	var stdoutBytes, stderrBytes bytes.Buffer
	cmd := exec.Command(fleetctlBinPath, args...)
	cmd.Stdout = &stdoutBytes
	cmd.Stderr = &stderrBytes
	err := cmd.Run()
	return stdoutBytes.String(), stderrBytes.String(), err
}

func waitForNMachines(count int) ([]string, error) {
	var machines []string
	for i := 0; i <= 7; i++ {
		if i == 7 {
			return nil, fmt.Errorf("Failed to find %d machines within the time limit", count)
		}

		log.Printf("Waiting 5s for %d fleet services to check in...", count)
		time.Sleep(5 * time.Second)

		stdout, _, err := fleetctl("list-machines", "--no-legend", "-l")
		if err != nil {
			continue
		}

		machines := strings.Split(strings.TrimSpace(stdout), "\n")
		if len(machines) != count {
			continue
		}

		for k, v := range machines {
			machines[k] = strings.SplitN(v, "\t", 2)[0]
		}

		break
	}

	return machines, nil
}

func waitForNActiveUnits(count int) error {
	for i := 0; i <= 6; i++ {
		if i == 6 {
			return fmt.Errorf("Failed to find %d active units within the time limit", count)
		}

		log.Printf("Waiting 1s for %d fleet units to become active...", count)
		time.Sleep(time.Second)

		stdout, _, err := fleetctl("list-units", "--no-legend")
		if err != nil {
			continue
		}

		units := strings.Split(strings.TrimSpace(stdout), "\n")
		states := parseUnitStates(units)
		if activeCount(states) != count {
			continue
		}

		break
	}

	return nil
}
<commit_msg>test(functional): Deal with empty return values properly<commit_after>package functional

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/coreos/fleet/functional/platform"
)

var fleetctlBinPath string

func init() {
	fleetctlBinPath = os.Getenv("FLEETCTL_BIN")
	if fleetctlBinPath == "" {
		fmt.Println("FLEETCTL_BIN environment variable must be set")
		os.Exit(1)
	} else if _, err := os.Stat(fleetctlBinPath); err != nil {
		fmt.Printf("%v\n", err)
		os.Exit(1)
	}

	if os.Getenv("SSH_AUTH_SOCK") == "" {
		fmt.Println("SSH_AUTH_SOCK environment variable must be set")
		os.Exit(1)
	}
}

func TestCluster(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatalf(err.Error())
	}
	defer cluster.DestroyAll()

	// Start with a simple three-node cluster
	if err := cluster.Create(3); err != nil {
		t.Fatalf(err.Error())
	}
	machines, err := waitForNMachines(3)
	if err != nil {
		t.Fatalf(err.Error())
	}

	// Ensure we can SSH into each machine using fleetctl
	for _, machine := range machines {
		if _, _, err := fleetctl("--strict-host-key-checking=false", "ssh", machine, "uptime"); err != nil {
			t.Errorf("Unable to SSH into fleet machine: %v", err)
		}
	}

	// Start the 5 services
	for i := 0; i < 5; i++ {
		unitName := fmt.Sprintf("fixtures/units/conflict.%d.service", i)
		_, _, err := fleetctl("start", "--no-block", unitName)
		if err != nil {
			t.Errorf("Failed starting %s: %v", unitName, err)
		}
	}

	// All 5 services should be visible immediately and become ACTIVE
	// shortly thereafter
	stdout, _, err := fleetctl("list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units := strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 5 {
		t.Fatalf("Did not find five units in cluster: \n%s", stdout)
	}
	if err := waitForNActiveUnits(3); err != nil {
		t.Fatalf(err.Error())
	}

	// Add two more machines to the cluster and ensure the remaining
	// unscheduled services are picked up.
	if err := cluster.Create(2); err != nil {
		t.Fatalf(err.Error())
	}
	machines, err = waitForNMachines(5)
	if err != nil {
		t.Fatalf(err.Error())
	}
	if err := waitForNActiveUnits(5); err != nil {
		t.Fatalf(err.Error())
	}
}

func parseUnitStates(units []string) []string {
	states := make([]string, len(units))
	for i, unit := range units {
		cols := strings.SplitN(unit, "\t", 6)
		if len(cols) == 6 {
			states[i] = cols[2]
		}
	}
	return states
}

func activeCount(states []string) (count int) {
	for _, state := range states {
		if state == "active" {
			count++
		}
	}
	return
}

func fleetctl(args ...string) (string, string, error) {
	log.Printf("%s %s", fleetctlBinPath, strings.Join(args, " "))
	var stdoutBytes, stderrBytes bytes.Buffer
	cmd := exec.Command(fleetctlBinPath, args...)
	cmd.Stdout = &stdoutBytes
	cmd.Stderr = &stderrBytes
	err := cmd.Run()
	return stdoutBytes.String(), stderrBytes.String(), err
}

func waitForNMachines(count int) ([]string, error) {
	var machines []string
	for i := 0; i <= 7; i++ {
		if i == 7 {
			return nil, fmt.Errorf("Failed to find %d machines within the time limit", count)
		}

		log.Printf("Waiting 5s for %d fleet services to check in...", count)
		time.Sleep(5 * time.Second)

		stdout, _, err := fleetctl("list-machines", "--no-legend", "-l")
		stdout = strings.TrimSpace(stdout)
		if stdout == "" || err != nil {
			continue
		}

		machines = strings.Split(stdout, "\n")
		if len(machines) != count {
			continue
		}

		for k, v := range machines {
			machines[k] = strings.SplitN(v, "\t", 2)[0]
		}

		break
	}

	return machines, nil
}

func waitForNActiveUnits(count int) error {
	for i := 0; i <= 6; i++ {
		if i == 6 {
			return fmt.Errorf("Failed to find %d active units within the time limit", count)
		}

		log.Printf("Waiting 1s for %d fleet units to become active...", count)
		time.Sleep(time.Second)

		stdout, _, err := fleetctl("list-units", "--no-legend")
		stdout = strings.TrimSpace(stdout)
		if stdout == "" || err != nil {
			continue
		}

		units := strings.Split(stdout, "\n")
		states := parseUnitStates(units)
		if activeCount(states) != count {
			continue
		}

		break
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"servant/conf"
	"servant/server"
	"fmt"
	"flag"
	"os"
	"github.com/davecgh/go-spew/spew"
)

type arrayFlags []string

func (self *arrayFlags) String() string {
	return fmt.Sprintf("%v", *self)
}

func (self *arrayFlags) Set(value string) error {
	*self = append(*self, value)
	return nil
}

func main() {
	var configs arrayFlags
	var configDirs arrayFlags
	var vars arrayFlags
	flag.Var(&configs, "conf", "config files path")
	flag.Var(&configDirs, "confdir", "config directories path")
	flag.Var(&vars, "var", "vars")
	var debug bool
	flag.BoolVar(&debug, "debug", false, "enable debug mode")
	flag.Parse()

	server.SetArgVars(vars)
	server.SetEnvVars()

	config, err := conf.LoadXmlConfig(configs, configDirs, server.CloneGlobalParams())
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(2)
	}
	if debug {
		config.Debug = true
		spew.Config.Indent = "    "
		spew.Config.MaxDepth = 100
		spew.Fdump(os.Stderr, config)
		//spew.Fprintf(os.Stderr, "%#v", config)
	}
	err = server.NewServer(&config).Run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(3)
	}
}

<commit_msg>remove debug code<commit_after>package main

import (
	"servant/conf"
	"servant/server"
	"fmt"
	"flag"
	"os"
//	"github.com/davecgh/go-spew/spew"
)

type arrayFlags []string

func (self *arrayFlags) String() string {
	return fmt.Sprintf("%v", *self)
}

func (self *arrayFlags) Set(value string) error {
	*self = append(*self, value)
	return nil
}

func main() {
	var configs arrayFlags
	var configDirs arrayFlags
	var vars arrayFlags
	flag.Var(&configs, "conf", "config files path")
	flag.Var(&configDirs, "confdir", "config directories path")
	flag.Var(&vars, "var", "vars")
	//var debug bool
	//flag.BoolVar(&debug, "debug", false, "enable debug mode")
	flag.Parse()

	server.SetArgVars(vars)
	server.SetEnvVars()

	config, err := conf.LoadXmlConfig(configs, configDirs, server.CloneGlobalParams())
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(2)
	}
	/*
	if debug {
		config.Debug = true
		spew.Config.Indent = "    "
		spew.Config.MaxDepth = 100
		spew.Fdump(os.Stderr, config)
	}*/
	err = server.NewServer(&config).Run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(3)
	}
}

<|endoftext|>


<commit_before>package config

import (
	"os"

	"github.com/BurntSushi/toml"
	"github.com/njpatel/loggo"

	"utils"
)

//go:generate bash bake_default_config.sh
const defaultTomlConfig = `
# This is where top level info is stored for the counter manager we
# could also use a boltDB in the DataDir but this would make it harder
# to sync over replicas since not all replicas will hold the all the
# counters.
info_dir = "~/.skizze"

# This is where the data is stored either as json or .count (pure bytes)
data_dir = "~/.skizze/data"

# The host interface for the server
host = "localhost"

# The port number for the server
port = 3596

# Treshold for saving a sketch to disk
save_threshold_seconds = 1
`

var logger = loggo.GetLogger("config")

// Config stores all configuration parameters for Go
type Config struct {
	InfoDir              string `toml:"info_dir"`
	DataDir              string `toml:"data_dir"`
	Host                 string `toml:"host"`
	Port                 int    `toml:"port"`
	SaveThresholdSeconds uint   `toml:"save_threshold_seconds"`
}

var config *Config
// InfoDir initialized from config file
var InfoDir              string
// DataDir initialized from config file
var DataDir              string
// Host initialized from config file
var Host                 string
// Port initialized from config file
var Port                 int
// SaveThresholdSeconds initialized from config file
var SaveThresholdSeconds uint

// MaxKeySize ...
const MaxKeySize int = 32768 // max key size BoltDB in bytes

func parseConfigTOML() *Config {
	cfg := &Config{}
	if _, err := toml.Decode(defaultTomlConfig, &cfg); err != nil {
		utils.PanicOnError(err)
	}

	configPath := os.Getenv("SKIZZE_CONFIG")
	if configPath != "" {
		_, err := os.Open(configPath)
		if err != nil {
			logger.Warningf("Unable to find config file, using defaults")
			return cfg
		}
		if _, err := toml.DecodeFile(configPath, &cfg); err != nil {
			logger.Warningf("Error parsing config file, using defaults")
		}
	}

	return cfg
}

// GetConfig returns a singleton Configuration
func GetConfig() *Config {
	if config == nil {
		config = parseConfigTOML()

		if err := os.MkdirAll(config.InfoDir, os.ModePerm); err != nil {
			panic(err)
		}

		if err := os.MkdirAll(config.DataDir, os.ModePerm); err != nil {
			panic(err)
		}

		InfoDir = config.InfoDir
		DataDir = config.DataDir
		Host = config.Host
		Port = config.Port
		SaveThresholdSeconds = config.SaveThresholdSeconds
	}
	return config
}

// init initializes a singleton Configuration
func init() {
	GetConfig()
}

// Reset ...
func Reset() {
	GetConfig()
}
<commit_msg>cleanup comment<commit_after>package config

import (
	"os"

	"github.com/BurntSushi/toml"
	"github.com/njpatel/loggo"

	"utils"
)

//go:generate bash bake_default_config.sh
const defaultTomlConfig = `
# This is where top level info is stored for the counter manager we
# could also use a boltDB in the DataDir but this would make it harder
# to sync over replicas since not all replicas will hold the all the
# counters.
info_dir = "~/.skizze"

# This is where the data is stored either as json or .count (pure bytes)
data_dir = "~/.skizze/data"

# The host interface for the server
host = "localhost"

# The port number for the server
port = 3596

# Treshold for saving a sketch to disk
save_threshold_seconds = 1
`

var logger = loggo.GetLogger("config")

// Config stores all configuration parameters for Go
type Config struct {
	InfoDir              string `toml:"info_dir"`
	DataDir              string `toml:"data_dir"`
	Host                 string `toml:"host"`
	Port                 int    `toml:"port"`
	SaveThresholdSeconds uint   `toml:"save_threshold_seconds"`
}

var config *Config
// InfoDir initialized from config file
var InfoDir              string
// DataDir initialized from config file
var DataDir              string
// Host initialized from config file
var Host                 string
// Port initialized from config file
var Port                 int
// SaveThresholdSeconds initialized from config file
var SaveThresholdSeconds uint

// MaxKeySize for BoltDB keys in bytes
const MaxKeySize int = 32768

func parseConfigTOML() *Config {
	cfg := &Config{}
	if _, err := toml.Decode(defaultTomlConfig, &cfg); err != nil {
		utils.PanicOnError(err)
	}

	configPath := os.Getenv("SKIZZE_CONFIG")
	if configPath != "" {
		_, err := os.Open(configPath)
		if err != nil {
			logger.Warningf("Unable to find config file, using defaults")
			return cfg
		}
		if _, err := toml.DecodeFile(configPath, &cfg); err != nil {
			logger.Warningf("Error parsing config file, using defaults")
		}
	}

	return cfg
}

// GetConfig returns a singleton Configuration
func GetConfig() *Config {
	if config == nil {
		config = parseConfigTOML()

		if err := os.MkdirAll(config.InfoDir, os.ModePerm); err != nil {
			panic(err)
		}

		if err := os.MkdirAll(config.DataDir, os.ModePerm); err != nil {
			panic(err)
		}

		InfoDir = config.InfoDir
		DataDir = config.DataDir
		Host = config.Host
		Port = config.Port
		SaveThresholdSeconds = config.SaveThresholdSeconds
	}
	return config
}

// init initializes a singleton Configuration
func init() {
	GetConfig()
}

// Reset ...
func Reset() {
	GetConfig()
}
<|endoftext|>