<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Bram Gruneir (bram+code@cockroachlabs.com)

package storage_test

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/keys"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

// loadNodes fetches a node and recursively all of its children.
func loadNodes(t *testing.T, db *client.DB, key roachpb.Key, nodes map[string]roachpb.RangeTreeNode) {
	node := new(roachpb.RangeTreeNode)
	if err := db.GetProto(keys.RangeTreeNodeKey(key), node); err != nil {
		t.Fatal(err)
	}
	nodes[node.Key.String()] = *node
	if node.LeftKey != nil {
		loadNodes(t, db, node.LeftKey, nodes)
	}
	if node.RightKey != nil {
		loadNodes(t, db, node.RightKey, nodes)
	}
}

// loadTree loads the tree root and all of its nodes. It puts all of the nodes
// into a map.
func loadTree(t *testing.T, db *client.DB) (*roachpb.RangeTree, map[string]roachpb.RangeTreeNode) {
	tree := new(roachpb.RangeTree)
	if err := db.GetProto(keys.RangeTreeRoot, tree); err != nil {
		t.Fatal(err)
	}
	nodes := make(map[string]roachpb.RangeTreeNode)
	if tree.RootKey != nil {
		loadNodes(t, db, tree.RootKey, nodes)
	}
	return tree, nodes
}

// VerifyTree checks to ensure that the tree is indeed balanced and a correct
// red-black tree. It does so by checking each of the red-black tree properties.
// These verify functions are similar to the those found in the range_tree_test
// but these use a map of nodes instead of a tree context.
func VerifyTree(t *testing.T, tree *roachpb.RangeTree, nodes map[string]roachpb.RangeTreeNode, testName string) {
	root, ok := nodes[tree.RootKey.String()]
	if !ok {
		t.Fatalf("%s: could not find root node with key %s", testName, tree.RootKey)
	}

	verifyBinarySearchTree(t, nodes, testName, &root, roachpb.KeyMin, roachpb.KeyMax)
	// Property 1 is always correct. All nodes are already colored.
	verifyProperty2(t, testName, &root)
	// Property 3 is always correct. All leaves are black.
	verifyProperty4(t, nodes, testName, &root)
	pathBlackCount := new(int)
	*pathBlackCount = -1
	verifyProperty5(t, nodes, testName, &root, 0, pathBlackCount)
}

// isRed will return true only if node exists and is not set to black. This is
// the same helper function as the one embedded in the range tree.
func isRed(node *roachpb.RangeTreeNode) bool {
	if node == nil {
		return false
	}
	return !node.Black
}

// getLeftAndRight returns the left and right nodes, if they exist, in order,
// from the passed in map of nodes.
func getLeftAndRight(t *testing.T, nodes map[string]roachpb.RangeTreeNode, testName string, node *roachpb.RangeTreeNode) (*roachpb.RangeTreeNode, *roachpb.RangeTreeNode) {
	var left *roachpb.RangeTreeNode
	var right *roachpb.RangeTreeNode
	var ok bool
	if node.LeftKey != nil {
		left = new(roachpb.RangeTreeNode)
		if *left, ok = nodes[node.LeftKey.String()]; !ok {
			t.Errorf("%s: could not locate node with key %s", testName, node.LeftKey)
		}
	}
	if node.RightKey != nil {
		right = new(roachpb.RangeTreeNode)
		if *right, ok = nodes[node.RightKey.String()]; !ok {
			t.Errorf("%s: could not locate node with key %s", testName, node.RightKey)
		}
	}
	return left, right
}

// verifyBinarySearchTree checks to ensure that all keys to the left of the root
// node are less than it, and all nodes to the right of the root node are
// greater than it. It recursively walks the tree to perform this same check.
func verifyBinarySearchTree(t *testing.T, nodes map[string]roachpb.RangeTreeNode, testName string, node *roachpb.RangeTreeNode, keyMin, keyMax roachpb.Key) {
	if node == nil {
		return
	}
	if !node.Key.Less(keyMax) {
		t.Errorf("%s: Failed Property BST - The key %s is not less than %s.", testName, node.Key, keyMax)
	}
	// We need the extra check since roachpb.KeyMin is actually a range start key.
	if !keyMin.Less(node.Key) && !node.Key.Equal(roachpb.KeyMin) {
		t.Errorf("%s: Failed Property BST - The key %s is not greater than %s.", testName, node.Key, keyMin)
	}
	left, right := getLeftAndRight(t, nodes, testName, node)
	verifyBinarySearchTree(t, nodes, testName, left, keyMin, node.Key)
	verifyBinarySearchTree(t, nodes, testName, right, node.Key, keyMax)
}

// verifyProperty2 ensures that the root node is black.
func verifyProperty2(t *testing.T, testName string, root *roachpb.RangeTreeNode) {
	if e, a := false, isRed(root); e != a {
		t.Errorf("%s: Failed Property 2 - The root node is not black.", testName)
	}
}

// verifyProperty4 ensures that the parent of every red node is black.
func verifyProperty4(t *testing.T, nodes map[string]roachpb.RangeTreeNode, testName string, node *roachpb.RangeTreeNode) {
	if node == nil {
		return
	}

	left, right := getLeftAndRight(t, nodes, testName, node)
	if isRed(node) {
		if e, a := false, isRed(left); e != a {
			t.Errorf("%s: Failed property 4 - Red Node %s's left child %s is also red.", testName, node.Key, left.Key)
		}
		if e, a := false, isRed(right); e != a {
			t.Errorf("%s: Failed property 4 - Red Node %s's right child %s is also red.", testName, node.Key, right.Key)
		}
	}
	verifyProperty4(t, nodes, testName, left)
	verifyProperty4(t, nodes, testName, right)
}

// verifyProperty5 ensures that all paths from any given node to its leaf nodes
// contain the same number of black nodes.
func verifyProperty5(t *testing.T, nodes map[string]roachpb.RangeTreeNode, testName string, node *roachpb.RangeTreeNode, blackCount int, pathBlackCount *int) {
	if !isRed(node) {
		blackCount++
	}
	if node == nil {
		if *pathBlackCount == -1 {
			*pathBlackCount = blackCount
		} else {
			if e, a := *pathBlackCount, blackCount; e != a {
				t.Errorf("%s: Failed property 5 - Expected a black count of %d but instead got %d.", testName, e, a)
			}
		}
		return
	}
	left, right := getLeftAndRight(t, nodes, testName, node)
	verifyProperty5(t, nodes, testName, left, blackCount, pathBlackCount)
	verifyProperty5(t, nodes, testName, right, blackCount, pathBlackCount)
}

// TestSetupRangeTree ensures that SetupRangeTree correctly setups up the range
// tree and first node. SetupRangeTree is called via store.BootstrapRange.
func TestSetupRangeTree(t *testing.T) {
	defer leaktest.AfterTest(t)
	store, stopper := createTestStore(t)
	defer stopper.Stop()
	db := store.DB()

	tree, nodes := loadTree(t, db)
	expectedTree := &roachpb.RangeTree{
		RootKey: roachpb.KeyMin,
	}
	if !reflect.DeepEqual(tree, expectedTree) {
		t.Fatalf("tree roots do not match - expected:%+v actual:%+v", expectedTree, tree)
	}
	VerifyTree(t, tree, nodes, "setup")
}

// TestTree is a similar to the TestTree test in range_tree_test but this one
// performs actual splits and merges.
func TestTree(t *testing.T) {
	defer leaktest.AfterTest(t)
	store, stopper := createTestStore(t)
	defer stopper.Stop()
	db := store.DB()

	keys := []string{"m",
		"f", "e", "d", "c", "b", "a",
		"g", "h", "i", "j", "k", "l",
		"s", "r", "q", "p", "o", "n",
		"t", "u", "v", "w", "x", "y", "z",
	}

	for _, key := range keys {
		if err := db.AdminSplit(key); err != nil {
			t.Fatal(err)
		}
		tree, nodes := loadTree(t, db)
		VerifyTree(t, tree, nodes, key)
	}

	// To test merging, we just call AdminMerge on the lowest key to merge all
	// ranges back into a single one.
	for i := 0; i < len(keys); i++ {
		if err := db.AdminMerge(roachpb.KeyMin); err != nil {
			t.Fatal(err)
		}
		tree, nodes := loadTree(t, db)
		VerifyTree(t, tree, nodes, fmt.Sprintf("remove %d", i))
	}

}
<commit_msg>Disable the merge half of TestTree.<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Bram Gruneir (bram+code@cockroachlabs.com)

package storage_test

import (
	"reflect"
	"testing"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/keys"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

// loadNodes fetches a node and recursively all of its children.
func loadNodes(t *testing.T, db *client.DB, key roachpb.Key, nodes map[string]roachpb.RangeTreeNode) {
	node := new(roachpb.RangeTreeNode)
	if err := db.GetProto(keys.RangeTreeNodeKey(key), node); err != nil {
		t.Fatal(err)
	}
	nodes[node.Key.String()] = *node
	if node.LeftKey != nil {
		loadNodes(t, db, node.LeftKey, nodes)
	}
	if node.RightKey != nil {
		loadNodes(t, db, node.RightKey, nodes)
	}
}

// loadTree loads the tree root and all of its nodes. It puts all of the nodes
// into a map.
func loadTree(t *testing.T, db *client.DB) (*roachpb.RangeTree, map[string]roachpb.RangeTreeNode) {
	tree := new(roachpb.RangeTree)
	if err := db.GetProto(keys.RangeTreeRoot, tree); err != nil {
		t.Fatal(err)
	}
	nodes := make(map[string]roachpb.RangeTreeNode)
	if tree.RootKey != nil {
		loadNodes(t, db, tree.RootKey, nodes)
	}
	return tree, nodes
}

// VerifyTree checks to ensure that the tree is indeed balanced and a correct
// red-black tree. It does so by checking each of the red-black tree properties.
// These verify functions are similar to the those found in the range_tree_test
// but these use a map of nodes instead of a tree context.
func VerifyTree(t *testing.T, tree *roachpb.RangeTree, nodes map[string]roachpb.RangeTreeNode, testName string) {
	root, ok := nodes[tree.RootKey.String()]
	if !ok {
		t.Fatalf("%s: could not find root node with key %s", testName, tree.RootKey)
	}

	verifyBinarySearchTree(t, nodes, testName, &root, roachpb.KeyMin, roachpb.KeyMax)
	// Property 1 is always correct. All nodes are already colored.
	verifyProperty2(t, testName, &root)
	// Property 3 is always correct. All leaves are black.
	verifyProperty4(t, nodes, testName, &root)
	pathBlackCount := new(int)
	*pathBlackCount = -1
	verifyProperty5(t, nodes, testName, &root, 0, pathBlackCount)
}

// isRed will return true only if node exists and is not set to black. This is
// the same helper function as the one embedded in the range tree.
func isRed(node *roachpb.RangeTreeNode) bool {
	if node == nil {
		return false
	}
	return !node.Black
}

// getLeftAndRight returns the left and right nodes, if they exist, in order,
// from the passed in map of nodes.
func getLeftAndRight(t *testing.T, nodes map[string]roachpb.RangeTreeNode, testName string, node *roachpb.RangeTreeNode) (*roachpb.RangeTreeNode, *roachpb.RangeTreeNode) {
	var left *roachpb.RangeTreeNode
	var right *roachpb.RangeTreeNode
	var ok bool
	if node.LeftKey != nil {
		left = new(roachpb.RangeTreeNode)
		if *left, ok = nodes[node.LeftKey.String()]; !ok {
			t.Errorf("%s: could not locate node with key %s", testName, node.LeftKey)
		}
	}
	if node.RightKey != nil {
		right = new(roachpb.RangeTreeNode)
		if *right, ok = nodes[node.RightKey.String()]; !ok {
			t.Errorf("%s: could not locate node with key %s", testName, node.RightKey)
		}
	}
	return left, right
}

// verifyBinarySearchTree checks to ensure that all keys to the left of the root
// node are less than it, and all nodes to the right of the root node are
// greater than it. It recursively walks the tree to perform this same check.
func verifyBinarySearchTree(t *testing.T, nodes map[string]roachpb.RangeTreeNode, testName string, node *roachpb.RangeTreeNode, keyMin, keyMax roachpb.Key) {
	if node == nil {
		return
	}
	if !node.Key.Less(keyMax) {
		t.Errorf("%s: Failed Property BST - The key %s is not less than %s.", testName, node.Key, keyMax)
	}
	// We need the extra check since roachpb.KeyMin is actually a range start key.
	if !keyMin.Less(node.Key) && !node.Key.Equal(roachpb.KeyMin) {
		t.Errorf("%s: Failed Property BST - The key %s is not greater than %s.", testName, node.Key, keyMin)
	}
	left, right := getLeftAndRight(t, nodes, testName, node)
	verifyBinarySearchTree(t, nodes, testName, left, keyMin, node.Key)
	verifyBinarySearchTree(t, nodes, testName, right, node.Key, keyMax)
}

// verifyProperty2 ensures that the root node is black.
func verifyProperty2(t *testing.T, testName string, root *roachpb.RangeTreeNode) {
	if e, a := false, isRed(root); e != a {
		t.Errorf("%s: Failed Property 2 - The root node is not black.", testName)
	}
}

// verifyProperty4 ensures that the parent of every red node is black.
func verifyProperty4(t *testing.T, nodes map[string]roachpb.RangeTreeNode, testName string, node *roachpb.RangeTreeNode) {
	if node == nil {
		return
	}

	left, right := getLeftAndRight(t, nodes, testName, node)
	if isRed(node) {
		if e, a := false, isRed(left); e != a {
			t.Errorf("%s: Failed property 4 - Red Node %s's left child %s is also red.", testName, node.Key, left.Key)
		}
		if e, a := false, isRed(right); e != a {
			t.Errorf("%s: Failed property 4 - Red Node %s's right child %s is also red.", testName, node.Key, right.Key)
		}
	}
	verifyProperty4(t, nodes, testName, left)
	verifyProperty4(t, nodes, testName, right)
}

// verifyProperty5 ensures that all paths from any given node to its leaf nodes
// contain the same number of black nodes.
func verifyProperty5(t *testing.T, nodes map[string]roachpb.RangeTreeNode, testName string, node *roachpb.RangeTreeNode, blackCount int, pathBlackCount *int) {
	if !isRed(node) {
		blackCount++
	}
	if node == nil {
		if *pathBlackCount == -1 {
			*pathBlackCount = blackCount
		} else {
			if e, a := *pathBlackCount, blackCount; e != a {
				t.Errorf("%s: Failed property 5 - Expected a black count of %d but instead got %d.", testName, e, a)
			}
		}
		return
	}
	left, right := getLeftAndRight(t, nodes, testName, node)
	verifyProperty5(t, nodes, testName, left, blackCount, pathBlackCount)
	verifyProperty5(t, nodes, testName, right, blackCount, pathBlackCount)
}

// TestSetupRangeTree ensures that SetupRangeTree correctly setups up the range
// tree and first node. SetupRangeTree is called via store.BootstrapRange.
func TestSetupRangeTree(t *testing.T) {
	defer leaktest.AfterTest(t)
	store, stopper := createTestStore(t)
	defer stopper.Stop()
	db := store.DB()

	tree, nodes := loadTree(t, db)
	expectedTree := &roachpb.RangeTree{
		RootKey: roachpb.KeyMin,
	}
	if !reflect.DeepEqual(tree, expectedTree) {
		t.Fatalf("tree roots do not match - expected:%+v actual:%+v", expectedTree, tree)
	}
	VerifyTree(t, tree, nodes, "setup")
}

// TestTree is a similar to the TestTree test in range_tree_test but this one
// performs actual splits and merges.
func TestTree(t *testing.T) {
	defer leaktest.AfterTest(t)
	store, stopper := createTestStore(t)
	defer stopper.Stop()
	db := store.DB()

	keys := []string{"m",
		"f", "e", "d", "c", "b", "a",
		"g", "h", "i", "j", "k", "l",
		"s", "r", "q", "p", "o", "n",
		"t", "u", "v", "w", "x", "y", "z",
	}

	for _, key := range keys {
		if err := db.AdminSplit(key); err != nil {
			t.Fatal(err)
		}
		tree, nodes := loadTree(t, db)
		VerifyTree(t, tree, nodes, key)
	}

	// To test merging, we just call AdminMerge on the lowest key to merge all
	// ranges back into a single one.
	// TODO(bdarnell): re-enable this when merging is more reliable.
	// https://github.com/cockroachdb/cockroach/issues/2433
	/*
		for i := 0; i < len(keys); i++ {
			if err := db.AdminMerge(roachpb.KeyMin); err != nil {
				t.Fatal(err)
			}
			tree, nodes := loadTree(t, db)
			VerifyTree(t, tree, nodes, fmt.Sprintf("remove %d", i))
		}
	*/
}
<|endoftext|>


<commit_before>package filesystem

import (
	"testing"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"

	. "gopkg.in/check.v1"
	"gopkg.in/src-d/go-git-fixtures.v3"
)

type FsSuite struct {
	fixtures.Suite
}

var objectTypes = []plumbing.ObjectType{
	plumbing.CommitObject,
	plumbing.TagObject,
	plumbing.TreeObject,
	plumbing.BlobObject,
}

var _ = Suite(&FsSuite{})

func (s *FsSuite) TestGetFromObjectFile(c *C) {
	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
	o, err := NewObjectStorage(dotgit.New(fs))
	c.Assert(err, IsNil)

	expected := plumbing.NewHash("f3dfe29d268303fc6e1bbce268605fc99573406e")
	obj, err := o.EncodedObject(plumbing.AnyObject, expected)
	c.Assert(err, IsNil)
	c.Assert(obj.Hash(), Equals, expected)
}

func (s *FsSuite) TestGetFromPackfile(c *C) {
	fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) {
		fs := f.DotGit()
		o, err := NewObjectStorage(dotgit.New(fs))
		c.Assert(err, IsNil)

		expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
		obj, err := o.EncodedObject(plumbing.AnyObject, expected)
		c.Assert(err, IsNil)
		c.Assert(obj.Hash(), Equals, expected)
	})
}

func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) {
	fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit()
	o, err := NewObjectStorage(dotgit.New(fs))
	c.Assert(err, IsNil)

	expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3")
	obj, err := o.getFromPackfile(expected, false)
	c.Assert(err, IsNil)
	c.Assert(obj.Hash(), Equals, expected)

	expected = plumbing.NewHash("e9cfa4c9ca160546efd7e8582ec77952a27b17db")
	obj, err = o.getFromPackfile(expected, false)
	c.Assert(err, IsNil)
	c.Assert(obj.Hash(), Equals, expected)
}

func (s *FsSuite) TestIter(c *C) {
	fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) {
		fs := f.DotGit()
		o, err := NewObjectStorage(dotgit.New(fs))
		c.Assert(err, IsNil)

		iter, err := o.IterEncodedObjects(plumbing.AnyObject)
		c.Assert(err, IsNil)

		var count int32
		err = iter.ForEach(func(o plumbing.EncodedObject) error {
			count++
			return nil
		})

		c.Assert(err, IsNil)
		c.Assert(count, Equals, f.ObjectsCount)
	})
}

func (s *FsSuite) TestIterWithType(c *C) {
	fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
		for _, t := range objectTypes {
			fs := f.DotGit()
			o, err := NewObjectStorage(dotgit.New(fs))
			c.Assert(err, IsNil)

			iter, err := o.IterEncodedObjects(t)
			c.Assert(err, IsNil)

			err = iter.ForEach(func(o plumbing.EncodedObject) error {
				c.Assert(o.Type(), Equals, t)
				return nil
			})

			c.Assert(err, IsNil)
		}

	})
}

func (s *FsSuite) TestPackfileIter(c *C) {
	fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
		fs := f.DotGit()
		dg := dotgit.New(fs)

		for _, t := range objectTypes {
			ph, err := dg.ObjectPacks()
			c.Assert(err, IsNil)

			for _, h := range ph {
				f, err := dg.ObjectPack(h)
				c.Assert(err, IsNil)

				idxf, err := dg.ObjectPackIdx(h)
				c.Assert(err, IsNil)

				iter, err := NewPackfileIter(f, idxf, t)
				c.Assert(err, IsNil)
				err = iter.ForEach(func(o plumbing.EncodedObject) error {
					c.Assert(o.Type(), Equals, t)
					return nil
				})

				c.Assert(err, IsNil)
			}
		}
	})

}

func BenchmarkPackfileIter(b *testing.B) {
	if err := fixtures.Init(); err != nil {
		b.Fatal(err)
	}

	defer func() {
		if err := fixtures.Clean(); err != nil {
			b.Fatal(err)
		}
	}()

	for _, f := range fixtures.ByTag(".git") {
		b.Run(f.URL, func(b *testing.B) {
			fs := f.DotGit()
			dg := dotgit.New(fs)

			for i := 0; i < b.N; i++ {
				for _, t := range objectTypes {
					ph, err := dg.ObjectPacks()
					if err != nil {
						b.Fatal(err)
					}

					for _, h := range ph {
						f, err := dg.ObjectPack(h)
						if err != nil {
							b.Fatal(err)
						}

						idxf, err := dg.ObjectPackIdx(h)
						if err != nil {
							b.Fatal(err)
						}

						iter, err := NewPackfileIter(f, idxf, t)
						if err != nil {
							b.Fatal(err)
						}

						err = iter.ForEach(func(o plumbing.EncodedObject) error {
							if o.Type() != t {
								b.Errorf("expecting %s, got %s", t, o.Type())
							}
							return nil
						})

						if err != nil {
							b.Fatal(err)
						}
					}
				}
			}
		})
	}
}
<commit_msg>storage: filesystem, add PackfileIter benchmark reading object content<commit_after>package filesystem

import (
	"io/ioutil"
	"testing"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"

	. "gopkg.in/check.v1"
	"gopkg.in/src-d/go-git-fixtures.v3"
)

type FsSuite struct {
	fixtures.Suite
}

var objectTypes = []plumbing.ObjectType{
	plumbing.CommitObject,
	plumbing.TagObject,
	plumbing.TreeObject,
	plumbing.BlobObject,
}

var _ = Suite(&FsSuite{})

func (s *FsSuite) TestGetFromObjectFile(c *C) {
	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
	o, err := NewObjectStorage(dotgit.New(fs))
	c.Assert(err, IsNil)

	expected := plumbing.NewHash("f3dfe29d268303fc6e1bbce268605fc99573406e")
	obj, err := o.EncodedObject(plumbing.AnyObject, expected)
	c.Assert(err, IsNil)
	c.Assert(obj.Hash(), Equals, expected)
}

func (s *FsSuite) TestGetFromPackfile(c *C) {
	fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) {
		fs := f.DotGit()
		o, err := NewObjectStorage(dotgit.New(fs))
		c.Assert(err, IsNil)

		expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
		obj, err := o.EncodedObject(plumbing.AnyObject, expected)
		c.Assert(err, IsNil)
		c.Assert(obj.Hash(), Equals, expected)
	})
}

func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) {
	fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit()
	o, err := NewObjectStorage(dotgit.New(fs))
	c.Assert(err, IsNil)

	expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3")
	obj, err := o.getFromPackfile(expected, false)
	c.Assert(err, IsNil)
	c.Assert(obj.Hash(), Equals, expected)

	expected = plumbing.NewHash("e9cfa4c9ca160546efd7e8582ec77952a27b17db")
	obj, err = o.getFromPackfile(expected, false)
	c.Assert(err, IsNil)
	c.Assert(obj.Hash(), Equals, expected)
}

func (s *FsSuite) TestIter(c *C) {
	fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) {
		fs := f.DotGit()
		o, err := NewObjectStorage(dotgit.New(fs))
		c.Assert(err, IsNil)

		iter, err := o.IterEncodedObjects(plumbing.AnyObject)
		c.Assert(err, IsNil)

		var count int32
		err = iter.ForEach(func(o plumbing.EncodedObject) error {
			count++
			return nil
		})

		c.Assert(err, IsNil)
		c.Assert(count, Equals, f.ObjectsCount)
	})
}

func (s *FsSuite) TestIterWithType(c *C) {
	fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
		for _, t := range objectTypes {
			fs := f.DotGit()
			o, err := NewObjectStorage(dotgit.New(fs))
			c.Assert(err, IsNil)

			iter, err := o.IterEncodedObjects(t)
			c.Assert(err, IsNil)

			err = iter.ForEach(func(o plumbing.EncodedObject) error {
				c.Assert(o.Type(), Equals, t)
				return nil
			})

			c.Assert(err, IsNil)
		}

	})
}

func (s *FsSuite) TestPackfileIter(c *C) {
	fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
		fs := f.DotGit()
		dg := dotgit.New(fs)

		for _, t := range objectTypes {
			ph, err := dg.ObjectPacks()
			c.Assert(err, IsNil)

			for _, h := range ph {
				f, err := dg.ObjectPack(h)
				c.Assert(err, IsNil)

				idxf, err := dg.ObjectPackIdx(h)
				c.Assert(err, IsNil)

				iter, err := NewPackfileIter(f, idxf, t)
				c.Assert(err, IsNil)
				err = iter.ForEach(func(o plumbing.EncodedObject) error {
					c.Assert(o.Type(), Equals, t)
					return nil
				})

				c.Assert(err, IsNil)
			}
		}
	})

}

func BenchmarkPackfileIter(b *testing.B) {
	if err := fixtures.Init(); err != nil {
		b.Fatal(err)
	}

	defer func() {
		if err := fixtures.Clean(); err != nil {
			b.Fatal(err)
		}
	}()

	for _, f := range fixtures.ByTag(".git") {
		b.Run(f.URL, func(b *testing.B) {
			fs := f.DotGit()
			dg := dotgit.New(fs)

			for i := 0; i < b.N; i++ {
				for _, t := range objectTypes {
					ph, err := dg.ObjectPacks()
					if err != nil {
						b.Fatal(err)
					}

					for _, h := range ph {
						f, err := dg.ObjectPack(h)
						if err != nil {
							b.Fatal(err)
						}

						idxf, err := dg.ObjectPackIdx(h)
						if err != nil {
							b.Fatal(err)
						}

						iter, err := NewPackfileIter(f, idxf, t)
						if err != nil {
							b.Fatal(err)
						}

						err = iter.ForEach(func(o plumbing.EncodedObject) error {
							if o.Type() != t {
								b.Errorf("expecting %s, got %s", t, o.Type())
							}
							return nil
						})

						if err != nil {
							b.Fatal(err)
						}
					}
				}
			}
		})
	}
}

func BenchmarkPackfileIterReadContent(b *testing.B) {
	if err := fixtures.Init(); err != nil {
		b.Fatal(err)
	}

	defer func() {
		if err := fixtures.Clean(); err != nil {
			b.Fatal(err)
		}
	}()

	for _, f := range fixtures.ByTag(".git") {
		b.Run(f.URL, func(b *testing.B) {
			fs := f.DotGit()
			dg := dotgit.New(fs)

			for i := 0; i < b.N; i++ {
				for _, t := range objectTypes {
					ph, err := dg.ObjectPacks()
					if err != nil {
						b.Fatal(err)
					}

					for _, h := range ph {
						f, err := dg.ObjectPack(h)
						if err != nil {
							b.Fatal(err)
						}

						idxf, err := dg.ObjectPackIdx(h)
						if err != nil {
							b.Fatal(err)
						}

						iter, err := NewPackfileIter(f, idxf, t)
						if err != nil {
							b.Fatal(err)
						}

						err = iter.ForEach(func(o plumbing.EncodedObject) error {
							if o.Type() != t {
								b.Errorf("expecting %s, got %s", t, o.Type())
							}

							r, err := o.Reader()
							if err != nil {
								b.Fatal(err)
							}

							if _, err := ioutil.ReadAll(r); err != nil {
								b.Fatal(err)
							}

							return r.Close()
						})

						if err != nil {
							b.Fatal(err)
						}
					}
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>// +build linux,!nonsystemd

package journald

import (
	"os"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/coreos/go-systemd/sdjournal"
	"github.com/inconshreveable/log15"
	"github.com/stephane-martin/skewer/model"
	"github.com/stephane-martin/skewer/services/base"
	"github.com/stephane-martin/skewer/utils"
	"github.com/stephane-martin/skewer/utils/eerrors"
)

// TODO: provide a way to link statically to libsystemd

var Supported = true

type Reader struct {
	journal        *sdjournal.Journal
	stopchan       chan struct{}
	shutdownchan   chan struct{}
	wgroup         sync.WaitGroup
	logger         log15.Logger
	stasher        base.Stasher
	fatalErrorChan chan struct{}
	fatalOnce      sync.Once
}

type Converter func(map[string]string) *model.FullMessage

func EntryToSyslog(entry map[string]string) (m *model.SyslogMessage) {
	m = model.CleanFactory()
	properties := map[string]string{}
	for k, v := range entry {
		k = strings.ToLower(k)
		switch k {
		case "syslog_identifier":
		case "_comm":
			m.AppName = v
		case "message":
			m.Message = v
		case "syslog_pid":
		case "_pid":
			m.ProcId = v
		case "priority":
			p, err := strconv.Atoi(v)
			if err == nil {
				m.Severity = model.Severity(p)
			}
		case "syslog_facility":
			f, err := strconv.Atoi(v)
			if err == nil {
				m.Facility = model.Facility(f)
			}
		case "_hostname":
			m.HostName = v
		case "_source_realtime_timestamp": // microseconds
			t, err := strconv.ParseInt(v, 10, 64)
			if err == nil {
				m.TimeReportedNum = t * 1000
			}
		default:
			if strings.HasPrefix(k, "_") {
				properties[k] = v
			}

		}
	}
	if len(m.AppName) == 0 {
		m.AppName = entry["SYSLOG_IDENTIFIER"]
	}
	if len(m.ProcId) == 0 {
		m.ProcId = entry["SYSLOG_PID"]
	}
	m.TimeGeneratedNum = time.Now().UnixNano()
	if m.TimeReportedNum == 0 {
		m.TimeReportedNum = m.TimeGeneratedNum
	}
	m.Priority = model.Priority(int(m.Facility)*8 + int(m.Severity))
	m.ClearDomain("journald")
	m.Properties.Map["journald"].Map = properties
	m.SetProperty("skewer", "client", m.HostName)
	return m
}

func makeMapConverter(coding string, confID utils.MyULID) Converter {
	decoder := utils.SelectDecoder(coding)
	generator := utils.NewGenerator()
	return func(m map[string]string) *model.FullMessage {
		dest := make(map[string]string)
		var k, k2, v, v2 string
		var err error
		for k, v = range m {
			k2, err = decoder.String(k)
			if err == nil {
				v2, err = decoder.String(v)
				if err == nil {
					dest[k2] = v2
				}
			}
		}
		full := model.FullFactoryFrom(EntryToSyslog(dest))
		full.Uid = generator.Uid()
		full.ConfId = confID
		return full
	}
}

func (r *Reader) FatalError() chan struct{} {
	return r.fatalErrorChan
}

func (r *Reader) dofatal() {
	r.fatalOnce.Do(func() { close(r.fatalErrorChan) })
}

func NewReader(stasher base.Stasher, logger log15.Logger) (*Reader, error) {
	var err error
	r := &Reader{
		logger:         logger,
		stasher:        stasher,
		shutdownchan:   make(chan struct{}),
		fatalErrorChan: make(chan struct{}),
	}
	r.journal, err = sdjournal.NewJournal()
	if err != nil {
		return nil, err
	}
	err = r.journal.SeekTail()
	if err != nil {
		r.journal.Close()
		return nil, err
	}
	_, err = r.journal.Previous()
	if err != nil {
		r.journal.Close()
		return nil, err
	}
	return r, nil
}

func (r *Reader) wait() chan struct{} {
	events := make(chan struct{})
	r.wgroup.Add(1)

	go func() {
		defer r.wgroup.Done()
		var ev int

		for {
			select {
			case <-r.stopchan:
				close(events)
				return
			case <-r.shutdownchan:
				close(events)
				return
			default:
				ev = r.journal.Wait(time.Second)
				if ev == sdjournal.SD_JOURNAL_APPEND || ev == sdjournal.SD_JOURNAL_INVALIDATE {
					close(events)
					return
				} else if ev == -int(syscall.EBADF) {
					r.logger.Debug("journal.Wait returned EBADF") // r.journal was closed
					close(events)
					return
				} else if ev != 0 {
					// r.logger.Debug("journal.Wait event", "code", ev)
				}
			}
		}
	}()
	return events
}

func (r *Reader) Start(confID utils.MyULID) {
	r.stopchan = make(chan struct{})
	hostname, err := os.Hostname()
	if err != nil {
		hostname = "unknown"
	}

	r.wgroup.Add(1)
	go func() {
		defer func() {
			r.wgroup.Done()
		}()

		var err error
		var nb uint64
		var entry *sdjournal.JournalEntry
		converter := makeMapConverter("utf8", confID)

		for {
			// get entries from journald
		LoopGetEntries:
			for {
				select {
				case <-r.stopchan:
					return
				default:
					nb, err = r.journal.Next()
					if err != nil {
						return
					}
					if nb == 0 {
						select {
						case <-r.shutdownchan:
							return
						default:
							break LoopGetEntries
						}
					}
					entry, err = r.journal.GetEntry()
					if err != nil {
						return
					}
					err = r.stasher.Stash(converter(entry.Fields))
					if eerrors.Is("Fatal", err) {
						r.logger.Error("Fatal error stashing journal message", "error", err)
						r.dofatal()
						return
					}
					if err != nil {
						r.logger.Warn("Non-fatal error stashing journal message", "error", err)
						continue
					}
					base.IncomingMsgsCounter.WithLabelValues("journald", hostname, "", "").Inc()

				}
			}

			// wait that journald has more entries
			events := r.wait()
			select {
			case <-events:
			case <-r.stopchan:
				return
			}
		}
	}()
}

func (r *Reader) WaitFinished() {
	r.wgroup.Wait()
}

func (r *Reader) Stop() {
	if r.stopchan != nil {
		close(r.stopchan)
		r.WaitFinished()
	}
}

func (r *Reader) Shutdown() {
	close(r.shutdownchan)
	r.WaitFinished()
	if r.stopchan != nil {
		close(r.stopchan)
	}
	// async close the low level journald reader
	go func() {
		r.journal.Close()
	}()
}
<commit_msg>simplify journald reader<commit_after>// +build linux,!nonsystemd

package journald

import (
	"context"
	"os"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/coreos/go-systemd/sdjournal"
	"github.com/inconshreveable/log15"
	"github.com/stephane-martin/skewer/model"
	"github.com/stephane-martin/skewer/services/base"
	"github.com/stephane-martin/skewer/utils"
	"github.com/stephane-martin/skewer/utils/eerrors"
)

var Supported = true

type Reader struct {
	journal        *sdjournal.Journal
	stop           context.CancelFunc
	wgroup         sync.WaitGroup
	logger         log15.Logger
	stasher        base.Stasher
	fatalErrorChan chan struct{}
	fatalOnce      sync.Once
}

type Converter func(*sdjournal.JournalEntry) *model.FullMessage

func EntryToSyslog(entry map[string]string) (m *model.SyslogMessage) {
	m = model.CleanFactory()
	properties := map[string]string{}
	for k, v := range entry {
		k = strings.ToLower(k)
		switch k {
		case "syslog_identifier":
		case "_comm":
			m.AppName = v
		case "message":
			m.Message = v
		case "syslog_pid":
		case "_pid":
			m.ProcId = v
		case "priority":
			p, err := strconv.Atoi(v)
			if err == nil {
				m.Severity = model.Severity(p)
			}
		case "syslog_facility":
			f, err := strconv.Atoi(v)
			if err == nil {
				m.Facility = model.Facility(f)
			}
		case "_hostname":
			m.HostName = v
		case "_source_realtime_timestamp": // microseconds
			t, err := strconv.ParseInt(v, 10, 64)
			if err == nil {
				m.TimeReportedNum = t * 1000
			}
		default:
			if strings.HasPrefix(k, "_") {
				properties[k] = v
			}

		}
	}
	if len(m.AppName) == 0 {
		m.AppName = entry["SYSLOG_IDENTIFIER"]
	}
	if len(m.ProcId) == 0 {
		m.ProcId = entry["SYSLOG_PID"]
	}
	m.TimeGeneratedNum = time.Now().UnixNano()
	if m.TimeReportedNum == 0 {
		m.TimeReportedNum = m.TimeGeneratedNum
	}
	m.Priority = model.Priority(int(m.Facility)*8 + int(m.Severity))
	m.ClearDomain("journald")
	m.Properties.Map["journald"].Map = properties
	m.SetProperty("skewer", "client", m.HostName)
	return m
}

func makeMapConverter(coding string, confID utils.MyULID) Converter {
	decoder := utils.SelectDecoder(coding)
	generator := utils.NewGenerator()

	return func(m *sdjournal.JournalEntry) *model.FullMessage {
		dest := make(map[string]string, len(m.Fields))
		for k, v := range m.Fields {
			k2, err := decoder.String(k)
			if err == nil {
				v2, err := decoder.String(v)
				if err == nil {
					dest[k2] = v2
				}
			}
		}
		full := model.FullFactoryFrom(EntryToSyslog(dest))
		full.Uid = generator.Uid()
		full.ConfId = confID
		return full
	}
}

func (r *Reader) FatalError() chan struct{} {
	return r.fatalErrorChan
}

func (r *Reader) dofatal() {
	r.fatalOnce.Do(func() { close(r.fatalErrorChan) })
}

func NewReader(stasher base.Stasher, logger log15.Logger) (*Reader, error) {
	var err error
	r := &Reader{
		logger:         logger,
		stasher:        stasher,
		fatalErrorChan: make(chan struct{}),
	}
	r.journal, err = sdjournal.NewJournal()
	if err != nil {
		return nil, err
	}
	err = r.journal.SeekTail()
	if err != nil {
		r.journal.Close()
		return nil, err
	}
	_, err = r.journal.Previous()
	if err != nil {
		r.journal.Close()
		return nil, err
	}
	return r, nil
}

func wait(ctx context.Context, logger log15.Logger, j *sdjournal.Journal) {
	lctx, lcancel := context.WithCancel(ctx)

	go func() {
		for {
			// Wait() is a blocking call
			ev := j.Wait(time.Second)
			if ev == sdjournal.SD_JOURNAL_APPEND || ev == sdjournal.SD_JOURNAL_INVALIDATE {
				lcancel()
				return
			} else if ev == -int(syscall.EBADF) {
				logger.Debug("journal.Wait returned EBADF") // r.journal was closed
				lcancel()
				return
			} else if ev != 0 {
				// r.logger.Debug("journal.Wait event", "code", ev)
			}
		}
	}()
	<-lctx.Done()
}

func (r *Reader) Start(confID utils.MyULID) {
	var ctx context.Context
	ctx, r.stop = context.WithCancel(context.Background())
	hostname, err := os.Hostname()
	if err != nil {
		hostname = "unknown"
	}
	converter := makeMapConverter("utf8", confID)

	r.wgroup.Add(1)
	go func() {
		defer r.wgroup.Done()

	L:
		for {
			select {
			case <-ctx.Done():
				return
			default:
				nb, err := r.journal.Next()
				if err != nil {
					return
				}
				if nb == 0 {
					wait(ctx, r.logger, r.journal) // wait that journald has more entries
					continue L
				}
				entry, err := r.journal.GetEntry()
				if err != nil {
					return
				}
				err = r.stasher.Stash(converter(entry))
				if eerrors.IsFatal(err) {
					r.logger.Error("Fatal error stashing journal message", "error", err)
					r.dofatal()
					return
				}
				if err != nil {
					r.logger.Warn("Non-fatal error stashing journal message", "error", err)
					continue L
				}
				base.IncomingMsgsCounter.WithLabelValues("journald", hostname, "", "").Inc()

			}
		}

	}()
}

func (r *Reader) WaitFinished() {
	r.wgroup.Wait()
}

func (r *Reader) Stop() {
	if r.stop != nil {
		r.stop()
		r.stop = nil
	}
	r.WaitFinished()
}

func (r *Reader) Shutdown() {
	r.Stop()
	// async close the low level journald reader
	go func() {
		r.journal.Close()
	}()
}
<|endoftext|>


<commit_before>package json

import (
	"net/http"

	"github.com/bborbe/log"
	"reflect"
	"encoding/json"
	error_handler "github.com/bborbe/server/handler/error"
)

var logger = log.DefaultLogger

type jsonHandler struct {
	m interface{}
}

func NewJsonHandler(m interface{}) *jsonHandler {
	h := new(jsonHandler)
	h.m = m
	return h
}

func (m *jsonHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) {
	logger.Debug("write json")
	logger.Debugf("object to convert %v", m.m)
	b, err := json.Marshal(m.m)
	if err != nil {
		logger.Debugf("Marshal json failed: %v", err)
		e := error_handler.NewErrorMessage(http.StatusInternalServerError, err.Error())
		e.ServeHTTP(responseWriter, request)
		return
	}
	logger.Debugf("json string %s", string(b))
	responseWriter.Header().Set("Content-Type", "application/json")
	responseWriter.WriteHeader(http.StatusOK)

	logger.Debugf("object type %v", reflect.TypeOf(m.m).Kind())
	if reflect.TypeOf(m.m).Kind() == reflect.Slice && string(b) == "null" {
		responseWriter.Write([]byte("[]"))
	}   else {
		responseWriter.Write(b)
	}

}
<commit_msg>format<commit_after>package json

import (
	"net/http"

	"encoding/json"
	"reflect"

	"github.com/bborbe/log"
	error_handler "github.com/bborbe/server/handler/error"
)

var logger = log.DefaultLogger

type jsonHandler struct {
	m interface{}
}

func NewJsonHandler(m interface{}) *jsonHandler {
	h := new(jsonHandler)
	h.m = m
	return h
}

func (m *jsonHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) {
	logger.Debug("write json")
	logger.Debugf("object to convert %v", m.m)
	b, err := json.Marshal(m.m)
	if err != nil {
		logger.Debugf("Marshal json failed: %v", err)
		e := error_handler.NewErrorMessage(http.StatusInternalServerError, err.Error())
		e.ServeHTTP(responseWriter, request)
		return
	}
	logger.Debugf("json string %s", string(b))
	responseWriter.Header().Set("Content-Type", "application/json")
	responseWriter.WriteHeader(http.StatusOK)

	logger.Debugf("object type %v", reflect.TypeOf(m.m).Kind())
	if reflect.TypeOf(m.m).Kind() == reflect.Slice && string(b) == "null" {
		responseWriter.Write([]byte("[]"))
	} else {
		responseWriter.Write(b)
	}

}
<|endoftext|>


<commit_before>package model

import (
	"encoding/json"
	"fmt"
	"io"
	"strconv"

	"github.com/olivere/elastic"
	"github.com/pkg/errors"
)

// queryBinding represents information about where and how the data should be fetched.
type elasticQueryBinding struct {
	Index string
	Field string
}

// PageviewElastic is ElasticDB implementation of PageviewStorage.
type PageviewElastic struct {
	DB            *ElasticDB
	actionsCached map[string][]string
}

// Count returns number of Pageviews matching the filter defined by PageviewOptions.
func (pDB *PageviewElastic) Count(options AggregateOptions) (CountRowCollection, bool, error) {
	// pageview events are stored in multiple measurements which need to be resolved
	binding, err := pDB.resolveQueryBindings(options.Action)
	if err != nil {
		return nil, false, err
	}

	// action is not being tracked within separate measurements and we would get no records back
	// removing it before applying filter
	options.Action = ""

	extras := make(map[string]elastic.Aggregation)

	search := pDB.DB.Client.Search().
		Index(binding.Index).
		Type("_doc").
		Size(0) // return no specific results

	search, err = pDB.DB.addSearchFilters(search, binding.Index, options)
	if err != nil {
		return nil, false, err
	}

	if options.TimeHistogram != nil {
		extras["date_time_histogram"] = elastic.NewDateHistogramAggregation().
			Field("time").
			Interval(options.TimeHistogram.Interval).
			TimeZone("UTC").
			Offset(options.TimeHistogram.Offset)
	}

	search, err = pDB.DB.addGroupBy(search, binding.Index, options, extras)
	if err != nil {
		return nil, false, err
	}

	// get results
	result, err := search.Do(pDB.DB.Context)
	if err != nil {
		return nil, false, err
	}

	if len(options.GroupBy) == 0 && options.TimeHistogram == nil {
		// extract simplified results (no aggregation)
		return CountRowCollection{
			CountRow{
				Count: int(result.Hits.TotalHits),
			},
		}, true, nil
	}

	// extract results
	return pDB.DB.countRowCollectionFromAggregations(result, options)
}

// Sum returns number of Pageviews matching the filter defined by AggregateOptions.
func (pDB *PageviewElastic) Sum(options AggregateOptions) (SumRowCollection, bool, error) {
	// pageview events are stored in multiple measurements which need to be resolved
	binding, err := pDB.resolveQueryBindings(options.Action)
	if err != nil {
		return nil, false, err
	}

	// action is not being tracked within separate measurements and we would get no records back
	// removing it before applying filter
	options.Action = ""

	extras := make(map[string]elastic.Aggregation)
	targetAgg := fmt.Sprintf("%s_sum", binding.Field)
	extras[targetAgg] = elastic.NewSumAggregation().Field(binding.Field)

	search := pDB.DB.Client.Search().
		Index(binding.Index).
		Type("_doc").
		Size(0) // return no specific results

	search, err = pDB.DB.addSearchFilters(search, binding.Index, options)
	if err != nil {
		return nil, false, err
	}

	if options.TimeHistogram != nil {
		extras["date_time_histogram"] = elastic.NewDateHistogramAggregation().
			Field("time").
			Interval(options.TimeHistogram.Interval).
			TimeZone("UTC").
			Offset(options.TimeHistogram.Offset)
	}

	search, err = pDB.DB.addGroupBy(search, binding.Index, options, extras)
	if err != nil {
		return nil, false, err
	}

	// get results
	result, err := search.Do(pDB.DB.Context)
	if err != nil {
		return nil, false, err
	}

	return pDB.DB.sumRowCollectionFromAggregations(result, options, targetAgg)
}

// List returns list of all Pageviews based on given PageviewOptions.
func (pDB *PageviewElastic) List(options ListOptions) (PageviewRowCollection, error) {
	var prc PageviewRowCollection

	fsc := elastic.NewFetchSourceContext(true).Include(options.SelectFields...)
	scroll := pDB.DB.Client.Scroll("pageviews").
		Type("_doc").
		Size(1000).
		FetchSourceContext(fsc)

	scroll, err := pDB.DB.addScrollFilters(scroll, "pageviews", options.AggregateOptions)
	if err != nil {
		return nil, err
	}

	// prepare PageviewRow buckets
	prBuckets := make(map[string]*PageviewRow)

	// get results
	for {
		results, err := scroll.Do(pDB.DB.Context)
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, errors.Wrap(err, "error while reading list data from elastic")
		}

		// Send the hits to the hits channel
		for _, hit := range results.Hits.Hits {
			// populate pageview for collection
			pv := &Pageview{}
			if err := json.Unmarshal(*hit.Source, pv); err != nil {
				return nil, errors.Wrap(err, "error reading pageview record from elastic")
			}

			// extract raw pageview data to build tags map
			rawPv := make(map[string]interface{})
			if err := json.Unmarshal(*hit.Source, &rawPv); err != nil {
				return nil, errors.Wrap(err, "error reading pageview record from elastic")
			}

			// we need to get string value for tags by type casting
			tags := make(map[string]string)
			key := ""
			for _, field := range options.GroupBy {
				var tagVal string
				switch val := rawPv[field].(type) {
				case nil:
					tagVal = ""
				case bool:
					if val {
						tagVal = "1"
					} else {
						tagVal = "0"
					}
				case string:
					tagVal = val
				case float64:
					tagVal = strconv.FormatFloat(val, 'f', 0, 64)
				case int64:
					tagVal = strconv.FormatInt(val, 10)
				default:
					return nil, fmt.Errorf("unhandled tag type in pageview listing: %T", rawPv[field])
				}

				tags[field] = fmt.Sprintf("%s", tagVal)
				key = fmt.Sprintf("%s%s=%s_", key, field, tagVal)
			}

			// place Pageview instance into proper PageviewRow based on tags (key)
			pr, ok := prBuckets[key]
			if !ok {
				pr = &PageviewRow{
					Tags: tags,
				}
				prBuckets[key] = pr
			}
			pr.Pageviews = append(pr.Pageviews, pv)
		}
	}

	for _, pr := range prBuckets {
		prc = append(prc, pr)
	}

	return prc, nil
}

// Categories lists all tracked categories.
func (pDB *PageviewElastic) Categories() []string {
	return []string{
		CategoryPageview,
	}
}

// Flags lists all available flags.
func (pDB *PageviewElastic) Flags() []string {
	return []string{
		FlagArticle,
	}
}

// Actions lists all tracked actions under the given category.
func (pDB *PageviewElastic) Actions(category string) ([]string, error) {
	switch category {
	case CategoryPageview:
		return []string{
			ActionPageviewLoad,
		}, nil
	}
	return nil, fmt.Errorf("unknown pageview category: %s", category)
}

// Users lists all tracked users.
func (pDB *PageviewElastic) Users() ([]string, error) {
	// prepare aggregation
	search := pDB.DB.Client.Search().Index("Pageviews").Type("_doc").Size(0)
	agg := elastic.NewTermsAggregation().Field("user_id.keyword")
	search = search.Aggregation("buckets", agg)

	// get results
	result, err := search.Do(pDB.DB.Context)
	if err != nil {
		return nil, err
	}
	aggResult, _ := result.Aggregations.Terms("buckets")

	users := []string{}
	for _, bucket := range aggResult.Buckets {
		key, ok := bucket.Key.(string) // non-nested aggregation has string key
		if !ok {
			return nil, fmt.Errorf("unexpected type of bucket key: %T", bucket.Key)
		}
		users = append(users, key)
	}

	return users, nil
}

// resolveQueryBindings returns name of the table and field used within the aggregate function
// based on the provided action.
func (pDB *PageviewElastic) resolveQueryBindings(action string) (elasticQueryBinding, error) {
	switch action {
	case ActionPageviewLoad:
		return elasticQueryBinding{
			Index: TablePageviews,
			Field: "token",
		}, nil
	case ActionPageviewTimespent:
		return elasticQueryBinding{
			Index: TableTimespent,
			Field: "sum",
		}, nil
	}
	return elasticQueryBinding{}, fmt.Errorf("unable to resolve query bindings: action [%s] unknown", action)
}
<commit_msg>segments api fix timespent field <commit_after>package model

import (
	"encoding/json"
	"fmt"
	"io"
	"strconv"

	"github.com/olivere/elastic"
	"github.com/pkg/errors"
)

// queryBinding represents information about where and how the data should be fetched.
type elasticQueryBinding struct {
	Index string
	Field string
}

// PageviewElastic is ElasticDB implementation of PageviewStorage.
type PageviewElastic struct {
	DB            *ElasticDB
	actionsCached map[string][]string
}

// Count returns number of Pageviews matching the filter defined by PageviewOptions.
func (pDB *PageviewElastic) Count(options AggregateOptions) (CountRowCollection, bool, error) {
	// pageview events are stored in multiple measurements which need to be resolved
	binding, err := pDB.resolveQueryBindings(options.Action)
	if err != nil {
		return nil, false, err
	}

	// action is not being tracked within separate measurements and we would get no records back
	// removing it before applying filter
	options.Action = ""

	extras := make(map[string]elastic.Aggregation)

	search := pDB.DB.Client.Search().
		Index(binding.Index).
		Type("_doc").
		Size(0) // return no specific results

	search, err = pDB.DB.addSearchFilters(search, binding.Index, options)
	if err != nil {
		return nil, false, err
	}

	if options.TimeHistogram != nil {
		extras["date_time_histogram"] = elastic.NewDateHistogramAggregation().
			Field("time").
			Interval(options.TimeHistogram.Interval).
			TimeZone("UTC").
			Offset(options.TimeHistogram.Offset)
	}

	search, err = pDB.DB.addGroupBy(search, binding.Index, options, extras)
	if err != nil {
		return nil, false, err
	}

	// get results
	result, err := search.Do(pDB.DB.Context)
	if err != nil {
		return nil, false, err
	}

	if len(options.GroupBy) == 0 && options.TimeHistogram == nil {
		// extract simplified results (no aggregation)
		return CountRowCollection{
			CountRow{
				Count: int(result.Hits.TotalHits),
			},
		}, true, nil
	}

	// extract results
	return pDB.DB.countRowCollectionFromAggregations(result, options)
}

// Sum returns number of Pageviews matching the filter defined by AggregateOptions.
func (pDB *PageviewElastic) Sum(options AggregateOptions) (SumRowCollection, bool, error) {
	// pageview events are stored in multiple measurements which need to be resolved
	binding, err := pDB.resolveQueryBindings(options.Action)
	if err != nil {
		return nil, false, err
	}

	// action is not being tracked within separate measurements and we would get no records back
	// removing it before applying filter
	options.Action = ""

	extras := make(map[string]elastic.Aggregation)
	targetAgg := fmt.Sprintf("%s_sum", binding.Field)
	extras[targetAgg] = elastic.NewSumAggregation().Field(binding.Field)

	search := pDB.DB.Client.Search().
		Index(binding.Index).
		Type("_doc").
		Size(0) // return no specific results

	search, err = pDB.DB.addSearchFilters(search, binding.Index, options)
	if err != nil {
		return nil, false, err
	}

	if options.TimeHistogram != nil {
		extras["date_time_histogram"] = elastic.NewDateHistogramAggregation().
			Field("time").
			Interval(options.TimeHistogram.Interval).
			TimeZone("UTC").
			Offset(options.TimeHistogram.Offset)
	}

	search, err = pDB.DB.addGroupBy(search, binding.Index, options, extras)
	if err != nil {
		return nil, false, err
	}

	// get results
	result, err := search.Do(pDB.DB.Context)
	if err != nil {
		return nil, false, err
	}

	return pDB.DB.sumRowCollectionFromAggregations(result, options, targetAgg)
}

// List returns list of all Pageviews based on given PageviewOptions.
func (pDB *PageviewElastic) List(options ListOptions) (PageviewRowCollection, error) {
	var prc PageviewRowCollection

	fsc := elastic.NewFetchSourceContext(true).Include(options.SelectFields...)
	scroll := pDB.DB.Client.Scroll("pageviews").
		Type("_doc").
		Size(1000).
		FetchSourceContext(fsc)

	scroll, err := pDB.DB.addScrollFilters(scroll, "pageviews", options.AggregateOptions)
	if err != nil {
		return nil, err
	}

	// prepare PageviewRow buckets
	prBuckets := make(map[string]*PageviewRow)

	// get results
	for {
		results, err := scroll.Do(pDB.DB.Context)
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, errors.Wrap(err, "error while reading list data from elastic")
		}

		// Send the hits to the hits channel
		for _, hit := range results.Hits.Hits {
			// populate pageview for collection
			pv := &Pageview{}
			if err := json.Unmarshal(*hit.Source, pv); err != nil {
				return nil, errors.Wrap(err, "error reading pageview record from elastic")
			}

			// extract raw pageview data to build tags map
			rawPv := make(map[string]interface{})
			if err := json.Unmarshal(*hit.Source, &rawPv); err != nil {
				return nil, errors.Wrap(err, "error reading pageview record from elastic")
			}

			// we need to get string value for tags by type casting
			tags := make(map[string]string)
			key := ""
			for _, field := range options.GroupBy {
				var tagVal string
				switch val := rawPv[field].(type) {
				case nil:
					tagVal = ""
				case bool:
					if val {
						tagVal = "1"
					} else {
						tagVal = "0"
					}
				case string:
					tagVal = val
				case float64:
					tagVal = strconv.FormatFloat(val, 'f', 0, 64)
				case int64:
					tagVal = strconv.FormatInt(val, 10)
				default:
					return nil, fmt.Errorf("unhandled tag type in pageview listing: %T", rawPv[field])
				}

				tags[field] = fmt.Sprintf("%s", tagVal)
				key = fmt.Sprintf("%s%s=%s_", key, field, tagVal)
			}

			// place Pageview instance into proper PageviewRow based on tags (key)
			pr, ok := prBuckets[key]
			if !ok {
				pr = &PageviewRow{
					Tags: tags,
				}
				prBuckets[key] = pr
			}
			pr.Pageviews = append(pr.Pageviews, pv)
		}
	}

	for _, pr := range prBuckets {
		prc = append(prc, pr)
	}

	return prc, nil
}

// Categories lists all tracked categories.
func (pDB *PageviewElastic) Categories() []string {
	return []string{
		CategoryPageview,
	}
}

// Flags lists all available flags.
func (pDB *PageviewElastic) Flags() []string {
	return []string{
		FlagArticle,
	}
}

// Actions lists all tracked actions under the given category.
func (pDB *PageviewElastic) Actions(category string) ([]string, error) {
	switch category {
	case CategoryPageview:
		return []string{
			ActionPageviewLoad,
		}, nil
	}
	return nil, fmt.Errorf("unknown pageview category: %s", category)
}

// Users lists all tracked users.
func (pDB *PageviewElastic) Users() ([]string, error) {
	// prepare aggregation
	search := pDB.DB.Client.Search().Index("Pageviews").Type("_doc").Size(0)
	agg := elastic.NewTermsAggregation().Field("user_id.keyword")
	search = search.Aggregation("buckets", agg)

	// get results
	result, err := search.Do(pDB.DB.Context)
	if err != nil {
		return nil, err
	}
	aggResult, _ := result.Aggregations.Terms("buckets")

	users := []string{}
	for _, bucket := range aggResult.Buckets {
		key, ok := bucket.Key.(string) // non-nested aggregation has string key
		if !ok {
			return nil, fmt.Errorf("unexpected type of bucket key: %T", bucket.Key)
		}
		users = append(users, key)
	}

	return users, nil
}

// resolveQueryBindings returns name of the table and field used within the aggregate function
// based on the provided action.
func (pDB *PageviewElastic) resolveQueryBindings(action string) (elasticQueryBinding, error) {
	switch action {
	case ActionPageviewLoad:
		return elasticQueryBinding{
			Index: TablePageviews,
			Field: "token",
		}, nil
	case ActionPageviewTimespent:
		return elasticQueryBinding{
			Index: TableTimespent,
			Field: "timespent",
		}, nil
	}
	return elasticQueryBinding{}, fmt.Errorf("unable to resolve query bindings: action [%s] unknown", action)
}
<|endoftext|>


<commit_before>/*
Use this package's methods to write logs any where
*/
package log

import (
	"log"
	"os"
)

const (
	DEBUG = 0
	INFO = 1
	WARN = 2
	ERROR = 3
	FATAL = 4
)

var (
	debug_logger *log.Logger
	info_logger *log.Logger
	warn_logger *log.Logger
	error_logger *log.Logger
	fatal_logger *log.Logger
)

func init() {
	debug_logger = log.New(os.Stdout, "DEBUG:", log.Ldate + log.Ltime)
	info_logger = log.New(os.Stdout, "INFO:", log.Ldate + log.Ltime)
	warn_logger = log.New(os.Stdout, "WARN:", log.Ldate + log.Ltime)
	error_logger = log.New(os.Stdout, "ERROR:", log.Ldate + log.Ltime)
	fatal_logger = log.New(os.Stdout, "FATAL:", log.Ldate + log.Ltime)
}

// Prints debug level statements
func Debug(v ...interface{}) {
	debug_logger.Print(v...)
}

// same as Debug function with formatted string
func Debugf(format string, v ...interface{}) {
	debug_logger.Printf(format, v...)
}

// Prints info level statements
func Info(v ...interface{}) {
	info_logger.Print(v...)
}

// same as Info function with formatted string
func Infof(format string, v ...interface{}) {
	info_logger.Printf(format, v...)
}

// Prints warn level statements
func Warn(v ...interface{}) {
	warn_logger.Print(v...)
}

// same as Warn function with formatted string
func Warnf(format string, v ...interface{}) {
	warn_logger.Printf(format, v...)
}

// Prints error level statements
func Error(v ...interface{}) {
	error_logger.Print(v...)
}

// same as Error function with formatted string
func Errorf(format string, v ...interface{}) {
	error_logger.Printf(format, v...)
}

// Prints fatal level statements
func Fatal(v ...interface{}) {
	fatal_logger.Print(v...)
}

// same as Fatal function with formatted string
func Fatalf(format string, v ...interface{}) {
	fatal_logger.Printf(format, v...)
}

<commit_msg>added verbose logging support<commit_after>/*
Use this package's methods to write logs any where
*/
package log

import (
	"log"
	"os"
)

const (
	VERBOSE = 0
	DEBUG = 1
	INFO = 2
	WARN = 3
	ERROR = 4
	FATAL = 5
)

var (
	verbose_logger *log.Logger
	debug_logger *log.Logger
	info_logger *log.Logger
	warn_logger *log.Logger
	error_logger *log.Logger
	fatal_logger *log.Logger
)

func init() {
	verbose_logger = log.New(os.Stdout, "VERBOSE:", log.Ldate + log.Ltime)
	debug_logger = log.New(os.Stdout, "DEBUG:", log.Ldate + log.Ltime)
	info_logger = log.New(os.Stdout, "INFO:", log.Ldate + log.Ltime)
	warn_logger = log.New(os.Stdout, "WARN:", log.Ldate + log.Ltime)
	error_logger = log.New(os.Stdout, "ERROR:", log.Ldate + log.Ltime)
	fatal_logger = log.New(os.Stdout, "FATAL:", log.Ldate + log.Ltime)
}

// Prints debug level statements
func Verbose(v ...interface{}) {
	verbose_logger.Print(v...)
}

// same as Debug function with formatted string
func Verbosef(format string, v ...interface{}) {
	verbose_logger.Printf(format, v...)
}

// Prints debug level statements
func Debug(v ...interface{}) {
	debug_logger.Print(v...)
}

// same as Debug function with formatted string
func Debugf(format string, v ...interface{}) {
	debug_logger.Printf(format, v...)
}

// Prints info level statements
func Info(v ...interface{}) {
	info_logger.Print(v...)
}

// same as Info function with formatted string
func Infof(format string, v ...interface{}) {
	info_logger.Printf(format, v...)
}

// Prints warn level statements
func Warn(v ...interface{}) {
	warn_logger.Print(v...)
}

// same as Warn function with formatted string
func Warnf(format string, v ...interface{}) {
	warn_logger.Printf(format, v...)
}

// Prints error level statements
func Error(v ...interface{}) {
	error_logger.Print(v...)
}

// same as Error function with formatted string
func Errorf(format string, v ...interface{}) {
	error_logger.Printf(format, v...)
}

// Prints fatal level statements
func Fatal(v ...interface{}) {
	fatal_logger.Print(v...)
}

// same as Fatal function with formatted string
func Fatalf(format string, v ...interface{}) {
	fatal_logger.Printf(format, v...)
}

<|endoftext|>


<commit_before>package neutron_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"

	"github.com/markstgodard/go-neutron/neutron"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Neutron API", func() {
	var (
		client *neutron.Client
		server *httptest.Server
	)

	const networks = `{
  "networks": [
    {
      "id": "e53a3b67-0074-404c-90b5-52ae217c3587",
      "name": "public",
      "description": "public network",
      "status": "ACTIVE",
      "subnets": [
        "3cc622c0-1ed8-470b-8d2b-081305df63b5",
        "9081fc4f-2415-4d99-ae99-0abb262ada90"
      ],
      "tenant_id": "1f77bad08b454898803a3d9f9e3799ec",
      "mtu": 1500,
      "project_id": "1f77bad08b454898803a3d9f9e3799ec"
    }
  ]
}`

	const networksByName = `{
  "networks": [
    {
      "provider:physical_network": null,
      "ipv6_address_scope": null,
      "revision_number": 5,
      "port_security_enabled": true,
      "mtu": 1450,
      "id": "bd62af4c-bbe7-43fb-af21-29f3082fd734",
      "router:external": false,
      "availability_zone_hints": [],
      "availability_zones": [],
      "ipv4_address_scope": null,
      "shared": false,
      "project_id": "1f77bad08b454898803a3d9f9e3799ec",
      "status": "ACTIVE",
      "subnets": [
        "d087782e-3779-4982-b7ca-a4bde71b5aa5"
      ],
      "description": "",
      "tags": [],
      "updated_at": "2016-11-07T03:24:33Z",
      "provider:segmentation_id": 16,
      "name": "network1",
      "admin_state_up": true,
      "tenant_id": "1f77bad08b454898803a3d9f9e3799ec",
      "created_at": "2016-11-07T03:24:33Z",
      "provider:network_type": "vxlan"
    }
  ]
}`

	Describe("NewClient", func() {
		var err error

		It("requires a URL and token", func() {
			client, err = neutron.NewClient("http://192.168.56.101:9696", "some-token")
			Expect(err).ToNot(HaveOccurred())
		})

		Context("when URL is missing", func() {
			It("returns an error", func() {
				client, err = neutron.NewClient("", "some-token")
				Expect(err).To(MatchError("missing URL"))
			})
		})

		Context("when token is missing", func() {
			It("returns an error", func() {
				client, err = neutron.NewClient("http://192.168.56.101:9696", "")
				Expect(err).To(MatchError("missing token"))
			})
		})

	})

	Describe("Networks", func() {

		BeforeEach(func() {
			var err error
			server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if r.URL.Query().Get("name") != "" {
					fmt.Fprintln(w, networksByName)
				} else {
					fmt.Fprintln(w, networks)
				}
			}))

			client, err = neutron.NewClient(server.URL, "some-token")
			Expect(err).ToNot(HaveOccurred())
		})

		AfterEach(func() {
			server.Close()
		})

		It("can list networks", func() {
			networks, err := client.Networks()
			Expect(err).ToNot(HaveOccurred())
			Expect(networks).To(HaveLen(1))
			Expect(networks[0].Name).To(Equal("public"))
			Expect(networks[0].ID).To(Equal("e53a3b67-0074-404c-90b5-52ae217c3587"))
			Expect(networks[0].Description).To(Equal("public network"))
			Expect(networks[0].Status).To(Equal("ACTIVE"))
			Expect(networks[0].Subnets).To(HaveLen(2))
			Expect(networks[0].TenantID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
			Expect(networks[0].MTU).To(Equal(1500))
			Expect(networks[0].ProjectID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
		})

		It("can list networks by name", func() {
			networks, err := client.NetworksByName("network1")
			Expect(err).ToNot(HaveOccurred())
			Expect(networks).To(HaveLen(1))
			Expect(networks[0].Name).To(Equal("network1"))
			Expect(networks[0].ID).To(Equal("bd62af4c-bbe7-43fb-af21-29f3082fd734"))
			Expect(networks[0].Description).To(Equal(""))
			Expect(networks[0].Status).To(Equal("ACTIVE"))
			Expect(networks[0].Subnets).To(HaveLen(1))
			Expect(networks[0].TenantID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
			Expect(networks[0].MTU).To(Equal(1450))
			Expect(networks[0].ProjectID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
		})
	})

	// Describe("Subnets", func() {
	// 	BeforeEach(func() {
	// 		var err error
	// 		server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	// 			fmt.Fprintln(w, subnets)
	// 		}))

	// 		client, err = neutron.NewClient(server.URL, "some-token")
	// 		Expect(err).ToNot(HaveOccurred())
	// 	})

	// 	AfterEach(func() {
	// 		server.Close()
	// 	})

	// 	It("can list subnets", func() {
	// 		subnets, err := client.Subnets()
	// 		Expect(err).ToNot(HaveOccurred())
	// 		Expect(subnets).To(HaveLen(1))
	// 		Expect(subnets[0].Name).To(Equal("public"))
	// 		Expect(subnets[0].ID).To(Equal("e53a3b67-0074-404c-90b5-52ae217c3587"))
	// 		Expect(subnets[0].Description).To(Equal("public network"))
	// 		Expect(networks[0].ProjectID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
	// 	})
	// })

})
<commit_msg>networks by name and empty<commit_after>package neutron_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"

	"github.com/markstgodard/go-neutron/neutron"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Neutron API", func() {
	var (
		client *neutron.Client
		server *httptest.Server
	)

	const networks = `{
  "networks": [
    {
      "id": "e53a3b67-0074-404c-90b5-52ae217c3587",
      "name": "public",
      "description": "public network",
      "status": "ACTIVE",
      "subnets": [
        "3cc622c0-1ed8-470b-8d2b-081305df63b5",
        "9081fc4f-2415-4d99-ae99-0abb262ada90"
      ],
      "tenant_id": "1f77bad08b454898803a3d9f9e3799ec",
      "mtu": 1500,
      "project_id": "1f77bad08b454898803a3d9f9e3799ec"
    }
  ]
}`

	const networksByName = `{
  "networks": [
    {
      "provider:physical_network": null,
      "ipv6_address_scope": null,
      "revision_number": 5,
      "port_security_enabled": true,
      "mtu": 1450,
      "id": "bd62af4c-bbe7-43fb-af21-29f3082fd734",
      "router:external": false,
      "availability_zone_hints": [],
      "availability_zones": [],
      "ipv4_address_scope": null,
      "shared": false,
      "project_id": "1f77bad08b454898803a3d9f9e3799ec",
      "status": "ACTIVE",
      "subnets": [
        "d087782e-3779-4982-b7ca-a4bde71b5aa5"
      ],
      "description": "",
      "tags": [],
      "updated_at": "2016-11-07T03:24:33Z",
      "provider:segmentation_id": 16,
      "name": "network1",
      "admin_state_up": true,
      "tenant_id": "1f77bad08b454898803a3d9f9e3799ec",
      "created_at": "2016-11-07T03:24:33Z",
      "provider:network_type": "vxlan"
    }
  ]
}`

	const networksEmpty = `{
  "networks": []
}`

	Describe("NewClient", func() {
		var err error

		It("requires a URL and token", func() {
			client, err = neutron.NewClient("http://192.168.56.101:9696", "some-token")
			Expect(err).ToNot(HaveOccurred())
		})

		Context("when URL is missing", func() {
			It("returns an error", func() {
				client, err = neutron.NewClient("", "some-token")
				Expect(err).To(MatchError("missing URL"))
			})
		})

		Context("when token is missing", func() {
			It("returns an error", func() {
				client, err = neutron.NewClient("http://192.168.56.101:9696", "")
				Expect(err).To(MatchError("missing token"))
			})
		})

	})

	Describe("Networks", func() {
		Describe("Networks", func() {
			BeforeEach(func() {
				server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					fmt.Fprintln(w, networks)
				}))
				var err error
				client, err = neutron.NewClient(server.URL, "some-token")
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				server.Close()
			})

			It("lists networks", func() {
				networks, err := client.Networks()
				Expect(err).ToNot(HaveOccurred())
				Expect(networks).To(HaveLen(1))
				Expect(networks[0].Name).To(Equal("public"))
				Expect(networks[0].ID).To(Equal("e53a3b67-0074-404c-90b5-52ae217c3587"))
				Expect(networks[0].Description).To(Equal("public network"))
				Expect(networks[0].Status).To(Equal("ACTIVE"))
				Expect(networks[0].Subnets).To(HaveLen(2))
				Expect(networks[0].TenantID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
				Expect(networks[0].MTU).To(Equal(1500))
				Expect(networks[0].ProjectID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
			})
		})

		Describe("NetworksByName", func() {
			BeforeEach(func() {
				server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					switch r.URL.Query().Get("name") {
					case "network1":
						fmt.Fprintln(w, networksByName)
					default:
						fmt.Fprintln(w, networksEmpty)
					}
				}))
				var err error
				client, err = neutron.NewClient(server.URL, "some-token")
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				server.Close()
			})

			It("list networks by name", func() {
				networks, err := client.NetworksByName("network1")
				Expect(err).ToNot(HaveOccurred())
				Expect(networks).To(HaveLen(1))
				Expect(networks[0].Name).To(Equal("network1"))
				Expect(networks[0].ID).To(Equal("bd62af4c-bbe7-43fb-af21-29f3082fd734"))
				Expect(networks[0].Description).To(Equal(""))
				Expect(networks[0].Status).To(Equal("ACTIVE"))
				Expect(networks[0].Subnets).To(HaveLen(1))
				Expect(networks[0].TenantID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
				Expect(networks[0].MTU).To(Equal(1450))
				Expect(networks[0].ProjectID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
			})

			Context("when network name is invalid", func() {
				It("returns an error", func() {
					_, err := client.NetworksByName("")
					Expect(err).To(MatchError("empty 'name' parameter"))
				})
			})

			Context("when network does not exist", func() {
				It("returns empty when not found by name", func() {
					networks, err := client.NetworksByName("does-not-exist")
					Expect(err).ToNot(HaveOccurred())
					Expect(networks).To(HaveLen(0))
				})
			})
		})
	})

	// Describe("Subnets", func() {
	// 	BeforeEach(func() {
	// 		var err error
	// 		server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	// 			fmt.Fprintln(w, subnets)
	// 		}))

	// 		client, err = neutron.NewClient(server.URL, "some-token")
	// 		Expect(err).ToNot(HaveOccurred())
	// 	})

	// 	AfterEach(func() {
	// 		server.Close()
	// 	})

	// 	It("can list subnets", func() {
	// 		subnets, err := client.Subnets()
	// 		Expect(err).ToNot(HaveOccurred())
	// 		Expect(subnets).To(HaveLen(1))
	// 		Expect(subnets[0].Name).To(Equal("public"))
	// 		Expect(subnets[0].ID).To(Equal("e53a3b67-0074-404c-90b5-52ae217c3587"))
	// 		Expect(subnets[0].Description).To(Equal("public network"))
	// 		Expect(networks[0].ProjectID).To(Equal("1f77bad08b454898803a3d9f9e3799ec"))
	// 	})
	// })

})
<|endoftext|>


<commit_before>package newznab

import (
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"regexp"
	"testing"

	log "github.com/Sirupsen/logrus"
	. "github.com/smartystreets/goconvey/convey"
)

func TestUsenetCrawlerClient(t *testing.T) {
	log.SetLevel(log.DebugLevel)
	apiKey := "gibberish"

	// Set up our mock server
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var f []byte
		var err error

		reg := regexp.MustCompile(`\W`)
		fixedPath := reg.ReplaceAllString(r.URL.RawQuery, "_")

		log.Info("Local fixture path: tests/fixtures" + r.URL.Path + "/" + fixedPath)

		if r.URL.Query()["t"][0] == "get" {
			// Fetch nzb
			nzbID := r.URL.Query()["id"][0]
			filePath := fmt.Sprintf("../tests/fixtures/nzbs/%v.nzb", nzbID)
			f, err = ioutil.ReadFile(filePath)
		} else {
			// Get xml
			filePath := fmt.Sprintf("../tests/fixtures%v/%v.xml", r.URL.Path, fixedPath)
			f, err = ioutil.ReadFile(filePath)
		}

		if err != nil {
			log.Error(err)
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte("File not found"))
		} else {
			w.Write(f)
		}
	}))

	defer ts.Close()

	Convey("I have setup a torznab client", t, func() {
		client := New(ts.URL, apiKey, 1234, true)

		Convey("I can search using simple query", func() {
			categories := []int{CategoryTVHD}
			results, err := client.SearchWithQuery(categories, "Supernatural S11E01", "tvshows")
			//for _, result := range results {
			//	log.Info(result.JSONString())
			//}

			So(err, ShouldBeNil)
			So(len(results), ShouldBeGreaterThan, 0)
		})
	})

	Convey("I have setup a nzb client", t, func() {
		client := New(ts.URL, apiKey, 1234, false)
		categories := []int{CategoryTVSD}

		Convey("Handle errors", func() {

			Convey("Return an error for an invalid search.", func() {
				_, err := client.SearchWithTVDB(categories, 1234, 9, 2)
				So(err, ShouldNotBeNil)
			})

			Convey("Return an error for invalid api usage.", func() {
				_, err := client.SearchWithTVDB(categories, 5678, 9, 2)
				So(err.Error(), ShouldEqual, "newznab api error 100: Invalid API Key")
			})
		})

		Convey("When getting TV show information", func() {

			Convey("Given a category and a TheTVDB id", func() {
				results, err := client.SearchWithTVDB(categories, 75682, 10, 1)

				Convey("A valid result is returned.", func() {
					So(err, ShouldBeNil)
					So(len(results), ShouldBeGreaterThan, 0)
				})
			})

			Convey("When given a category and a tvrage id", func() {
				results, err := client.SearchWithTVRage(categories, 2870, 10, 1)

				Convey("A valid result is returned.", func() {
					So(err, ShouldBeNil)
					So(len(results), ShouldBeGreaterThan, 0)
				})

				Convey("I can populate the comments for an NZB.", func() {
					nzb := results[1]
					So(len(nzb.Comments), ShouldEqual, 0)
					So(nzb.NumComments, ShouldBeGreaterThan, 0)
					err := client.PopulateComments(&nzb)
					So(err, ShouldBeNil)

					for _, comment := range nzb.Comments {
						log.Info(comment.JSONString())
					}

					So(len(nzb.Comments), ShouldBeGreaterThan, 0)
				})

				Convey("I can get the download url.", func() {
					url := client.NZBDownloadURL(results[0])
					So(len(url), ShouldBeGreaterThan, 0)
					log.Infof("URL: %s", url)
				})

				Convey("I can download the NZB.", func() {
					bytes, err := client.DownloadNZB(results[0])
					So(err, ShouldBeNil)

					md5Sum := md5.Sum(bytes)
					log.WithFields(log.Fields{
						"num_bytes": len(bytes),
						"md5":       base64.StdEncoding.EncodeToString(md5Sum[:]),
					}).Info("downloaded")

					So(len(bytes), ShouldBeGreaterThan, 0)
				})
			})
		})

		Convey("When getting movie information", func() {
			Convey("Given multiple categories and an IMDB id", func() {
				cats := []int{
					CategoryMovieHD,
					CategoryMovieBluRay,
				}
				results, err := client.SearchWithIMDB(cats, "0371746")

				So(err, ShouldBeNil)
				So(len(results), ShouldBeGreaterThan, 0)

				Convey("The results have different categories.", func() {
					So(results[0].Category[1], ShouldEqual, "2040")
					So(results[22].Category[1], ShouldEqual, "2050")
				})
			})

			Convey("Given a single category and an IMDB id", func() {
				cats := []int{CategoryMovieHD}
				results, err := client.SearchWithIMDB(cats, "0364569")

				So(err, ShouldBeNil)
				So(len(results), ShouldBeGreaterThan, 0)

				Convey("I can get movie specific fields", func() {

					Convey("An IMDB id.", func() {
						imdbAttr := results[0].IMDBID
						So(imdbAttr, ShouldEqual, "0364569")
					})

					Convey("An IMDB title.", func() {
						imdbAttr := results[0].IMDBTitle
						So(imdbAttr, ShouldEqual, "Oldboy")
					})

					Convey("An IMDB year.", func() {
						imdbAttr := results[0].IMDBYear
						So(imdbAttr, ShouldEqual, 2003)
					})

					Convey("An IMDB score.", func() {
						imdbAttr := results[0].IMDBScore
						So(imdbAttr, ShouldEqual, 8.4)
					})

					Convey("A cover URL.", func() {
						imdbAttr := results[0].CoverURL
						So(imdbAttr, ShouldEqual, "https://dognzb.cr/content/covers/movies/thumbs/364569.jpg")
					})
				})
			})
		})

		Convey("When getting recent items via RSS", func() {
			num := 50
			categories := []int{CategoryMovieAll, CategoryTVAll}

			Convey("I can load the current RSS feed.", func() {
				results, err := client.LoadRSSFeed(categories, num)

				Convey("A valid result is returned.", func() {
					So(err, ShouldBeNil)
					So(len(results), ShouldEqual, num)
				})

				Convey("A TV result is present.", func() {
					guid := results[0].ID
					So(guid, ShouldEqual, "bcdbf3f1e7a1ef964527f1d40d5ec639")
				})

				Convey("A Movie result is present.", func() {
					title := results[6].Title
					So(title, ShouldEqual, "030517-VSHS0101720WDA20H264V")
				})

				Convey("An airdate with RFC1123Z format is parsed.", func() {
					year := results[7].AirDate.Year()
					So(year, ShouldEqual, 2017)
				})

				Convey("An usenetdate with RFC3339 format is parsed.", func() {
					year := results[7].UsenetDate.Year()
					So(year, ShouldEqual, 2017)
				})

			})

			Convey("I can load the RSS feed up to a given NZB ID.", func() {
				results, err := client.LoadRSSFeedUntilNZBID(categories, num, "29527a54ac54bb7533abacd7dad66a6a", 0)

				Convey("A valid result is returned.", func() {
					So(err, ShouldBeNil)
					So(len(results), ShouldEqual, 101)
				})

				Convey("Everything up to the given ID is returned.", func() {
					firstID := results[0].ID
					So(firstID, ShouldEqual, "8841b21c4d2fb96f0d47ca24cae9a5b7")

					lastID := results[len(results)-1].ID
					So(lastID, ShouldEqual, "2c6c0e2ac562db69d8b3646deaf2d0cd")
				})
			})

			Convey("I can load the RSS feed up to a given NZB ID but will stop after N tries", func() {
				results, err := client.LoadRSSFeedUntilNZBID(categories, num, "does-not-exist", 2)

				Convey("100 results with 2 requests were fetched.", func() {
					So(err, ShouldBeNil)
					So(len(results), ShouldEqual, 100)
				})
			})
		})
	})
}
<commit_msg>switched from convey to std go subtests<commit_after>package newznab

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"regexp"
	"testing"

	"github.com/stretchr/testify/require"
)

func TestUsenetCrawlerClient(t *testing.T) {
	//log.SetLevel(log.DebugLevel)
	apiKey := "gibberish"

	// Set up our mock server
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var f []byte
		var err error

		reg := regexp.MustCompile(`\W`)
		fixedPath := reg.ReplaceAllString(r.URL.RawQuery, "_")

		if r.URL.Query()["t"][0] == "get" {
			// Fetch nzb
			nzbID := r.URL.Query()["id"][0]
			filePath := fmt.Sprintf("../tests/fixtures/nzbs/%v.nzb", nzbID)
			f, err = ioutil.ReadFile(filePath)
		} else {
			// Get xml
			filePath := fmt.Sprintf("../tests/fixtures%v/%v.xml", r.URL.Path, fixedPath)
			f, err = ioutil.ReadFile(filePath)
		}

		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte("File not found"))
		} else {
			w.Write(f)
		}
	}))

	defer ts.Close()

	t.Run("torznab client", func(t *testing.T) {
		client := New(ts.URL, apiKey, 1234, true)

		t.Run("Simple query search", func(t *testing.T) {
			categories := []int{CategoryTVHD}
			results, err := client.SearchWithQuery(categories, "Supernatural S11E01", "tvshows")
			require.NoError(t, err)
			require.NotEmpty(t, results, "expected results")
		})
	})

	t.Run("nzb client", func(t *testing.T) {
		client := New(ts.URL, apiKey, 1234, false)
		categories := []int{CategoryTVSD}

		t.Run("invalid search", func(t *testing.T) {
			_, err := client.SearchWithTVDB(categories, 1234, 9, 2)
			require.Error(t, err, "expected an error")
		})

		t.Run("invalid api usage", func(t *testing.T) {
			_, err := client.SearchWithTVDB(categories, 5678, 9, 2)
			require.Error(t, err, "expected an error")
			require.EqualError(t, err, "newznab api error 100: Invalid API Key")
		})

		t.Run("valid category and TheTVDB id", func(t *testing.T) {
			results, err := client.SearchWithTVDB(categories, 75682, 10, 1)
			require.NoError(t, err)
			require.NotEmpty(t, results, "expected results")
		})

		t.Run("valid category and tvrage id", func(t *testing.T) {
			results, err := client.SearchWithTVRage(categories, 2870, 10, 1)
			require.NoError(t, err)
			require.NotEmpty(t, results, "expected results")

			t.Run("populate comments", func(t *testing.T) {
				nzb := results[1]
				require.Empty(t, nzb.Comments)
				require.NotZero(t, nzb.NumComments)
				err := client.PopulateComments(&nzb)
				require.NoError(t, err)
				require.NotEmpty(t, nzb.Comments, "expected at least one comment")
				for _, comment := range nzb.Comments {
					require.NotEmpty(t, comment, "comment should not be empty")
				}
			})

			t.Run("download url", func(t *testing.T) {
				url := client.NZBDownloadURL(results[0])
				require.NotEmpty(t, url, "expected a url")
			})

			t.Run("download nzb", func(t *testing.T) {
				bytes, err := client.DownloadNZB(results[0])
				require.NoError(t, err)
				require.NotEmpty(t, bytes, "expected to download something")
			})
		})

		t.Run("multiple categories and IMDB id", func(t *testing.T) {
			cats := []int{
				CategoryMovieHD,
				CategoryMovieBluRay,
			}
			results, err := client.SearchWithIMDB(cats, "0371746")
			require.NoError(t, err)
			require.NotEmpty(t, results, "expected results")

			require.Equal(t, "2040", results[0].Category[1])
			require.Equal(t, "2050", results[22].Category[1])
		})

		t.Run("single category and IMDB id", func(t *testing.T) {
			cats := []int{CategoryMovieHD}
			results, err := client.SearchWithIMDB(cats, "0364569")
			require.NoError(t, err)
			require.NotEmpty(t, results, "expected results")

			t.Run("movie specific fields", func(t *testing.T) {
				require.Equal(t, "0364569", results[0].IMDBID)
				require.Equal(t, "Oldboy", results[0].IMDBTitle)
				require.Equal(t, 2003, results[0].IMDBYear)
				require.Equal(t, float32(8.4), results[0].IMDBScore)
				require.Equal(t, "https://dognzb.cr/content/covers/movies/thumbs/364569.jpg", results[0].CoverURL)
			})
		})

		t.Run("recent items via RSS", func(t *testing.T) {
			num := 50
			categories := []int{CategoryMovieAll, CategoryTVAll}

			t.Run("recent items", func(t *testing.T) {
				results, err := client.LoadRSSFeed(categories, num)
				require.NoError(t, err)
				require.Len(t, results, num)
				require.Equal(t, "bcdbf3f1e7a1ef964527f1d40d5ec639", results[0].ID)
				require.Equal(t, "030517-VSHS0101720WDA20H264V", results[6].Title)

				t.Run("airdate with RFC1123Z format", func(t *testing.T) {
					require.Equal(t, 2017, results[7].AirDate.Year())
				})

				t.Run("usenetdate with RFC3339 format", func(t *testing.T) {
					require.Equal(t, 2017, results[7].UsenetDate.Year())
				})
			})

			t.Run("up until", func(t *testing.T) {
				results, err := client.LoadRSSFeedUntilNZBID(categories, num, "29527a54ac54bb7533abacd7dad66a6a", 0)
				require.NoError(t, err)
				require.Len(t, results, 101)

				t.Run("boundary results", func(t *testing.T) {
					require.Equal(t, "8841b21c4d2fb96f0d47ca24cae9a5b7", results[0].ID)
					require.Equal(t, "2c6c0e2ac562db69d8b3646deaf2d0cd", results[len(results)-1].ID)
				})

				t.Run("RSS up until with failures/retries", func(t *testing.T) {
					results, err := client.LoadRSSFeedUntilNZBID(categories, num, "does-not-exist", 2)
					require.NoError(t, err)
					require.Len(t, results, 100)
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: goinstall importpath...\n")
	fmt.Fprintf(os.Stderr, "\tgoinstall -a\n")
	flag.PrintDefaults()
	os.Exit(2)
}

const logfile = "goinstall.log"

var (
	fset          = token.NewFileSet()
	argv0         = os.Args[0]
	errors        = false
	parents       = make(map[string]string)
	visit         = make(map[string]status)
	installedPkgs = make(map[string]map[string]bool)
	schemeRe      = regexp.MustCompile(`^[a-z]+://`)

	allpkg            = flag.Bool("a", false, "install all previously installed packages")
	reportToDashboard = flag.Bool("dashboard", true, "report public packages at "+dashboardURL)
	update            = flag.Bool("u", false, "update already-downloaded packages")
	doInstall         = flag.Bool("install", true, "build and install")
	clean             = flag.Bool("clean", false, "clean the package directory before installing")
	nuke              = flag.Bool("nuke", false, "clean the package directory and target before installing")
	useMake           = flag.Bool("make", true, "use make to build and install")
	verbose           = flag.Bool("v", false, "verbose")
)

type status int // status for visited map
const (
	unvisited status = iota
	visiting
	done
)

func logf(format string, args ...interface{}) {
	format = "%s: " + format
	args = append([]interface{}{argv0}, args...)
	fmt.Fprintf(os.Stderr, format, args...)
}

func printf(format string, args ...interface{}) {
	if *verbose {
		logf(format, args...)
	}
}

func errorf(format string, args ...interface{}) {
	errors = true
	logf(format, args...)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if runtime.GOROOT() == "" {
		fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0)
		os.Exit(1)
	}
	readPackageList()

	// special case - "unsafe" is already installed
	visit["unsafe"] = done

	args := flag.Args()
	if *allpkg {
		if len(args) != 0 {
			usage() // -a and package list both provided
		}
		// install all packages that were ever installed
		n := 0
		for _, pkgs := range installedPkgs {
			for pkg := range pkgs {
				args = append(args, pkg)
				n++
			}
		}
		if n == 0 {
			logf("no installed packages\n")
			os.Exit(1)
		}
	}
	if len(args) == 0 {
		usage()
	}
	for _, path := range args {
		if s := schemeRe.FindString(path); s != "" {
			errorf("%q used in import path, try %q\n", s, path[len(s):])
			continue
		}

		install(path, "")
	}
	if errors {
		os.Exit(1)
	}
}

// printDeps prints the dependency path that leads to pkg.
func printDeps(pkg string) {
	if pkg == "" {
		return
	}
	if visit[pkg] != done {
		printDeps(parents[pkg])
	}
	fmt.Fprintf(os.Stderr, "\t%s ->\n", pkg)
}

// readPackageList reads the list of installed packages from the
// goinstall.log files in GOROOT and the GOPATHs and initalizes
// the installedPkgs variable.
func readPackageList() {
	for _, t := range build.Path {
		installedPkgs[t.Path] = make(map[string]bool)
		name := filepath.Join(t.Path, logfile)
		pkglistdata, err := ioutil.ReadFile(name)
		if err != nil {
			printf("%s\n", err)
			continue
		}
		pkglist := strings.Fields(string(pkglistdata))
		for _, pkg := range pkglist {
			installedPkgs[t.Path][pkg] = true
		}
	}
}

// logPackage logs the named package as installed in the goinstall.log file
// in the given tree if the package is not already in that file.
func logPackage(pkg string, tree *build.Tree) (logged bool) {
	if installedPkgs[tree.Path][pkg] {
		return false
	}
	name := filepath.Join(tree.Path, logfile)
	fout, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		logf("%s\n", err)
		return false
	}
	fmt.Fprintf(fout, "%s\n", pkg)
	fout.Close()
	return true
}

// install installs the package named by path, which is needed by parent.
func install(pkg, parent string) {
	// Make sure we're not already trying to install pkg.
	switch visit[pkg] {
	case done:
		return
	case visiting:
		fmt.Fprintf(os.Stderr, "%s: package dependency cycle\n", argv0)
		printDeps(parent)
		fmt.Fprintf(os.Stderr, "\t%s\n", pkg)
		os.Exit(2)
	}
	parents[pkg] = parent
	visit[pkg] = visiting
	defer func() {
		visit[pkg] = done
	}()

	// Check whether package is local or remote.
	// If remote, download or update it.
	tree, pkg, err := build.FindTree(pkg)
	// Don't build the standard library.
	if err == nil && tree.Goroot && isStandardPath(pkg) {
		if parent == "" {
			errorf("%s: can not goinstall the standard library\n", pkg)
		} else {
			printf("%s: skipping standard library\n", pkg)
		}
		return
	}
	// Download remote packages if not found or forced with -u flag.
	remote, public := isRemote(pkg), false
	if remote {
		if err == build.ErrNotFound || (err == nil && *update) {
			// Download remote package.
			printf("%s: download\n", pkg)
			public, err = download(pkg, tree.SrcDir())
		} else {
			// Test if this is a public repository
			// (for reporting to dashboard).
			m, _ := findPublicRepo(pkg)
			public = m != nil
		}
	}
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	dir := filepath.Join(tree.SrcDir(), pkg)

	// Install prerequisites.
	dirInfo, err := build.ScanDir(dir, parent == "")
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
		errorf("%s: package has no files\n", pkg)
		return
	}
	for _, p := range dirInfo.Imports {
		if p != "C" {
			install(p, pkg)
		}
	}
	if errors {
		return
	}

	// Install this package.
	if *useMake {
		err := domake(dir, pkg, tree, dirInfo.IsCommand())
		if err != nil {
			errorf("%s: install: %v\n", pkg, err)
			return
		}
	} else {
		script, err := build.Build(tree, pkg, dirInfo)
		if err != nil {
			errorf("%s: install: %v\n", pkg, err)
			return
		}
		if *nuke {
			printf("%s: nuke\n", pkg)
			script.Nuke()
		} else if *clean {
			printf("%s: clean\n", pkg)
			script.Clean()
		}
		if *doInstall {
			if script.Stale() {
				printf("%s: install\n", pkg)
				if err := script.Run(); err != nil {
					errorf("%s: install: %v\n", pkg, err)
					return
				}
			} else {
				printf("%s: up-to-date\n", pkg)
			}
		}
	}

	if remote {
		// mark package as installed in goinstall.log
		logged := logPackage(pkg, tree)

		// report installation to the dashboard if this is the first
		// install from a public repository.
		if logged && public {
			maybeReportToDashboard(pkg)
		}
	}
}

// Is this a standard package path?  strings container/vector etc.
// Assume that if the first element has a dot, it's a domain name
// and is not the standard package path.
func isStandardPath(s string) bool {
	dot := strings.Index(s, ".")
	slash := strings.Index(s, "/")
	return dot < 0 || 0 < slash && slash < dot
}

// run runs the command cmd in directory dir with standard input stdin.
// If the command fails, run prints the command and output on standard error
// in addition to returning a non-nil os.Error.
func run(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, false)
}

// quietRun is like run but prints nothing on failure unless -v is used.
func quietRun(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, true)
}

// genRun implements run and quietRun.
func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error {
	cmd := exec.Command(arg[0], arg[1:]...)
	cmd.Stdin = bytes.NewBuffer(stdin)
	cmd.Dir = dir
	printf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		if !quiet || *verbose {
			if dir != "" {
				dir = "cd " + dir + "; "
			}
			fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " "))
			os.Stderr.Write(out)
			fmt.Fprintf(os.Stderr, "--- %s\n", err)
		}
		return os.NewError("running " + arg[0] + ": " + err.String())
	}
	return nil
}
<commit_msg>goinstall: error out with paths that end with '/'<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: goinstall importpath...\n")
	fmt.Fprintf(os.Stderr, "\tgoinstall -a\n")
	flag.PrintDefaults()
	os.Exit(2)
}

const logfile = "goinstall.log"

var (
	fset          = token.NewFileSet()
	argv0         = os.Args[0]
	errors        = false
	parents       = make(map[string]string)
	visit         = make(map[string]status)
	installedPkgs = make(map[string]map[string]bool)
	schemeRe      = regexp.MustCompile(`^[a-z]+://`)

	allpkg            = flag.Bool("a", false, "install all previously installed packages")
	reportToDashboard = flag.Bool("dashboard", true, "report public packages at "+dashboardURL)
	update            = flag.Bool("u", false, "update already-downloaded packages")
	doInstall         = flag.Bool("install", true, "build and install")
	clean             = flag.Bool("clean", false, "clean the package directory before installing")
	nuke              = flag.Bool("nuke", false, "clean the package directory and target before installing")
	useMake           = flag.Bool("make", true, "use make to build and install")
	verbose           = flag.Bool("v", false, "verbose")
)

type status int // status for visited map
const (
	unvisited status = iota
	visiting
	done
)

func logf(format string, args ...interface{}) {
	format = "%s: " + format
	args = append([]interface{}{argv0}, args...)
	fmt.Fprintf(os.Stderr, format, args...)
}

func printf(format string, args ...interface{}) {
	if *verbose {
		logf(format, args...)
	}
}

func errorf(format string, args ...interface{}) {
	errors = true
	logf(format, args...)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if runtime.GOROOT() == "" {
		fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0)
		os.Exit(1)
	}
	readPackageList()

	// special case - "unsafe" is already installed
	visit["unsafe"] = done

	args := flag.Args()
	if *allpkg {
		if len(args) != 0 {
			usage() // -a and package list both provided
		}
		// install all packages that were ever installed
		n := 0
		for _, pkgs := range installedPkgs {
			for pkg := range pkgs {
				args = append(args, pkg)
				n++
			}
		}
		if n == 0 {
			logf("no installed packages\n")
			os.Exit(1)
		}
	}
	if len(args) == 0 {
		usage()
	}
	for _, path := range args {
		if s := schemeRe.FindString(path); s != "" {
			errorf("%q used in import path, try %q\n", s, path[len(s):])
			continue
		}

		install(path, "")
	}
	if errors {
		os.Exit(1)
	}
}

// printDeps prints the dependency path that leads to pkg.
func printDeps(pkg string) {
	if pkg == "" {
		return
	}
	if visit[pkg] != done {
		printDeps(parents[pkg])
	}
	fmt.Fprintf(os.Stderr, "\t%s ->\n", pkg)
}

// readPackageList reads the list of installed packages from the
// goinstall.log files in GOROOT and the GOPATHs and initalizes
// the installedPkgs variable.
func readPackageList() {
	for _, t := range build.Path {
		installedPkgs[t.Path] = make(map[string]bool)
		name := filepath.Join(t.Path, logfile)
		pkglistdata, err := ioutil.ReadFile(name)
		if err != nil {
			printf("%s\n", err)
			continue
		}
		pkglist := strings.Fields(string(pkglistdata))
		for _, pkg := range pkglist {
			installedPkgs[t.Path][pkg] = true
		}
	}
}

// logPackage logs the named package as installed in the goinstall.log file
// in the given tree if the package is not already in that file.
func logPackage(pkg string, tree *build.Tree) (logged bool) {
	if installedPkgs[tree.Path][pkg] {
		return false
	}
	name := filepath.Join(tree.Path, logfile)
	fout, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		logf("%s\n", err)
		return false
	}
	fmt.Fprintf(fout, "%s\n", pkg)
	fout.Close()
	return true
}

// install installs the package named by path, which is needed by parent.
func install(pkg, parent string) {
	// Make sure we're not already trying to install pkg.
	switch visit[pkg] {
	case done:
		return
	case visiting:
		fmt.Fprintf(os.Stderr, "%s: package dependency cycle\n", argv0)
		printDeps(parent)
		fmt.Fprintf(os.Stderr, "\t%s\n", pkg)
		os.Exit(2)
	}
	parents[pkg] = parent
	visit[pkg] = visiting
	defer func() {
		visit[pkg] = done
	}()

	// Don't allow trailing '/'
	if _, f := filepath.Split(pkg); f == "" {
		errorf("%s should not have trailing '/'\n", pkg)
		return
	}

	// Check whether package is local or remote.
	// If remote, download or update it.
	tree, pkg, err := build.FindTree(pkg)
	// Don't build the standard library.
	if err == nil && tree.Goroot && isStandardPath(pkg) {
		if parent == "" {
			errorf("%s: can not goinstall the standard library\n", pkg)
		} else {
			printf("%s: skipping standard library\n", pkg)
		}
		return
	}
	// Download remote packages if not found or forced with -u flag.
	remote, public := isRemote(pkg), false
	if remote {
		if err == build.ErrNotFound || (err == nil && *update) {
			// Download remote package.
			printf("%s: download\n", pkg)
			public, err = download(pkg, tree.SrcDir())
		} else {
			// Test if this is a public repository
			// (for reporting to dashboard).
			m, _ := findPublicRepo(pkg)
			public = m != nil
		}
	}
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	dir := filepath.Join(tree.SrcDir(), pkg)

	// Install prerequisites.
	dirInfo, err := build.ScanDir(dir, parent == "")
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
		errorf("%s: package has no files\n", pkg)
		return
	}
	for _, p := range dirInfo.Imports {
		if p != "C" {
			install(p, pkg)
		}
	}
	if errors {
		return
	}

	// Install this package.
	if *useMake {
		err := domake(dir, pkg, tree, dirInfo.IsCommand())
		if err != nil {
			errorf("%s: install: %v\n", pkg, err)
			return
		}
	} else {
		script, err := build.Build(tree, pkg, dirInfo)
		if err != nil {
			errorf("%s: install: %v\n", pkg, err)
			return
		}
		if *nuke {
			printf("%s: nuke\n", pkg)
			script.Nuke()
		} else if *clean {
			printf("%s: clean\n", pkg)
			script.Clean()
		}
		if *doInstall {
			if script.Stale() {
				printf("%s: install\n", pkg)
				if err := script.Run(); err != nil {
					errorf("%s: install: %v\n", pkg, err)
					return
				}
			} else {
				printf("%s: up-to-date\n", pkg)
			}
		}
	}

	if remote {
		// mark package as installed in goinstall.log
		logged := logPackage(pkg, tree)

		// report installation to the dashboard if this is the first
		// install from a public repository.
		if logged && public {
			maybeReportToDashboard(pkg)
		}
	}
}

// Is this a standard package path?  strings container/vector etc.
// Assume that if the first element has a dot, it's a domain name
// and is not the standard package path.
func isStandardPath(s string) bool {
	dot := strings.Index(s, ".")
	slash := strings.Index(s, "/")
	return dot < 0 || 0 < slash && slash < dot
}

// run runs the command cmd in directory dir with standard input stdin.
// If the command fails, run prints the command and output on standard error
// in addition to returning a non-nil os.Error.
func run(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, false)
}

// quietRun is like run but prints nothing on failure unless -v is used.
func quietRun(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, true)
}

// genRun implements run and quietRun.
func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error {
	cmd := exec.Command(arg[0], arg[1:]...)
	cmd.Stdin = bytes.NewBuffer(stdin)
	cmd.Dir = dir
	printf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		if !quiet || *verbose {
			if dir != "" {
				dir = "cd " + dir + "; "
			}
			fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " "))
			os.Stderr.Write(out)
			fmt.Fprintf(os.Stderr, "--- %s\n", err)
		}
		return os.NewError("running " + arg[0] + ": " + err.String())
	}
	return nil
}
<|endoftext|>