<commit_before>package fslock_test

import (
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/utils/fslock"
)

func Test(t *testing.T) {
	TestingT(t)
}

type fslockSuite struct{}

var _ = Suite(fslockSuite{})

func (fslockSuite) SetUpSuite(c *C) {
	fslock.SetLockWaitDelay(1 * time.Millisecond)
}

func (fslockSuite) TearDownSuite(c *C) {
	fslock.SetLockWaitDelay(1 * time.Second)
}

// This test also happens to test that locks can get created when the parent
// lock directory doesn't exist.
func (fslockSuite) TestValidNamesLockDir(c *C) {

	for _, name := range []string{
		"a",
		"longer",
		"longer-with.special-characters",
	} {
		dir := c.MkDir()
		_, err := fslock.NewLock(dir, name)
		c.Assert(err, IsNil)
	}
}

func (fslockSuite) TestInvalidNames(c *C) {

	for _, name := range []string{
		"NoCapitals",
		"no+plus",
		"no/slash",
		"no\\backslash",
		"no$dollar",
	} {
		dir := c.MkDir()
		_, err := fslock.NewLock(dir, name)
		c.Assert(err, ErrorMatches, "Invalid lock name .*")
	}
}

func (fslockSuite) TestNewLockWithExistingDir(c *C) {
	dir := c.MkDir()
	err := os.MkdirAll(dir, 0755)
	c.Assert(err, IsNil)
	_, err = fslock.NewLock(dir, "special")
	c.Assert(err, IsNil)
}

func (fslockSuite) TestNewLockWithExistingFileInPlace(c *C) {
	dir := c.MkDir()
	err := os.MkdirAll(dir, 0755)
	c.Assert(err, IsNil)
	path := path.Join(dir, "locks")
	err = ioutil.WriteFile(path, []byte("foo"), 0644)
	c.Assert(err, IsNil)

	_, err = fslock.NewLock(path, "special")
	c.Assert(err, ErrorMatches, `.* not a directory`)
}

func (fslockSuite) TestIsLockHeldBasics(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, false)

	err = lock.Lock()
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, true)

	err = lock.Unlock()
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, false)
}

func (fslockSuite) TestIsLockHeldTwoLocks(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)
	c.Assert(lock2.IsLockHeld(), Equals, false)
}

func (fslockSuite) TestLockBlocks(c *C) {

	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	acquired := make(chan struct{})
	err = lock1.Lock()
	c.Assert(err, IsNil)

	go func() {
		lock2.Lock()
		acquired <- struct{}{}
		close(acquired)
	}()

	// Waiting for something not to happen is inherintly hard...
	select {
	case <-acquired:
		c.Fatalf("Unexpected lock acquisition")
	case <-time.After(50 * time.Millisecond):
		// all good
	}

	err = lock1.Unlock()
	c.Assert(err, IsNil)

	select {
	case <-acquired:
		// all good
	case <-time.After(50 * time.Millisecond):
		c.Fatalf("Expected lock acquisition")
	}

	c.Assert(lock2.IsLockHeld(), Equals, true)
}

func (fslockSuite) TestLockWithTimeoutUnlocked(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock.LockWithTimeout(10 * time.Millisecond)
	c.Assert(err, IsNil)
}

func (fslockSuite) TestLockWithTimeoutLocked(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)

	err = lock2.LockWithTimeout(10 * time.Millisecond)
	c.Assert(err, Equals, fslock.ErrTimeout)
}

func (fslockSuite) TestUnlock(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock.Unlock()
	c.Assert(err, Equals, fslock.ErrLockNotHeld)
}
<commit_msg>Fix spelling.<commit_after>package fslock_test

import (
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/utils/fslock"
)

func Test(t *testing.T) {
	TestingT(t)
}

type fslockSuite struct{}

var _ = Suite(fslockSuite{})

func (fslockSuite) SetUpSuite(c *C) {
	fslock.SetLockWaitDelay(1 * time.Millisecond)
}

func (fslockSuite) TearDownSuite(c *C) {
	fslock.SetLockWaitDelay(1 * time.Second)
}

// This test also happens to test that locks can get created when the parent
// lock directory doesn't exist.
func (fslockSuite) TestValidNamesLockDir(c *C) {

	for _, name := range []string{
		"a",
		"longer",
		"longer-with.special-characters",
	} {
		dir := c.MkDir()
		_, err := fslock.NewLock(dir, name)
		c.Assert(err, IsNil)
	}
}

func (fslockSuite) TestInvalidNames(c *C) {

	for _, name := range []string{
		"NoCapitals",
		"no+plus",
		"no/slash",
		"no\\backslash",
		"no$dollar",
	} {
		dir := c.MkDir()
		_, err := fslock.NewLock(dir, name)
		c.Assert(err, ErrorMatches, "Invalid lock name .*")
	}
}

func (fslockSuite) TestNewLockWithExistingDir(c *C) {
	dir := c.MkDir()
	err := os.MkdirAll(dir, 0755)
	c.Assert(err, IsNil)
	_, err = fslock.NewLock(dir, "special")
	c.Assert(err, IsNil)
}

func (fslockSuite) TestNewLockWithExistingFileInPlace(c *C) {
	dir := c.MkDir()
	err := os.MkdirAll(dir, 0755)
	c.Assert(err, IsNil)
	path := path.Join(dir, "locks")
	err = ioutil.WriteFile(path, []byte("foo"), 0644)
	c.Assert(err, IsNil)

	_, err = fslock.NewLock(path, "special")
	c.Assert(err, ErrorMatches, `.* not a directory`)
}

func (fslockSuite) TestIsLockHeldBasics(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, false)

	err = lock.Lock()
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, true)

	err = lock.Unlock()
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, false)
}

func (fslockSuite) TestIsLockHeldTwoLocks(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)
	c.Assert(lock2.IsLockHeld(), Equals, false)
}

func (fslockSuite) TestLockBlocks(c *C) {

	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	acquired := make(chan struct{})
	err = lock1.Lock()
	c.Assert(err, IsNil)

	go func() {
		lock2.Lock()
		acquired <- struct{}{}
		close(acquired)
	}()

	// Waiting for something not to happen is inherently hard...
	select {
	case <-acquired:
		c.Fatalf("Unexpected lock acquisition")
	case <-time.After(50 * time.Millisecond):
		// all good
	}

	err = lock1.Unlock()
	c.Assert(err, IsNil)

	select {
	case <-acquired:
		// all good
	case <-time.After(50 * time.Millisecond):
		c.Fatalf("Expected lock acquisition")
	}

	c.Assert(lock2.IsLockHeld(), Equals, true)
}

func (fslockSuite) TestLockWithTimeoutUnlocked(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock.LockWithTimeout(10 * time.Millisecond)
	c.Assert(err, IsNil)
}

func (fslockSuite) TestLockWithTimeoutLocked(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)

	err = lock2.LockWithTimeout(10 * time.Millisecond)
	c.Assert(err, Equals, fslock.ErrTimeout)
}

func (fslockSuite) TestUnlock(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock.Unlock()
	c.Assert(err, Equals, fslock.ErrLockNotHeld)
}
<|endoftext|>


<commit_before>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

/*
#cgo LDFLAGS: -lnetsnmp -L/usr/local/lib
#cgo CFLAGS: -I/usr/local/include
#include <net-snmp/net-snmp-config.h>
#include <net-snmp/mib_api.h>
#include <unistd.h>
// From parse.c
#define MAXTC   4096
  struct tc {
  int             type;
  int             modid;
  char           *descriptor;
  char           *hint;
  struct enum_list *enums;
  struct range_list *ranges;
  char           *description;
} tclist[MAXTC];

// Return the size of a fixed, or 0 if it is not fixed.
int get_tc_fixed_size(int tc_index) {
	if (tc_index < 0 || tc_index >= MAXTC) {
    return 0;
  }
  struct range_list *ranges;
  ranges = tclist[tc_index].ranges;
  // Look for one range with only one possible value.
  if (ranges == NULL || ranges->low != ranges->high || ranges->next != NULL) {
    return 0;
  }
  return ranges->low;
}

*/
import "C"

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
)

// One entry in the tree of the MIB.
type Node struct {
	Oid               string
	subid             int64
	Label             string
	Augments          string
	Children          []*Node
	Description       string
	Type              string
	Hint              string
	TextualConvention string
	FixedSize         int
	Units             string
	Access            string
	EnumValues        map[int]string

	Indexes      []string
	ImpliedIndex bool
}

// Copy returns a deep copy of the tree underneath the current Node.
func (n *Node) Copy() *Node {
	newNode := *n
	newNode.Children = make([]*Node, 0, len(n.Children))
	newNode.EnumValues = make(map[int]string, len(n.EnumValues))
	newNode.Indexes = make([]string, len(n.Indexes))
	copy(newNode.Indexes, n.Indexes)
	// Deep copy children and enums.
	for _, child := range n.Children {
		newNode.Children = append(newNode.Children, child.Copy())
	}
	for k, v := range n.EnumValues {
		newNode.EnumValues[k] = v
	}
	return &newNode
}

// Adapted from parse.h.
var (
	netSnmptypeMap = map[int]string{
		0:  "OTHER",
		1:  "OBJID",
		2:  "OCTETSTR",
		3:  "INTEGER",
		4:  "NETADDR",
		5:  "IPADDR",
		6:  "COUNTER",
		7:  "GAUGE",
		8:  "TIMETICKS",
		9:  "OPAQUE",
		10: "NULL",
		11: "COUNTER64",
		12: "BITSTRING",
		13: "NSAPADDRESS",
		14: "UINTEGER",
		15: "UNSIGNED32",
		16: "INTEGER32",
		20: "TRAPTYPE",
		21: "NOTIFTYPE",
		22: "OBJGROUP",
		23: "NOTIFGROUP",
		24: "MODID",
		25: "AGENTCAP",
		26: "MODCOMP",
		27: "OBJIDENTITY",
	}
	netSnmpaccessMap = map[int]string{
		18: "ACCESS_READONLY",
		19: "ACCESS_READWRITE",
		20: "ACCESS_WRITEONLY",
		21: "ACCESS_NOACCESS",
		67: "ACCESS_NOTIFY",
		48: "ACCESS_CREATE",
	}
)

// Initialize NetSNMP. Returns MIB parse errors.
//
// Warning: This function plays with the stderr file descriptor.
func initSNMP(logger log.Logger) (string, error) {
	// Load all the MIBs.
	os.Setenv("MIBS", "ALL")
	// Help the user find their MIB directories.
	level.Info(logger).Log("msg", "Loading MIBs", "from", C.GoString(C.netsnmp_get_mib_directory()))
	// We want the descriptions.
	C.snmp_set_save_descriptions(1)

	// Make stderr go to a pipe, as netsnmp tends to spew a
	// lot of errors on startup that there's no apparent
	// way to disable or redirect.
	r, w, err := os.Pipe()
	if err != nil {
		return "", fmt.Errorf("error creating pipe: %s", err)
	}
	defer r.Close()
	defer w.Close()
	savedStderrFd := C.dup(2)
	C.close(2)
	C.dup2(C.int(w.Fd()), 2)
	ch := make(chan string)
	errch := make(chan error)
	go func() {
		data, err := ioutil.ReadAll(r)
		if err != nil {
			errch <- fmt.Errorf("error reading from pipe: %s", err)
			return
		}
		errch <- nil
		ch <- string(data)
	}()

	// Do the initialization.
	C.netsnmp_init_mib()

	// Restore stderr to normal.
	w.Close()
	C.close(2)
	C.dup2(savedStderrFd, 2)
	C.close(savedStderrFd)
	if err := <-errch; err != nil {
		return "", err
	}
	return <-ch, nil
}

// Walk NetSNMP MIB tree, building a Go tree from it.
func buildMIBTree(t *C.struct_tree, n *Node, oid string) {
	n.subid = int64(t.subid)
	if oid != "" {
		n.Oid = fmt.Sprintf("%s.%d", oid, t.subid)
	} else {
		n.Oid = fmt.Sprintf("%d", t.subid)
	}
	n.Label = C.GoString(t.label)
	if typ, ok := netSnmptypeMap[int(t._type)]; ok {
		n.Type = typ
	} else {
		n.Type = "unknown"
	}

	if access, ok := netSnmpaccessMap[int(t.access)]; ok {
		n.Access = access
	} else {
		n.Access = "unknown"
	}

	n.Augments = C.GoString(t.augments)
	n.Description = C.GoString(t.description)
	n.Hint = C.GoString(t.hint)
	n.TextualConvention = C.GoString(C.get_tc_descriptor(t.tc_index))
	n.FixedSize = int(C.get_tc_fixed_size(t.tc_index))
	n.Units = C.GoString(t.units)

	n.EnumValues = map[int]string{}
	enum := t.enums
	for enum != nil {
		n.EnumValues[int(enum.value)] = C.GoString(enum.label)
		enum = enum.next
	}

	if t.child_list == nil {
		return
	}

	head := t.child_list
	n.Children = []*Node{}
	for head != nil {
		child := &Node{}
		// Prepend, as nodes are backwards.
		n.Children = append([]*Node{child}, n.Children...)
		buildMIBTree(head, child, n.Oid)
		head = head.next_peer
	}

	// Ensure things are consistently ordered.
	sort.Slice(n.Children, func(i, j int) bool {
		return n.Children[i].subid < n.Children[j].subid
	})

	// Set names of indexes on each child.
	// In practice this means only the entry will have it.
	index := t.indexes
	indexes := []string{}
	for index != nil {
		indexes = append(indexes, C.GoString(index.ilabel))
		if index.isimplied != 0 {
			n.ImpliedIndex = true
		}
		index = index.next
	}
	n.Indexes = indexes
}

// Convert the NetSNMP MIB tree to a Go data structure.
func getMIBTree() *Node {

	tree := C.get_tree_head()
	head := &Node{}
	buildMIBTree(tree, head, "")
	return head
}
<commit_msg>Hacky workaround for net-snmp version detection (#704)<commit_after>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

/*
#cgo LDFLAGS: -lnetsnmp -L/usr/local/lib
#cgo CFLAGS: -I/usr/local/include
#include <net-snmp/net-snmp-config.h>
#include <net-snmp/mib_api.h>
#include <net-snmp/agent/agent_callbacks.h>
#include <net-snmp/library/default_store.h>
#include <unistd.h>
// From parse.c
// Hacky workarounds to detect which version of net-snmp this
// based on various headers that appear in 5.8 and 5.9.
#if !defined(NETSNMP_DS_LIB_ADD_FORWARDER_INFO)
#if defined(SNMPD_CALLBACK_UNREGISTER_NOTIFICATIONS)
#define MAXTC   16384
#else
#define MAXTC   4096
#endif
#endif

struct tc {
  int             type;
  int             modid;
  char           *descriptor;
  char           *hint;
  struct enum_list *enums;
  struct range_list *ranges;
  char           *description;
#if !defined(NETSNMP_DS_LIB_ADD_FORWARDER_INFO)
} tclist[MAXTC];
int tc_alloc = MAXTC;
#else
} *tclist;
int tc_alloc;
#endif

// Return the size of a fixed, or 0 if it is not fixed.
int get_tc_fixed_size(int tc_index) {
  if (tc_index < 0 || tc_index >= tc_alloc) {
    return 0;
  }
  struct range_list *ranges;
  ranges = tclist[tc_index].ranges;
  // Look for one range with only one possible value.
  if (ranges == NULL || ranges->low != ranges->high || ranges->next != NULL) {
    return 0;
  }
  return ranges->low;
}

*/
import "C"

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
)

// One entry in the tree of the MIB.
type Node struct {
	Oid               string
	subid             int64
	Label             string
	Augments          string
	Children          []*Node
	Description       string
	Type              string
	Hint              string
	TextualConvention string
	FixedSize         int
	Units             string
	Access            string
	EnumValues        map[int]string

	Indexes      []string
	ImpliedIndex bool
}

// Copy returns a deep copy of the tree underneath the current Node.
func (n *Node) Copy() *Node {
	newNode := *n
	newNode.Children = make([]*Node, 0, len(n.Children))
	newNode.EnumValues = make(map[int]string, len(n.EnumValues))
	newNode.Indexes = make([]string, len(n.Indexes))
	copy(newNode.Indexes, n.Indexes)
	// Deep copy children and enums.
	for _, child := range n.Children {
		newNode.Children = append(newNode.Children, child.Copy())
	}
	for k, v := range n.EnumValues {
		newNode.EnumValues[k] = v
	}
	return &newNode
}

// Adapted from parse.h.
var (
	netSnmptypeMap = map[int]string{
		0:  "OTHER",
		1:  "OBJID",
		2:  "OCTETSTR",
		3:  "INTEGER",
		4:  "NETADDR",
		5:  "IPADDR",
		6:  "COUNTER",
		7:  "GAUGE",
		8:  "TIMETICKS",
		9:  "OPAQUE",
		10: "NULL",
		11: "COUNTER64",
		12: "BITSTRING",
		13: "NSAPADDRESS",
		14: "UINTEGER",
		15: "UNSIGNED32",
		16: "INTEGER32",
		20: "TRAPTYPE",
		21: "NOTIFTYPE",
		22: "OBJGROUP",
		23: "NOTIFGROUP",
		24: "MODID",
		25: "AGENTCAP",
		26: "MODCOMP",
		27: "OBJIDENTITY",
	}
	netSnmpaccessMap = map[int]string{
		18: "ACCESS_READONLY",
		19: "ACCESS_READWRITE",
		20: "ACCESS_WRITEONLY",
		21: "ACCESS_NOACCESS",
		67: "ACCESS_NOTIFY",
		48: "ACCESS_CREATE",
	}
)

// Initialize NetSNMP. Returns MIB parse errors.
//
// Warning: This function plays with the stderr file descriptor.
func initSNMP(logger log.Logger) (string, error) {
	// Load all the MIBs.
	os.Setenv("MIBS", "ALL")
	// Help the user find their MIB directories.
	level.Info(logger).Log("msg", "Loading MIBs", "from", C.GoString(C.netsnmp_get_mib_directory()))
	// We want the descriptions.
	C.snmp_set_save_descriptions(1)

	// Make stderr go to a pipe, as netsnmp tends to spew a
	// lot of errors on startup that there's no apparent
	// way to disable or redirect.
	r, w, err := os.Pipe()
	if err != nil {
		return "", fmt.Errorf("error creating pipe: %s", err)
	}
	defer r.Close()
	defer w.Close()
	savedStderrFd := C.dup(2)
	C.close(2)
	C.dup2(C.int(w.Fd()), 2)
	ch := make(chan string)
	errch := make(chan error)
	go func() {
		data, err := ioutil.ReadAll(r)
		if err != nil {
			errch <- fmt.Errorf("error reading from pipe: %s", err)
			return
		}
		errch <- nil
		ch <- string(data)
	}()

	// Do the initialization.
	C.netsnmp_init_mib()

	// Restore stderr to normal.
	w.Close()
	C.close(2)
	C.dup2(savedStderrFd, 2)
	C.close(savedStderrFd)
	if err := <-errch; err != nil {
		return "", err
	}
	return <-ch, nil
}

// Walk NetSNMP MIB tree, building a Go tree from it.
func buildMIBTree(t *C.struct_tree, n *Node, oid string) {
	n.subid = int64(t.subid)
	if oid != "" {
		n.Oid = fmt.Sprintf("%s.%d", oid, t.subid)
	} else {
		n.Oid = fmt.Sprintf("%d", t.subid)
	}
	n.Label = C.GoString(t.label)
	if typ, ok := netSnmptypeMap[int(t._type)]; ok {
		n.Type = typ
	} else {
		n.Type = "unknown"
	}

	if access, ok := netSnmpaccessMap[int(t.access)]; ok {
		n.Access = access
	} else {
		n.Access = "unknown"
	}

	n.Augments = C.GoString(t.augments)
	n.Description = C.GoString(t.description)
	n.Hint = C.GoString(t.hint)
	n.TextualConvention = C.GoString(C.get_tc_descriptor(t.tc_index))
	n.FixedSize = int(C.get_tc_fixed_size(t.tc_index))
	n.Units = C.GoString(t.units)

	n.EnumValues = map[int]string{}
	enum := t.enums
	for enum != nil {
		n.EnumValues[int(enum.value)] = C.GoString(enum.label)
		enum = enum.next
	}

	if t.child_list == nil {
		return
	}

	head := t.child_list
	n.Children = []*Node{}
	for head != nil {
		child := &Node{}
		// Prepend, as nodes are backwards.
		n.Children = append([]*Node{child}, n.Children...)
		buildMIBTree(head, child, n.Oid)
		head = head.next_peer
	}

	// Ensure things are consistently ordered.
	sort.Slice(n.Children, func(i, j int) bool {
		return n.Children[i].subid < n.Children[j].subid
	})

	// Set names of indexes on each child.
	// In practice this means only the entry will have it.
	index := t.indexes
	indexes := []string{}
	for index != nil {
		indexes = append(indexes, C.GoString(index.ilabel))
		if index.isimplied != 0 {
			n.ImpliedIndex = true
		}
		index = index.next
	}
	n.Indexes = indexes
}

// Convert the NetSNMP MIB tree to a Go data structure.
func getMIBTree() *Node {

	tree := C.get_tree_head()
	head := &Node{}
	buildMIBTree(tree, head, "")
	return head
}
<|endoftext|>


<commit_before>package gimporter

import (
	"github.com/bjwbell/gir/gst"
	"go/types"
	"go/token"
)

func ParseFuncDecl(fnDecl *gst.FuncDecl) (*types.Func, bool) {
	var fn *types.Func
	var pkg *types.Package
	pkg = nil
	name := fnDecl.Name
	var sig *types.Signature
	sig = nil
	var pos token.Pos
	fn = types.NewFunc(pos, pkg, name, sig)
	return fn, true
}
<commit_msg>Create dummy Signature<commit_after>package gimporter

import (
	"github.com/bjwbell/gir/gst"
	"go/types"
	"go/token"
)

func ParseFuncDecl(fnDecl *gst.FuncDecl) (*types.Func, bool) {
	var fn *types.Func
	var pkg *types.Package
	pkg = nil
	name := fnDecl.Name
	var sig *types.Signature
	sig = types.NewSignature(nil, nil, nil, false)
	var pos token.Pos
	fn = types.NewFunc(pos, pkg, name, sig)
	return fn, true
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"regexp"
	"strings"
	"time"

	"io/ioutil"
	"path/filepath"

	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/ginkgo/interrupthandler"
	"github.com/onsi/ginkgo/ginkgo/testrunner"
	"github.com/onsi/ginkgo/types"
)

func BuildRunCommand() *Command {
	commandFlags := NewRunCommandFlags(flag.NewFlagSet("ginkgo", flag.ExitOnError))
	notifier := NewNotifier(commandFlags)
	interruptHandler := interrupthandler.NewInterruptHandler()
	runner := &SpecRunner{
		commandFlags:     commandFlags,
		notifier:         notifier,
		interruptHandler: interruptHandler,
		suiteRunner:      NewSuiteRunner(notifier, interruptHandler),
	}

	return &Command{
		Name:         "",
		FlagSet:      commandFlags.FlagSet,
		UsageCommand: "ginkgo <FLAGS> <PACKAGES> -- <PASS-THROUGHS>",
		Usage: []string{
			"Run the tests in the passed in <PACKAGES> (or the package in the current directory if left blank).",
			"Any arguments after -- will be passed to the test.",
			"Accepts the following flags:",
		},
		Command: runner.RunSpecs,
	}
}

type SpecRunner struct {
	commandFlags     *RunWatchAndBuildCommandFlags
	notifier         *Notifier
	interruptHandler *interrupthandler.InterruptHandler
	suiteRunner      *SuiteRunner
}

func (r *SpecRunner) RunSpecs(args []string, additionalArgs []string) {
	r.commandFlags.computeNodes()
	r.notifier.VerifyNotificationsAreAvailable()

	suites, skippedPackages := findSuites(args, r.commandFlags.Recurse, r.commandFlags.SkipPackage, true)
	if len(skippedPackages) > 0 {
		fmt.Println("Will skip:")
		for _, skippedPackage := range skippedPackages {
			fmt.Println("  " + skippedPackage)
		}
	}

	if len(skippedPackages) > 0 && len(suites) == 0 {
		fmt.Println("All tests skipped!  Exiting...")
		os.Exit(0)
	}

	if len(suites) == 0 {
		complainAndQuit("Found no test suites")
	}

	r.ComputeSuccinctMode(len(suites))

	t := time.Now()

	runners := []*testrunner.TestRunner{}
	for _, suite := range suites {
		runners = append(runners, testrunner.New(suite, r.commandFlags.NumCPU, r.commandFlags.ParallelStream, r.commandFlags.Timeout, r.commandFlags.GoOpts, additionalArgs))
	}

	numSuites := 0
	runResult := testrunner.PassingRunResult()
	if r.commandFlags.UntilItFails {
		iteration := 0
		for {
			r.UpdateSeed()
			randomizedRunners := r.randomizeOrder(runners)
			runResult, numSuites = r.suiteRunner.RunSuites(randomizedRunners, r.commandFlags.NumCompilers, r.commandFlags.KeepGoing, nil)
			iteration++

			if r.interruptHandler.WasInterrupted() {
				break
			}

			if runResult.Passed {
				fmt.Printf("\nAll tests passed...\nWill keep running them until they fail.\nThis was attempt #%d\n%s\n", iteration, orcMessage(iteration))
			} else {
				fmt.Printf("\nTests failed on attempt #%d\n\n", iteration)
				break
			}
		}
	} else {
		randomizedRunners := r.randomizeOrder(runners)
		runResult, numSuites = r.suiteRunner.RunSuites(randomizedRunners, r.commandFlags.NumCompilers, r.commandFlags.KeepGoing, nil)
	}

	for _, runner := range runners {
		runner.CleanUp()
	}

	if r.isInCoverageMode() {
		if r.getOutputDir() != "" {
			// If coverprofile is set, combine coverages
			if r.getCoverprofile() != "" {
				if err := r.combineCoverprofiles(runners); err != nil {
					fmt.Println(err.Error())
					os.Exit(1)
				}
			} else {
				// Just move them
				r.moveCoverprofiles(runners)
			}
		}
	}

	fmt.Printf("\nGinkgo ran %d %s in %s\n", numSuites, pluralizedWord("suite", "suites", numSuites), time.Since(t))

	if runResult.Passed {
		if runResult.HasProgrammaticFocus && strings.TrimSpace(os.Getenv("GINKGO_EDITOR_INTEGRATION")) == "" {
			fmt.Printf("Test Suite Passed\n")
			fmt.Printf("Detected Programmatic Focus - setting exit status to %d\n", types.GINKGO_FOCUS_EXIT_CODE)
			os.Exit(types.GINKGO_FOCUS_EXIT_CODE)
		} else {
			fmt.Printf("Test Suite Passed\n")
			os.Exit(0)
		}
	} else {
		fmt.Printf("Test Suite Failed\n")
		os.Exit(1)
	}
}

// Moves all generated profiles to specified directory
func (r *SpecRunner) moveCoverprofiles(runners []*testrunner.TestRunner) {
	for _, runner := range runners {
		_, filename := filepath.Split(runner.CoverageFile)
		err := os.Rename(runner.CoverageFile, filepath.Join(r.getOutputDir(), filename))

		if err != nil {
			fmt.Printf("Unable to move coverprofile %s, %v\n", runner.CoverageFile, err)
			return
		}
	}
}

// Combines all generated profiles in the specified directory
func (r *SpecRunner) combineCoverprofiles(runners []*testrunner.TestRunner) error {

	path, _ := filepath.Abs(r.getOutputDir())
	if !fileExists(path) {
		return fmt.Errorf("Unable to create combined profile, outputdir does not exist: %s", r.getOutputDir())
	}

	fmt.Println("path is " + path)

	combined, err := os.OpenFile(
		filepath.Join(path, r.getCoverprofile()),
		os.O_WRONLY|os.O_CREATE,
		0666,
	)

	if err != nil {
		fmt.Printf("Unable to create combined profile, %v\n", err)
		return nil // non-fatal error
	}

	modeRegex := regexp.MustCompile(`^mode: .*\n`)
	for index, runner := range runners {
		contents, err := ioutil.ReadFile(runner.CoverageFile)

		// remove the cover mode line from every file
		// except the first one
		if index > 0 {
			contents = modeRegex.ReplaceAll(contents, []byte{})
		}

		if err != nil {
			fmt.Printf("Unable to read coverage file %s to combine, %v\n", runner.CoverageFile, err)
			return nil // non-fatal error
		}

		_, err = combined.Write(contents)

		if err != nil {
			fmt.Printf("Unable to append to coverprofile, %v\n", err)
			return nil // non-fatal error
		}
	}

	fmt.Println("All profiles combined")
	return nil
}

func (r *SpecRunner) isInCoverageMode() bool {
	opts := r.commandFlags.GoOpts
	return *opts["cover"].(*bool) || *opts["coverpkg"].(*string) != "" || *opts["covermode"].(*string) != ""
}

func (r *SpecRunner) getCoverprofile() string {
	return *r.commandFlags.GoOpts["coverprofile"].(*string)
}

func (r *SpecRunner) getOutputDir() string {
	return *r.commandFlags.GoOpts["outputdir"].(*string)
}

func (r *SpecRunner) ComputeSuccinctMode(numSuites int) {
	if config.DefaultReporterConfig.Verbose {
		config.DefaultReporterConfig.Succinct = false
		return
	}

	if numSuites == 1 {
		return
	}

	if numSuites > 1 && !r.commandFlags.wasSet("succinct") {
		config.DefaultReporterConfig.Succinct = true
	}
}

func (r *SpecRunner) UpdateSeed() {
	if !r.commandFlags.wasSet("seed") {
		config.GinkgoConfig.RandomSeed = time.Now().Unix()
	}
}

func (r *SpecRunner) randomizeOrder(runners []*testrunner.TestRunner) []*testrunner.TestRunner {
	if !r.commandFlags.RandomizeSuites {
		return runners
	}

	if len(runners) <= 1 {
		return runners
	}

	randomizedRunners := make([]*testrunner.TestRunner, len(runners))
	randomizer := rand.New(rand.NewSource(config.GinkgoConfig.RandomSeed))
	permutation := randomizer.Perm(len(runners))
	for i, j := range permutation {
		randomizedRunners[i] = runners[j]
	}
	return randomizedRunners
}

func orcMessage(iteration int) string {
	if iteration < 10 {
		return ""
	} else if iteration < 30 {
		return []string{
			"If at first you succeed...",
			"...try, try again.",
			"Looking good!",
			"Still good...",
			"I think your tests are fine....",
			"Yep, still passing",
			"Oh boy, here I go testin' again!",
			"Even the gophers are getting bored",
			"Did you try -race?",
			"Maybe you should stop now?",
			"I'm getting tired...",
			"What if I just made you a sandwich?",
			"Hit ^C, hit ^C, please hit ^C",
			"Make it stop. Please!",
			"Come on!  Enough is enough!",
			"Dave, this conversation can serve no purpose anymore. Goodbye.",
			"Just what do you think you're doing, Dave? ",
			"I, Sisyphus",
			"Insanity: doing the same thing over and over again and expecting different results. -Einstein",
			"I guess Einstein never tried to churn butter",
		}[iteration-10] + "\n"
	} else {
		return "No, seriously... you can probably stop now.\n"
	}
}
<commit_msg>Fix missing newline in combined coverage file (#641)<commit_after>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"regexp"
	"strings"
	"time"

	"io/ioutil"
	"path/filepath"

	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/ginkgo/interrupthandler"
	"github.com/onsi/ginkgo/ginkgo/testrunner"
	"github.com/onsi/ginkgo/types"
)

func BuildRunCommand() *Command {
	commandFlags := NewRunCommandFlags(flag.NewFlagSet("ginkgo", flag.ExitOnError))
	notifier := NewNotifier(commandFlags)
	interruptHandler := interrupthandler.NewInterruptHandler()
	runner := &SpecRunner{
		commandFlags:     commandFlags,
		notifier:         notifier,
		interruptHandler: interruptHandler,
		suiteRunner:      NewSuiteRunner(notifier, interruptHandler),
	}

	return &Command{
		Name:         "",
		FlagSet:      commandFlags.FlagSet,
		UsageCommand: "ginkgo <FLAGS> <PACKAGES> -- <PASS-THROUGHS>",
		Usage: []string{
			"Run the tests in the passed in <PACKAGES> (or the package in the current directory if left blank).",
			"Any arguments after -- will be passed to the test.",
			"Accepts the following flags:",
		},
		Command: runner.RunSpecs,
	}
}

type SpecRunner struct {
	commandFlags     *RunWatchAndBuildCommandFlags
	notifier         *Notifier
	interruptHandler *interrupthandler.InterruptHandler
	suiteRunner      *SuiteRunner
}

func (r *SpecRunner) RunSpecs(args []string, additionalArgs []string) {
	r.commandFlags.computeNodes()
	r.notifier.VerifyNotificationsAreAvailable()

	suites, skippedPackages := findSuites(args, r.commandFlags.Recurse, r.commandFlags.SkipPackage, true)
	if len(skippedPackages) > 0 {
		fmt.Println("Will skip:")
		for _, skippedPackage := range skippedPackages {
			fmt.Println("  " + skippedPackage)
		}
	}

	if len(skippedPackages) > 0 && len(suites) == 0 {
		fmt.Println("All tests skipped!  Exiting...")
		os.Exit(0)
	}

	if len(suites) == 0 {
		complainAndQuit("Found no test suites")
	}

	r.ComputeSuccinctMode(len(suites))

	t := time.Now()

	runners := []*testrunner.TestRunner{}
	for _, suite := range suites {
		runners = append(runners, testrunner.New(suite, r.commandFlags.NumCPU, r.commandFlags.ParallelStream, r.commandFlags.Timeout, r.commandFlags.GoOpts, additionalArgs))
	}

	numSuites := 0
	runResult := testrunner.PassingRunResult()
	if r.commandFlags.UntilItFails {
		iteration := 0
		for {
			r.UpdateSeed()
			randomizedRunners := r.randomizeOrder(runners)
			runResult, numSuites = r.suiteRunner.RunSuites(randomizedRunners, r.commandFlags.NumCompilers, r.commandFlags.KeepGoing, nil)
			iteration++

			if r.interruptHandler.WasInterrupted() {
				break
			}

			if runResult.Passed {
				fmt.Printf("\nAll tests passed...\nWill keep running them until they fail.\nThis was attempt #%d\n%s\n", iteration, orcMessage(iteration))
			} else {
				fmt.Printf("\nTests failed on attempt #%d\n\n", iteration)
				break
			}
		}
	} else {
		randomizedRunners := r.randomizeOrder(runners)
		runResult, numSuites = r.suiteRunner.RunSuites(randomizedRunners, r.commandFlags.NumCompilers, r.commandFlags.KeepGoing, nil)
	}

	for _, runner := range runners {
		runner.CleanUp()
	}

	if r.isInCoverageMode() {
		if r.getOutputDir() != "" {
			// If coverprofile is set, combine coverages
			if r.getCoverprofile() != "" {
				if err := r.combineCoverprofiles(runners); err != nil {
					fmt.Println(err.Error())
					os.Exit(1)
				}
			} else {
				// Just move them
				r.moveCoverprofiles(runners)
			}
		}
	}

	fmt.Printf("\nGinkgo ran %d %s in %s\n", numSuites, pluralizedWord("suite", "suites", numSuites), time.Since(t))

	if runResult.Passed {
		if runResult.HasProgrammaticFocus && strings.TrimSpace(os.Getenv("GINKGO_EDITOR_INTEGRATION")) == "" {
			fmt.Printf("Test Suite Passed\n")
			fmt.Printf("Detected Programmatic Focus - setting exit status to %d\n", types.GINKGO_FOCUS_EXIT_CODE)
			os.Exit(types.GINKGO_FOCUS_EXIT_CODE)
		} else {
			fmt.Printf("Test Suite Passed\n")
			os.Exit(0)
		}
	} else {
		fmt.Printf("Test Suite Failed\n")
		os.Exit(1)
	}
}

// Moves all generated profiles to specified directory
func (r *SpecRunner) moveCoverprofiles(runners []*testrunner.TestRunner) {
	for _, runner := range runners {
		_, filename := filepath.Split(runner.CoverageFile)
		err := os.Rename(runner.CoverageFile, filepath.Join(r.getOutputDir(), filename))

		if err != nil {
			fmt.Printf("Unable to move coverprofile %s, %v\n", runner.CoverageFile, err)
			return
		}
	}
}

// Combines all generated profiles in the specified directory
func (r *SpecRunner) combineCoverprofiles(runners []*testrunner.TestRunner) error {

	path, _ := filepath.Abs(r.getOutputDir())
	if !fileExists(path) {
		return fmt.Errorf("Unable to create combined profile, outputdir does not exist: %s", r.getOutputDir())
	}

	fmt.Println("path is " + path)

	combined, err := os.OpenFile(
		filepath.Join(path, r.getCoverprofile()),
		os.O_WRONLY|os.O_CREATE,
		0666,
	)

	if err != nil {
		fmt.Printf("Unable to create combined profile, %v\n", err)
		return nil // non-fatal error
	}

	modeRegex := regexp.MustCompile(`^mode: .*\n`)
	for index, runner := range runners {
		contents, err := ioutil.ReadFile(runner.CoverageFile)

		if err != nil {
			fmt.Printf("Unable to read coverage file %s to combine, %v\n", runner.CoverageFile, err)
			return nil // non-fatal error
		}

		// remove the cover mode line from every file
		// except the first one
		if index > 0 {
			contents = modeRegex.ReplaceAll(contents, []byte{})
		}

		_, err = combined.Write(contents)

		// Add a newline to the end of every file if missing.
		if err == nil && contents[len(contents)-1] != '\n' {
			_, err = combined.Write([]byte("\n"))
		}

		if err != nil {
			fmt.Printf("Unable to append to coverprofile, %v\n", err)
			return nil // non-fatal error
		}
	}

	fmt.Println("All profiles combined")
	return nil
}

func (r *SpecRunner) isInCoverageMode() bool {
	opts := r.commandFlags.GoOpts
	return *opts["cover"].(*bool) || *opts["coverpkg"].(*string) != "" || *opts["covermode"].(*string) != ""
}

func (r *SpecRunner) getCoverprofile() string {
	return *r.commandFlags.GoOpts["coverprofile"].(*string)
}

func (r *SpecRunner) getOutputDir() string {
	return *r.commandFlags.GoOpts["outputdir"].(*string)
}

func (r *SpecRunner) ComputeSuccinctMode(numSuites int) {
	if config.DefaultReporterConfig.Verbose {
		config.DefaultReporterConfig.Succinct = false
		return
	}

	if numSuites == 1 {
		return
	}

	if numSuites > 1 && !r.commandFlags.wasSet("succinct") {
		config.DefaultReporterConfig.Succinct = true
	}
}

func (r *SpecRunner) UpdateSeed() {
	if !r.commandFlags.wasSet("seed") {
		config.GinkgoConfig.RandomSeed = time.Now().Unix()
	}
}

func (r *SpecRunner) randomizeOrder(runners []*testrunner.TestRunner) []*testrunner.TestRunner {
	if !r.commandFlags.RandomizeSuites {
		return runners
	}

	if len(runners) <= 1 {
		return runners
	}

	randomizedRunners := make([]*testrunner.TestRunner, len(runners))
	randomizer := rand.New(rand.NewSource(config.GinkgoConfig.RandomSeed))
	permutation := randomizer.Perm(len(runners))
	for i, j := range permutation {
		randomizedRunners[i] = runners[j]
	}
	return randomizedRunners
}

func orcMessage(iteration int) string {
	if iteration < 10 {
		return ""
	} else if iteration < 30 {
		return []string{
			"If at first you succeed...",
			"...try, try again.",
			"Looking good!",
			"Still good...",
			"I think your tests are fine....",
			"Yep, still passing",
			"Oh boy, here I go testin' again!",
			"Even the gophers are getting bored",
			"Did you try -race?",
			"Maybe you should stop now?",
			"I'm getting tired...",
			"What if I just made you a sandwich?",
			"Hit ^C, hit ^C, please hit ^C",
			"Make it stop. Please!",
			"Come on!  Enough is enough!",
			"Dave, this conversation can serve no purpose anymore. Goodbye.",
			"Just what do you think you're doing, Dave? ",
			"I, Sisyphus",
			"Insanity: doing the same thing over and over again and expecting different results. -Einstein",
			"I guess Einstein never tried to churn butter",
		}[iteration-10] + "\n"
	} else {
		return "No, seriously... you can probably stop now.\n"
	}
}
<|endoftext|>


<commit_before>package gitManip

import (
	"fmt"

	"github.com/fatih/color"

	git "gopkg.in/libgit2/git2go.v25"
)

/*Map to match the RepositoryState enum type with a string
 */
var repositoryStateToString = map[git.RepositoryState]string{
	git.RepositoryStateNone:                 "None",
	git.RepositoryStateMerge:                "Merge",
	git.RepositoryStateRevert:               "Revert",
	git.RepositoryStateCherrypick:           "Cherrypick",
	git.RepositoryStateBisect:               "Bisect",
	git.RepositoryStateRebase:               "Rebase",
	git.RepositoryStateRebaseInteractive:    "Rebase Interactive",
	git.RepositoryStateRebaseMerge:          "Rebase Merge",
	git.RepositoryStateApplyMailbox:         "Apply Mailbox",
	git.RepositoryStateApplyMailboxOrRebase: "Apply Mailbox or Rebase",
}

var fileStateToString = map[git.Status]string{
	git.StatusIndexNew:     "You forgot to commit a new file!",
	git.StatusIgnored:      "Ignored",
	git.StatusConflicted:   "Conflicted",
	git.StatusWtNew:        "New file in your working tree!",
	git.StatusWtModified:   "Modified file in your working tree!",
	git.StatusWtDeleted:    "Deleted file in your working tree!",
	git.StatusWtTypeChange: "Type change detected in your working tree!",
	git.StatusWtRenamed:    "Renamed file in your working tree!",
}

/*Global variable to set the StatusOption parameter, in order to list each file status
 */
var statusOption = git.StatusOptions{
	Show:     git.StatusShowIndexAndWorkdir,
	Flags:    git.StatusOptIncludeUntracked,
	Pathspec: []string{},
}

/*GitObject contains informations about the current git repository
 *
 *The structure is:
 *  accessible:
 *		Is the repository still exists in the hard drive?
 *	path:
 *		The path file.
 *	repository:
 *		The object repository.
 */
type GitObject struct {
	accessible error
	path       string
	repository git.Repository
}

/*New is a constructor for GitObject
 *
 * It neeeds:
 *	path:
 *		The path of the current repository.
 */
func New(path string) *GitObject {
	r, err := git.OpenRepository(path)
	return &GitObject{accessible: err, path: path, repository: *r}
}

/*isAccesible returns the information that is the current git repository is existing or not.
 *This method returns a boolean value: true if the git repository is still accesible (still exists), or false if not.
 */
func (g *GitObject) isAccessible() bool {
	return g.accessible == nil
}

/*Status prints the current status of the repository, accessible via the structure path field.
 *This method works only if the repository is accessible.
 */
func (g *GitObject) Status() {
	if g.isAccessible() {
		if err := g.printChanges(); err != nil {
			color.RedString("Impossible to get stats from %s, due to error %s", g.path, err)
		}
	} else {
		color.RedString("Repository %s not found!", g.path)
	}
}

/*getDiffWithWT returns the difference between the working tree and the index, for the current git repository.
 *If there is an error processing the request, it returns an error.
 */
func (g *GitObject) getDiffWithWT() (*git.Diff, error) {
	// Get the index of the repository
	currentIndex, err := g.repository.Index()
	if err != nil {
		return nil, err
	}
	// Get the default diff options, and add it custom flags
	defaultDiffOptions, err := git.DefaultDiffOptions()
	if err != nil {
		return nil, err
	}
	defaultDiffOptions.Flags = defaultDiffOptions.Flags | git.DiffIncludeUntracked | git.DiffIncludeTypeChange
	// Check the difference between the working directory and the index
	diff, err := g.repository.DiffIndexToWorkdir(currentIndex, &defaultDiffOptions)
	if err != nil {
		return nil, err
	}
	return diff, nil
}

/*printChanges prints out all changes for the current git repository.
 *If there is an error processing the request, it returns this one.
 */
func (g *GitObject) printChanges() error {
	diff, err := g.getDiffWithWT()
	if err != nil {
		return err
	}
	numDeltas, err := diff.NumDeltas()
	if err != nil {
		return err
	}
	if numDeltas > 0 {
		fmt.Printf("%s %s\t[%d modification(s)]\n", color.RedString("/!\\"), g.path, numDeltas)
		for i := 0; i < numDeltas; i++ {
			delta, _ := diff.GetDelta(i)
			currentStatus := delta.Status
			newFile := delta.NewFile.Path
			oldFile := delta.OldFile.Path
			switch currentStatus {
			case git.DeltaAdded:
				fmt.Printf("\t===> %s has been added!\n", color.MagentaString(newFile))
			case git.DeltaDeleted:
				fmt.Printf("\t===> %s has been deleted!\n", color.MagentaString(newFile))
			case git.DeltaModified:
				fmt.Printf("\t===> %s has been modified!\n", color.MagentaString(newFile))
			case git.DeltaRenamed:
				fmt.Printf("\t===> %s has been renamed to %s!\n", color.MagentaString(oldFile), color.MagentaString(newFile))
			case git.DeltaUntracked:
				fmt.Printf("\t===> %s is untracked - please to add it or update the gitignore file!\n", color.MagentaString(newFile))
			case git.DeltaTypeChange:
				fmt.Printf("\t===> the type of %s has been changed from %d to %d!", color.MagentaString(newFile), delta.OldFile.Mode, delta.NewFile.Mode)
			}
		}
	}
	return nil
}

/*List lists the path and the accessibility of a list of git repositories
 */
func List(repositories *[]GitObject) {
	for _, object := range *repositories {
		fmt.Printf("* %s ", object.path)
		if object.isAccessible() {
			fmt.Println(color.GreenString(" [accessible]"))
		} else {
			fmt.Println(color.RedString(" [not accessible]"))
		}
	}
}
<commit_msg>Add unicode characters to display OK/NOT OK<commit_after>package gitManip

import (
	"fmt"

	"github.com/fatih/color"

	git "gopkg.in/libgit2/git2go.v25"
)

/*Map to match the RepositoryState enum type with a string
 */
var repositoryStateToString = map[git.RepositoryState]string{
	git.RepositoryStateNone:                 "None",
	git.RepositoryStateMerge:                "Merge",
	git.RepositoryStateRevert:               "Revert",
	git.RepositoryStateCherrypick:           "Cherrypick",
	git.RepositoryStateBisect:               "Bisect",
	git.RepositoryStateRebase:               "Rebase",
	git.RepositoryStateRebaseInteractive:    "Rebase Interactive",
	git.RepositoryStateRebaseMerge:          "Rebase Merge",
	git.RepositoryStateApplyMailbox:         "Apply Mailbox",
	git.RepositoryStateApplyMailboxOrRebase: "Apply Mailbox or Rebase",
}

var fileStateToString = map[git.Status]string{
	git.StatusIndexNew:     "You forgot to commit a new file!",
	git.StatusIgnored:      "Ignored",
	git.StatusConflicted:   "Conflicted",
	git.StatusWtNew:        "New file in your working tree!",
	git.StatusWtModified:   "Modified file in your working tree!",
	git.StatusWtDeleted:    "Deleted file in your working tree!",
	git.StatusWtTypeChange: "Type change detected in your working tree!",
	git.StatusWtRenamed:    "Renamed file in your working tree!",
}

/*Global variable to set the StatusOption parameter, in order to list each file status
 */
var statusOption = git.StatusOptions{
	Show:     git.StatusShowIndexAndWorkdir,
	Flags:    git.StatusOptIncludeUntracked,
	Pathspec: []string{},
}

/*GitObject contains informations about the current git repository
 *
 *The structure is:
 *  accessible:
 *		Is the repository still exists in the hard drive?
 *	path:
 *		The path file.
 *	repository:
 *		The object repository.
 */
type GitObject struct {
	accessible error
	path       string
	repository git.Repository
}

/*New is a constructor for GitObject
 *
 * It neeeds:
 *	path:
 *		The path of the current repository.
 */
func New(path string) *GitObject {
	r, err := git.OpenRepository(path)
	return &GitObject{accessible: err, path: path, repository: *r}
}

/*isAccesible returns the information that is the current git repository is existing or not.
 *This method returns a boolean value: true if the git repository is still accesible (still exists), or false if not.
 */
func (g *GitObject) isAccessible() bool {
	return g.accessible == nil
}

/*Status prints the current status of the repository, accessible via the structure path field.
 *This method works only if the repository is accessible.
 */
func (g *GitObject) Status() {
	if g.isAccessible() {
		if err := g.printChanges(); err != nil {
			color.RedString("Impossible to get stats from %s, due to error %s", g.path, err)
		}
	} else {
		color.RedString("Repository %s not found!", g.path)
	}
}

/*getDiffWithWT returns the difference between the working tree and the index, for the current git repository.
 *If there is an error processing the request, it returns an error.
 */
func (g *GitObject) getDiffWithWT() (*git.Diff, error) {
	// Get the index of the repository
	currentIndex, err := g.repository.Index()
	if err != nil {
		return nil, err
	}
	// Get the default diff options, and add it custom flags
	defaultDiffOptions, err := git.DefaultDiffOptions()
	if err != nil {
		return nil, err
	}
	defaultDiffOptions.Flags = defaultDiffOptions.Flags | git.DiffIncludeUntracked | git.DiffIncludeTypeChange
	// Check the difference between the working directory and the index
	diff, err := g.repository.DiffIndexToWorkdir(currentIndex, &defaultDiffOptions)
	if err != nil {
		return nil, err
	}
	return diff, nil
}

/*printChanges prints out all changes for the current git repository.
 *If there is an error processing the request, it returns this one.
 */
func (g *GitObject) printChanges() error {
	diff, err := g.getDiffWithWT()
	if err != nil {
		return err
	}
	numDeltas, err := diff.NumDeltas()
	if err != nil {
		return err
	}
	if numDeltas > 0 {
		fmt.Printf("%s %s\t[%d modification(s)]\n", color.RedString("✘"), g.path, numDeltas)
		for i := 0; i < numDeltas; i++ {
			delta, _ := diff.GetDelta(i)
			currentStatus := delta.Status
			newFile := delta.NewFile.Path
			oldFile := delta.OldFile.Path
			switch currentStatus {
			case git.DeltaAdded:
				fmt.Printf("\t===> %s has been added!\n", color.MagentaString(newFile))
			case git.DeltaDeleted:
				fmt.Printf("\t===> %s has been deleted!\n", color.MagentaString(newFile))
			case git.DeltaModified:
				fmt.Printf("\t===> %s has been modified!\n", color.MagentaString(newFile))
			case git.DeltaRenamed:
				fmt.Printf("\t===> %s has been renamed to %s!\n", color.MagentaString(oldFile), color.MagentaString(newFile))
			case git.DeltaUntracked:
				fmt.Printf("\t===> %s is untracked - please to add it or update the gitignore file!\n", color.MagentaString(newFile))
			case git.DeltaTypeChange:
				fmt.Printf("\t===> the type of %s has been changed from %d to %d!", color.MagentaString(newFile), delta.OldFile.Mode, delta.NewFile.Mode)
			}
		}
	} else {
		fmt.Printf("%s %s\n", color.GreenString("✔"), g.path)
	}
	return nil
}

/*List lists the path and the accessibility of a list of git repositories
 */
func List(repositories *[]GitObject) {
	for _, object := range *repositories {
		fmt.Printf("* %s ", object.path)
		if object.isAccessible() {
			fmt.Println(color.GreenString(" [accessible]"))
		} else {
			fmt.Println(color.RedString(" [not accessible]"))
		}
	}
}
<|endoftext|>


<commit_before>package issues

import (
	"testing"

	"go.skia.org/infra/go/issues"
	"go.skia.org/infra/go/mockhttpclient"
	"go.skia.org/infra/go/testutils"
)

func TestCreateBadBugURL(t *testing.T) {
	testutils.SmallTest(t)
	// No http calls need to be mocked up, as none should be used.
	urlMock := mockhttpclient.NewURLMock()
	im := NewManager(urlMock.Client())
	p := IssueReportingPackage{
		FuzzName:       "1234567890abcdef",
		CommitRevision: "fedcba9876543210",
		Category:       "color_icc",
	}
	url, err := im.CreateBadBugURL(p)
	if err != nil {
		t.Errorf("Should not have returned error: %s", err)
	}
	expectedURL := `https://bugs.chromium.org/p/skia/issues/entry?cc=kjlubick%40google.com&comment=%23+Description+here+about+fuzz+found+in+SkColorSpace+-+ICC%0A%0A%0ATo+replicate%2C+build+target+%22fuzz%22+at+the+specified+commit+and+run%3A%0Aout%2FRelease%2Ffuzz+--type+icc+--bytes+~%2FDownloads%2F1234567890abcdef%0A%0AThe+problem+may+only+be+revealed+by+an+ASAN+build%2C+in+which+case+you+would+need+to+run%3A%0Agn+gen+out%2FASAN+--args%3D%27cc%3D%22%2Fusr%2Fbin%2Fclang%22+cxx%3D%22%2Fusr%2Fbin%2Fclang%2B%2B%22+sanitize%3D%22ASAN%22%27%0Aor%3A%0Agn+gen+out%2FASAN+--args%3D%27cc%3D%22%2Fusr%2Fbin%2Fclang%22+cxx%3D%22%2Fusr%2Fbin%2Fclang%2B%2B%22+sanitize%3D%22ASAN%22+is_debug%3Dfalse%27%0A%0Aprior+to+building.%0A%0A%23+tracking+metadata+below%3A%0Afuzz_category%3A+color_icc%0Afuzz_commit%3A+fedcba9876543210%0Arelated_fuzz%3A+https%3A%2F%2Ffuzzer.skia.org%2Fcategory%2Fcolor_icc%2Fname%2F1234567890abcdef%0Afuzz_download%3A+https%3A%2F%2Ffuzzer.skia.org%2Ffuzz%2F1234567890abcdef%0A&labels=Area-ImageDecoder%2CFromSkiaFuzzer%2CRestrict-View-Google%2CType-Defect%2CPriority-Medium&owner=msarett%40google.com&status=New&summary=New+crash+found+in+SkColorSpace+-+ICC+by+fuzzer`
	if url != expectedURL {
		t.Errorf("URL does not match.  Expected: %s\n\nWas: %s\n", expectedURL, url)
	}

}

func TestCreateBadBugIssue(t *testing.T) {
	testutils.SmallTest(t)
	urlMock := mockhttpclient.NewURLMock()
	im := NewManager(urlMock.Client())
	p := IssueReportingPackage{
		FuzzName:       "1234567890abcdef",
		CommitRevision: "fedcba9876543210",
		Category:       "api_parse_path",
	}

	urlMock.MockOnce(issues.MONORAIL_BASE_URL, mockhttpclient.MockPostDialogue("application/json", expectedIssueRequest, []byte(exampleMonorailResponse)))

	err := im.CreateBadBugIssue(p, "Mock fuzzer found a problem")
	if err != nil {
		t.Errorf("Should not have returned error: %s", err)
	}
}

var expectedIssueRequest = []byte(`{"status":"New","owner":{"name":"caryclark@google.com","htmlLink":"","kind":""},"cc":[{"name":"kjlubick@google.com","htmlLink":"","kind":""}],"labels":["FromSkiaFuzzer","Restrict-View-Google","Type-Defect","Priority-Medium"],"summary":"New crash found in API - ParsePath by fuzzer","description":"# Description here about fuzz found in API - ParsePath\nMock fuzzer found a problem\n\nTo replicate, build target \"fuzz\" at the specified commit and run:\nout/Release/fuzz --type api --name ParsePath --bytes ~/Downloads/1234567890abcdef\n\nThe problem may only be revealed by an ASAN build, in which case you would need to run:\ngn gen out/ASAN --args='cc=\"/usr/bin/clang\" cxx=\"/usr/bin/clang++\" sanitize=\"ASAN\"'\nor:\ngn gen out/ASAN --args='cc=\"/usr/bin/clang\" cxx=\"/usr/bin/clang++\" sanitize=\"ASAN\" is_debug=false'\n\nprior to building.\n\n# tracking metadata below:\nfuzz_category: api_parse_path\nfuzz_commit: fedcba9876543210\nrelated_fuzz: https://fuzzer.skia.org/category/api_parse_path/name/1234567890abcdef\nfuzz_download: https://fuzzer.skia.org/fuzz/1234567890abcdef\n"}
`)

var exampleMonorailResponse = `{
 "status": "New",
 "updated": "2016-05-09T14:37:43",
 "canEdit": true,
 "author": {
  "kind": "monorail#issuePerson",
  "htmlLink": "https://bugs.chromium.org/u/redactedNumbers",
  "name": "service-account-@redacted.com"
 },
 "projectId": "skia",
 "labels": [
  "Type-Defect",
  "Priority-Medium",
  "Restrict-View-Google"
 ],
 "kind": "monorail#issue",
 "canComment": true,
 "state": "open",
 "stars": 0,
 "published": "2016-05-09T14:37:43",
 "title": "Another test bug",
 "starred": false,
 "summary": "Another test bug",
 "id": 5268,
 "etag": "\"FCnnF6QwisNABmHbGpwISZgQNXk/D3OWSf3kqXOPmm4kavoM01N4mLc\""
}`
<commit_msg>Fix fuzzer test<commit_after>package issues

import (
	"testing"

	"go.skia.org/infra/go/issues"
	"go.skia.org/infra/go/mockhttpclient"
	"go.skia.org/infra/go/testutils"
)

func TestCreateBadBugURL(t *testing.T) {
	testutils.SmallTest(t)
	// No http calls need to be mocked up, as none should be used.
	urlMock := mockhttpclient.NewURLMock()
	im := NewManager(urlMock.Client())
	p := IssueReportingPackage{
		FuzzName:       "1234567890abcdef",
		CommitRevision: "fedcba9876543210",
		Category:       "color_icc",
	}
	url, err := im.CreateBadBugURL(p)
	if err != nil {
		t.Errorf("Should not have returned error: %s", err)
	}
	expectedURL := `https://bugs.chromium.org/p/skia/issues/entry?cc=kjlubick%40google.com&comment=%23+Description+here+about+fuzz+found+in+SkColorSpace+-+ICC%0A%0A%0ATo+replicate%2C+build+target+%22fuzz%22+at+the+specified+commit+and+run%3A%0Aout%2FRelease%2Ffuzz+--type+icc+--bytes+~%2FDownloads%2F1234567890abcdef%0A%0AThe+problem+may+only+be+revealed+by+an+ASAN+build%2C+in+which+case+you+would+need+to+run%3A%0Agn+gen+out%2FASAN+--args%3D%27cc%3D%22%2Fusr%2Fbin%2Fclang%22+cxx%3D%22%2Fusr%2Fbin%2Fclang%2B%2B%22+sanitize%3D%22ASAN%22%27%0Aor%3A%0Agn+gen+out%2FASAN+--args%3D%27cc%3D%22%2Fusr%2Fbin%2Fclang%22+cxx%3D%22%2Fusr%2Fbin%2Fclang%2B%2B%22+sanitize%3D%22ASAN%22+is_debug%3Dfalse%27%0A%0Aprior+to+building.%0A%0A%23+tracking+metadata+below%3A%0Afuzz_category%3A+color_icc%0Afuzz_commit%3A+fedcba9876543210%0Arelated_fuzz%3A+https%3A%2F%2Ffuzzer.skia.org%2Fcategory%2Fcolor_icc%2Fname%2F1234567890abcdef%0Afuzz_download%3A+https%3A%2F%2Ffuzzer.skia.org%2Ffuzz%2F1234567890abcdef%0A&labels=Area-ImageDecoder%2CFromSkiaFuzzer%2CRestrict-View-Google%2CType-Defect%2CPriority-Medium&owner=scroggo%40google.com&status=New&summary=New+crash+found+in+SkColorSpace+-+ICC+by+fuzzer`
	if url != expectedURL {
		t.Errorf("URL does not match.  Expected: %s\n\nWas: %s\n", expectedURL, url)
	}

}

func TestCreateBadBugIssue(t *testing.T) {
	testutils.SmallTest(t)
	urlMock := mockhttpclient.NewURLMock()
	im := NewManager(urlMock.Client())
	p := IssueReportingPackage{
		FuzzName:       "1234567890abcdef",
		CommitRevision: "fedcba9876543210",
		Category:       "api_parse_path",
	}

	urlMock.MockOnce(issues.MONORAIL_BASE_URL, mockhttpclient.MockPostDialogue("application/json", expectedIssueRequest, []byte(exampleMonorailResponse)))

	err := im.CreateBadBugIssue(p, "Mock fuzzer found a problem")
	if err != nil {
		t.Errorf("Should not have returned error: %s", err)
	}
}

var expectedIssueRequest = []byte(`{"status":"New","owner":{"name":"caryclark@google.com","htmlLink":"","kind":""},"cc":[{"name":"kjlubick@google.com","htmlLink":"","kind":""}],"labels":["FromSkiaFuzzer","Restrict-View-Google","Type-Defect","Priority-Medium"],"summary":"New crash found in API - ParsePath by fuzzer","description":"# Description here about fuzz found in API - ParsePath\nMock fuzzer found a problem\n\nTo replicate, build target \"fuzz\" at the specified commit and run:\nout/Release/fuzz --type api --name ParsePath --bytes ~/Downloads/1234567890abcdef\n\nThe problem may only be revealed by an ASAN build, in which case you would need to run:\ngn gen out/ASAN --args='cc=\"/usr/bin/clang\" cxx=\"/usr/bin/clang++\" sanitize=\"ASAN\"'\nor:\ngn gen out/ASAN --args='cc=\"/usr/bin/clang\" cxx=\"/usr/bin/clang++\" sanitize=\"ASAN\" is_debug=false'\n\nprior to building.\n\n# tracking metadata below:\nfuzz_category: api_parse_path\nfuzz_commit: fedcba9876543210\nrelated_fuzz: https://fuzzer.skia.org/category/api_parse_path/name/1234567890abcdef\nfuzz_download: https://fuzzer.skia.org/fuzz/1234567890abcdef\n"}
`)

var exampleMonorailResponse = `{
 "status": "New",
 "updated": "2016-05-09T14:37:43",
 "canEdit": true,
 "author": {
  "kind": "monorail#issuePerson",
  "htmlLink": "https://bugs.chromium.org/u/redactedNumbers",
  "name": "service-account-@redacted.com"
 },
 "projectId": "skia",
 "labels": [
  "Type-Defect",
  "Priority-Medium",
  "Restrict-View-Google"
 ],
 "kind": "monorail#issue",
 "canComment": true,
 "state": "open",
 "stars": 0,
 "published": "2016-05-09T14:37:43",
 "title": "Another test bug",
 "starred": false,
 "summary": "Another test bug",
 "id": 5268,
 "etag": "\"FCnnF6QwisNABmHbGpwISZgQNXk/D3OWSf3kqXOPmm4kavoM01N4mLc\""
}`
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/colinmarc/hdfs"
)

func ls(paths []string, long, all, humanReadable bool) {
	paths, client, err := getClientAndExpandedPaths(paths)
	if err != nil {
		fatal(err)
	}

	if len(paths) == 0 {
		paths = []string{userDir()}
	}

	files := make([]string, 0, len(paths))
	fileInfos := make([]os.FileInfo, 0, len(paths))
	dirs := make([]string, 0, len(paths))
	for _, p := range paths {
		fi, err := client.Stat(p)
		if err != nil {
			fatal(err)
		}

		if fi.IsDir() {
			dirs = append(dirs, p)
		} else {
			files = append(files, p)
			fileInfos = append(fileInfos, fi)
		}
	}

	if len(files) == 0 && len(dirs) == 1 {
		printDir(client, dirs[0], long, all, humanReadable)
	} else {
		if long {
			tw := lsTabWriter()
			for i, p := range files {
				printLong(tw, p, fileInfos[i], humanReadable)
			}

			tw.Flush()
		} else {
			for _, p := range files {
				fmt.Println(p)
			}
		}

		for i, dir := range dirs {
			if i > 0 {
				fmt.Println()
			}

			fmt.Printf("%s/:\n", dir)
			printDir(client, dir, long, all, humanReadable)
		}
	}
}

func printDir(client *hdfs.Client, dir string, long, all, humanReadable bool) {
	dirReader, err := client.Open(dir)
	if err != nil {
		fatal(err)
	}

	var tw *tabwriter.Writer
	if long {
		tw = lsTabWriter()
		defer tw.Flush()
	}

	if all {
		if long {
			dirInfo, err := client.Stat(dir)
			if err != nil {
				fatal(err)
			}

			parentPath := path.Join(dir, "..")
			parentInfo, err := client.Stat(parentPath)
			if err != nil {
				fatal(err)
			}

			printLong(tw, ".", dirInfo, humanReadable)
			printLong(tw, "..", parentInfo, humanReadable)
		} else {
			fmt.Println(".")
			fmt.Println("..")
		}
	}

	var partial []os.FileInfo
	for ; err != io.EOF; partial, err = dirReader.Readdir(100) {
		if err != nil {
			fatal(err)
		}

		printFiles(tw, partial, long, all, humanReadable)
	}

	if long {
		tw.Flush()
	}
}

func printFiles(tw *tabwriter.Writer, files []os.FileInfo, long, all, humanReadable bool) {
	for _, file := range files {
		if !all && strings.HasPrefix(file.Name(), ".") {
			continue
		}

		if long {
			printLong(tw, file.Name(), file, humanReadable)
		} else {
			fmt.Println(file.Name())
		}
	}
}

func printLong(tw *tabwriter.Writer, name string, info os.FileInfo, humanReadable bool) {
	fi := info.(*hdfs.FileInfo)
	// mode owner group size date(\w tab) time/year name
	mode := fi.Mode().String()
	owner := fi.Owner()
	group := fi.OwnerGroup()
	size := strconv.FormatInt(fi.Size(), 10)
	if humanReadable {
		size = formatBytes(fi.Size())
	}

	modtime := fi.ModTime()
	date := modtime.Format("Jan _2")
	var timeOrYear string
	if modtime.Year() == time.Now().Year() {
		timeOrYear = modtime.Format("15:04")
	} else {
		timeOrYear = string(modtime.Year())
	}

	fmt.Fprintf(tw, "%s \t%s \t %s \t %s \t%s \t%s \t%s\n",
		mode, owner, group, size, date, timeOrYear, name)
}

func lsTabWriter() *tabwriter.Writer {
	return tabwriter.NewWriter(os.Stdout, 3, 8, 0, ' ', tabwriter.AlignRight|tabwriter.TabIndent)
}
<commit_msg>Format ls -l year correctly<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/colinmarc/hdfs"
)

func ls(paths []string, long, all, humanReadable bool) {
	paths, client, err := getClientAndExpandedPaths(paths)
	if err != nil {
		fatal(err)
	}

	if len(paths) == 0 {
		paths = []string{userDir()}
	}

	files := make([]string, 0, len(paths))
	fileInfos := make([]os.FileInfo, 0, len(paths))
	dirs := make([]string, 0, len(paths))
	for _, p := range paths {
		fi, err := client.Stat(p)
		if err != nil {
			fatal(err)
		}

		if fi.IsDir() {
			dirs = append(dirs, p)
		} else {
			files = append(files, p)
			fileInfos = append(fileInfos, fi)
		}
	}

	if len(files) == 0 && len(dirs) == 1 {
		printDir(client, dirs[0], long, all, humanReadable)
	} else {
		if long {
			tw := lsTabWriter()
			for i, p := range files {
				printLong(tw, p, fileInfos[i], humanReadable)
			}

			tw.Flush()
		} else {
			for _, p := range files {
				fmt.Println(p)
			}
		}

		for i, dir := range dirs {
			if i > 0 {
				fmt.Println()
			}

			fmt.Printf("%s/:\n", dir)
			printDir(client, dir, long, all, humanReadable)
		}
	}
}

func printDir(client *hdfs.Client, dir string, long, all, humanReadable bool) {
	dirReader, err := client.Open(dir)
	if err != nil {
		fatal(err)
	}

	var tw *tabwriter.Writer
	if long {
		tw = lsTabWriter()
		defer tw.Flush()
	}

	if all {
		if long {
			dirInfo, err := client.Stat(dir)
			if err != nil {
				fatal(err)
			}

			parentPath := path.Join(dir, "..")
			parentInfo, err := client.Stat(parentPath)
			if err != nil {
				fatal(err)
			}

			printLong(tw, ".", dirInfo, humanReadable)
			printLong(tw, "..", parentInfo, humanReadable)
		} else {
			fmt.Println(".")
			fmt.Println("..")
		}
	}

	var partial []os.FileInfo
	for ; err != io.EOF; partial, err = dirReader.Readdir(100) {
		if err != nil {
			fatal(err)
		}

		printFiles(tw, partial, long, all, humanReadable)
	}

	if long {
		tw.Flush()
	}
}

func printFiles(tw *tabwriter.Writer, files []os.FileInfo, long, all, humanReadable bool) {
	for _, file := range files {
		if !all && strings.HasPrefix(file.Name(), ".") {
			continue
		}

		if long {
			printLong(tw, file.Name(), file, humanReadable)
		} else {
			fmt.Println(file.Name())
		}
	}
}

func printLong(tw *tabwriter.Writer, name string, info os.FileInfo, humanReadable bool) {
	fi := info.(*hdfs.FileInfo)
	// mode owner group size date(\w tab) time/year name
	mode := fi.Mode().String()
	owner := fi.Owner()
	group := fi.OwnerGroup()
	size := strconv.FormatInt(fi.Size(), 10)
	if humanReadable {
		size = formatBytes(fi.Size())
	}

	modtime := fi.ModTime()
	date := modtime.Format("Jan _2")
	var timeOrYear string
	if modtime.Year() == time.Now().Year() {
		timeOrYear = modtime.Format("15:04")
	} else {
		timeOrYear = modtime.Format("2006")
	}

	fmt.Fprintf(tw, "%s \t%s \t %s \t %s \t%s \t%s \t%s\n",
		mode, owner, group, size, date, timeOrYear, name)
}

func lsTabWriter() *tabwriter.Writer {
	return tabwriter.NewWriter(os.Stdout, 3, 8, 0, ' ', tabwriter.AlignRight|tabwriter.TabIndent)
}
<|endoftext|>


<commit_before>package cmdutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/motemen/ghq/logger"
)

// Run the command
func Run(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return RunCommand(cmd, false)
}

// RunSilently runs the command silently
func RunSilently(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard

	return RunCommand(cmd, true)
}

// RunInDir runs the command in the specified directory
func RunInDir(dir, command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Dir = dir

	return RunCommand(cmd, false)
}

// RunInDirSilently run the command in the specified directory silently
func RunInDirSilently(dir, command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard
	cmd.Dir = dir

	return RunCommand(cmd, true)
}

// RunFunc for the type command execution
type RunFunc func(*exec.Cmd) error

// CommandRunner is for running the command
var CommandRunner = func(cmd *exec.Cmd) error {
	return cmd.Run()
}

// RunCommand run the command
func RunCommand(cmd *exec.Cmd, silent bool) error {
	if !silent {
		logger.Log(cmd.Args[0], strings.Join(cmd.Args[1:], " "))
	}
	err := CommandRunner(cmd)
	if err != nil {
		if execErr, ok := err.(*exec.Error); ok {
			logger.Log("warning", fmt.Sprintf("%q: %s", execErr.Name, execErr.Err))
		}
		return &RunError{cmd, err}
	}

	return nil
}

// RunError is the error type for cmdutil
type RunError struct {
	Command   *exec.Cmd
	ExecError error
}

// Error to implement error interface
func (e *RunError) Error() string {
	return fmt.Sprintf("%s: %s", e.Command.Path, e.ExecError)
}
<commit_msg>Revert "warn if executable was not found when RunCommand"<commit_after>package cmdutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/motemen/ghq/logger"
)

// Run the command
func Run(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return RunCommand(cmd, false)
}

// RunSilently runs the command silently
func RunSilently(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard

	return RunCommand(cmd, true)
}

// RunInDir runs the command in the specified directory
func RunInDir(dir, command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Dir = dir

	return RunCommand(cmd, false)
}

// RunInDirSilently run the command in the specified directory silently
func RunInDirSilently(dir, command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdout = ioutil.Discard
	cmd.Stderr = ioutil.Discard
	cmd.Dir = dir

	return RunCommand(cmd, true)
}

// RunFunc for the type command execution
type RunFunc func(*exec.Cmd) error

// CommandRunner is for running the command
var CommandRunner = func(cmd *exec.Cmd) error {
	return cmd.Run()
}

// RunCommand run the command
func RunCommand(cmd *exec.Cmd, silent bool) error {
	if !silent {
		logger.Log(cmd.Args[0], strings.Join(cmd.Args[1:], " "))
	}
	err := CommandRunner(cmd)
	if err != nil {
		return &RunError{cmd, err}
	}

	return nil
}

// RunError is the error type for cmdutil
type RunError struct {
	Command   *exec.Cmd
	ExecError error
}

// Error to implement error interface
func (e *RunError) Error() string {
	return fmt.Sprintf("%s: %s", e.Command.Path, e.ExecError)
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build darwin

package libkb

import (
	"encoding/base64"

	keychain "github.com/keybase/go-keychain"
)

type KeychainSecretStore struct {
	context SecretStoreContext
}

var _ SecretStoreAll = KeychainSecretStore{}

func (k KeychainSecretStore) serviceName() string {
	return k.context.GetStoredSecretServiceName()
}

func (k KeychainSecretStore) StoreSecret(accountName NormalizedUsername, secret LKSecFullSecret) (err error) {
	// Base64 encode to make it easy to work with Keychain Access (since we are using a password item and secret is not utf-8)
	encodedSecret := base64.StdEncoding.EncodeToString(secret.Bytes())
	item := keychain.NewGenericPassword(k.serviceName(), string(accountName), "", []byte(encodedSecret), k.accessGroup())
	item.SetSynchronizable(k.synchronizable())
	item.SetAccessible(k.accessible())
	k.context.GetLog().Debug("KeychainSecretStore.StoreSecret(%s): deleting item before adding new one", accountName)
	err = keychain.DeleteItem(item)
	if err != nil {
		// error probably ok here?
		k.context.GetLog().Debug("KeychainSecretStore.StoreSecret(%s): DeleteItem error: %s", accountName, err)
	}
	k.context.GetLog().Debug("KeychainSecretStore.StoreSecret(%s): adding item", accountName)
	err = keychain.AddItem(item)
	if err != nil {
		k.context.GetLog().Warning("KeychainSecretStore.StoreSecret(%s): AddItem error: %s", accountName, err)
		return err
	}
	k.context.GetLog().Debug("KeychainSecretStore.StoreSecret(%s): AddItem success", accountName)

	return nil
}

func (k KeychainSecretStore) RetrieveSecret(accountName NormalizedUsername) (LKSecFullSecret, error) {
	k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s)", accountName)
	encodedSecret, err := keychain.GetGenericPassword(k.serviceName(), string(accountName), "", "")
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) error: %s", accountName, err)
		return LKSecFullSecret{}, err
	}
	if encodedSecret == nil {
		k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) nil encodedSecret", accountName)
		return LKSecFullSecret{}, SecretStoreError{Msg: "No secret for " + string(accountName)}
	}

	secret, err := base64.StdEncoding.DecodeString(string(encodedSecret))
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) base64.Decode error: %s", accountName, err)
		return LKSecFullSecret{}, err
	}

	k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) got secret, creating lksec", accountName)

	lk, err := newLKSecFullSecretFromBytes(secret)
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) error creating lksec: %s", accountName, err)
		return LKSecFullSecret{}, err
	}

	k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) success", accountName)

	return lk, nil
}

func (k KeychainSecretStore) ClearSecret(accountName NormalizedUsername) error {
	k.context.GetLog().Debug("KeychainSecretStore.ClearSecret(%s)", accountName)
	query := keychain.NewGenericPassword(k.serviceName(), string(accountName), "", nil, "")
	query.SetMatchLimit(keychain.MatchLimitAll)
	err := keychain.DeleteItem(query)
	if err == keychain.ErrorItemNotFound {
		k.context.GetLog().Debug("KeychainSecretStore.ClearSecret(%s), item not found", accountName)
		return nil
	}
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.ClearSecret(%s), DeleteItem error: %s", accountName, err)
	}

	k.context.GetLog().Debug("KeychainSecretStore.ClearSecret(%s) success", accountName)

	return err
}

func NewSecretStoreAll(g *GlobalContext) SecretStoreAll {
	return KeychainSecretStore{
		context: g,
	}
}

func NewTestSecretStoreAll(c SecretStoreContext, g *GlobalContext) SecretStoreAll {
	return nil
}

func HasSecretStore() bool {
	return true
}

func (k KeychainSecretStore) GetUsersWithStoredSecrets() ([]string, error) {
	users, err := keychain.GetAccountsForService(k.context.GetStoredSecretServiceName())
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.GetUsersWithStoredSecrets() error: %s", err)
		return nil, err
	}

	k.context.GetLog().Debug("KeychainSecretStore.GetUsersWithStoredSecrets() -> %d users", len(users))
	return users, nil
}

func (k KeychainSecretStore) GetApprovalPrompt() string {
	return "Save in Keychain"
}

func (k KeychainSecretStore) GetTerminalPrompt() string {
	return "Store your key in Apple's local keychain?"
}
<commit_msg>Mobile iOS doesn't support MatchLimitAll on delete (#6317)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build darwin

package libkb

import (
	"encoding/base64"

	keychain "github.com/keybase/go-keychain"
)

type KeychainSecretStore struct {
	context SecretStoreContext
}

var _ SecretStoreAll = KeychainSecretStore{}

func (k KeychainSecretStore) serviceName() string {
	return k.context.GetStoredSecretServiceName()
}

func (k KeychainSecretStore) StoreSecret(accountName NormalizedUsername, secret LKSecFullSecret) (err error) {
	// Base64 encode to make it easy to work with Keychain Access (since we are using a password item and secret is not utf-8)
	encodedSecret := base64.StdEncoding.EncodeToString(secret.Bytes())
	item := keychain.NewGenericPassword(k.serviceName(), string(accountName), "", []byte(encodedSecret), k.accessGroup())
	item.SetSynchronizable(k.synchronizable())
	item.SetAccessible(k.accessible())
	k.context.GetLog().Debug("KeychainSecretStore.StoreSecret(%s): deleting item before adding new one", accountName)
	err = keychain.DeleteItem(item)
	if err != nil {
		// error probably ok here?
		k.context.GetLog().Debug("KeychainSecretStore.StoreSecret(%s): DeleteItem error: %s", accountName, err)
	}
	k.context.GetLog().Debug("KeychainSecretStore.StoreSecret(%s): adding item", accountName)
	err = keychain.AddItem(item)
	if err != nil {
		k.context.GetLog().Warning("KeychainSecretStore.StoreSecret(%s): AddItem error: %s", accountName, err)
		return err
	}
	k.context.GetLog().Debug("KeychainSecretStore.StoreSecret(%s): AddItem success", accountName)

	return nil
}

func (k KeychainSecretStore) RetrieveSecret(accountName NormalizedUsername) (LKSecFullSecret, error) {
	k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s)", accountName)
	encodedSecret, err := keychain.GetGenericPassword(k.serviceName(), string(accountName), "", "")
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) error: %s", accountName, err)
		return LKSecFullSecret{}, err
	}
	if encodedSecret == nil {
		k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) nil encodedSecret", accountName)
		return LKSecFullSecret{}, SecretStoreError{Msg: "No secret for " + string(accountName)}
	}

	secret, err := base64.StdEncoding.DecodeString(string(encodedSecret))
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) base64.Decode error: %s", accountName, err)
		return LKSecFullSecret{}, err
	}

	k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) got secret, creating lksec", accountName)

	lk, err := newLKSecFullSecretFromBytes(secret)
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) error creating lksec: %s", accountName, err)
		return LKSecFullSecret{}, err
	}

	k.context.GetLog().Debug("KeychainSecretStore.RetrieveSecret(%s) success", accountName)

	return lk, nil
}

func (k KeychainSecretStore) ClearSecret(accountName NormalizedUsername) error {
	k.context.GetLog().Debug("KeychainSecretStore.ClearSecret(%s)", accountName)
	var query keychain.Item
	if isIOS {
		query = keychain.NewGenericPassword(k.serviceName(), string(accountName), "", nil, k.accessGroup())
	} else {
		query = keychain.NewGenericPassword(k.serviceName(), string(accountName), "", nil, "")
		query.SetMatchLimit(keychain.MatchLimitAll)
	}
	err := keychain.DeleteItem(query)
	if err == keychain.ErrorItemNotFound {
		k.context.GetLog().Debug("KeychainSecretStore.ClearSecret(%s), item not found", accountName)
		return nil
	}
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.ClearSecret(%s), DeleteItem error: %s", accountName, err)
	}

	k.context.GetLog().Debug("KeychainSecretStore.ClearSecret(%s) success", accountName)

	return err
}

func NewSecretStoreAll(g *GlobalContext) SecretStoreAll {
	return KeychainSecretStore{
		context: g,
	}
}

func NewTestSecretStoreAll(c SecretStoreContext, g *GlobalContext) SecretStoreAll {
	return nil
}

func HasSecretStore() bool {
	return true
}

func (k KeychainSecretStore) GetUsersWithStoredSecrets() ([]string, error) {
	users, err := keychain.GetAccountsForService(k.context.GetStoredSecretServiceName())
	if err != nil {
		k.context.GetLog().Debug("KeychainSecretStore.GetUsersWithStoredSecrets() error: %s", err)
		return nil, err
	}

	k.context.GetLog().Debug("KeychainSecretStore.GetUsersWithStoredSecrets() -> %d users", len(users))
	return users, nil
}

func (k KeychainSecretStore) GetApprovalPrompt() string {
	return "Save in Keychain"
}

func (k KeychainSecretStore) GetTerminalPrompt() string {
	return "Store your key in Apple's local keychain?"
}
<|endoftext|>


<commit_before>package data

import (
	"errors"
	"bufio"
	"io"
	"sync"
	"log"
	"net"

	"github.com/golang/protobuf/proto"

	riakprotobuf "github.com/likemindnetworks/riakdataportation/protobuf"
	riakcli "github.com/likemindnetworks/riakdataportation/client"
)

var (
	ImportAlreadyRunning = errors.New("Import is already running")
)

type Import struct {
	cli *riakcli.Client
	input *bufio.Reader
	bucketOverrideFn func([]byte) []byte
	wg sync.WaitGroup
	errorChan chan error
	byteTransferedChan chan int
	isDeleteFirst bool
	isDryRun bool
	isForceOriVclock bool
	isNoVclock bool
	bucketTypes []string
	dtBucketTypes []string
}

type reqRespPair struct {
	reqCode byte
	reqBuf []byte
	resCode byte
	resBuf []byte
}

func NewImport(
		cli *riakcli.Client,
		input *bufio.Reader,
		bucketOverrideFn func([]byte) []byte,
		isDryRun bool,
		isDeleteFirst bool,
		isForceOriVclock bool,
		isNoVclock bool,
		bucketTypes []string,
		dtBucketTypes []string,
) *Import {
	if (bucketOverrideFn == nil) {
		bucketOverrideFn = func(x []byte) []byte { return x }
	}

	return &Import{
		cli: cli,
		input: input,
		bucketOverrideFn: bucketOverrideFn,
		isDeleteFirst: isDeleteFirst,
		isDryRun: isDryRun,
		isForceOriVclock: isForceOriVclock,
		isNoVclock: isNoVclock,
		bucketTypes: bucketTypes,
		dtBucketTypes: dtBucketTypes,
	}
}

func (imp *Import) Run(progressChan chan float64) (byteCnt int, err error) {

	if imp.errorChan != nil {
		return 0, ImportAlreadyRunning
	}

	imp.errorChan = make(chan error)
	imp.byteTransferedChan = make(chan int)

	headerBuf := make([]byte, 5, 5)
	quit := make(chan int)
	byteTransferedCnt := 0

	// abort on error
	go func() {
		for {
			select {
			case b := <- imp.byteTransferedChan:
				byteTransferedCnt += b

				// update progress
				if progressChan != nil {
					if byteCnt > 0 {
						progressChan <- float64(byteTransferedCnt) / float64(byteCnt)
					} else {
						progressChan <- 0
					}
				}
			case err, _ = <- imp.errorChan:
				// either there is an error, or the channel ended

				if err != nil {
					log.Printf("Error during import: %s", err);
				}

				quit <- 0
			}
		}
	}()

	for {
		// load a request

		err = imp.fillBuffer(headerBuf)
		if err != nil {
			break
		}

		msgSize, msgReqCode := parseHeader(headerBuf)

		reqData := make([]byte, msgSize, msgSize)
		err = imp.fillBuffer(reqData)
		if err != nil {
			break
		}

		byteCnt += 4 + 1 + int(msgSize)

		// load a response

		err = imp.fillBuffer(headerBuf)
		if err != nil {
			break
		}

		msgSize, msgResCode := parseHeader(headerBuf)

		resData := make([]byte, msgSize, msgSize)
		err = imp.fillBuffer(resData)
		if err != nil {
			break
		}

		byteCnt += 4 + 1 + int(msgSize)

		// process
		imp.wg.Add(1)
		go imp.processPair(&reqRespPair{
			reqCode: msgReqCode,
			reqBuf: reqData,
			resCode: msgResCode,
			resBuf: resData,
		})
	}

	imp.wg.Wait()

	// no more data after this point
	close(imp.errorChan)

	<-quit

	if err == io.EOF {
		err = nil
	}

	imp.errorChan  = nil
	imp.byteTransferedChan = nil

	// close progress
	if progressChan != nil {
		close(progressChan)
	}
	return
}

func parseHeader(headerBuf []byte) (int, byte) {
	// minus 1 because of the msgcode
	msgSize := -1 + int(headerBuf[0]) << 24 +
		int(headerBuf[1]) << 16 +
		int(headerBuf[2]) << 8 +
		int(headerBuf[3])

	msgCode := headerBuf[4]

	return msgSize, msgCode
}

func (imp *Import) fillBuffer(buf []byte) (err error) {
	c := 0;
	size := len(buf)

	for i := 0; i < size; {
		c, err = imp.input.Read(buf[i:size])
		i += c

		if err != nil {
			break
		}
	}

	return
}

func (imp *Import) isBucketTypeEligible(bt []byte) bool {
	for _, b := range imp.bucketTypes {
		if b == string(bt) {
			return true
		}
	}

	return false
}

func (imp *Import) isDtBucketTypeEligible(bt []byte) bool {
	for _, b := range imp.dtBucketTypes {
		if b == string(bt) {
			return true
		}
	}

	return false
}

func (imp *Import) processPair(pair *reqRespPair) {
	defer imp.wg.Done()

	var reqMsg, resMsg proto.Message

	// unmarshal req

	switch pair.reqCode {
		case riakprotobuf.CodeRpbGetReq:
			reqMsg = &riakprotobuf.RpbGetReq{}
		case riakprotobuf.CodeDtFetchReq:
			reqMsg = &riakprotobuf.DtFetchReq{}
	}

	err := proto.Unmarshal(pair.reqBuf, reqMsg)
	if err != nil {
		imp.errorChan <- err
		return
	}

	// unmarshal resp

	switch pair.resCode {
		case riakprotobuf.CodeRpbGetResp:
			resMsg = &riakprotobuf.RpbGetResp{}
		case riakprotobuf.CodeDtFetchResp:
			resMsg = &riakprotobuf.DtFetchResp{}
	}

	err = proto.Unmarshal(pair.resBuf, resMsg)
	if err != nil {
		imp.errorChan <- err
		return
	}

	// perform import

	switch pair.resCode {
		case riakprotobuf.CodeRpbGetResp:
			imp.importKV(
				reqMsg.(*riakprotobuf.RpbGetReq),
				resMsg.(*riakprotobuf.RpbGetResp),
			)
		case riakprotobuf.CodeDtFetchResp:
			imp.importDT(
				reqMsg.(*riakprotobuf.DtFetchReq),
				resMsg.(*riakprotobuf.DtFetchResp),
			)
	}

	imp.byteTransferedChan <- len(pair.reqBuf) + len(pair.resBuf) + 10
}

func (imp *Import) fetchKVVclock(bt []byte, bucket []byte, key []byte) (vk []byte, err error) {
	var conn *net.TCPConn

	req := riakprotobuf.RpbGetReq{
		Type: bt,
		Bucket: bucket,
		Key: key,
	}

	err, conn, _ = imp.cli.SendMessage(&req, riakprotobuf.CodeRpbGetReq)

	if err != nil {
		return
	}

	err, _, responsebuf := imp.cli.ReceiveRawMessage(conn, false)

	resMsg := &riakprotobuf.RpbGetResp{}
	err = proto.Unmarshal(responsebuf, resMsg)

	if err != nil {
		return
	} else {
		vk = resMsg.Vclock
	}

	return
}

func (imp *Import) importKV(
	req *riakprotobuf.RpbGetReq,
	res *riakprotobuf.RpbGetResp,
) {
	if len(res.Content) == 0 {
		return
	}

	if !imp.isBucketTypeEligible(req.Type) {
		// log.Printf("skipping bucket type: %s", req.Type)
		return
	}

	if imp.isDryRun {
		return
	}

	if imp.isDeleteFirst {
		err := imp.cli.ClearKey(
			req.Type,
			imp.bucketOverrideFn(req.Bucket),
			req.Key,
			false,
		)

		if err != nil {
			imp.errorChan <- err
			return
		}
	}

	var (
		newReq riakprotobuf.RpbPutReq
		vk []byte
		err error
	)

	if imp.isForceOriVclock {
		vk = res.Vclock
	} else if imp.isNoVclock {
		vk = nil
	} else {
		vk, err = imp.fetchKVVclock(req.Type, imp.bucketOverrideFn(req.Bucket), req.Key);
	}

	if err != nil {
		imp.errorChan <- err
		return
	}

	if vk != nil && len(vk) > 0 {
		newReq = riakprotobuf.RpbPutReq{
			Type: req.Type,
			Bucket: imp.bucketOverrideFn(req.Bucket),
			Key: req.Key,
			Content: res.Content[0],
			Vclock: vk,
		}
	} else {
		newReq = riakprotobuf.RpbPutReq{
			Type: req.Type,
			Bucket: imp.bucketOverrideFn(req.Bucket),
			Key: req.Key,
			Content: res.Content[0],
		}
	}

	err, conn, _ := imp.cli.SendMessage(&newReq, riakprotobuf.CodeRpbPutReq)
	if err != nil {
		imp.errorChan <- err
		return
	}

	newRes := &riakprotobuf.RpbPutResp{}
	err = imp.cli.ReceiveMessage(conn, newRes, false)
	if (err != nil) {
		imp.errorChan <- err
		return
	}
}

func (imp *Import) fetchCounter(bt []byte, bucket []byte, key []byte) (val int64, err error) {
	var conn *net.TCPConn

	req := riakprotobuf.DtFetchReq{
		Type: bt,
		Bucket: bucket,
		Key: key,
	}

	err, conn, _ = imp.cli.SendMessage(&req, riakprotobuf.CodeDtFetchReq)

	if err != nil {
		return
	}

	err, _, responsebuf := imp.cli.ReceiveRawMessage(conn, false)

	resMsg := &riakprotobuf.DtFetchResp{}
	err = proto.Unmarshal(responsebuf, resMsg)

	if err != nil {
		return
	} else {
		val = *resMsg.Value.CounterValue
	}

	return
}

func (imp *Import) importDT(
	req *riakprotobuf.DtFetchReq,
	res *riakprotobuf.DtFetchResp,
) {
	dtop := &riakprotobuf.DtOp{}

	switch *res.Type {
	case riakprotobuf.DtFetchResp_COUNTER:
		count, err := imp.fetchCounter(req.Type, imp.bucketOverrideFn(req.Bucket), req.Key)

		if err != nil {
			return
		} else {
			count = *res.Value.CounterValue - count

			dtop.CounterOp = &riakprotobuf.CounterOp{ Increment: &count }
		}
	case riakprotobuf.DtFetchResp_SET:
		dtop.SetOp = &riakprotobuf.SetOp{ Adds: res.Value.SetValue }
	case riakprotobuf.DtFetchResp_MAP:
		dtop.MapOp = createMapOp(res.Value.MapValue)
	}

	w := uint32(1)

	newReq := riakprotobuf.DtUpdateReq{
		Type: req.Type,
		Bucket: imp.bucketOverrideFn(req.Bucket),
		Key: req.Key,
		Op: dtop,
		W: &w,
	}

	if !imp.isDtBucketTypeEligible(req.Type) {
		// log.Printf("skipping crdt bucket type: %s", req.Type)
		return
	}

	if imp.isDryRun {
		return
	}

	if imp.isDeleteFirst {
		err := imp.cli.ClearKey(
			req.Type,
			imp.bucketOverrideFn(req.Bucket),
			req.Key,
			true,
		)

		if err != nil {
			imp.errorChan <- err
			return
		}
	}

	err, conn, _ := imp.cli.SendMessage(&newReq, riakprotobuf.CodeDtUpdateReq)
	if err != nil {
		imp.errorChan <- err
		return
	}

	newRes := &riakprotobuf.DtUpdateResp{}
	err = imp.cli.ReceiveMessage(conn, newRes, false)
	if (err != nil) {
		imp.errorChan <- err
		return
	}
}

func createMapOp(entries []*riakprotobuf.MapEntry) *riakprotobuf.MapOp {
	mapEntryCnt := len(entries)
	mapOp := &riakprotobuf.MapOp{}
	mapOp.Updates = make([]*riakprotobuf.MapUpdate, mapEntryCnt, mapEntryCnt)

	for i, mapEntry := range entries {
		mapOp.Updates[i] = &riakprotobuf.MapUpdate{ Field: mapEntry.Field }

		switch *mapEntry.Field.Type {
		case riakprotobuf.MapField_COUNTER:
			mapOp.Updates[i].CounterOp =
					&riakprotobuf.CounterOp{ Increment: mapEntry.CounterValue }
		case riakprotobuf.MapField_SET:
			mapOp.Updates[i].SetOp = &riakprotobuf.SetOp{ Adds: mapEntry.SetValue }
		case riakprotobuf.MapField_REGISTER:
			mapOp.Updates[i].RegisterOp = mapEntry.RegisterValue
		case riakprotobuf.MapField_FLAG:
			var flapOp riakprotobuf.MapUpdate_FlagOp

			if *mapEntry.FlagValue {
				flapOp = riakprotobuf.MapUpdate_ENABLE
			} else {
				flapOp = riakprotobuf.MapUpdate_DISABLE
			}

			mapOp.Updates[i].FlagOp = &flapOp
		case riakprotobuf.MapField_MAP:
			mapOp.Updates[i].MapOp = createMapOp(mapEntry.MapValue)
		}
	}

	return mapOp
}
<commit_msg>m: handle cases for deleted counter<commit_after>package data

import (
	"errors"
	"bufio"
	"io"
	"sync"
	"log"
	"net"

	"github.com/golang/protobuf/proto"

	riakprotobuf "github.com/likemindnetworks/riakdataportation/protobuf"
	riakcli "github.com/likemindnetworks/riakdataportation/client"
)

var (
	ImportAlreadyRunning = errors.New("Import is already running")
)

type Import struct {
	cli *riakcli.Client
	input *bufio.Reader
	bucketOverrideFn func([]byte) []byte
	wg sync.WaitGroup
	errorChan chan error
	byteTransferedChan chan int
	isDeleteFirst bool
	isDryRun bool
	isForceOriVclock bool
	isNoVclock bool
	bucketTypes []string
	dtBucketTypes []string
}

type reqRespPair struct {
	reqCode byte
	reqBuf []byte
	resCode byte
	resBuf []byte
}

func NewImport(
		cli *riakcli.Client,
		input *bufio.Reader,
		bucketOverrideFn func([]byte) []byte,
		isDryRun bool,
		isDeleteFirst bool,
		isForceOriVclock bool,
		isNoVclock bool,
		bucketTypes []string,
		dtBucketTypes []string,
) *Import {
	if (bucketOverrideFn == nil) {
		bucketOverrideFn = func(x []byte) []byte { return x }
	}

	return &Import{
		cli: cli,
		input: input,
		bucketOverrideFn: bucketOverrideFn,
		isDeleteFirst: isDeleteFirst,
		isDryRun: isDryRun,
		isForceOriVclock: isForceOriVclock,
		isNoVclock: isNoVclock,
		bucketTypes: bucketTypes,
		dtBucketTypes: dtBucketTypes,
	}
}

func (imp *Import) Run(progressChan chan float64) (byteCnt int, err error) {

	if imp.errorChan != nil {
		return 0, ImportAlreadyRunning
	}

	imp.errorChan = make(chan error)
	imp.byteTransferedChan = make(chan int)

	headerBuf := make([]byte, 5, 5)
	quit := make(chan int)
	byteTransferedCnt := 0

	// abort on error
	go func() {
		for {
			select {
			case b := <- imp.byteTransferedChan:
				byteTransferedCnt += b

				// update progress
				if progressChan != nil {
					if byteCnt > 0 {
						progressChan <- float64(byteTransferedCnt) / float64(byteCnt)
					} else {
						progressChan <- 0
					}
				}
			case err, _ = <- imp.errorChan:
				// either there is an error, or the channel ended

				if err != nil {
					log.Printf("Error during import: %s", err);
				}

				quit <- 0
			}
		}
	}()

	for {
		// load a request

		err = imp.fillBuffer(headerBuf)
		if err != nil {
			break
		}

		msgSize, msgReqCode := parseHeader(headerBuf)

		reqData := make([]byte, msgSize, msgSize)
		err = imp.fillBuffer(reqData)
		if err != nil {
			break
		}

		byteCnt += 4 + 1 + int(msgSize)

		// load a response

		err = imp.fillBuffer(headerBuf)
		if err != nil {
			break
		}

		msgSize, msgResCode := parseHeader(headerBuf)

		resData := make([]byte, msgSize, msgSize)
		err = imp.fillBuffer(resData)
		if err != nil {
			break
		}

		byteCnt += 4 + 1 + int(msgSize)

		// process
		imp.wg.Add(1)
		go imp.processPair(&reqRespPair{
			reqCode: msgReqCode,
			reqBuf: reqData,
			resCode: msgResCode,
			resBuf: resData,
		})
	}

	imp.wg.Wait()

	// no more data after this point
	close(imp.errorChan)

	<-quit

	if err == io.EOF {
		err = nil
	}

	imp.errorChan  = nil
	imp.byteTransferedChan = nil

	// close progress
	if progressChan != nil {
		close(progressChan)
	}
	return
}

func parseHeader(headerBuf []byte) (int, byte) {
	// minus 1 because of the msgcode
	msgSize := -1 + int(headerBuf[0]) << 24 +
		int(headerBuf[1]) << 16 +
		int(headerBuf[2]) << 8 +
		int(headerBuf[3])

	msgCode := headerBuf[4]

	return msgSize, msgCode
}

func (imp *Import) fillBuffer(buf []byte) (err error) {
	c := 0;
	size := len(buf)

	for i := 0; i < size; {
		c, err = imp.input.Read(buf[i:size])
		i += c

		if err != nil {
			break
		}
	}

	return
}

func (imp *Import) isBucketTypeEligible(bt []byte) bool {
	for _, b := range imp.bucketTypes {
		if b == string(bt) {
			return true
		}
	}

	return false
}

func (imp *Import) isDtBucketTypeEligible(bt []byte) bool {
	for _, b := range imp.dtBucketTypes {
		if b == string(bt) {
			return true
		}
	}

	return false
}

func (imp *Import) processPair(pair *reqRespPair) {
	defer imp.wg.Done()

	var reqMsg, resMsg proto.Message

	// unmarshal req

	switch pair.reqCode {
		case riakprotobuf.CodeRpbGetReq:
			reqMsg = &riakprotobuf.RpbGetReq{}
		case riakprotobuf.CodeDtFetchReq:
			reqMsg = &riakprotobuf.DtFetchReq{}
	}

	err := proto.Unmarshal(pair.reqBuf, reqMsg)
	if err != nil {
		imp.errorChan <- err
		return
	}

	// unmarshal resp

	switch pair.resCode {
		case riakprotobuf.CodeRpbGetResp:
			resMsg = &riakprotobuf.RpbGetResp{}
		case riakprotobuf.CodeDtFetchResp:
			resMsg = &riakprotobuf.DtFetchResp{}
	}

	err = proto.Unmarshal(pair.resBuf, resMsg)
	if err != nil {
		imp.errorChan <- err
		return
	}

	// perform import

	switch pair.resCode {
		case riakprotobuf.CodeRpbGetResp:
			imp.importKV(
				reqMsg.(*riakprotobuf.RpbGetReq),
				resMsg.(*riakprotobuf.RpbGetResp),
			)
		case riakprotobuf.CodeDtFetchResp:
			imp.importDT(
				reqMsg.(*riakprotobuf.DtFetchReq),
				resMsg.(*riakprotobuf.DtFetchResp),
			)
	}

	imp.byteTransferedChan <- len(pair.reqBuf) + len(pair.resBuf) + 10
}

func (imp *Import) fetchKVVclock(bt []byte, bucket []byte, key []byte) (vk []byte, err error) {
	var conn *net.TCPConn

	req := riakprotobuf.RpbGetReq{
		Type: bt,
		Bucket: bucket,
		Key: key,
	}

	err, conn, _ = imp.cli.SendMessage(&req, riakprotobuf.CodeRpbGetReq)

	if err != nil {
		return
	}

	err, _, responsebuf := imp.cli.ReceiveRawMessage(conn, false)

	resMsg := &riakprotobuf.RpbGetResp{}
	err = proto.Unmarshal(responsebuf, resMsg)

	if err != nil {
		return
	} else {
		vk = resMsg.Vclock
	}

	return
}

func (imp *Import) importKV(
	req *riakprotobuf.RpbGetReq,
	res *riakprotobuf.RpbGetResp,
) {
	if len(res.Content) == 0 {
		return
	}

	if !imp.isBucketTypeEligible(req.Type) {
		// log.Printf("skipping bucket type: %s", req.Type)
		return
	}

	if imp.isDryRun {
		return
	}

	if imp.isDeleteFirst {
		err := imp.cli.ClearKey(
			req.Type,
			imp.bucketOverrideFn(req.Bucket),
			req.Key,
			false,
		)

		if err != nil {
			imp.errorChan <- err
			return
		}
	}

	var (
		newReq riakprotobuf.RpbPutReq
		vk []byte
		err error
	)

	if imp.isForceOriVclock {
		vk = res.Vclock
	} else if imp.isNoVclock {
		vk = nil
	} else {
		vk, err = imp.fetchKVVclock(req.Type, imp.bucketOverrideFn(req.Bucket), req.Key);
	}

	if err != nil {
		imp.errorChan <- err
		return
	}

	if vk != nil && len(vk) > 0 {
		newReq = riakprotobuf.RpbPutReq{
			Type: req.Type,
			Bucket: imp.bucketOverrideFn(req.Bucket),
			Key: req.Key,
			Content: res.Content[0],
			Vclock: vk,
		}
	} else {
		newReq = riakprotobuf.RpbPutReq{
			Type: req.Type,
			Bucket: imp.bucketOverrideFn(req.Bucket),
			Key: req.Key,
			Content: res.Content[0],
		}
	}

	err, conn, _ := imp.cli.SendMessage(&newReq, riakprotobuf.CodeRpbPutReq)
	if err != nil {
		imp.errorChan <- err
		return
	}

	newRes := &riakprotobuf.RpbPutResp{}
	err = imp.cli.ReceiveMessage(conn, newRes, false)
	if (err != nil) {
		imp.errorChan <- err
		return
	}
}

func (imp *Import) fetchCounter(bt []byte, bucket []byte, key []byte) (val int64, err error) {
	var conn *net.TCPConn

	req := riakprotobuf.DtFetchReq{
		Type: bt,
		Bucket: bucket,
		Key: key,
	}

	err, conn, _ = imp.cli.SendMessage(&req, riakprotobuf.CodeDtFetchReq)

	if err != nil {
		return
	}

	err, _, responsebuf := imp.cli.ReceiveRawMessage(conn, false)

	resMsg := &riakprotobuf.DtFetchResp{}
	err = proto.Unmarshal(responsebuf, resMsg)

	if err != nil {
		return
	} else if resMsg.Value != nil {
		val = *resMsg.Value.CounterValue
	}

	return
}

func (imp *Import) importDT(
	req *riakprotobuf.DtFetchReq,
	res *riakprotobuf.DtFetchResp,
) {
	dtop := &riakprotobuf.DtOp{}

	switch *res.Type {
	case riakprotobuf.DtFetchResp_COUNTER:
		count, err := imp.fetchCounter(req.Type, imp.bucketOverrideFn(req.Bucket), req.Key)

		if err != nil {
			return
		} else {
			count = *res.Value.CounterValue - count

			dtop.CounterOp = &riakprotobuf.CounterOp{ Increment: &count }
		}
	case riakprotobuf.DtFetchResp_SET:
		dtop.SetOp = &riakprotobuf.SetOp{ Adds: res.Value.SetValue }
	case riakprotobuf.DtFetchResp_MAP:
		dtop.MapOp = createMapOp(res.Value.MapValue)
	}

	w := uint32(1)

	newReq := riakprotobuf.DtUpdateReq{
		Type: req.Type,
		Bucket: imp.bucketOverrideFn(req.Bucket),
		Key: req.Key,
		Op: dtop,
		W: &w,
	}

	if !imp.isDtBucketTypeEligible(req.Type) {
		// log.Printf("skipping crdt bucket type: %s", req.Type)
		return
	}

	if imp.isDryRun {
		return
	}

	if imp.isDeleteFirst {
		err := imp.cli.ClearKey(
			req.Type,
			imp.bucketOverrideFn(req.Bucket),
			req.Key,
			true,
		)

		if err != nil {
			imp.errorChan <- err
			return
		}
	}

	err, conn, _ := imp.cli.SendMessage(&newReq, riakprotobuf.CodeDtUpdateReq)
	if err != nil {
		imp.errorChan <- err
		return
	}

	newRes := &riakprotobuf.DtUpdateResp{}
	err = imp.cli.ReceiveMessage(conn, newRes, false)
	if (err != nil) {
		imp.errorChan <- err
		return
	}
}

func createMapOp(entries []*riakprotobuf.MapEntry) *riakprotobuf.MapOp {
	mapEntryCnt := len(entries)
	mapOp := &riakprotobuf.MapOp{}
	mapOp.Updates = make([]*riakprotobuf.MapUpdate, mapEntryCnt, mapEntryCnt)

	for i, mapEntry := range entries {
		mapOp.Updates[i] = &riakprotobuf.MapUpdate{ Field: mapEntry.Field }

		switch *mapEntry.Field.Type {
		case riakprotobuf.MapField_COUNTER:
			mapOp.Updates[i].CounterOp =
					&riakprotobuf.CounterOp{ Increment: mapEntry.CounterValue }
		case riakprotobuf.MapField_SET:
			mapOp.Updates[i].SetOp = &riakprotobuf.SetOp{ Adds: mapEntry.SetValue }
		case riakprotobuf.MapField_REGISTER:
			mapOp.Updates[i].RegisterOp = mapEntry.RegisterValue
		case riakprotobuf.MapField_FLAG:
			var flapOp riakprotobuf.MapUpdate_FlagOp

			if *mapEntry.FlagValue {
				flapOp = riakprotobuf.MapUpdate_ENABLE
			} else {
				flapOp = riakprotobuf.MapUpdate_DISABLE
			}

			mapOp.Updates[i].FlagOp = &flapOp
		case riakprotobuf.MapField_MAP:
			mapOp.Updates[i].MapOp = createMapOp(mapEntry.MapValue)
		}
	}

	return mapOp
}
<|endoftext|>