<commit_before>// +build !windows

package checklog

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestFindFileByInode(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	testFileExist := func() {
		logfi, err := os.Stat(logf)
		inode := detectInode(logfi)
		f, err := findFileByInode(inode, dir)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, logf, f)
	}
	testFileExist()

	testFileNotExist := func() {
		f, err := findFileByInode(100, dir)
		assert.Equal(t, err, errFileNotFoundByInode, "err should be errFileNotFoundByInode")
		assert.Equal(t, "", f)
	}
	testFileNotExist()
}

func TestOpenOldFile(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Fatalf("TempDir failed: %s", err)
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	testFoundOldFile := func() {
		ologf := filepath.Join(dir, "dummy.1")
		ofh, _ := os.Create(ologf)
		ofi, _ := ofh.Stat()
		defer ofh.Close()

		l1 := "FATAL\n"
		ofh.WriteString(l1 + l1)

		state := &state{SkipBytes: int64(len(l1)), Inode: detectInode(ofi)}
		f, err := openOldFile(logf, state)
		defer f.Close()
		pos, _ := f.Seek(-1, io.SeekCurrent) // get current offset

		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, ofh.Name(), f.Name(), "openOldFile should be return old file")
		assert.Equal(t, len(l1)-1, int(pos))
	}
	testFoundOldFile()

	testNotFoundOldFile := func() {
		state := &state{SkipBytes: 0, Inode: 0}
		f, err := openOldFile(logf, state)

		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, f, (*os.File)(nil), "file should be nil")
	}
	testNotFoundOldFile()

	testFoundOldFileByInode := func() {
		// dir different from logf
		dir, err := ioutil.TempDir("", "check-log-test")
		if err != nil {
			t.Fatalf("TempDir failed: %s", err)
		}
		defer os.RemoveAll(dir)
		ologf := filepath.Join(dir, "dummy.1")

		ofh, _ := os.Create(ologf)
		ofi, _ := ofh.Stat()
		defer ofh.Close()

		state := &state{SkipBytes: 0, Inode: detectInode(ofi)}
		f, err := openOldFile(logf, state)
		defer f.Close()

		assert.Equal(t, err, nil, "err should be nil")
	}
	testFoundOldFileByInode()
}

func TestRunTraceInode(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	ptn := `FATAL`
	opts, _ := parseArgs([]string{"-s", dir, "-f", logf, "-p", ptn})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, logf, opts.origArgs)

	bytes, _ := getBytesToSkip(stateFile)
	assert.Equal(t, int64(0), bytes, "something went wrong")

	l1 := "SUCCESS\n"
	l2 := "FATAL\n"
	l3 := "SUCCESS\n"
	testRotate := func() {
		// first check
		fh.WriteString(l1)
		opts.searchLog(logf)

		// write FATAL
		fh.WriteString(l2)
		fh.Close()

		// logrotate
		rotatedLogf := filepath.Join(dir, "dummy.1")
		os.Rename(logf, rotatedLogf)
		fh, _ = os.Create(logf)

		fh.WriteString(l3)
		// second check
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "FATAL\n", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l3)), bytes, "should not include oldfile skip bytes")

		os.Remove(rotatedLogf)
	}
	testRotate()
}
<commit_msg>add test case for errFileNotFoundByInode<commit_after>// +build !windows

package checklog

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestFindFileByInode(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	testFileExist := func() {
		logfi, err := os.Stat(logf)
		inode := detectInode(logfi)
		f, err := findFileByInode(inode, dir)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, logf, f)
	}
	testFileExist()

	testFileNotExist := func() {
		f, err := findFileByInode(100, dir)
		assert.Equal(t, err, errFileNotFoundByInode, "err should be errFileNotFoundByInode")
		assert.Equal(t, "", f)
	}
	testFileNotExist()
}

func TestOpenOldFile(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Fatalf("TempDir failed: %s", err)
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	testFoundOldFile := func() {
		ologf := filepath.Join(dir, "dummy.1")
		ofh, _ := os.Create(ologf)
		ofi, _ := ofh.Stat()
		defer ofh.Close()

		l1 := "FATAL\n"
		ofh.WriteString(l1 + l1)

		state := &state{SkipBytes: int64(len(l1)), Inode: detectInode(ofi)}
		f, err := openOldFile(logf, state)
		defer f.Close()
		pos, _ := f.Seek(-1, io.SeekCurrent) // get current offset

		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, ofh.Name(), f.Name(), "openOldFile should be return old file")
		assert.Equal(t, len(l1)-1, int(pos))
	}
	testFoundOldFile()

	testNotFoundOldFile := func() {
		state := &state{SkipBytes: 0, Inode: 0}
		f, err := openOldFile(logf, state)

		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, f, (*os.File)(nil), "file should be nil")
	}
	testNotFoundOldFile()

	testFoundOldFileByInode := func() {
		// dir different from logf
		dir, err := ioutil.TempDir("", "check-log-test")
		if err != nil {
			t.Fatalf("TempDir failed: %s", err)
		}
		defer os.RemoveAll(dir)
		ologf := filepath.Join(dir, "dummy.1")

		ofh, _ := os.Create(ologf)
		ofi, _ := ofh.Stat()
		defer ofh.Close()

		state := &state{SkipBytes: 0, Inode: detectInode(ofi)}
		f, err := openOldFile(logf, state)
		defer f.Close()

		assert.Equal(t, err, nil, "err should be nil")
	}
	testFoundOldFileByInode()
}

func TestRunTraceInode(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	ptn := `FATAL`
	opts, _ := parseArgs([]string{"-s", dir, "-f", logf, "-p", ptn})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, logf, opts.origArgs)

	bytes, _ := getBytesToSkip(stateFile)
	assert.Equal(t, int64(0), bytes, "something went wrong")

	l1 := "SUCCESS\n"
	l2 := "FATAL\n"
	l3 := "SUCCESS\n"
	testRotate := func() {
		// first check
		fh.WriteString(l1)
		opts.searchLog(logf)

		// write FATAL
		fh.WriteString(l2)
		fh.Close()

		// logrotate
		rotatedLogf := filepath.Join(dir, "dummy.1")
		os.Rename(logf, rotatedLogf)
		fh, _ = os.Create(logf)

		fh.WriteString(l3)
		// second check
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "FATAL\n", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l3)), bytes, "should not include oldfile skip bytes")

		os.Remove(rotatedLogf)
	}
	testRotate()

	// case of renaming to different directory
	// from the directory of the target logfile
	testRotateDifferentDir := func() {
		// first check
		fh.WriteString(l1)
		opts.searchLog(logf)

		// write FATAL
		fh.WriteString(l2)
		fh.Close()

		// logrotate to <dir>/dummy/dummy.1
		newDir := filepath.Join(dir, "dummy")
		os.MkdirAll(newDir, 0755)
		rotatedLogf := filepath.Join(newDir, "dummy.1")
		os.Rename(logf, rotatedLogf)
		fh, _ = os.Create(logf)

		fh.WriteString(l3)
		// second check
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil when the old file is not found")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l3)), bytes, "should not include oldfile skip bytes")

		os.Remove(rotatedLogf)
	}
	testRotateDifferentDir()
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gc

//	case OADD:
//		if(n->right->op == OLITERAL) {
//			v = n->right->vconst;
//			naddr(n->left, a, canemitcode);
//		} else
//		if(n->left->op == OLITERAL) {
//			v = n->left->vconst;
//			naddr(n->right, a, canemitcode);
//		} else
//			goto bad;
//		a->offset += v;
//		break;

// a function named init is a special case.
// it is called by the initialization before
// main is run. to make it unique within a
// package and also uncallable, the name,
// normally "pkg.init", is altered to "pkg.init.1".

var renameinit_initgen int

func renameinit() *Sym {
	renameinit_initgen++
	return LookupN("init.", renameinit_initgen)
}

// hand-craft the following initialization code
//	var initdone· uint8 				(1)
//	func init()					(2)
//              if initdone· > 1 {                      (3)
//                      return                          (3a)
//		if initdone· == 1 {			(4)
//			throw();			(4a)
//		}
//		initdone· = 1;				(6)
//		// over all matching imported symbols
//			<pkg>.init()			(7)
//		{ <init stmts> }			(8)
//		init.<n>() // if any			(9)
//		initdone· = 2;				(10)
//		return					(11)
//	}
func anyinit(n []*Node) bool {
	// are there any interesting init statements
	for _, ln := range n {
		switch ln.Op {
		case ODCLFUNC, ODCLCONST, ODCLTYPE, OEMPTY:
			break

		case OAS, OASWB:
			if isblank(ln.Left) && candiscard(ln.Right) {
				break
			}
			fallthrough
		default:
			return true
		}
	}

	// is this main
	if localpkg.Name == "main" {
		return true
	}

	// is there an explicit init function
	s := Lookup("init.1")

	if s.Def != nil {
		return true
	}

	// are there any imported init functions
	for _, s := range initSyms {
		if s.Def != nil {
			return true
		}
	}

	// then none
	return false
}

func fninit(n []*Node) {
	if Debug['A'] != 0 {
		// sys.go or unsafe.go during compiler build
		return
	}

	nf := initfix(n)
	if !anyinit(nf) {
		return
	}

	var r []*Node

	// (1)
	gatevar := newname(Lookup("initdone·"))
	addvar(gatevar, Types[TUINT8], PEXTERN)

	// (2)
	Maxarg = 0

	fn := Nod(ODCLFUNC, nil, nil)
	initsym := Lookup("init")
	fn.Func.Nname = newname(initsym)
	fn.Func.Nname.Name.Defn = fn
	fn.Func.Nname.Name.Param.Ntype = Nod(OTFUNC, nil, nil)
	declare(fn.Func.Nname, PFUNC)
	funchdr(fn)

	// (3)
	a := Nod(OIF, nil, nil)
	a.Left = Nod(OGT, gatevar, Nodintconst(1))
	a.Likely = 1
	r = append(r, a)
	// (3a)
	a.Nbody.Set1(Nod(ORETURN, nil, nil))

	// (4)
	b := Nod(OIF, nil, nil)
	b.Left = Nod(OEQ, gatevar, Nodintconst(1))
	// this actually isn't likely, but code layout is better
	// like this: no JMP needed after the call.
	b.Likely = 1
	r = append(r, b)
	// (4a)
	b.Nbody.Set1(Nod(OCALL, syslook("throwinit"), nil))

	// (6)
	a = Nod(OAS, gatevar, Nodintconst(1))

	r = append(r, a)

	// (7)
	for _, s := range initSyms {
		if s.Def != nil && s != initsym {
			// could check that it is fn of no args/returns
			a = Nod(OCALL, s.Def, nil)
			r = append(r, a)
		}
	}

	// (8)
	r = append(r, nf...)

	// (9)
	// could check that it is fn of no args/returns
	for i := 1; ; i++ {
		s := LookupN("init.", i)
		if s.Def == nil {
			break
		}
		a = Nod(OCALL, s.Def, nil)
		r = append(r, a)
	}

	// (10)
	a = Nod(OAS, gatevar, Nodintconst(2))

	r = append(r, a)

	// (11)
	a = Nod(ORETURN, nil, nil)

	r = append(r, a)
	exportsym(fn.Func.Nname)

	fn.Nbody.Set(r)
	funcbody(fn)

	Curfn = fn
	fn = typecheck(fn, Etop)
	typecheckslice(r, Etop)
	Curfn = nil
	funccompile(fn)
}
<commit_msg>cmd/compile/internal/gc: minor cleanup of init.go comments<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gc

//	case OADD:
//		if(n->right->op == OLITERAL) {
//			v = n->right->vconst;
//			naddr(n->left, a, canemitcode);
//		} else
//		if(n->left->op == OLITERAL) {
//			v = n->left->vconst;
//			naddr(n->right, a, canemitcode);
//		} else
//			goto bad;
//		a->offset += v;
//		break;

// a function named init is a special case.
// it is called by the initialization before
// main is run. to make it unique within a
// package and also uncallable, the name,
// normally "pkg.init", is altered to "pkg.init.1".

var renameinit_initgen int

func renameinit() *Sym {
	renameinit_initgen++
	return LookupN("init.", renameinit_initgen)
}

// hand-craft the following initialization code
//      var initdone· uint8                             (1)
//      func init() {                                   (2)
//              if initdone· > 1 {                      (3)
//                      return                          (3a)
//              }
//              if initdone· == 1 {                     (4)
//                      throw()                         (4a)
//              }
//              initdone· = 1                           (5)
//              // over all matching imported symbols
//                      <pkg>.init()                    (6)
//              { <init stmts> }                        (7)
//              init.<n>() // if any                    (8)
//              initdone· = 2                           (9)
//              return                                  (10)
//      }
func anyinit(n []*Node) bool {
	// are there any interesting init statements
	for _, ln := range n {
		switch ln.Op {
		case ODCLFUNC, ODCLCONST, ODCLTYPE, OEMPTY:
			break

		case OAS, OASWB:
			if isblank(ln.Left) && candiscard(ln.Right) {
				break
			}
			fallthrough
		default:
			return true
		}
	}

	// is this main
	if localpkg.Name == "main" {
		return true
	}

	// is there an explicit init function
	s := Lookup("init.1")

	if s.Def != nil {
		return true
	}

	// are there any imported init functions
	for _, s := range initSyms {
		if s.Def != nil {
			return true
		}
	}

	// then none
	return false
}

func fninit(n []*Node) {
	if Debug['A'] != 0 {
		// sys.go or unsafe.go during compiler build
		return
	}

	nf := initfix(n)
	if !anyinit(nf) {
		return
	}

	var r []*Node

	// (1)
	gatevar := newname(Lookup("initdone·"))
	addvar(gatevar, Types[TUINT8], PEXTERN)

	// (2)
	Maxarg = 0

	fn := Nod(ODCLFUNC, nil, nil)
	initsym := Lookup("init")
	fn.Func.Nname = newname(initsym)
	fn.Func.Nname.Name.Defn = fn
	fn.Func.Nname.Name.Param.Ntype = Nod(OTFUNC, nil, nil)
	declare(fn.Func.Nname, PFUNC)
	funchdr(fn)

	// (3)
	a := Nod(OIF, nil, nil)
	a.Left = Nod(OGT, gatevar, Nodintconst(1))
	a.Likely = 1
	r = append(r, a)
	// (3a)
	a.Nbody.Set1(Nod(ORETURN, nil, nil))

	// (4)
	b := Nod(OIF, nil, nil)
	b.Left = Nod(OEQ, gatevar, Nodintconst(1))
	// this actually isn't likely, but code layout is better
	// like this: no JMP needed after the call.
	b.Likely = 1
	r = append(r, b)
	// (4a)
	b.Nbody.Set1(Nod(OCALL, syslook("throwinit"), nil))

	// (5)
	a = Nod(OAS, gatevar, Nodintconst(1))

	r = append(r, a)

	// (6)
	for _, s := range initSyms {
		if s.Def != nil && s != initsym {
			// could check that it is fn of no args/returns
			a = Nod(OCALL, s.Def, nil)
			r = append(r, a)
		}
	}

	// (7)
	r = append(r, nf...)

	// (8)
	// could check that it is fn of no args/returns
	for i := 1; ; i++ {
		s := LookupN("init.", i)
		if s.Def == nil {
			break
		}
		a = Nod(OCALL, s.Def, nil)
		r = append(r, a)
	}

	// (9)
	a = Nod(OAS, gatevar, Nodintconst(2))

	r = append(r, a)

	// (10)
	a = Nod(ORETURN, nil, nil)

	r = append(r, a)
	exportsym(fn.Func.Nname)

	fn.Nbody.Set(r)
	funcbody(fn)

	Curfn = fn
	fn = typecheck(fn, Etop)
	typecheckslice(r, Etop)
	Curfn = nil
	funccompile(fn)
}
<|endoftext|>


<commit_before>package query_test

import (
	"testing"

	"github.com/codelingo/platform/controller/graphdb"
	"github.com/codelingo/platform/tests/setup"
	jc "github.com/juju/testing/checkers"
	. "gopkg.in/check.v1"
)

type lSomeSuite struct {
}

var _ = Suite(&lSomeSuite{
	doIngest:   false,
	buildStore: false,
})

func Test(t *testing.T) {
	TestingT(t)
}

func (s *SomeSuite) SetUpSuite(c *C) {
	var err error
	if s.buildStore {
		s.ts, err = setup.NewTestStore()
	} else {
		s.ts, err = setup.NewDgraphTestClient()
	}
	c.Assert(err, jc.ErrorIsNil)

	store, err := graphdb.NewStore()
	c.Assert(err, jc.ErrorIsNil)
	s.store = store
	s.isIngested = make(map[string]bool)
}

func (s *SomeSuite) SetUpTest(c *C) {} // Issue

func (s *SomeSuite) TearDownTest(c *C) {} // Issue
<commit_msg>Remove platform dependency<commit_after>package query_test

import (
	"testing"

	jc "github.com/juju/testing/checkers"
	. "gopkg.in/check.v1"
)

type lSomeSuite struct {
}

var _ = Suite(&lSomeSuite{
	doIngest:   false,
	buildStore: false,
})

func Test(t *testing.T) {
	TestingT(t)
}

func (s *SomeSuite) SetUpSuite(c *C) {
	var err error
	c.Assert(err, jc.ErrorIsNil)
}

func (s *SomeSuite) SetUpTest(c *C) {} // Issue

func (s *SomeSuite) TearDownTest(c *C) {} // Issue
<|endoftext|>


<commit_before>// Copyright 2018 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package util_test

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/open-policy-agent/opa/util"
)

func TestRoundTrip(t *testing.T) {
	cases := []interface{}{
		nil,
		1,
		1.1,
		false,
		[]int{1},
		[]bool{true},
		[]string{"foo"},
		map[string]string{"foo": "bar"},
		struct {
			F string `json:"foo"`
			B int    `json:"bar"`
		}{"x", 32},
		map[string][]int{
			"ones": {1, 1, 1},
		},
	}
	for _, tc := range cases {
		t.Run(fmt.Sprintf("input %v", tc), func(t *testing.T) {
			err := util.RoundTrip(&tc)
			if err != nil {
				t.Errorf("expected error=nil, got %s", err.Error())
			}
			switch x := tc.(type) {
			// These are the output types we want, nothing else
			case nil, bool, json.Number, int64, float64, int, string, []interface{},
				[]string, map[string]interface{}, map[string]string:
			default:
				t.Error("unexpected type %T", x)
			}
		})
	}
}
<commit_msg>Fix error formatting in test.<commit_after>// Copyright 2018 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package util_test

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/open-policy-agent/opa/util"
)

func TestRoundTrip(t *testing.T) {
	cases := []interface{}{
		nil,
		1,
		1.1,
		false,
		[]int{1},
		[]bool{true},
		[]string{"foo"},
		map[string]string{"foo": "bar"},
		struct {
			F string `json:"foo"`
			B int    `json:"bar"`
		}{"x", 32},
		map[string][]int{
			"ones": {1, 1, 1},
		},
	}
	for _, tc := range cases {
		t.Run(fmt.Sprintf("input %v", tc), func(t *testing.T) {
			err := util.RoundTrip(&tc)
			if err != nil {
				t.Errorf("expected error=nil, got %s", err.Error())
			}
			switch x := tc.(type) {
			// These are the output types we want, nothing else
			case nil, bool, json.Number, int64, float64, int, string, []interface{},
				[]string, map[string]interface{}, map[string]string:
			default:
				t.Errorf("unexpected type %T", x)
			}
		})
	}
}
<|endoftext|>


<commit_before>package rollbar

import (
	"bytes"
	"encoding/json"
	"fmt"
	"hash/adler32"
	"net/http"
	"os"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"time"
)

const (
	NAME    = "go-rollbar"
	VERSION = "0.0.4"

	// Severity levels
	CRIT  = "critical"
	ERR   = "error"
	WARN  = "warning"
	INFO  = "info"
	DEBUG = "debug"
)

var (
	// Rollbar access token. If this is blank, no errors will be reported to
	// Rollbar.
	Token = ""

	// All errors and messages will be submitted under this environment.
	Environment = "development"

	// API endpoint for Rollbar.
	Endpoint = "https://api.rollbar.com/api/1/item/"

	// Maximum number of errors allowed in the sending queue before we start
	// dropping new errors on the floor.
	Buffer = 1000

	// Queue of messages to be sent.
	bodyChannel chan map[string]interface{}
	waitGroup   sync.WaitGroup
)

// -- Setup

func init() {
	bodyChannel = make(chan map[string]interface{}, Buffer)

	go func() {
		for body := range bodyChannel {
			post(body)
			waitGroup.Done()
		}
	}()
}

// -- Error reporting

// Error asynchronously sends an error to Rollbar with the given severity level.
func Error(level string, err error) {
	ErrorWithStackSkip(level, err, 1)
}

// ErrorWithStackSkip asynchronously sends an error to Rollbar with the given
// severity level and a given number of stack trace frames skipped.
func ErrorWithStackSkip(level string, err error, skip int) {
	body := buildBody(level, err.Error())
	data := body["data"].(map[string]interface{})
	errBody, fingerprint := errorBody(err, skip)
	data["body"] = errBody
	data["fingerprint"] = fingerprint

	push(body)
}

// -- Message reporting

// Message asynchronously sends a message to Rollbar with the given severity
// level. Rollbar request is asynchronous.
func Message(level string, msg string) {
	body := buildBody(level, msg)
	data := body["data"].(map[string]interface{})
	data["body"] = messageBody(msg)

	push(body)
}

// -- Misc.

// Wait will block until the queue of errors / messages is empty.
func Wait() {
	waitGroup.Wait()
}

// Build the main JSON structure that will be sent to Rollbar with the
// appropriate metadata.
func buildBody(level, title string) map[string]interface{} {
	timestamp := time.Now().Unix()
	hostname, _ := os.Hostname()

	return map[string]interface{}{
		"access_token": Token,
		"data": map[string]interface{}{
			"environment": Environment,
			"title":       title,
			"level":       level,
			"timestamp":   timestamp,
			"platform":    runtime.GOOS,
			"language":    "go",
			"server": map[string]interface{}{
				"host": hostname,
			},
			"notifier": map[string]interface{}{
				"name":    NAME,
				"version": VERSION,
			},
		},
	}
}

// Build an error inner-body for the given error. If skip is provided, that
// number of stack trace frames will be skipped.
func errorBody(err error, skip int) (map[string]interface{}, string) {
	stack := BuildStack(3 + skip)
	fingerprint := stack.Fingerprint()
	errBody := map[string]interface{}{
		"trace": map[string]interface{}{
			"frames": stack,
			"exception": map[string]interface{}{
				"class":   errorClass(err),
				"message": err.Error(),
			},
		},
	}
	return errBody, fingerprint
}

// Build a message inner-body for the given message string.
func messageBody(s string) map[string]interface{} {
	return map[string]interface{}{
		"message": map[string]interface{}{
			"body": s,
		},
	}
}

func errorClass(err error) string {
	class := reflect.TypeOf(err).String()
	if class == "" {
		return "panic"
	} else if class == "*errors.errorString" {
		checksum := adler32.Checksum([]byte(err.Error()))
		return fmt.Sprintf("{%x}", checksum)
	} else {
		return strings.TrimPrefix(class, "*")
	}
}

// -- POST handling

// Queue the given JSON body to be POSTed to Rollbar.
func push(body map[string]interface{}) {
	if len(bodyChannel) < Buffer {
		waitGroup.Add(1)
		bodyChannel <- body
	} else {
		stderr("buffer full, dropping error on the floor")
	}
}

// POST the given JSON body to Rollbar synchronously.
func post(body map[string]interface{}) {
	if len(Token) == 0 {
		stderr("empty token")
		return
	}

	jsonBody, err := json.Marshal(body)
	if err != nil {
		stderr("failed to encode payload: %s", err.Error())
		return
	}

	resp, err := http.Post(Endpoint, "application/json", bytes.NewReader(jsonBody))
	if err == nil {
		resp.Body.Close()
		if resp.StatusCode != 200 {
			stderr("received response: %s", resp.Status)
		}
	} else {
		stderr("POST failed: %s", err.Error())
	}
}

// -- stderr

func stderr(format string, args ...interface{}) {
	format = "Rollbar error: " + format + "\n"
	fmt.Fprintf(os.Stderr, format, args...)
}
<commit_msg>Clean up response body closing.<commit_after>package rollbar

import (
	"bytes"
	"encoding/json"
	"fmt"
	"hash/adler32"
	"net/http"
	"os"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"time"
)

const (
	NAME    = "go-rollbar"
	VERSION = "0.0.4"

	// Severity levels
	CRIT  = "critical"
	ERR   = "error"
	WARN  = "warning"
	INFO  = "info"
	DEBUG = "debug"
)

var (
	// Rollbar access token. If this is blank, no errors will be reported to
	// Rollbar.
	Token = ""

	// All errors and messages will be submitted under this environment.
	Environment = "development"

	// API endpoint for Rollbar.
	Endpoint = "https://api.rollbar.com/api/1/item/"

	// Maximum number of errors allowed in the sending queue before we start
	// dropping new errors on the floor.
	Buffer = 1000

	// Queue of messages to be sent.
	bodyChannel chan map[string]interface{}
	waitGroup   sync.WaitGroup
)

// -- Setup

func init() {
	bodyChannel = make(chan map[string]interface{}, Buffer)

	go func() {
		for body := range bodyChannel {
			post(body)
			waitGroup.Done()
		}
	}()
}

// -- Error reporting

// Error asynchronously sends an error to Rollbar with the given severity level.
func Error(level string, err error) {
	ErrorWithStackSkip(level, err, 1)
}

// ErrorWithStackSkip asynchronously sends an error to Rollbar with the given
// severity level and a given number of stack trace frames skipped.
func ErrorWithStackSkip(level string, err error, skip int) {
	body := buildBody(level, err.Error())
	data := body["data"].(map[string]interface{})
	errBody, fingerprint := errorBody(err, skip)
	data["body"] = errBody
	data["fingerprint"] = fingerprint

	push(body)
}

// -- Message reporting

// Message asynchronously sends a message to Rollbar with the given severity
// level. Rollbar request is asynchronous.
func Message(level string, msg string) {
	body := buildBody(level, msg)
	data := body["data"].(map[string]interface{})
	data["body"] = messageBody(msg)

	push(body)
}

// -- Misc.

// Wait will block until the queue of errors / messages is empty.
func Wait() {
	waitGroup.Wait()
}

// Build the main JSON structure that will be sent to Rollbar with the
// appropriate metadata.
func buildBody(level, title string) map[string]interface{} {
	timestamp := time.Now().Unix()
	hostname, _ := os.Hostname()

	return map[string]interface{}{
		"access_token": Token,
		"data": map[string]interface{}{
			"environment": Environment,
			"title":       title,
			"level":       level,
			"timestamp":   timestamp,
			"platform":    runtime.GOOS,
			"language":    "go",
			"server": map[string]interface{}{
				"host": hostname,
			},
			"notifier": map[string]interface{}{
				"name":    NAME,
				"version": VERSION,
			},
		},
	}
}

// Build an error inner-body for the given error. If skip is provided, that
// number of stack trace frames will be skipped.
func errorBody(err error, skip int) (map[string]interface{}, string) {
	stack := BuildStack(3 + skip)
	fingerprint := stack.Fingerprint()
	errBody := map[string]interface{}{
		"trace": map[string]interface{}{
			"frames": stack,
			"exception": map[string]interface{}{
				"class":   errorClass(err),
				"message": err.Error(),
			},
		},
	}
	return errBody, fingerprint
}

// Build a message inner-body for the given message string.
func messageBody(s string) map[string]interface{} {
	return map[string]interface{}{
		"message": map[string]interface{}{
			"body": s,
		},
	}
}

func errorClass(err error) string {
	class := reflect.TypeOf(err).String()
	if class == "" {
		return "panic"
	} else if class == "*errors.errorString" {
		checksum := adler32.Checksum([]byte(err.Error()))
		return fmt.Sprintf("{%x}", checksum)
	} else {
		return strings.TrimPrefix(class, "*")
	}
}

// -- POST handling

// Queue the given JSON body to be POSTed to Rollbar.
func push(body map[string]interface{}) {
	if len(bodyChannel) < Buffer {
		waitGroup.Add(1)
		bodyChannel <- body
	} else {
		stderr("buffer full, dropping error on the floor")
	}
}

// POST the given JSON body to Rollbar synchronously.
func post(body map[string]interface{}) {
	if len(Token) == 0 {
		stderr("empty token")
		return
	}

	jsonBody, err := json.Marshal(body)
	if err != nil {
		stderr("failed to encode payload: %s", err.Error())
		return
	}

	resp, err := http.Post(Endpoint, "application/json", bytes.NewReader(jsonBody))
	if err != nil {
		stderr("POST failed: %s", err.Error())
	} else if resp.StatusCode != 200 {
		stderr("received response: %s", resp.Status)
	}
	if resp != nil {
		resp.Body.Close()
	}
}

// -- stderr

func stderr(format string, args ...interface{}) {
	format = "Rollbar error: " + format + "\n"
	fmt.Fprintf(os.Stderr, format, args...)
}
<|endoftext|>


<commit_before>package plugin_repo

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/cloudfoundry/cli/cf/command_metadata"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/codegangsta/cli"

	clipr "github.com/cloudfoundry-incubator/cli-plugin-repo/models"

	. "github.com/cloudfoundry/cli/cf/i18n"
)

type AddPluginRepo struct {
	ui     terminal.UI
	config core_config.ReadWriter
}

func NewAddPluginRepo(ui terminal.UI, config core_config.ReadWriter) AddPluginRepo {
	return AddPluginRepo{
		ui:     ui,
		config: config,
	}
}

func (cmd AddPluginRepo) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "add-plugin-repo",
		Description: T("Add a new plugin repository"),
		Usage: T(`CF_NAME add-plugin-repo [REPO_NAME] [URL]

EXAMPLE:
   cf add-plugin-repo PrivateRepo http://myprivaterepo.com/repo/
`),
		TotalArgs: 2,
	}
}

func (cmd AddPluginRepo) GetRequirements(_ requirements.Factory, c *cli.Context) (req []requirements.Requirement, err error) {
	if len(c.Args()) != 2 {
		cmd.ui.FailWithUsage(c)
	}
	return
}

func (cmd AddPluginRepo) Run(c *cli.Context) {

	cmd.ui.Say("")
	repoUrl := strings.ToLower(c.Args()[1])
	repoName := strings.Trim(c.Args()[0], " ")

	cmd.checkIfRepoExists(repoName, repoUrl)

	repoUrl = cmd.verifyUrl(repoUrl)

	resp, err := http.Get(repoUrl)
	if err != nil {
		cmd.ui.Failed(T("There is an error performing request on '{{.repoUrl}}': ", map[string]interface{}{"repoUrl": repoUrl}), err.Error())
	}
	defer resp.Body.Close()

	if resp.StatusCode == 404 {
		cmd.ui.Failed(repoUrl + T(" is not responding. Please make sure it is a valid plugin repo."))
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		cmd.ui.Failed(T("Error reading response from server: ") + err.Error())
	}

	result := clipr.PluginsJson{}
	err = json.Unmarshal(body, &result)
	if err != nil {
		cmd.ui.Failed(T("Error processing data from server: ") + err.Error())
	}

	if result.Plugins == nil {
		cmd.ui.Failed(T(`"Plugins" object not found in the responded data.`))
	}

	cmd.config.SetPluginRepo(models.PluginRepo{
		Name: c.Args()[0],
		Url:  c.Args()[1],
	})

	cmd.ui.Ok()
	cmd.ui.Say(repoUrl + T(" added as '") + c.Args()[0] + "'")
	cmd.ui.Say("")
}

func (cmd AddPluginRepo) checkIfRepoExists(repoName, repoUrl string) {
	repos := cmd.config.PluginRepos()
	for _, repo := range repos {
		if repo.Name == repoName {
			cmd.ui.Failed(T(`Plugin repo named "{{.repoName}}" already exists, please use another name.`, map[string]interface{}{"repoName": repoName}))
		} else if repo.Url == repoUrl {
			cmd.ui.Failed(repo.Url + ` (` + repo.Name + T(`) already exists.`))
		}
	}
}

func (cmd AddPluginRepo) verifyUrl(repoUrl string) string {
	if !strings.HasPrefix(repoUrl, "http://") && !strings.HasPrefix(repoUrl, "https://") {
		cmd.ui.Failed(repoUrl + T(" is not a valid url, please provide a url, e.g. http://your_repo.com"))
	}

	if strings.HasSuffix(repoUrl, "/") {
		repoUrl = repoUrl + "list"
	} else {
		repoUrl = repoUrl + "/list"
	}

	return repoUrl
}
<commit_msg>Repo name comparisons in add-plugin-repo are case-insensitive.<commit_after>package plugin_repo

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/cloudfoundry/cli/cf/command_metadata"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/codegangsta/cli"

	clipr "github.com/cloudfoundry-incubator/cli-plugin-repo/models"

	. "github.com/cloudfoundry/cli/cf/i18n"
)

type AddPluginRepo struct {
	ui     terminal.UI
	config core_config.ReadWriter
}

func NewAddPluginRepo(ui terminal.UI, config core_config.ReadWriter) AddPluginRepo {
	return AddPluginRepo{
		ui:     ui,
		config: config,
	}
}

func (cmd AddPluginRepo) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "add-plugin-repo",
		Description: T("Add a new plugin repository"),
		Usage: T(`CF_NAME add-plugin-repo [REPO_NAME] [URL]

EXAMPLE:
   cf add-plugin-repo PrivateRepo http://myprivaterepo.com/repo/
`),
		TotalArgs: 2,
	}
}

func (cmd AddPluginRepo) GetRequirements(_ requirements.Factory, c *cli.Context) (req []requirements.Requirement, err error) {
	if len(c.Args()) != 2 {
		cmd.ui.FailWithUsage(c)
	}
	return
}

func (cmd AddPluginRepo) Run(c *cli.Context) {

	cmd.ui.Say("")
	repoUrl := strings.ToLower(c.Args()[1])
	repoName := strings.Trim(c.Args()[0], " ")

	cmd.checkIfRepoExists(repoName, repoUrl)

	repoUrl = cmd.verifyUrl(repoUrl)

	resp, err := http.Get(repoUrl)
	if err != nil {
		cmd.ui.Failed(T("There is an error performing request on '{{.repoUrl}}': ", map[string]interface{}{"repoUrl": repoUrl}), err.Error())
	}
	defer resp.Body.Close()

	if resp.StatusCode == 404 {
		cmd.ui.Failed(repoUrl + T(" is not responding. Please make sure it is a valid plugin repo."))
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		cmd.ui.Failed(T("Error reading response from server: ") + err.Error())
	}

	result := clipr.PluginsJson{}
	err = json.Unmarshal(body, &result)
	if err != nil {
		cmd.ui.Failed(T("Error processing data from server: ") + err.Error())
	}

	if result.Plugins == nil {
		cmd.ui.Failed(T(`"Plugins" object not found in the responded data.`))
	}

	cmd.config.SetPluginRepo(models.PluginRepo{
		Name: c.Args()[0],
		Url:  c.Args()[1],
	})

	cmd.ui.Ok()
	cmd.ui.Say(repoUrl + T(" added as '") + c.Args()[0] + "'")
	cmd.ui.Say("")
}

func (cmd AddPluginRepo) checkIfRepoExists(repoName, repoUrl string) {
	repos := cmd.config.PluginRepos()
	for _, repo := range repos {
		if strings.ToLower(repo.Name) == strings.ToLower(repoName) {
			cmd.ui.Failed(T(`Plugin repo named "{{.repoName}}" already exists, please use another name.`, map[string]interface{}{"repoName": repoName}))
		} else if repo.Url == repoUrl {
			cmd.ui.Failed(repo.Url + ` (` + repo.Name + T(`) already exists.`))
		}
	}
}

func (cmd AddPluginRepo) verifyUrl(repoUrl string) string {
	if !strings.HasPrefix(repoUrl, "http://") && !strings.HasPrefix(repoUrl, "https://") {
		cmd.ui.Failed(repoUrl + T(" is not a valid url, please provide a url, e.g. http://your_repo.com"))
	}

	if strings.HasSuffix(repoUrl, "/") {
		repoUrl = repoUrl + "list"
	} else {
		repoUrl = repoUrl + "/list"
	}

	return repoUrl
}
<|endoftext|>


<commit_before>package testrunner

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/ginkgo/testsuite"
	"github.com/onsi/ginkgo/internal/remote"
	"github.com/onsi/ginkgo/reporters/stenographer"
	"github.com/onsi/ginkgo/types"
)

type TestRunner struct {
	Suite testsuite.TestSuite

	compiled              bool
	compilationTargetPath string

	numCPU         int
	parallelStream bool
	race           bool
	cover          bool
	coverPkg       string
	tags           string
	additionalArgs []string
}

func New(suite testsuite.TestSuite, numCPU int, parallelStream bool, race bool, cover bool, coverPkg string, tags string, additionalArgs []string) *TestRunner {
	runner := &TestRunner{
		Suite:          suite,
		numCPU:         numCPU,
		parallelStream: parallelStream,
		race:           race,
		cover:          cover,
		coverPkg:       coverPkg,
		tags:           tags,
		additionalArgs: additionalArgs,
	}

	if !suite.Precompiled {
		dir, err := ioutil.TempDir("", "ginkgo")
		if err != nil {
			panic(fmt.Sprintf("coulnd't create temporary directory... might be time to rm -rf:\n%s", err.Error()))
		}
		runner.compilationTargetPath = filepath.Join(dir, suite.PackageName+".test")
	}

	return runner
}

func (t *TestRunner) Compile() error {
	return t.CompileTo(t.compilationTargetPath)
}

func (t *TestRunner) CompileTo(path string) error {
	if t.compiled {
		return nil
	}

	if t.Suite.Precompiled {
		return nil
	}

	args := []string{"test", "-c", "-i", "-o", path}
	if t.race {
		args = append(args, "-race")
	}
	if t.cover || t.coverPkg != "" {
		args = append(args, "-cover", "-covermode=atomic")
	}
	if t.coverPkg != "" {
		args = append(args, fmt.Sprintf("-coverpkg=%s", t.coverPkg))
	}
	if t.tags != "" {
		args = append(args, fmt.Sprintf("-tags=%s", t.tags))
	}

	cmd := exec.Command("go", args...)

	cmd.Dir = t.Suite.Path

	output, err := cmd.CombinedOutput()

	if err != nil {
		fixedOutput := fixCompilationOutput(string(output), t.Suite.Path)
		if len(output) > 0 {
			return fmt.Errorf("Failed to compile %s:\n\n%s", t.Suite.PackageName, fixedOutput)
		}
		return fmt.Errorf("Failed to compile %s", t.Suite.PackageName)
	}

	if fileExists(path) == false {
		compiledFile := filepath.Join(t.Suite.Path, t.Suite.PackageName+".test")
		if fileExists(compiledFile) {
			// seems like we are on an old go version that does not support the -o flag on go test
			// move the compiled test file to the desired location by hand
			err = os.Rename(compiledFile, path)
			if err != nil {
				// We cannot move the file, perhaps because the source and destination
				// are on different partitions. We can copy the file, however.
				err = copyFile(compiledFile, path)
				return fmt.Errorf("Failed to copy compiled file: %s", err)
			}
		} else {
			return fmt.Errorf("Failed to compile %s: output file %q could not be found", t.Suite.PackageName, path)
		}
	}

	t.compiled = true

	return nil
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil || os.IsNotExist(err) == false
}

// copyFile copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}

	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return err
	}

	defer func() {
		closeErr := out.Close()
		if err == nil {
			err = closeErr
		}
	}()

	_, err = io.Copy(out, in)
	if err != nil {
		return err
	}

	err = out.Sync()
	return err
}

/*
go test -c -i spits package.test out into the cwd. there's no way to change this.

to make sure it doesn't generate conflicting .test files in the cwd, Compile() must switch the cwd to the test package.

unfortunately, this causes go test's compile output to be expressed *relative to the test package* instead of the cwd.

this makes it hard to reason about what failed, and also prevents iterm's Cmd+click from working.

fixCompilationOutput..... rewrites the output to fix the paths.

yeah......
*/
func fixCompilationOutput(output string, relToPath string) string {
	re := regexp.MustCompile(`^(\S.*\.go)\:\d+\:`)
	lines := strings.Split(output, "\n")
	for i, line := range lines {
		indices := re.FindStringSubmatchIndex(line)
		if len(indices) == 0 {
			continue
		}

		path := line[indices[2]:indices[3]]
		path = filepath.Join(relToPath, path)
		lines[i] = path + line[indices[3]:]
	}
	return strings.Join(lines, "\n")
}

func (t *TestRunner) Run() RunResult {
	if t.Suite.IsGinkgo {
		if t.numCPU > 1 {
			if t.parallelStream {
				return t.runAndStreamParallelGinkgoSuite()
			} else {
				return t.runParallelGinkgoSuite()
			}
		} else {
			return t.runSerialGinkgoSuite()
		}
	} else {
		return t.runGoTestSuite()
	}
}

func (t *TestRunner) CleanUp() {
	if t.Suite.Precompiled {
		return
	}
	os.RemoveAll(filepath.Dir(t.compilationTargetPath))
}

func (t *TestRunner) runSerialGinkgoSuite() RunResult {
	ginkgoArgs := config.BuildFlagArgs("ginkgo", config.GinkgoConfig, config.DefaultReporterConfig)
	return t.run(t.cmd(ginkgoArgs, os.Stdout, 1), nil)
}

func (t *TestRunner) runGoTestSuite() RunResult {
	return t.run(t.cmd([]string{"-test.v"}, os.Stdout, 1), nil)
}

func (t *TestRunner) runAndStreamParallelGinkgoSuite() RunResult {
	completions := make(chan RunResult)
	writers := make([]*logWriter, t.numCPU)

	server, err := remote.NewServer(t.numCPU)
	if err != nil {
		panic("Failed to start parallel spec server")
	}

	server.Start()
	defer server.Close()

	for cpu := 0; cpu < t.numCPU; cpu++ {
		config.GinkgoConfig.ParallelNode = cpu + 1
		config.GinkgoConfig.ParallelTotal = t.numCPU
		config.GinkgoConfig.SyncHost = server.Address()

		ginkgoArgs := config.BuildFlagArgs("ginkgo", config.GinkgoConfig, config.DefaultReporterConfig)

		writers[cpu] = newLogWriter(os.Stdout, cpu+1)

		cmd := t.cmd(ginkgoArgs, writers[cpu], cpu+1)

		server.RegisterAlive(cpu+1, func() bool {
			if cmd.ProcessState == nil {
				return true
			}
			return !cmd.ProcessState.Exited()
		})

		go t.run(cmd, completions)
	}

	res := PassingRunResult()

	for cpu := 0; cpu < t.numCPU; cpu++ {
		res = res.Merge(<-completions)
	}

	for _, writer := range writers {
		writer.Close()
	}

	os.Stdout.Sync()

	if t.cover || t.coverPkg != "" {
		t.combineCoverprofiles()
	}

	return res
}

func (t *TestRunner) runParallelGinkgoSuite() RunResult {
	result := make(chan bool)
	completions := make(chan RunResult)
	writers := make([]*logWriter, t.numCPU)
	reports := make([]*bytes.Buffer, t.numCPU)

	stenographer := stenographer.New(!config.DefaultReporterConfig.NoColor)
	aggregator := remote.NewAggregator(t.numCPU, result, config.DefaultReporterConfig, stenographer)

	server, err := remote.NewServer(t.numCPU)
	if err != nil {
		panic("Failed to start parallel spec server")
	}
	server.RegisterReporters(aggregator)
	server.Start()
	defer server.Close()

	for cpu := 0; cpu < t.numCPU; cpu++ {
		config.GinkgoConfig.ParallelNode = cpu + 1
		config.GinkgoConfig.ParallelTotal = t.numCPU
		config.GinkgoConfig.SyncHost = server.Address()
		config.GinkgoConfig.StreamHost = server.Address()

		ginkgoArgs := config.BuildFlagArgs("ginkgo", config.GinkgoConfig, config.DefaultReporterConfig)

		reports[cpu] = &bytes.Buffer{}
		writers[cpu] = newLogWriter(reports[cpu], cpu+1)

		cmd := t.cmd(ginkgoArgs, writers[cpu], cpu+1)

		server.RegisterAlive(cpu+1, func() bool {
			if cmd.ProcessState == nil {
				return true
			}
			return !cmd.ProcessState.Exited()
		})

		go t.run(cmd, completions)
	}

	res := PassingRunResult()

	for cpu := 0; cpu < t.numCPU; cpu++ {
		res = res.Merge(<-completions)
	}

	//all test processes are done, at this point
	//we should be able to wait for the aggregator to tell us that it's done

	select {
	case <-result:
		fmt.Println("")
	case <-time.After(time.Second):
		//the aggregator never got back to us!  something must have gone wrong
		fmt.Println(`
	 -------------------------------------------------------------------
	|                                                                   |
	|  Ginkgo timed out waiting for all parallel nodes to report back!  |
	|                                                                   |
	 -------------------------------------------------------------------
`)

		os.Stdout.Sync()

		for _, writer := range writers {
			writer.Close()
		}

		for _, report := range reports {
			fmt.Print(report.String())
		}

		os.Stdout.Sync()
	}

	if t.cover || t.coverPkg != "" {
		t.combineCoverprofiles()
	}

	return res
}

func (t *TestRunner) cmd(ginkgoArgs []string, stream io.Writer, node int) *exec.Cmd {
	args := []string{"--test.timeout=24h"}
	if t.cover || t.coverPkg != "" {
		coverprofile := "--test.coverprofile=" + t.Suite.PackageName + ".coverprofile"
		if t.numCPU > 1 {
			coverprofile = fmt.Sprintf("%s.%d", coverprofile, node)
		}
		args = append(args, coverprofile)
	}

	args = append(args, ginkgoArgs...)
	args = append(args, t.additionalArgs...)

	path := t.compilationTargetPath
	if t.Suite.Precompiled {
		path, _ = filepath.Abs(filepath.Join(t.Suite.Path, fmt.Sprintf("%s.test", t.Suite.PackageName)))
	}

	cmd := exec.Command(path, args...)

	cmd.Dir = t.Suite.Path
	cmd.Stderr = stream
	cmd.Stdout = stream

	return cmd
}

func (t *TestRunner) run(cmd *exec.Cmd, completions chan RunResult) RunResult {
	var res RunResult

	defer func() {
		if completions != nil {
			completions <- res
		}
	}()

	err := cmd.Start()
	if err != nil {
		fmt.Printf("Failed to run test suite!\n\t%s", err.Error())
		return res
	}

	cmd.Wait()
	exitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
	res.Passed = (exitStatus == 0) || (exitStatus == types.GINKGO_FOCUS_EXIT_CODE)
	res.HasProgrammaticFocus = (exitStatus == types.GINKGO_FOCUS_EXIT_CODE)

	return res
}

func (t *TestRunner) combineCoverprofiles() {
	profiles := []string{}
	for cpu := 1; cpu <= t.numCPU; cpu++ {
		coverFile := fmt.Sprintf("%s.coverprofile.%d", t.Suite.PackageName, cpu)
		coverFile = filepath.Join(t.Suite.Path, coverFile)
		coverProfile, err := ioutil.ReadFile(coverFile)
		os.Remove(coverFile)

		if err == nil {
			profiles = append(profiles, string(coverProfile))
		}
	}

	if len(profiles) != t.numCPU {
		return
	}

	lines := map[string]int{}
	lineOrder := []string{}
	for i, coverProfile := range profiles {
		for _, line := range strings.Split(string(coverProfile), "\n")[1:] {
			if len(line) == 0 {
				continue
			}
			components := strings.Split(line, " ")
			count, _ := strconv.Atoi(components[len(components)-1])
			prefix := strings.Join(components[0:len(components)-1], " ")
			lines[prefix] += count
			if i == 0 {
				lineOrder = append(lineOrder, prefix)
			}
		}
	}

	output := []string{"mode: atomic"}
	for _, line := range lineOrder {
		output = append(output, fmt.Sprintf("%s %d", line, lines[line]))
	}
	finalOutput := strings.Join(output, "\n")
	ioutil.WriteFile(filepath.Join(t.Suite.Path, fmt.Sprintf("%s.coverprofile", t.Suite.PackageName)), []byte(finalOutput), 0666)
}
<commit_msg>Only return error from copying file if it is non-nil.<commit_after>package testrunner

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/ginkgo/testsuite"
	"github.com/onsi/ginkgo/internal/remote"
	"github.com/onsi/ginkgo/reporters/stenographer"
	"github.com/onsi/ginkgo/types"
)

type TestRunner struct {
	Suite testsuite.TestSuite

	compiled              bool
	compilationTargetPath string

	numCPU         int
	parallelStream bool
	race           bool
	cover          bool
	coverPkg       string
	tags           string
	additionalArgs []string
}

func New(suite testsuite.TestSuite, numCPU int, parallelStream bool, race bool, cover bool, coverPkg string, tags string, additionalArgs []string) *TestRunner {
	runner := &TestRunner{
		Suite:          suite,
		numCPU:         numCPU,
		parallelStream: parallelStream,
		race:           race,
		cover:          cover,
		coverPkg:       coverPkg,
		tags:           tags,
		additionalArgs: additionalArgs,
	}

	if !suite.Precompiled {
		dir, err := ioutil.TempDir("", "ginkgo")
		if err != nil {
			panic(fmt.Sprintf("coulnd't create temporary directory... might be time to rm -rf:\n%s", err.Error()))
		}
		runner.compilationTargetPath = filepath.Join(dir, suite.PackageName+".test")
	}

	return runner
}

func (t *TestRunner) Compile() error {
	return t.CompileTo(t.compilationTargetPath)
}

func (t *TestRunner) CompileTo(path string) error {
	if t.compiled {
		return nil
	}

	if t.Suite.Precompiled {
		return nil
	}

	args := []string{"test", "-c", "-i", "-o", path}
	if t.race {
		args = append(args, "-race")
	}
	if t.cover || t.coverPkg != "" {
		args = append(args, "-cover", "-covermode=atomic")
	}
	if t.coverPkg != "" {
		args = append(args, fmt.Sprintf("-coverpkg=%s", t.coverPkg))
	}
	if t.tags != "" {
		args = append(args, fmt.Sprintf("-tags=%s", t.tags))
	}

	cmd := exec.Command("go", args...)

	cmd.Dir = t.Suite.Path

	output, err := cmd.CombinedOutput()

	if err != nil {
		fixedOutput := fixCompilationOutput(string(output), t.Suite.Path)
		if len(output) > 0 {
			return fmt.Errorf("Failed to compile %s:\n\n%s", t.Suite.PackageName, fixedOutput)
		}
		return fmt.Errorf("Failed to compile %s", t.Suite.PackageName)
	}

	if fileExists(path) == false {
		compiledFile := filepath.Join(t.Suite.Path, t.Suite.PackageName+".test")
		if fileExists(compiledFile) {
			// seems like we are on an old go version that does not support the -o flag on go test
			// move the compiled test file to the desired location by hand
			err = os.Rename(compiledFile, path)
			if err != nil {
				// We cannot move the file, perhaps because the source and destination
				// are on different partitions. We can copy the file, however.
				err = copyFile(compiledFile, path)
				if err != nil {
					return fmt.Errorf("Failed to copy compiled file: %s", err)
				}
			}
		} else {
			return fmt.Errorf("Failed to compile %s: output file %q could not be found", t.Suite.PackageName, path)
		}
	}

	t.compiled = true

	return nil
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil || os.IsNotExist(err) == false
}

// copyFile copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}

	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return err
	}

	defer func() {
		closeErr := out.Close()
		if err == nil {
			err = closeErr
		}
	}()

	_, err = io.Copy(out, in)
	if err != nil {
		return err
	}

	err = out.Sync()
	return err
}

/*
go test -c -i spits package.test out into the cwd. there's no way to change this.

to make sure it doesn't generate conflicting .test files in the cwd, Compile() must switch the cwd to the test package.

unfortunately, this causes go test's compile output to be expressed *relative to the test package* instead of the cwd.

this makes it hard to reason about what failed, and also prevents iterm's Cmd+click from working.

fixCompilationOutput..... rewrites the output to fix the paths.

yeah......
*/
func fixCompilationOutput(output string, relToPath string) string {
	re := regexp.MustCompile(`^(\S.*\.go)\:\d+\:`)
	lines := strings.Split(output, "\n")
	for i, line := range lines {
		indices := re.FindStringSubmatchIndex(line)
		if len(indices) == 0 {
			continue
		}

		path := line[indices[2]:indices[3]]
		path = filepath.Join(relToPath, path)
		lines[i] = path + line[indices[3]:]
	}
	return strings.Join(lines, "\n")
}

func (t *TestRunner) Run() RunResult {
	if t.Suite.IsGinkgo {
		if t.numCPU > 1 {
			if t.parallelStream {
				return t.runAndStreamParallelGinkgoSuite()
			} else {
				return t.runParallelGinkgoSuite()
			}
		} else {
			return t.runSerialGinkgoSuite()
		}
	} else {
		return t.runGoTestSuite()
	}
}

func (t *TestRunner) CleanUp() {
	if t.Suite.Precompiled {
		return
	}
	os.RemoveAll(filepath.Dir(t.compilationTargetPath))
}

func (t *TestRunner) runSerialGinkgoSuite() RunResult {
	ginkgoArgs := config.BuildFlagArgs("ginkgo", config.GinkgoConfig, config.DefaultReporterConfig)
	return t.run(t.cmd(ginkgoArgs, os.Stdout, 1), nil)
}

func (t *TestRunner) runGoTestSuite() RunResult {
	return t.run(t.cmd([]string{"-test.v"}, os.Stdout, 1), nil)
}

func (t *TestRunner) runAndStreamParallelGinkgoSuite() RunResult {
	completions := make(chan RunResult)
	writers := make([]*logWriter, t.numCPU)

	server, err := remote.NewServer(t.numCPU)
	if err != nil {
		panic("Failed to start parallel spec server")
	}

	server.Start()
	defer server.Close()

	for cpu := 0; cpu < t.numCPU; cpu++ {
		config.GinkgoConfig.ParallelNode = cpu + 1
		config.GinkgoConfig.ParallelTotal = t.numCPU
		config.GinkgoConfig.SyncHost = server.Address()

		ginkgoArgs := config.BuildFlagArgs("ginkgo", config.GinkgoConfig, config.DefaultReporterConfig)

		writers[cpu] = newLogWriter(os.Stdout, cpu+1)

		cmd := t.cmd(ginkgoArgs, writers[cpu], cpu+1)

		server.RegisterAlive(cpu+1, func() bool {
			if cmd.ProcessState == nil {
				return true
			}
			return !cmd.ProcessState.Exited()
		})

		go t.run(cmd, completions)
	}

	res := PassingRunResult()

	for cpu := 0; cpu < t.numCPU; cpu++ {
		res = res.Merge(<-completions)
	}

	for _, writer := range writers {
		writer.Close()
	}

	os.Stdout.Sync()

	if t.cover || t.coverPkg != "" {
		t.combineCoverprofiles()
	}

	return res
}

func (t *TestRunner) runParallelGinkgoSuite() RunResult {
	result := make(chan bool)
	completions := make(chan RunResult)
	writers := make([]*logWriter, t.numCPU)
	reports := make([]*bytes.Buffer, t.numCPU)

	stenographer := stenographer.New(!config.DefaultReporterConfig.NoColor)
	aggregator := remote.NewAggregator(t.numCPU, result, config.DefaultReporterConfig, stenographer)

	server, err := remote.NewServer(t.numCPU)
	if err != nil {
		panic("Failed to start parallel spec server")
	}
	server.RegisterReporters(aggregator)
	server.Start()
	defer server.Close()

	for cpu := 0; cpu < t.numCPU; cpu++ {
		config.GinkgoConfig.ParallelNode = cpu + 1
		config.GinkgoConfig.ParallelTotal = t.numCPU
		config.GinkgoConfig.SyncHost = server.Address()
		config.GinkgoConfig.StreamHost = server.Address()

		ginkgoArgs := config.BuildFlagArgs("ginkgo", config.GinkgoConfig, config.DefaultReporterConfig)

		reports[cpu] = &bytes.Buffer{}
		writers[cpu] = newLogWriter(reports[cpu], cpu+1)

		cmd := t.cmd(ginkgoArgs, writers[cpu], cpu+1)

		server.RegisterAlive(cpu+1, func() bool {
			if cmd.ProcessState == nil {
				return true
			}
			return !cmd.ProcessState.Exited()
		})

		go t.run(cmd, completions)
	}

	res := PassingRunResult()

	for cpu := 0; cpu < t.numCPU; cpu++ {
		res = res.Merge(<-completions)
	}

	//all test processes are done, at this point
	//we should be able to wait for the aggregator to tell us that it's done

	select {
	case <-result:
		fmt.Println("")
	case <-time.After(time.Second):
		//the aggregator never got back to us!  something must have gone wrong
		fmt.Println(`
	 -------------------------------------------------------------------
	|                                                                   |
	|  Ginkgo timed out waiting for all parallel nodes to report back!  |
	|                                                                   |
	 -------------------------------------------------------------------
`)

		os.Stdout.Sync()

		for _, writer := range writers {
			writer.Close()
		}

		for _, report := range reports {
			fmt.Print(report.String())
		}

		os.Stdout.Sync()
	}

	if t.cover || t.coverPkg != "" {
		t.combineCoverprofiles()
	}

	return res
}

func (t *TestRunner) cmd(ginkgoArgs []string, stream io.Writer, node int) *exec.Cmd {
	args := []string{"--test.timeout=24h"}
	if t.cover || t.coverPkg != "" {
		coverprofile := "--test.coverprofile=" + t.Suite.PackageName + ".coverprofile"
		if t.numCPU > 1 {
			coverprofile = fmt.Sprintf("%s.%d", coverprofile, node)
		}
		args = append(args, coverprofile)
	}

	args = append(args, ginkgoArgs...)
	args = append(args, t.additionalArgs...)

	path := t.compilationTargetPath
	if t.Suite.Precompiled {
		path, _ = filepath.Abs(filepath.Join(t.Suite.Path, fmt.Sprintf("%s.test", t.Suite.PackageName)))
	}

	cmd := exec.Command(path, args...)

	cmd.Dir = t.Suite.Path
	cmd.Stderr = stream
	cmd.Stdout = stream

	return cmd
}

func (t *TestRunner) run(cmd *exec.Cmd, completions chan RunResult) RunResult {
	var res RunResult

	defer func() {
		if completions != nil {
			completions <- res
		}
	}()

	err := cmd.Start()
	if err != nil {
		fmt.Printf("Failed to run test suite!\n\t%s", err.Error())
		return res
	}

	cmd.Wait()
	exitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
	res.Passed = (exitStatus == 0) || (exitStatus == types.GINKGO_FOCUS_EXIT_CODE)
	res.HasProgrammaticFocus = (exitStatus == types.GINKGO_FOCUS_EXIT_CODE)

	return res
}

func (t *TestRunner) combineCoverprofiles() {
	profiles := []string{}
	for cpu := 1; cpu <= t.numCPU; cpu++ {
		coverFile := fmt.Sprintf("%s.coverprofile.%d", t.Suite.PackageName, cpu)
		coverFile = filepath.Join(t.Suite.Path, coverFile)
		coverProfile, err := ioutil.ReadFile(coverFile)
		os.Remove(coverFile)

		if err == nil {
			profiles = append(profiles, string(coverProfile))
		}
	}

	if len(profiles) != t.numCPU {
		return
	}

	lines := map[string]int{}
	lineOrder := []string{}
	for i, coverProfile := range profiles {
		for _, line := range strings.Split(string(coverProfile), "\n")[1:] {
			if len(line) == 0 {
				continue
			}
			components := strings.Split(line, " ")
			count, _ := strconv.Atoi(components[len(components)-1])
			prefix := strings.Join(components[0:len(components)-1], " ")
			lines[prefix] += count
			if i == 0 {
				lineOrder = append(lineOrder, prefix)
			}
		}
	}

	output := []string{"mode: atomic"}
	for _, line := range lineOrder {
		output = append(output, fmt.Sprintf("%s %d", line, lines[line]))
	}
	finalOutput := strings.Join(output, "\n")
	ioutil.WriteFile(filepath.Join(t.Suite.Path, fmt.Sprintf("%s.coverprofile", t.Suite.PackageName)), []byte(finalOutput), 0666)
}
<|endoftext|>


<commit_before>package tes_server

import (
	"fmt"
	"github.com/boltdb/bolt"
	proto "github.com/golang/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	"log"
	"strings"
	"tes/ga4gh"
	"tes/server/proto"
)

// job ID -> ga4gh_task_exec.Task struct
var TaskBucket = []byte("tasks")

// job ID -> JWT token string
var TaskAuthBucket = []byte("tasks-auth")

// job ID -> job state string
var JobsQueued = []byte("jobs-queued")

// job ID -> job state string
var JobsActive = []byte("jobs-active")

// job ID -> job state string
var JobsComplete = []byte("jobs-complete")

// job ID -> ga4gh_task_exec.JobLog struct
var JobsLog = []byte("jobs-log")

// worker ID -> job ID
var WorkerJobs = []byte("worker-jobs")

// job ID -> worker ID
var JobWorker = []byte("job-worker")

// TaskBolt provides handlers for gRPC endpoints.
// Data is stored/retrieved from the BoltDB key-value database.
type TaskBolt struct {
	db           *bolt.DB
	serverConfig ga4gh_task_ref.ServerConfig
}

// NewTaskBolt returns a new instance of TaskBolt, accessing the database at
// the given path, and including the given ServerConfig.
func NewTaskBolt(path string, config ga4gh_task_ref.ServerConfig) *TaskBolt {
	db, _ := bolt.Open(path, 0600, nil)
	//Check to make sure all the required buckets have been created
	db.Update(func(tx *bolt.Tx) error {
		if tx.Bucket(TaskBucket) == nil {
			tx.CreateBucket(TaskBucket)
		}
		if tx.Bucket(TaskAuthBucket) == nil {
			tx.CreateBucket(TaskAuthBucket)
		}
		if tx.Bucket(JobsQueued) == nil {
			tx.CreateBucket(JobsQueued)
		}
		if tx.Bucket(JobsActive) == nil {
			tx.CreateBucket(JobsActive)
		}
		if tx.Bucket(JobsComplete) == nil {
			tx.CreateBucket(JobsComplete)
		}
		if tx.Bucket(JobsLog) == nil {
			tx.CreateBucket(JobsLog)
		}
		if tx.Bucket(WorkerJobs) == nil {
			tx.CreateBucket(WorkerJobs)
		}
		if tx.Bucket(JobWorker) == nil {
			tx.CreateBucket(JobWorker)
		}
		return nil
	})
	return &TaskBolt{db: db, serverConfig: config}
}

// ReadQueue returns a slice of queued Jobs. Up to "n" jobs are returned.
func (taskBolt *TaskBolt) ReadQueue(n int) []*ga4gh_task_exec.Job {
	jobs := make([]*ga4gh_task_exec.Job, 0)
	taskBolt.db.View(func(tx *bolt.Tx) error {

		// Iterate over the JobsQueued bucket, reading the first `n` jobs
		c := tx.Bucket(JobsQueued).Cursor()
		for k, _ := c.First(); k != nil && len(jobs) < n; k, _ = c.Next() {
			id := string(k)
			job := taskBolt.getJob(tx, id)
			jobs = append(jobs, job)
		}
		return nil
	})
	return jobs
}

// getJWT
// This function extracts the JWT token from the rpc header and returns the string
func getJWT(ctx context.Context) string {
	jwt := ""
	v, _ := metadata.FromContext(ctx)
	auth, ok := v["authorization"]
	if !ok {
		return jwt
	}
	for _, i := range auth {
		if strings.HasPrefix(i, "JWT ") {
			jwt = strings.TrimPrefix(i, "JWT ")
		}
	}
	return jwt
}

// RunTask documentation
// TODO: documentation
func (taskBolt *TaskBolt) RunTask(ctx context.Context, task *ga4gh_task_exec.Task) (*ga4gh_task_exec.JobID, error) {
	log.Println("Receiving Task for Queue", task)

	jobID, _ := uuid.NewV4()

	if len(task.Docker) == 0 {
		return nil, fmt.Errorf("No docker commands found")
	}

	// Check inputs of the task
	for _, input := range task.GetInputs() {
		diskFound := false
		for _, res := range task.Resources.Volumes {
			if strings.HasPrefix(input.Path, res.MountPoint) {
				diskFound = true
			}
		}
		if !diskFound {
			return nil, fmt.Errorf("Required volume '%s' not found in resources", input.Path)
		}
		//Fixing blank value to File by default... Is this too much hand holding?
		if input.Class == "" {
			input.Class = "File"
		}
	}

	for _, output := range task.GetOutputs() {
		if output.Class == "" {
			output.Class = "File"
		}
	}

	jwt := getJWT(ctx)
	log.Printf("JWT: %s", jwt)

	ch := make(chan *ga4gh_task_exec.JobID, 1)
	err := taskBolt.db.Update(func(tx *bolt.Tx) error {

		taskopB := tx.Bucket(TaskBucket)
		v, _ := proto.Marshal(task)
		taskopB.Put([]byte(jobID.String()), v)

		taskopA := tx.Bucket(TaskAuthBucket)
		taskopA.Put([]byte(jobID.String()), []byte(jwt))

		queueB := tx.Bucket(JobsQueued)
		queueB.Put([]byte(jobID.String()), []byte(ga4gh_task_exec.State_Queued.String()))
		ch <- &ga4gh_task_exec.JobID{Value: jobID.String()}
		return nil
	})
	if err != nil {
		return nil, err
	}
	a := <-ch
	return a, err
}

func (taskBolt *TaskBolt) getJobState(jobID string) (ga4gh_task_exec.State, error) {

	ch := make(chan ga4gh_task_exec.State, 1)
	err := taskBolt.db.View(func(tx *bolt.Tx) error {
		bQ := tx.Bucket(JobsQueued)
		bA := tx.Bucket(JobsActive)
		bC := tx.Bucket(JobsComplete)

		if v := bQ.Get([]byte(jobID)); v != nil {
			//if its queued
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else if v := bA.Get([]byte(jobID)); v != nil {
			//if its active
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else if v := bC.Get([]byte(jobID)); v != nil {
			//if its complete
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else {
			ch <- ga4gh_task_exec.State_Unknown
		}
		return nil
	})
	a := <-ch
	return a, err
}

func (taskBolt *TaskBolt) getJob(tx *bolt.Tx, jobID string) *ga4gh_task_exec.Job {
	bT := tx.Bucket(TaskBucket)
	v := bT.Get([]byte(jobID))
	task := &ga4gh_task_exec.Task{}
	proto.Unmarshal(v, task)

	job := ga4gh_task_exec.Job{}
	job.JobID = jobID
	job.Task = task
	job.State, _ = taskBolt.getJobState(jobID)

	//if there is logging info
	bL := tx.Bucket(JobsLog)
	out := make([]*ga4gh_task_exec.JobLog, len(job.Task.Docker), len(job.Task.Docker))
	for i := range job.Task.Docker {
		o := bL.Get([]byte(fmt.Sprint(jobID, i)))
		if o != nil {
			var log ga4gh_task_exec.JobLog
			proto.Unmarshal(o, &log)
			out[i] = &log
		} else {
			out[i] = &ga4gh_task_exec.JobLog{}
		}
	}
	job.Logs = out
	return &job
}

// GetJob documentation
// TODO: documentation
// Get info about a running task
func (taskBolt *TaskBolt) GetJob(ctx context.Context, id *ga4gh_task_exec.JobID) (*ga4gh_task_exec.Job, error) {
	log.Printf("Getting Task Info")
	var job *ga4gh_task_exec.Job
	err := taskBolt.db.View(func(tx *bolt.Tx) error {
		job = taskBolt.getJob(tx, id.Value)
		return nil
	})
	return job, err
}

// ListJobs returns a list of jobIDs
func (taskBolt *TaskBolt) ListJobs(ctx context.Context, in *ga4gh_task_exec.JobListRequest) (*ga4gh_task_exec.JobListResponse, error) {
	log.Printf("Getting Task List")
	ch := make(chan ga4gh_task_exec.JobDesc, 1)
	go taskBolt.db.View(func(tx *bolt.Tx) error {
		taskopB := tx.Bucket(TaskBucket)
		c := taskopB.Cursor()
		log.Println("Scanning")
		for k, _ := c.First(); k != nil; k, _ = c.Next() {
			jobID := string(k)
			jobState, _ := taskBolt.getJobState(jobID)
			ch <- ga4gh_task_exec.JobDesc{JobID: jobID, State: jobState}
		}
		close(ch)
		return nil
	})

	jobDescArray := make([]*ga4gh_task_exec.JobDesc, 0, 10)
	for t := range ch {
		jobDescArray = append(jobDescArray, &t)
	}

	out := ga4gh_task_exec.JobListResponse{
		Jobs: jobDescArray,
	}

	log.Println("Returning", out)
	return &out, nil
}

// CancelJob documentation
// TODO: documentation
// Cancel a running task
func (taskBolt *TaskBolt) CancelJob(ctx context.Context, taskop *ga4gh_task_exec.JobID) (*ga4gh_task_exec.JobID, error) {
	taskBolt.db.Update(func(tx *bolt.Tx) error {
		bQ := tx.Bucket(JobsQueued)
		bQ.Delete([]byte(taskop.Value))
		bjw := tx.Bucket(JobWorker)

		bA := tx.Bucket(JobsActive)
		bA.Delete([]byte(taskop.Value))

		workerID := bjw.Get([]byte(taskop.Value))
		bjw.Delete([]byte(taskop.Value))

		bW := tx.Bucket(WorkerJobs)
		bW.Delete([]byte(workerID))

		bC := tx.Bucket(JobsComplete)
		bC.Put([]byte(taskop.Value), []byte(ga4gh_task_exec.State_Canceled.String()))
		return nil
	})
	return taskop, nil
}

// GetServiceInfo provides an endpoint for TES clients to get information about this server.
// Could include:
// - resource availability
// - support storage systems
// - versions
// - etc.
func (taskBolt *TaskBolt) GetServiceInfo(ctx context.Context, info *ga4gh_task_exec.ServiceInfoRequest) (*ga4gh_task_exec.ServiceInfo, error) {
	//BUG: this isn't the best translation, probably lossy.
	//     Maybe ServiceInfo data structure schema needs to be refactored
	//     For example, you can't have multiple S3 endpoints
	out := map[string]string{}
	for _, i := range taskBolt.serverConfig.Storage {
		if i.Local != nil {
			out["Local.AllowedDirs"] = strings.Join(i.Local.AllowedDirs, ",")
		}

		if i.S3 != nil {
			out["S3.Endpoint"] = i.S3.Endpoint
		}
	}
	return &ga4gh_task_exec.ServiceInfo{StorageConfig: out}, nil
}
<commit_msg>Fix bug in job list<commit_after>package tes_server

import (
	"fmt"
	"github.com/boltdb/bolt"
	proto "github.com/golang/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	"log"
	"strings"
	"tes/ga4gh"
	"tes/server/proto"
)

// job ID -> ga4gh_task_exec.Task struct
var TaskBucket = []byte("tasks")

// job ID -> JWT token string
var TaskAuthBucket = []byte("tasks-auth")

// job ID -> job state string
var JobsQueued = []byte("jobs-queued")

// job ID -> job state string
var JobsActive = []byte("jobs-active")

// job ID -> job state string
var JobsComplete = []byte("jobs-complete")

// job ID -> ga4gh_task_exec.JobLog struct
var JobsLog = []byte("jobs-log")

// worker ID -> job ID
var WorkerJobs = []byte("worker-jobs")

// job ID -> worker ID
var JobWorker = []byte("job-worker")

// TaskBolt provides handlers for gRPC endpoints.
// Data is stored/retrieved from the BoltDB key-value database.
type TaskBolt struct {
	db           *bolt.DB
	serverConfig ga4gh_task_ref.ServerConfig
}

// NewTaskBolt returns a new instance of TaskBolt, accessing the database at
// the given path, and including the given ServerConfig.
func NewTaskBolt(path string, config ga4gh_task_ref.ServerConfig) *TaskBolt {
	db, _ := bolt.Open(path, 0600, nil)
	//Check to make sure all the required buckets have been created
	db.Update(func(tx *bolt.Tx) error {
		if tx.Bucket(TaskBucket) == nil {
			tx.CreateBucket(TaskBucket)
		}
		if tx.Bucket(TaskAuthBucket) == nil {
			tx.CreateBucket(TaskAuthBucket)
		}
		if tx.Bucket(JobsQueued) == nil {
			tx.CreateBucket(JobsQueued)
		}
		if tx.Bucket(JobsActive) == nil {
			tx.CreateBucket(JobsActive)
		}
		if tx.Bucket(JobsComplete) == nil {
			tx.CreateBucket(JobsComplete)
		}
		if tx.Bucket(JobsLog) == nil {
			tx.CreateBucket(JobsLog)
		}
		if tx.Bucket(WorkerJobs) == nil {
			tx.CreateBucket(WorkerJobs)
		}
		if tx.Bucket(JobWorker) == nil {
			tx.CreateBucket(JobWorker)
		}
		return nil
	})
	return &TaskBolt{db: db, serverConfig: config}
}

// ReadQueue returns a slice of queued Jobs. Up to "n" jobs are returned.
func (taskBolt *TaskBolt) ReadQueue(n int) []*ga4gh_task_exec.Job {
	jobs := make([]*ga4gh_task_exec.Job, 0)
	taskBolt.db.View(func(tx *bolt.Tx) error {

		// Iterate over the JobsQueued bucket, reading the first `n` jobs
		c := tx.Bucket(JobsQueued).Cursor()
		for k, _ := c.First(); k != nil && len(jobs) < n; k, _ = c.Next() {
			id := string(k)
			job := taskBolt.getJob(tx, id)
			jobs = append(jobs, job)
		}
		return nil
	})
	return jobs
}

// getJWT
// This function extracts the JWT token from the rpc header and returns the string
func getJWT(ctx context.Context) string {
	jwt := ""
	v, _ := metadata.FromContext(ctx)
	auth, ok := v["authorization"]
	if !ok {
		return jwt
	}
	for _, i := range auth {
		if strings.HasPrefix(i, "JWT ") {
			jwt = strings.TrimPrefix(i, "JWT ")
		}
	}
	return jwt
}

// RunTask documentation
// TODO: documentation
func (taskBolt *TaskBolt) RunTask(ctx context.Context, task *ga4gh_task_exec.Task) (*ga4gh_task_exec.JobID, error) {
	log.Println("Receiving Task for Queue", task)

	jobID, _ := uuid.NewV4()
	log.Printf("Assigning job ID, %s", jobID)

	if len(task.Docker) == 0 {
		return nil, fmt.Errorf("No docker commands found")
	}

	// Check inputs of the task
	for _, input := range task.GetInputs() {
		diskFound := false
		for _, res := range task.Resources.Volumes {
			if strings.HasPrefix(input.Path, res.MountPoint) {
				diskFound = true
			}
		}
		if !diskFound {
			return nil, fmt.Errorf("Required volume '%s' not found in resources", input.Path)
		}
		//Fixing blank value to File by default... Is this too much hand holding?
		if input.Class == "" {
			input.Class = "File"
		}
	}

	for _, output := range task.GetOutputs() {
		if output.Class == "" {
			output.Class = "File"
		}
	}

	jwt := getJWT(ctx)
	log.Printf("JWT: %s", jwt)

	ch := make(chan *ga4gh_task_exec.JobID, 1)
	err := taskBolt.db.Update(func(tx *bolt.Tx) error {

		taskopB := tx.Bucket(TaskBucket)
		v, _ := proto.Marshal(task)
		taskopB.Put([]byte(jobID.String()), v)

		taskopA := tx.Bucket(TaskAuthBucket)
		taskopA.Put([]byte(jobID.String()), []byte(jwt))

		queueB := tx.Bucket(JobsQueued)
		queueB.Put([]byte(jobID.String()), []byte(ga4gh_task_exec.State_Queued.String()))
		ch <- &ga4gh_task_exec.JobID{Value: jobID.String()}
		return nil
	})
	if err != nil {
		return nil, err
	}
	a := <-ch
	return a, err
}

func (taskBolt *TaskBolt) getJobState(jobID string) (ga4gh_task_exec.State, error) {

	ch := make(chan ga4gh_task_exec.State, 1)
	err := taskBolt.db.View(func(tx *bolt.Tx) error {
		bQ := tx.Bucket(JobsQueued)
		bA := tx.Bucket(JobsActive)
		bC := tx.Bucket(JobsComplete)

		if v := bQ.Get([]byte(jobID)); v != nil {
			//if its queued
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else if v := bA.Get([]byte(jobID)); v != nil {
			//if its active
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else if v := bC.Get([]byte(jobID)); v != nil {
			//if its complete
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else {
			ch <- ga4gh_task_exec.State_Unknown
		}
		return nil
	})
	a := <-ch
	return a, err
}

func (taskBolt *TaskBolt) getJob(tx *bolt.Tx, jobID string) *ga4gh_task_exec.Job {
	bT := tx.Bucket(TaskBucket)
	v := bT.Get([]byte(jobID))
	task := &ga4gh_task_exec.Task{}
	proto.Unmarshal(v, task)

	job := ga4gh_task_exec.Job{}
	job.JobID = jobID
	job.Task = task
	job.State, _ = taskBolt.getJobState(jobID)

	//if there is logging info
	bL := tx.Bucket(JobsLog)
	out := make([]*ga4gh_task_exec.JobLog, len(job.Task.Docker), len(job.Task.Docker))
	for i := range job.Task.Docker {
		o := bL.Get([]byte(fmt.Sprint(jobID, i)))
		if o != nil {
			var log ga4gh_task_exec.JobLog
			proto.Unmarshal(o, &log)
			out[i] = &log
		} else {
			out[i] = &ga4gh_task_exec.JobLog{}
		}
	}
	job.Logs = out
	return &job
}

// GetJob documentation
// TODO: documentation
// Get info about a running task
func (taskBolt *TaskBolt) GetJob(ctx context.Context, id *ga4gh_task_exec.JobID) (*ga4gh_task_exec.Job, error) {
	log.Printf("Getting Task Info")
	var job *ga4gh_task_exec.Job
	err := taskBolt.db.View(func(tx *bolt.Tx) error {
		job = taskBolt.getJob(tx, id.Value)
		return nil
	})
	return job, err
}

// ListJobs returns a list of jobIDs
func (taskBolt *TaskBolt) ListJobs(ctx context.Context, in *ga4gh_task_exec.JobListRequest) (*ga4gh_task_exec.JobListResponse, error) {
	log.Printf("Getting Task List")

	jobs := make([]*ga4gh_task_exec.JobDesc, 0, 10)

	taskBolt.db.View(func(tx *bolt.Tx) error {
		taskopB := tx.Bucket(TaskBucket)
		c := taskopB.Cursor()
		log.Println("Scanning")

		for k, _ := c.First(); k != nil; k, _ = c.Next() {
			jobID := string(k)
			jobState, _ := taskBolt.getJobState(jobID)
			job := &ga4gh_task_exec.JobDesc{JobID: jobID, State: jobState}
			jobs = append(jobs, job)
		}
		return nil
	})

	out := ga4gh_task_exec.JobListResponse{
		Jobs: jobs,
	}

	log.Println("Returning", out)
	return &out, nil
}

// CancelJob documentation
// TODO: documentation
// Cancel a running task
func (taskBolt *TaskBolt) CancelJob(ctx context.Context, taskop *ga4gh_task_exec.JobID) (*ga4gh_task_exec.JobID, error) {
	taskBolt.db.Update(func(tx *bolt.Tx) error {
		bQ := tx.Bucket(JobsQueued)
		bQ.Delete([]byte(taskop.Value))
		bjw := tx.Bucket(JobWorker)

		bA := tx.Bucket(JobsActive)
		bA.Delete([]byte(taskop.Value))

		workerID := bjw.Get([]byte(taskop.Value))
		bjw.Delete([]byte(taskop.Value))

		bW := tx.Bucket(WorkerJobs)
		bW.Delete([]byte(workerID))

		bC := tx.Bucket(JobsComplete)
		bC.Put([]byte(taskop.Value), []byte(ga4gh_task_exec.State_Canceled.String()))
		return nil
	})
	return taskop, nil
}

// GetServiceInfo provides an endpoint for TES clients to get information about this server.
// Could include:
// - resource availability
// - support storage systems
// - versions
// - etc.
func (taskBolt *TaskBolt) GetServiceInfo(ctx context.Context, info *ga4gh_task_exec.ServiceInfoRequest) (*ga4gh_task_exec.ServiceInfo, error) {
	//BUG: this isn't the best translation, probably lossy.
	//     Maybe ServiceInfo data structure schema needs to be refactored
	//     For example, you can't have multiple S3 endpoints
	out := map[string]string{}
	for _, i := range taskBolt.serverConfig.Storage {
		if i.Local != nil {
			out["Local.AllowedDirs"] = strings.Join(i.Local.AllowedDirs, ",")
		}

		if i.S3 != nil {
			out["S3.Endpoint"] = i.S3.Endpoint
		}
	}
	return &ga4gh_task_exec.ServiceInfo{StorageConfig: out}, nil
}
<|endoftext|>


<commit_before>package app

import (
	"github.com/toonsevrin/simplechain/types"
	"net/http"
	"github.com/gorilla/mux"

	"encoding/json"
	"io/ioutil"
	"fmt"
	"strings"
	"os"
)

type Server struct {
	App App
}
func (server *Server) Init(){

	router := mux.NewRouter().StrictSlash(true)

	router.Methods("GET").Path("/blocks").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		if isLocalhostOrPeer(*server, *request){
			json.NewEncoder(writer).Encode(server.App.Blockchain)
		}else {
			json.NewEncoder(writer).Encode(Success{false, "Unauthorized"})
		}
	})
	router.Methods("POST").Path("/mineBlock").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		if isLocalhostOrPeer(*server, *request) {
			body, err := ioutil.ReadAll(request.Body)
			if err != nil {
				json.NewEncoder(writer).Encode(Success{false, "An error occurred reading request body"})
				fmt.Println(err.Error())
				return
			}
			data := string(body)
			block := server.App.createAndAddNextBlock(data)
			json.NewEncoder(writer).Encode(Success{Success:true})
			server.App.broadcast(block)
			fmt.Println("log")
		}else {
			json.NewEncoder(writer).Encode(Success{false, "Unauthorized"})
		}
	})
	router.Methods("POST").Path("/addBlock").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		if isLocalhostOrPeer(*server, *request){
			block := types.Block{}
			body, err := ioutil.ReadAll(request.Body)
			if err != nil {
				json.NewEncoder(writer).Encode(Success{false,"An error occurred reading request body"})
				fmt.Println(err.Error())
				return
			}
			if err := json.Unmarshal(body, block); err != nil {
				json.NewEncoder(writer).Encode(Success{false,"An error occurred parsing request body"})
				fmt.Println(err.Error())
				return
			}
			if server.App.HasBlock(block){
				json.NewEncoder(writer).Encode(Success{false,"Block already exists"})
				fmt.Println("Received block that already exists in db.")
				return
			}
			if !block.IsValid() {
				json.NewEncoder(writer).Encode(Success{false,"Received invalid block"})
				fmt.Println("Received invalid block")
				return
			}
			if uint32(len(server.App.Blockchain)) == block.Index {//next block
				if block.PreviousHash == server.App.getLatestBlock().Hash {//next block references your chain
					server.App.AddBlock(block)
					json.NewEncoder(writer).Encode(Success{Success: true})
					server.App.broadcast(block)
				}
			}else if uint32(len(server.App.Blockchain)) < block.Index {//block is in the future
				RemoteChain := []types.Block{}
				response, err := http.NewRequest("GET", server.App.Peers[request.RemoteAddr].getUrl() + "/blocks", nil)
				if err != nil {
					json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
					fmt.Println(err.Error())
					return
				}
				body, err := ioutil.ReadAll(response.Body)
				if err != nil {
					json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
					fmt.Println(err.Error())
					return
				}
				if err := json.Unmarshal(body, RemoteChain); err != nil {
					json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
					fmt.Println(err.Error())
					return
				}
				if(server.App.pickLongestChain(RemoteChain)){
					json.NewEncoder(writer).Encode(Success{Success: true})
					server.App.broadcast(block)
				}
			}
		}else {
			json.NewEncoder(writer).Encode(Success{Success: false, Error: "Unauthorized"})
		}
	})
	router.Methods("POST").Path("/addPeer").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		if isLocalhost(*request){
			body, err := ioutil.ReadAll(request.Body)
			if err != nil {
				json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
				fmt.Println(err.Error())
				return
			}
			peer := Peer{}
			if err := json.Unmarshal(body, peer); err != nil {
				json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
				fmt.Println(err.Error())
				return
			}
			server.App.Peers[peer.getUrl()] = &peer
			server.App.PeerAddresses[peer.Ip] = true
			json.NewEncoder(writer).Encode(Success{Success: true})
		}else{
			writer.Write([]byte("Only localhost can add peers"))
		}
	})
	http.ListenAndServe(":" + getPort(), router)
}
func getPort() string{
	port := os.Getenv("PORT")
	if(port == ""){
		return "8080"
	}else {
		return port
	}
}
func isLocalhostOrPeer(server Server, request http.Request) bool{
	_, isPeer := server.App.PeerAddresses[request.RemoteAddr]
	return isPeer || isLocalhost(request)
}
func isLocalhost(req http.Request) bool{
	fmt.Println(req.RemoteAddr)
	return strings.Contains(req.RemoteAddr, "127.0.0.1") || strings.Contains(req.RemoteAddr, "[::1]")//::1 is ipv6
}

type Success struct {
	Success bool `json:"success"`
	Error string `json:"error"`
}<commit_msg>Fixed unmarshalling pointers<commit_after>package app

import (
	"github.com/toonsevrin/simplechain/types"
	"net/http"
	"github.com/gorilla/mux"

	"encoding/json"
	"io/ioutil"
	"fmt"
	"strings"
	"os"
)

type Server struct {
	App App
}
func (server *Server) Init(){

	router := mux.NewRouter().StrictSlash(true)

	router.Methods("GET").Path("/blocks").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		if isLocalhostOrPeer(*server, *request){
			json.NewEncoder(writer).Encode(server.App.Blockchain)
		}else {
			json.NewEncoder(writer).Encode(Success{false, "Unauthorized"})
		}
	})
	router.Methods("POST").Path("/mineBlock").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		if isLocalhostOrPeer(*server, *request) {
			body, err := ioutil.ReadAll(request.Body)
			if err != nil {
				json.NewEncoder(writer).Encode(Success{false, "An error occurred reading request body"})
				fmt.Println(err.Error())
				return
			}
			data := string(body)
			block := server.App.createAndAddNextBlock(data)
			json.NewEncoder(writer).Encode(Success{Success:true})
			server.App.broadcast(block)
		}else {
			json.NewEncoder(writer).Encode(Success{false, "Unauthorized"})
		}
	})
	router.Methods("POST").Path("/addBlock").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		if isLocalhostOrPeer(*server, *request){
			block := types.Block{}
			body, err := ioutil.ReadAll(request.Body)
			if err != nil {
				json.NewEncoder(writer).Encode(Success{false,"An error occurred reading request body"})
				fmt.Println(err.Error())
				return
			}
			if err := json.Unmarshal(body, &block); err != nil {
				json.NewEncoder(writer).Encode(Success{false,"An error occurred parsing request body"})
				fmt.Println(err.Error())
				return
			}
			if server.App.HasBlock(block){
				json.NewEncoder(writer).Encode(Success{false,"Block already exists"})
				fmt.Println("Received block that already exists in db.")
				return
			}
			if !block.IsValid() {
				json.NewEncoder(writer).Encode(Success{false,"Received invalid block"})
				fmt.Println("Received invalid block")
				return
			}
			if uint32(len(server.App.Blockchain)) == block.Index {//next block
				if block.PreviousHash == server.App.getLatestBlock().Hash {//next block references your chain
					server.App.AddBlock(block)
					json.NewEncoder(writer).Encode(Success{Success: true})
					server.App.broadcast(block)
				}
			}else if uint32(len(server.App.Blockchain)) < block.Index {//block is in the future
				RemoteChain := []types.Block{}
				response, err := http.NewRequest("GET", server.App.Peers[request.RemoteAddr].getUrl() + "/blocks", nil)
				if err != nil {
					json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
					fmt.Println(err.Error())
					return
				}
				body, err := ioutil.ReadAll(response.Body)
				if err != nil {
					json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
					fmt.Println(err.Error())
					return
				}
				if err := json.Unmarshal(body, &RemoteChain); err != nil {
					json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
					fmt.Println(err.Error())
					return
				}
				if(server.App.pickLongestChain(RemoteChain)){
					json.NewEncoder(writer).Encode(Success{Success: true})
					server.App.broadcast(block)
				}
			}
		}else {
			json.NewEncoder(writer).Encode(Success{Success: false, Error: "Unauthorized"})
		}
	})
	router.Methods("POST").Path("/addPeer").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
		if isLocalhost(*request){
			body, err := ioutil.ReadAll(request.Body)
			if err != nil {
				json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
				fmt.Println(err.Error())
				return
			}
			peer := Peer{}
			if err := json.Unmarshal(body, &peer); err != nil {
				json.NewEncoder(writer).Encode(Success{Success: false, Error: string(err.Error())})
				fmt.Println(err.Error())
				return
			}
			server.App.Peers[peer.getUrl()] = &peer
			server.App.PeerAddresses[peer.Ip] = true
			json.NewEncoder(writer).Encode(Success{Success: true})
		}else{
			writer.Write([]byte("Only localhost can add peers"))
		}
	})
	http.ListenAndServe(":" + getPort(), router)
}
func getPort() string{
	port := os.Getenv("PORT")
	if(port == ""){
		return "8080"
	}else {
		return port
	}
}
func isLocalhostOrPeer(server Server, request http.Request) bool{
	_, isPeer := server.App.PeerAddresses[request.RemoteAddr]
	return isPeer || isLocalhost(request)
}
func isLocalhost(req http.Request) bool{
	fmt.Println(req.RemoteAddr)
	return strings.Contains(req.RemoteAddr, "127.0.0.1") || strings.Contains(req.RemoteAddr, "[::1]")//::1 is ipv6
}

type Success struct {
	Success bool `json:"success"`
	Error string `json:"error"`
}<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/types"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/pkg/api/v1"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/util/removeall"
	"k8s.io/kubernetes/pkg/volume"
	volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
)

// ListVolumesForPod returns a map of the mounted volumes for the given pod.
// The key in the map is the OuterVolumeSpecName (i.e. pod.Spec.Volumes[x].Name)
func (kl *Kubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
	volumesToReturn := make(map[string]volume.Volume)
	podVolumes := kl.volumeManager.GetMountedVolumesForPod(
		volumetypes.UniquePodName(podUID))
	for outerVolumeSpecName, volume := range podVolumes {
		// TODO: volume.Mounter could be nil if volume object is recovered
		// from reconciler's sync state process. PR 33616 will fix this problem
		// to create Mounter object when recovering volume state.
		if volume.Mounter == nil {
			continue
		}
		volumesToReturn[outerVolumeSpecName] = volume.Mounter
	}

	return volumesToReturn, len(volumesToReturn) > 0
}

// podVolumesExist checks with the volume manager and returns true any of the
// pods for the specified volume are mounted.
func (kl *Kubelet) podVolumesExist(podUID types.UID) bool {
	if mountedVolumes :=
		kl.volumeManager.GetMountedVolumesForPod(
			volumetypes.UniquePodName(podUID)); len(mountedVolumes) > 0 {
		return true
	}

	return false
}

// newVolumeMounterFromPlugins attempts to find a plugin by volume spec, pod
// and volume options and then creates a Mounter.
// Returns a valid Unmounter or an error.
func (kl *Kubelet) newVolumeMounterFromPlugins(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	plugin, err := kl.volumePluginMgr.FindPluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("can't use volume plugins for %s: %v", spec.Name(), err)
	}
	physicalMounter, err := plugin.NewMounter(spec, pod, opts)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate mounter for volume: %s using plugin: %s with a root cause: %v", spec.Name(), plugin.GetPluginName(), err)
	}
	glog.V(10).Infof("Using volume plugin %q to mount %s", plugin.GetPluginName(), spec.Name())
	return physicalMounter, nil
}

// cleanupOrphanedPodDirs removes the volumes of pods that should not be
// running and that have no containers running.  Note that we roll up logs here since it runs in the main loop.
func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecontainer.Pod) error {
	allPods := sets.NewString()
	for _, pod := range pods {
		allPods.Insert(string(pod.UID))
	}
	for _, pod := range runningPods {
		allPods.Insert(string(pod.ID))
	}

	found, err := kl.listPodsFromDisk()
	if err != nil {
		return err
	}

	orphanRemovalErrors := []error{}
	orphanVolumeErrors := []error{}

	for _, uid := range found {
		if allPods.Has(string(uid)) {
			continue
		}
		// If volumes have not been unmounted/detached, do not delete directory.
		// Doing so may result in corruption of data.
		if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist {
			glog.V(3).Infof("Orphaned pod %q found, but volumes are not cleaned up", uid)
			continue
		}
		// If there are still volume directories, do not delete directory
		volumePaths, err := kl.getPodVolumePathListFromDisk(uid)
		if err != nil {
			orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but error %v occurred during reading volume dir from disk", uid, err))
			continue
		}
		if len(volumePaths) > 0 {
			orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but volume paths are still present on disk.", uid))
			continue
		}
		glog.V(3).Infof("Orphaned pod %q found, removing", uid)
		if err := removeall.RemoveAllOneFilesystem(kl.mounter, kl.getPodDir(uid)); err != nil {
			glog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err)
			orphanRemovalErrors = append(orphanRemovalErrors, err)
		}
	}

	logSpew := func(errs []error) {
		if len(errs) > 0 {
			glog.Errorf("%v : There were a total of %v errors similar to this.  Turn up verbosity to see them.", errs[0], len(errs))
			for _, err := range errs {
				glog.V(5).Infof("Orphan pod: %v", err)
			}
		}
	}
	logSpew(orphanVolumeErrors)
	logSpew(orphanRemovalErrors)
	return utilerrors.NewAggregate(orphanRemovalErrors)
}
<commit_msg>fix comment error in function newVolumeMounterFromPlugins<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/types"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/pkg/api/v1"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/util/removeall"
	"k8s.io/kubernetes/pkg/volume"
	volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
)

// ListVolumesForPod returns a map of the mounted volumes for the given pod.
// The key in the map is the OuterVolumeSpecName (i.e. pod.Spec.Volumes[x].Name)
func (kl *Kubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
	volumesToReturn := make(map[string]volume.Volume)
	podVolumes := kl.volumeManager.GetMountedVolumesForPod(
		volumetypes.UniquePodName(podUID))
	for outerVolumeSpecName, volume := range podVolumes {
		// TODO: volume.Mounter could be nil if volume object is recovered
		// from reconciler's sync state process. PR 33616 will fix this problem
		// to create Mounter object when recovering volume state.
		if volume.Mounter == nil {
			continue
		}
		volumesToReturn[outerVolumeSpecName] = volume.Mounter
	}

	return volumesToReturn, len(volumesToReturn) > 0
}

// podVolumesExist checks with the volume manager and returns true any of the
// pods for the specified volume are mounted.
func (kl *Kubelet) podVolumesExist(podUID types.UID) bool {
	if mountedVolumes :=
		kl.volumeManager.GetMountedVolumesForPod(
			volumetypes.UniquePodName(podUID)); len(mountedVolumes) > 0 {
		return true
	}

	return false
}

// newVolumeMounterFromPlugins attempts to find a plugin by volume spec, pod
// and volume options and then creates a Mounter.
// Returns a valid mounter or an error.
func (kl *Kubelet) newVolumeMounterFromPlugins(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	plugin, err := kl.volumePluginMgr.FindPluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("can't use volume plugins for %s: %v", spec.Name(), err)
	}
	physicalMounter, err := plugin.NewMounter(spec, pod, opts)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate mounter for volume: %s using plugin: %s with a root cause: %v", spec.Name(), plugin.GetPluginName(), err)
	}
	glog.V(10).Infof("Using volume plugin %q to mount %s", plugin.GetPluginName(), spec.Name())
	return physicalMounter, nil
}

// cleanupOrphanedPodDirs removes the volumes of pods that should not be
// running and that have no containers running.  Note that we roll up logs here since it runs in the main loop.
func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecontainer.Pod) error {
	allPods := sets.NewString()
	for _, pod := range pods {
		allPods.Insert(string(pod.UID))
	}
	for _, pod := range runningPods {
		allPods.Insert(string(pod.ID))
	}

	found, err := kl.listPodsFromDisk()
	if err != nil {
		return err
	}

	orphanRemovalErrors := []error{}
	orphanVolumeErrors := []error{}

	for _, uid := range found {
		if allPods.Has(string(uid)) {
			continue
		}
		// If volumes have not been unmounted/detached, do not delete directory.
		// Doing so may result in corruption of data.
		if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist {
			glog.V(3).Infof("Orphaned pod %q found, but volumes are not cleaned up", uid)
			continue
		}
		// If there are still volume directories, do not delete directory
		volumePaths, err := kl.getPodVolumePathListFromDisk(uid)
		if err != nil {
			orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but error %v occurred during reading volume dir from disk", uid, err))
			continue
		}
		if len(volumePaths) > 0 {
			orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but volume paths are still present on disk.", uid))
			continue
		}
		glog.V(3).Infof("Orphaned pod %q found, removing", uid)
		if err := removeall.RemoveAllOneFilesystem(kl.mounter, kl.getPodDir(uid)); err != nil {
			glog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err)
			orphanRemovalErrors = append(orphanRemovalErrors, err)
		}
	}

	logSpew := func(errs []error) {
		if len(errs) > 0 {
			glog.Errorf("%v : There were a total of %v errors similar to this.  Turn up verbosity to see them.", errs[0], len(errs))
			for _, err := range errs {
				glog.V(5).Infof("Orphan pod: %v", err)
			}
		}
	}
	logSpew(orphanVolumeErrors)
	logSpew(orphanRemovalErrors)
	return utilerrors.NewAggregate(orphanRemovalErrors)
}
<|endoftext|>