<commit_before>package bubbles

import (
	"io/ioutil"
	"log"
	"testing"
	"time"

	"github.com/realzeitmedia/bubbles/loges"
)

func init() {
	log.SetOutput(ioutil.Discard)
}

func TestIndex(t *testing.T) {
	es := newMockES(t, func() string {
		return `{"took":7,"items":[{"create":{"_index":"test","_type":"type1","_id":"1","_version":1}}]}`
	})
	defer es.Stop()

	c := &count{}
	b := New([]string{es.Addr()}, OptConnCount(2), OptFlush(10*time.Millisecond), OptCounter(c))

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue() <- ins
	time.Sleep(100 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if have, want := *c, (count{
		Retries:    0,
		Sends:      1,
		SendTotals: val{1, len(ins.Buf())},
		Troubles:   0,
	}); have != want {
		t.Fatalf("counts: have %v, want %v", have, want)
	}
}

func TestIndexNoES(t *testing.T) {
	// Index without an ES
	c := &count{}
	b := New([]string{"localhost:4321"}, OptConnCount(2), OptFlush(10*time.Millisecond), OptCounter(c))

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue() <- ins
	time.Sleep(20 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 1; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if pending[0] != ins {
		t.Errorf("Wrong pending object returned")
	}
	if have, want := c.Retries, 1; have < want {
		t.Fatalf("retries: have %v, want at least %v", have, want)
	}
}

type ErrorChan chan ActionError

func (e ErrorChan) Error(err error) {
	switch t := err.(type) {
	case ActionError:
		e <- t
	default:
		log.Fatal(err)
	}
}

func (e ErrorChan) Warning(err error) {
	log.Fatal(err)
}

func TestIndexErr(t *testing.T) {
	es := newMockES(
		t,
		func() string {
			return `{"took":8,"errors":true,"items":[{"index":{"_index":"index","_type":"type1","_id":"1","_version":5,"status":200}},{"index":{"_index":"index","_type":"type1","_id":"2","status":400,"error":"MapperParsingException[failed to parse]; nested: JsonParseException[Unexpected end-of-input within/between OBJECT entries\n at [Source: [B@5f72a900; line: 1, column: 160]]; "}}]}`
		},
	)
	defer es.Stop()

	errs := ErrorChan(make(chan ActionError))
	c := &count{}
	b := New([]string{es.Addr()},
		OptConnCount(2),
		OptFlush(10*time.Millisecond),
		OptErrer(errs),
		OptCounter(c),
	)

	ins1 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	ins2 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "2",
		},
		Document: `{"field1": `, // fake an error
	}

	b.Enqueue() <- ins1
	b.Enqueue() <- ins2
	var aerr ActionError
	select {
	case <-time.After(1 * time.Second):
		t.Fatalf("timeout")
	case aerr = <-errs:
	}
	if have, want := aerr.Action, ins2; have != want {
		t.Fatalf("wrong err. have %v, want %v", have, want)
	}
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if have, want := *c, (count{
		Sends:      1,
		Retries:    0,
		Errors:     1,
		SendTotals: val{1, len(ins1.Buf()) + len(ins2.Buf())},
		Troubles:   0,
	}); have != want {
		t.Fatalf("counts: have %v, want %v", have, want)
	}
}

func TestShutdownTimeout(t *testing.T) {
	es := newMockES(t, func() string {
		time.Sleep(10 * time.Second)
		return "{}"
	})
	defer es.Stop()

	maxDocs := 5
	b := New([]string{es.Addr()},
		OptConnCount(1),
		OptFlush(10*time.Millisecond),
		OptServerTimeout(5*time.Second),
		OptMaxDocs(5),
	)

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	docs := maxDocs
	for i := 0; i < docs; i++ {
		b.Enqueue() <- ins
	}

	time.Sleep(20 * time.Millisecond)
	p := make(chan []Action)
	go func() {
		p <- b.Stop()
	}()
	var pending []Action
	select {
	case pending = <-p:
	case <-time.After(1 * time.Second):
		t.Fatalf("Stop() took too long")
	}
	if have, want := len(pending), docs; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if pending[0] != ins {
		t.Errorf("Wrong pending object returned")
	}
}

type val struct{ C, T int }

type count struct {
	Sends      int
	Retries    int
	Errors     int
	SendTotals val
	Troubles   int
}

func (c *count) Actions(s, r, e int) {
	c.Sends += s
	c.Retries += r
	c.Errors += e
}

func (c *count) SendTotal(l int) {
	c.SendTotals.C++
	c.SendTotals.T += l
}

func (c *count) Trouble() {
	c.Troubles++
}

func (c *count) BatchTime(time.Duration) {
}

var _ loges.Counter = &count{}
<commit_msg>Improve error output for error tests.<commit_after>package bubbles

import (
	"io/ioutil"
	"log"
	"testing"
	"time"

	"github.com/realzeitmedia/bubbles/loges"
)

func init() {
	log.SetOutput(ioutil.Discard)
}

func TestIndex(t *testing.T) {
	es := newMockES(t, func() string {
		return `{"took":7,"items":[{"create":{"_index":"test","_type":"type1","_id":"1","_version":1}}]}`
	})
	defer es.Stop()

	c := &count{}
	b := New([]string{es.Addr()}, OptConnCount(2), OptFlush(10*time.Millisecond), OptCounter(c))

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue() <- ins
	time.Sleep(100 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if have, want := *c, (count{
		Retries:    0,
		Sends:      1,
		SendTotals: val{1, len(ins.Buf())},
		Troubles:   0,
	}); have != want {
		t.Fatalf("counts: have %v, want %v", have, want)
	}
}

func TestIndexNoES(t *testing.T) {
	// Index without an ES
	c := &count{}
	b := New([]string{"localhost:4321"}, OptConnCount(2), OptFlush(10*time.Millisecond), OptCounter(c))

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue() <- ins
	time.Sleep(20 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 1; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if pending[0] != ins {
		t.Errorf("Wrong pending object returned")
	}
	if have, want := c.Retries, 1; have < want {
		t.Fatalf("retries: have %v, want at least %v", have, want)
	}
}

type TestErrs struct {
	C chan ActionError
	t *testing.T
}

func NewTestErrs(t *testing.T) *TestErrs {
	return &TestErrs{
		C: make(chan ActionError),
		t: t,
	}
}

func (e *TestErrs) Error(err error) {
	switch t := err.(type) {
	case ActionError:
		e.C <- t
	default:
		e.t.Fatal(err)
	}
}

func (e *TestErrs) Warning(err error) {
	e.t.Fatal(err)
}

func TestIndexErr(t *testing.T) {
	es := newMockES(
		t,
		func() string {
			return `{"took":8,"errors":true,"items":[{"index":{"_index":"index","_type":"type1","_id":"1","_version":5,"status":200}},{"index":{"_index":"index","_type":"type1","_id":"2","status":400,"error":"MapperParsingException[failed to parse]; nested: JsonParseException[Unexpected end-of-input within/between OBJECT entries\n at [Source: [B@5f72a900; line: 1, column: 160]]; "}}]}`
		},
	)
	defer es.Stop()

	errs := NewTestErrs(t)
	c := &count{}
	b := New([]string{es.Addr()},
		OptConnCount(2),
		OptFlush(10*time.Millisecond),
		OptErrer(errs),
		OptCounter(c),
	)

	ins1 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	ins2 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "2",
		},
		Document: `{"field1": `, // fake an error
	}

	b.Enqueue() <- ins1
	b.Enqueue() <- ins2
	var aerr ActionError
	select {
	case <-time.After(1 * time.Second):
		t.Fatalf("timeout")
	case aerr = <-errs.C:
	}
	if have, want := aerr.Action, ins2; have != want {
		t.Fatalf("wrong err. have %v, want %v", have, want)
	}
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if have, want := *c, (count{
		Sends:      1,
		Retries:    0,
		Errors:     1,
		SendTotals: val{1, len(ins1.Buf()) + len(ins2.Buf())},
		Troubles:   0,
	}); have != want {
		t.Fatalf("counts: have %v, want %v", have, want)
	}
}

func TestShutdownTimeout(t *testing.T) {
	es := newMockES(t, func() string {
		time.Sleep(10 * time.Second)
		return "{}"
	})
	defer es.Stop()

	maxDocs := 5
	b := New([]string{es.Addr()},
		OptConnCount(1),
		OptFlush(10*time.Millisecond),
		OptServerTimeout(5*time.Second),
		OptMaxDocs(5),
	)

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: "test",
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	docs := maxDocs
	for i := 0; i < docs; i++ {
		b.Enqueue() <- ins
	}

	time.Sleep(20 * time.Millisecond)
	p := make(chan []Action)
	go func() {
		p <- b.Stop()
	}()
	var pending []Action
	select {
	case pending = <-p:
	case <-time.After(1 * time.Second):
		t.Fatalf("Stop() took too long")
	}
	if have, want := len(pending), docs; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
	if pending[0] != ins {
		t.Errorf("Wrong pending object returned")
	}
}

type val struct{ C, T int }

type count struct {
	Sends      int
	Retries    int
	Errors     int
	SendTotals val
	Troubles   int
}

func (c *count) Actions(s, r, e int) {
	c.Sends += s
	c.Retries += r
	c.Errors += e
}

func (c *count) SendTotal(l int) {
	c.SendTotals.C++
	c.SendTotals.T += l
}

func (c *count) Trouble() {
	c.Troubles++
}

func (c *count) BatchTime(time.Duration) {
}

var _ loges.Counter = &count{}
<|endoftext|>


<commit_before>package main

const releaseVersion = "0.1.0"
<commit_msg>Releasing 0.1.1.<commit_after>package main

const releaseVersion = "0.1.1"
<|endoftext|>


<commit_before>package hashtree

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	pathlib "path"
	"sort"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pfs"
)

type Names []string

func (n Names) Len() int {
	return len(n)
}

func (n Names) Less(i, j int) bool {
	return n[i] < n[j]
}

func (n Names) Swap(i, j int) {
	tmp := n[i]
	n[i] = n[j]
	n[j] = tmp
}

// cleanPath converts a path into a form that we use internally
// Basically we make sure that it has a leading slash and no trailing slash.
func cleanPath(path string) string {
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	return pathlib.Clean(path)
}

// UpdateHash uses the node's internal state to update the hash
// of the node
func (n *Node) UpdateHash() error {
	if n.DirNode != nil {
		sort.Sort(Names(n.DirNode.Children))
		var buf bytes.Buffer
		for _, child := range n.DirNode.Children {
			if _, err := buf.WriteString(child); err != nil {
				return fmt.Errorf(
					"error updating the hash of %s: \"%s\"; this is likely a bug", n.Name, err)
			}
		}
		cksum := sha256.Sum256(buf.Bytes())
		n.Hash = cksum[:]
	} else if n.FileNode != nil {
		var buf bytes.Buffer
		for _, blockRef := range n.FileNode.BlockRefs {
			if _, err := buf.WriteString(fmt.Sprintf("%s:%d:%d", blockRef.Block.Hash, blockRef.Range.Lower, blockRef.Range.Upper)); err != nil {
				return fmt.Errorf(
					"error updating the hash of %s: \"%s\"; this is likely a bug", n.Name, err)
			}
		}
	} else {
		return fmt.Errorf("malformed node %s: it's neither a file nor a directory", n.Name)
	}
	return nil
}

// GlobFile returns a list of nodes that match the given glob pattern
func (h *HashTree) GlobFile(pattern string) ([]*Node, error) {
	// "*" should be an allowed pattern, but our paths always start with "/", so
	// modify the pattern to fit our path structure.
	pattern = cleanPath(pattern)

	var res []*Node
	for p, node := range h.Fs {
		matched, err := pathlib.Match(pattern, p)
		if err != nil {
			if err == pathlib.ErrBadPattern {
				return nil, MalformedGlobErr
			}
			return nil, err
		}
		if matched {
			res = append(res, node)
		}
	}
	return res, nil
}

// ListFile returns the Nodes corresponding to the files and directories under 'path'
func (h *HashTree) ListFile(path string) ([]*Node, error) {
	path = cleanPath(path)
	node, ok := h.Fs[path]
	if !ok {
		return nil, PathNotFoundErr
	}
	d := node.DirNode
	if d == nil {
		return nil, fmt.Errorf("the file at %s is not a directory", path)
	}
	var result []*Node
	for _, childName := range d.Children {
		childPath := pathlib.Join(path, childName)
		child, ok := h.Fs[pathlib.Join(path, childPath)]
		if !ok {
			return nil, fmt.Errorf("malformed hash tree; the node %s is expected to exist but is not found; this is likely a bug", childPath)
		}
		result = append(result, child)
	}
	return result, nil
}

// PutFile inserts a file into the hierarchy
func (h *HashTree) PutFile(path string, blockRefs []*pfs.BlockRef) error {
	path = cleanPath(path)

	// Update/create the file node
	node, ok := h.Fs[path]
	if !ok {
		name := pathlib.Base(path)
		node = &Node{
			Name: name,
			FileNode: &FileNode{
				BlockRefs: blockRefs,
			},
		}
		h.Fs[path] = node
	} else {
		node.FileNode.BlockRefs = append(node.FileNode.BlockRefs)
	}
	if err := node.UpdateHash(); err != nil {
		return err
	}

	// Update/create parent directory nodes
	for path != "/" {
		dir, child := pathlib.Split(path)
		node, ok := h.Fs[dir]
		if !ok {
			node = &Node{
				Name: pathlib.Base(dir),
				DirNode: &DirectoryNode{
					Children: []string{child},
				},
			}
			h.Fs[dir] = node
		} else {
			node.DirNode.Children = append(node.DirNode.Children, child)
		}
		if err := node.UpdateHash(); err != nil {
			return err
		}
		path = cleanPath(dir)
	}
	return nil
}
<commit_msg>Fix PutFile<commit_after>package hashtree

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	pathlib "path"
	"sort"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pfs"
)

type Names []string

func (n Names) Len() int {
	return len(n)
}

func (n Names) Less(i, j int) bool {
	return n[i] < n[j]
}

func (n Names) Swap(i, j int) {
	tmp := n[i]
	n[i] = n[j]
	n[j] = tmp
}

// cleanPath converts a path into a form that we use internally
// Basically we make sure that it has a leading slash and no trailing slash.
func cleanPath(path string) string {
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	return pathlib.Clean(path)
}

// UpdateHash uses the node's internal state to update the hash
// of the node
func (n *Node) UpdateHash() error {
	if n.DirNode != nil {
		sort.Sort(Names(n.DirNode.Children))
		var buf bytes.Buffer
		for _, child := range n.DirNode.Children {
			if _, err := buf.WriteString(child); err != nil {
				return fmt.Errorf(
					"error updating the hash of %s: \"%s\"; this is likely a bug", n.Name, err)
			}
		}
		cksum := sha256.Sum256(buf.Bytes())
		n.Hash = cksum[:]
	} else if n.FileNode != nil {
		var buf bytes.Buffer
		for _, blockRef := range n.FileNode.BlockRefs {
			if _, err := buf.WriteString(fmt.Sprintf("%s:%d:%d", blockRef.Block.Hash, blockRef.Range.Lower, blockRef.Range.Upper)); err != nil {
				return fmt.Errorf(
					"error updating the hash of %s: \"%s\"; this is likely a bug", n.Name, err)
			}
		}
	} else {
		return fmt.Errorf("malformed node %s: it's neither a file nor a directory", n.Name)
	}
	return nil
}

// GlobFile returns a list of nodes that match the given glob pattern
func (h *HashTree) GlobFile(pattern string) ([]*Node, error) {
	// "*" should be an allowed pattern, but our paths always start with "/", so
	// modify the pattern to fit our path structure.
	pattern = cleanPath(pattern)

	var res []*Node
	for p, node := range h.Fs {
		matched, err := pathlib.Match(pattern, p)
		if err != nil {
			if err == pathlib.ErrBadPattern {
				return nil, MalformedGlobErr
			}
			return nil, err
		}
		if matched {
			res = append(res, node)
		}
	}
	return res, nil
}

// ListFile returns the Nodes corresponding to the files and directories under 'path'
func (h *HashTree) ListFile(path string) ([]*Node, error) {
	path = cleanPath(path)
	node, ok := h.Fs[path]
	if !ok {
		return nil, PathNotFoundErr
	}
	d := node.DirNode
	if d == nil {
		return nil, fmt.Errorf("the file at %s is not a directory", path)
	}
	var result []*Node
	for _, childName := range d.Children {
		childPath := pathlib.Join(path, childName)
		child, ok := h.Fs[pathlib.Join(path, childPath)]
		if !ok {
			return nil, fmt.Errorf("malformed hash tree; the node %s is expected to exist but is not found; this is likely a bug", childPath)
		}
		result = append(result, child)
	}
	return result, nil
}

// PutFile inserts a file into the hierarchy
func (h *HashTree) PutFile(path string, blockRefs []*pfs.BlockRef) error {
	path = cleanPath(path)

	// Update/create the file node
	node, ok := h.Fs[path]
	if !ok {
		name := pathlib.Base(path)
		node = &Node{
			Name: name,
			FileNode: &FileNode{
				BlockRefs: blockRefs,
			},
		}
		h.Fs[path] = node
	} else {
		node.FileNode.BlockRefs = append(node.FileNode.BlockRefs, blockRefs...)
	}
	if err := node.UpdateHash(); err != nil {
		return err
	}

	// Update/create parent directory nodes
	for path != "/" {
		dir, child := pathlib.Split(path)
		// by default, Split() includes the trailing slash
		dir = cleanPath(dir)
		node, ok := h.Fs[dir]
		if !ok {
			node = &Node{
				Name: pathlib.Base(dir),
				DirNode: &DirectoryNode{
					Children: []string{child},
				},
			}
			h.Fs[dir] = node
		} else {
			node.DirNode.Children = append(node.DirNode.Children, child)
		}
		if err := node.UpdateHash(); err != nil {
			return err
		}
		path = dir
	}
	return nil
}
<|endoftext|>


<commit_before>package gobrake_test

import (
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/airbrake/gobrake"
)

func BenchmarkSendNotice(b *testing.B) {
	handler := func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(http.StatusCreated)
		w.Write([]byte(`{"id":"123"}`))
	}
	server := httptest.NewServer(http.HandlerFunc(handler))

	notifier := gobrake.NewNotifierWithOptions(&gobrake.NotifierOptions{
		ProjectId:  1,
		ProjectKey: "key",
		Host:       server.URL,
	})

	notice := notifier.Notice(errors.New("benchmark"), nil, 0)

	b.ResetTimer()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			id, err := notifier.SendNotice(notice)
			if err != nil {
				b.Fatal(err)
			}
			if id != "123" {
				b.Fatalf("got %q, wanted 123", id)
			}
		}
	})
}

func BenchmarkNotifyRequest(b *testing.B) {
	notifier := gobrake.NewNotifierWithOptions(&gobrake.NotifierOptions{
		ProjectId:  1,
		ProjectKey: "",
	})

	const n = 100
	reqs := make([]*gobrake.RouteTrace, n)
	for i := 0; i < n; i++ {
		reqs[i] = &gobrake.RouteTrace{
			Method:     "GET",
			Route:      fmt.Sprintf("/api/v4/groups/%d", i),
			StatusCode: 200,
		}
	}

	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		var i int
		for pb.Next() {
			err := notifier.Routes.Notify(nil, reqs[i%n])
			if err != nil {
				b.Fatal(err)
			}
			i++
		}
	})
}
<commit_msg>Fix benchmark<commit_after>package gobrake_test

import (
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/airbrake/gobrake"
)

func BenchmarkSendNotice(b *testing.B) {
	handler := func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(http.StatusCreated)
		w.Write([]byte(`{"id":"123"}`))
	}
	server := httptest.NewServer(http.HandlerFunc(handler))

	notifier := gobrake.NewNotifierWithOptions(&gobrake.NotifierOptions{
		ProjectId:  1,
		ProjectKey: "key",
		Host:       server.URL,
	})

	b.ResetTimer()

	b.RunParallel(func(pb *testing.PB) {
		notice := notifier.Notice(errors.New("benchmark"), nil, 0)
		for pb.Next() {
			id, err := notifier.SendNotice(notice)
			if err != nil {
				b.Fatal(err)
			}
			if id != "123" {
				b.Fatalf("got %q, wanted 123", id)
			}
		}
	})
}

func BenchmarkNotifyRequest(b *testing.B) {
	notifier := gobrake.NewNotifierWithOptions(&gobrake.NotifierOptions{
		ProjectId:  1,
		ProjectKey: "",
	})

	const n = 100
	reqs := make([]*gobrake.RouteTrace, n)
	for i := 0; i < n; i++ {
		reqs[i] = &gobrake.RouteTrace{
			Method:     "GET",
			Route:      fmt.Sprintf("/api/v4/groups/%d", i),
			StatusCode: 200,
		}
	}

	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		var i int
		for pb.Next() {
			err := notifier.Routes.Notify(nil, reqs[i%n])
			if err != nil {
				b.Fatal(err)
			}
			i++
		}
	})
}
<|endoftext|>


<commit_before>package workload

import (
	"fmt"
	"io"
	"math/rand"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
)

// RunWorkload runs a test workload against a Pachyderm cluster.
func RunWorkload(
	client *client.APIClient,
	rand *rand.Rand,
	size int,
) error {
	worker := newWorker(rand)
	for i := 0; i < size; i++ {
		if err := worker.work(client); err != nil {
			return err
		}
	}
	for _, job := range worker.startedJobs {
		jobInfo, err := client.InspectJob(job.ID, true)
		if err != nil {
			return err
		}
		if jobInfo.State != ppsclient.JobState_JOB_SUCCESS {
			return fmt.Errorf("job %s failed", job.ID)
		}
	}
	return nil
}

type worker struct {
	repos       []*pfs.Repo
	finished    []*pfs.Commit
	started     []*pfs.Commit
	files       []*pfs.File
	startedJobs []*ppsclient.Job
	jobs        []*ppsclient.Job
	pipelines   []*ppsclient.Pipeline
	rand        *rand.Rand
}

func newWorker(rand *rand.Rand) *worker {
	return &worker{
		rand: rand,
	}
}

const (
	repo     float64 = .02
	commit           = .1
	file             = .9
	job              = .98
	pipeline         = 1.0
)

const maxStartedCommits = 6
const maxStartedJobs = 6

func (w *worker) work(c *client.APIClient) error {
	opt := w.rand.Float64()
	switch {
	case opt < repo:
		repoName := w.randString(10)
		if err := c.CreateRepo(repoName); err != nil {
			return err
		}
		w.repos = append(w.repos, &pfs.Repo{Name: repoName})
		commit, err := c.StartCommit(repoName, "", "")
		if err != nil {
			return err
		}
		w.started = append(w.started, commit)
	case opt < commit:
		if len(w.started) >= maxStartedCommits || len(w.finished) == 0 {
			if len(w.started) == 0 {
				return nil
			}
			i := w.rand.Intn(len(w.started))
			commit := w.started[i]
			// before we finish a commit we add a file, this assures that there
			// won't be any empty commits which will later crash jobs
			if _, err := c.PutFile(commit.Repo.Name, commit.ID, w.randString(10), w.reader()); err != nil {
				return err
			}
			if err := c.FinishCommit(commit.Repo.Name, commit.ID); err != nil {
				return err
			}
			w.started = append(w.started[:i], w.started[i+1:]...)
			w.finished = append(w.finished, commit)
		} else {
			if len(w.finished) == 0 {
				return nil
			}
			commit := w.finished[w.rand.Intn(len(w.finished))]
			commit, err := c.StartCommit(commit.Repo.Name, commit.ID, uuid.NewWithoutDashes())
			if err != nil {
				return err
			}
			w.started = append(w.started, commit)
		}
	case opt < file:
		if len(w.started) == 0 {
			return nil
		}
		commit := w.started[w.rand.Intn(len(w.started))]
		if _, err := c.PutFile(commit.Repo.Name, commit.ID, w.randString(10), w.reader()); err != nil {
			return err
		}
	case opt < job:
		if len(w.startedJobs) >= maxStartedJobs {
			job := w.startedJobs[0]
			w.startedJobs = w.startedJobs[1:]
			jobInfo, err := c.InspectJob(job.ID, true)
			if err != nil {
				return err
			}
			if jobInfo.State != ppsclient.JobState_JOB_SUCCESS {
				return fmt.Errorf("job %s failed", job.ID)
			}
			w.jobs = append(w.jobs, job)
		} else {
			if len(w.finished) == 0 {
				return nil
			}
			inputs := [5]string{}
			var jobInputs []*ppsclient.JobInput
			repoSet := make(map[string]bool)
			for i := range inputs {
				commit := w.finished[w.rand.Intn(len(w.finished))]
				if _, ok := repoSet[commit.Repo.Name]; ok {
					continue
				}
				repoSet[commit.Repo.Name] = true
				inputs[i] = commit.Repo.Name
				jobInputs = append(jobInputs, &ppsclient.JobInput{Commit: commit})
			}
			outFilename := w.randString(10)
			job, err := c.CreateJob(
				"",
				[]string{"bash"},
				w.grepCmd(inputs, outFilename),
				1,
				jobInputs,
				"",
			)
			if err != nil {
				return err
			}
			w.startedJobs = append(w.startedJobs, job)
		}
	case opt < pipeline:
		if len(w.repos) == 0 {
			return nil
		}
		inputs := [5]string{}
		var pipelineInputs []*ppsclient.PipelineInput
		repoSet := make(map[string]bool)
		for i := range inputs {
			repo := w.repos[w.rand.Intn(len(w.repos))]
			if _, ok := repoSet[repo.Name]; ok {
				continue
			}
			repoSet[repo.Name] = true
			inputs[i] = repo.Name
			pipelineInputs = append(pipelineInputs, &ppsclient.PipelineInput{Repo: repo})
		}
		pipelineName := w.randString(10)
		outFilename := w.randString(10)
		if err := c.CreatePipeline(
			pipelineName,
			"",
			[]string{"bash"},
			w.grepCmd(inputs, outFilename),
			1,
			pipelineInputs,
			false,
		); err != nil {
			return err
		}
		w.pipelines = append(w.pipelines, client.NewPipeline(pipelineName))
	}
	return nil
}

const letters = "abcdefghijklmnopqrstuvwxyz"
const lettersAndSpaces = "abcdefghijklmnopqrstuvwxyz      "

func (w *worker) randString(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[w.rand.Intn(len(letters))]
	}
	return string(b)
}

type reader struct {
	rand  *rand.Rand
	bytes int
}

// NewReader returns a Reader which generates strings of characters.
func NewReader(rand *rand.Rand, bytes int) io.Reader {
	return &reader{
		rand:  rand,
		bytes: bytes,
	}
}

func (r *reader) Read(p []byte) (int, error) {
	for i := range p {
		if i > r.bytes {
			return r.bytes, io.EOF
		}
		if i%128 == 127 {
			p[i] = '\n'
		} else {
			p[i] = lettersAndSpaces[r.rand.Intn(len(lettersAndSpaces))]
		}
	}
	p[len(p)-1] = '\n'
	r.bytes -= len(p)
	if r.bytes <= 0 {
		return len(p), io.EOF
	}
	return len(p), nil
}

func (w *worker) reader() io.Reader {
	return NewReader(w.rand, 1000)
}

func (w *worker) grepCmd(inputs [5]string, outFilename string) []string {
	return []string{
		fmt.Sprintf(
			"grep %s /pfs/{%s,%s,%s,%s,%s}/* >/pfs/out/%s; true",
			w.randString(4),
			inputs[0],
			inputs[1],
			inputs[2],
			inputs[3],
			inputs[4],
			outFilename,
		),
	}
}
<commit_msg>Fix workload.NewReader; it wasn't take the given size into account<commit_after>package workload

import (
	"fmt"
	"io"
	"math/rand"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
)

// RunWorkload runs a test workload against a Pachyderm cluster.
func RunWorkload(
	client *client.APIClient,
	rand *rand.Rand,
	size int,
) error {
	worker := newWorker(rand)
	for i := 0; i < size; i++ {
		if err := worker.work(client); err != nil {
			return err
		}
	}
	for _, job := range worker.startedJobs {
		jobInfo, err := client.InspectJob(job.ID, true)
		if err != nil {
			return err
		}
		if jobInfo.State != ppsclient.JobState_JOB_SUCCESS {
			return fmt.Errorf("job %s failed", job.ID)
		}
	}
	return nil
}

type worker struct {
	repos       []*pfs.Repo
	finished    []*pfs.Commit
	started     []*pfs.Commit
	files       []*pfs.File
	startedJobs []*ppsclient.Job
	jobs        []*ppsclient.Job
	pipelines   []*ppsclient.Pipeline
	rand        *rand.Rand
}

func newWorker(rand *rand.Rand) *worker {
	return &worker{
		rand: rand,
	}
}

const (
	repo     float64 = .02
	commit           = .1
	file             = .9
	job              = .98
	pipeline         = 1.0
)

const maxStartedCommits = 6
const maxStartedJobs = 6

func (w *worker) work(c *client.APIClient) error {
	opt := w.rand.Float64()
	switch {
	case opt < repo:
		repoName := w.randString(10)
		if err := c.CreateRepo(repoName); err != nil {
			return err
		}
		w.repos = append(w.repos, &pfs.Repo{Name: repoName})
		commit, err := c.StartCommit(repoName, "", "")
		if err != nil {
			return err
		}
		w.started = append(w.started, commit)
	case opt < commit:
		if len(w.started) >= maxStartedCommits || len(w.finished) == 0 {
			if len(w.started) == 0 {
				return nil
			}
			i := w.rand.Intn(len(w.started))
			commit := w.started[i]
			// before we finish a commit we add a file, this assures that there
			// won't be any empty commits which will later crash jobs
			if _, err := c.PutFile(commit.Repo.Name, commit.ID, w.randString(10), w.reader()); err != nil {
				return err
			}
			if err := c.FinishCommit(commit.Repo.Name, commit.ID); err != nil {
				return err
			}
			w.started = append(w.started[:i], w.started[i+1:]...)
			w.finished = append(w.finished, commit)
		} else {
			if len(w.finished) == 0 {
				return nil
			}
			commit := w.finished[w.rand.Intn(len(w.finished))]
			commit, err := c.StartCommit(commit.Repo.Name, commit.ID, uuid.NewWithoutDashes())
			if err != nil {
				return err
			}
			w.started = append(w.started, commit)
		}
	case opt < file:
		if len(w.started) == 0 {
			return nil
		}
		commit := w.started[w.rand.Intn(len(w.started))]
		if _, err := c.PutFile(commit.Repo.Name, commit.ID, w.randString(10), w.reader()); err != nil {
			return err
		}
	case opt < job:
		if len(w.startedJobs) >= maxStartedJobs {
			job := w.startedJobs[0]
			w.startedJobs = w.startedJobs[1:]
			jobInfo, err := c.InspectJob(job.ID, true)
			if err != nil {
				return err
			}
			if jobInfo.State != ppsclient.JobState_JOB_SUCCESS {
				return fmt.Errorf("job %s failed", job.ID)
			}
			w.jobs = append(w.jobs, job)
		} else {
			if len(w.finished) == 0 {
				return nil
			}
			inputs := [5]string{}
			var jobInputs []*ppsclient.JobInput
			repoSet := make(map[string]bool)
			for i := range inputs {
				commit := w.finished[w.rand.Intn(len(w.finished))]
				if _, ok := repoSet[commit.Repo.Name]; ok {
					continue
				}
				repoSet[commit.Repo.Name] = true
				inputs[i] = commit.Repo.Name
				jobInputs = append(jobInputs, &ppsclient.JobInput{Commit: commit})
			}
			outFilename := w.randString(10)
			job, err := c.CreateJob(
				"",
				[]string{"bash"},
				w.grepCmd(inputs, outFilename),
				1,
				jobInputs,
				"",
			)
			if err != nil {
				return err
			}
			w.startedJobs = append(w.startedJobs, job)
		}
	case opt < pipeline:
		if len(w.repos) == 0 {
			return nil
		}
		inputs := [5]string{}
		var pipelineInputs []*ppsclient.PipelineInput
		repoSet := make(map[string]bool)
		for i := range inputs {
			repo := w.repos[w.rand.Intn(len(w.repos))]
			if _, ok := repoSet[repo.Name]; ok {
				continue
			}
			repoSet[repo.Name] = true
			inputs[i] = repo.Name
			pipelineInputs = append(pipelineInputs, &ppsclient.PipelineInput{Repo: repo})
		}
		pipelineName := w.randString(10)
		outFilename := w.randString(10)
		if err := c.CreatePipeline(
			pipelineName,
			"",
			[]string{"bash"},
			w.grepCmd(inputs, outFilename),
			1,
			pipelineInputs,
			false,
		); err != nil {
			return err
		}
		w.pipelines = append(w.pipelines, client.NewPipeline(pipelineName))
	}
	return nil
}

const letters = "abcdefghijklmnopqrstuvwxyz"
const lettersAndSpaces = "abcdefghijklmnopqrstuvwxyz      "

func (w *worker) randString(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[w.rand.Intn(len(letters))]
	}
	return string(b)
}

type reader struct {
	rand      *rand.Rand
	bytes     int
	bytesRead int
}

// NewReader returns a Reader which generates strings of characters.
func NewReader(rand *rand.Rand, bytes int) io.Reader {
	return &reader{
		rand:  rand,
		bytes: bytes,
	}
}

func (r *reader) Read(p []byte) (int, error) {
	var bytesReadThisTime int
	for i := range p {
		if r.bytesRead+bytesReadThisTime == r.bytes {
			break
		}
		if i%128 == 127 {
			p[i] = '\n'
		} else {
			p[i] = lettersAndSpaces[r.rand.Intn(len(lettersAndSpaces))]
		}
		bytesReadThisTime++
	}
	r.bytesRead += bytesReadThisTime
	if r.bytesRead == r.bytes {
		return bytesReadThisTime, io.EOF
	}
	return bytesReadThisTime, nil
}

func (w *worker) reader() io.Reader {
	return NewReader(w.rand, 1000)
}

func (w *worker) grepCmd(inputs [5]string, outFilename string) []string {
	return []string{
		fmt.Sprintf(
			"grep %s /pfs/{%s,%s,%s,%s,%s}/* >/pfs/out/%s; true",
			w.randString(4),
			inputs[0],
			inputs[1],
			inputs[2],
			inputs[3],
			inputs[4],
			outFilename,
		),
	}
}
<|endoftext|>


<commit_before>package server

import (
	"context"
	"encoding/json"
	"os"
	"strings"
	"sync"
	"testing"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pps"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
	"gopkg.in/pachyderm/yaml.v3"
)

var pachClient *client.APIClient
var getPachClientOnce sync.Once

func getPachClient(t testing.TB) *client.APIClient {
	getPachClientOnce.Do(func() {
		var err error
		if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
			pachClient, err = client.NewInCluster()
		} else {
			pachClient, err = client.NewForTest()
		}
		require.NoError(t, err)
	})
	return pachClient
}

func YAMLToJSONString(t *testing.T, yamlStr string) string {
	holder := make(map[string]interface{})
	d := yaml.NewDecoder(strings.NewReader(yamlStr))
	if err := d.Decode(&holder); err != nil {
		t.Fatalf("error parsing TFJob: %v", err)
	}
	result, err := json.Marshal(holder)
	if err != nil {
		t.Fatalf("error marshalling TFJob to JSON: %v", err)
	}
	return string(result)
}

func TestTFJobBasic(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	dataRepo := tu.UniqueString("TestSimplePipeline_data")
	require.NoError(t, c.CreateRepo(dataRepo))

	pipeline := tu.UniqueString("pipeline1")
	tfJobString := YAMLToJSONString(t, `
    apiVersion: kubeflow.org/v1
    kind: TFJob
    metadata:
      generateName: tfjob
      namespace: kubeflow
    spec:
      tfReplicaSpecs:
        PS:
          replicas: 1
          restartPolicy: OnFailure
          template:
            spec:
              containers:
              - name: tensorflow
                image: gcr.io/your-project/your-image
                command:
                  - python
                  - -m
                  - trainer.task
                  - --batch_size=32
                  - --training_steps=1000
        Worker:
          replicas: 3
          restartPolicy: OnFailure
          template:
            spec:
              containers:
              - name: tensorflow
                image: gcr.io/your-project/your-image
                command:
                  - python
                  - -m
                  - trainer.task
                  - --batch_size=32
                  - --training_steps=1000
    `)
	_, err := c.PpsAPIClient.CreatePipeline(
		context.Background(),
		&pps.CreatePipelineRequest{
			Pipeline: client.NewPipeline(pipeline),
			Input:    client.NewPFSInput(dataRepo, "/*"),
			TFJob:    &pps.TFJob{TFJob: tfJobString},
		})
	require.YesError(t, err)
	require.Matches(t, "not supported yet", err.Error())
}
<commit_msg>Use serde in tfjob_test<commit_after>package server

import (
	"context"
	"encoding/json"
	"os"
	"sync"
	"testing"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/serde"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
)

var pachClient *client.APIClient
var getPachClientOnce sync.Once

func getPachClient(t testing.TB) *client.APIClient {
	getPachClientOnce.Do(func() {
		var err error
		if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
			pachClient, err = client.NewInCluster()
		} else {
			pachClient, err = client.NewForTest()
		}
		require.NoError(t, err)
	})
	return pachClient
}

func YAMLToJSONString(t *testing.T, yamlStr string) string {
	holder := make(map[string]interface{})
	if err := serde.DecodeYAML([]byte(yamlStr), &holder); err != nil {
		t.Fatalf("error parsing TFJob: %v", err)
	}
	result, err := json.Marshal(holder)
	if err != nil {
		t.Fatalf("error marshalling TFJob to JSON: %v", err)
	}
	return string(result)
}

func TestTFJobBasic(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	dataRepo := tu.UniqueString("TestSimplePipeline_data")
	require.NoError(t, c.CreateRepo(dataRepo))

	pipeline := tu.UniqueString("pipeline1")
	tfJobString := YAMLToJSONString(t, `
    apiVersion: kubeflow.org/v1
    kind: TFJob
    metadata:
      generateName: tfjob
      namespace: kubeflow
    spec:
      tfReplicaSpecs:
        PS:
          replicas: 1
          restartPolicy: OnFailure
          template:
            spec:
              containers:
              - name: tensorflow
                image: gcr.io/your-project/your-image
                command:
                  - python
                  - -m
                  - trainer.task
                  - --batch_size=32
                  - --training_steps=1000
        Worker:
          replicas: 3
          restartPolicy: OnFailure
          template:
            spec:
              containers:
              - name: tensorflow
                image: gcr.io/your-project/your-image
                command:
                  - python
                  - -m
                  - trainer.task
                  - --batch_size=32
                  - --training_steps=1000
    `)
	_, err := c.PpsAPIClient.CreatePipeline(
		context.Background(),
		&pps.CreatePipelineRequest{
			Pipeline: client.NewPipeline(pipeline),
			Input:    client.NewPFSInput(dataRepo, "/*"),
			TFJob:    &pps.TFJob{TFJob: tfJobString},
		})
	require.YesError(t, err)
	require.Matches(t, "not supported yet", err.Error())
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestBadRequest(t *testing.T) {
	var tests = []struct {
		err  error
		want string
	}{
		{
			fmt.Errorf(`BadRequest`),
			`BadRequest
`,
		},
	}

	for _, test := range tests {
		writer := httptest.NewRecorder()
		badRequest(writer, test.err)

		if result := writer.Result().StatusCode; result != http.StatusBadRequest {
			t.Errorf(`badRequest(%v) = %v, want %v`, test.err, result, http.StatusBadRequest)
		}

		if result, _ := readBody(writer.Result().Body); string(result) != string(test.want) {
			t.Errorf(`badRequest(%v) = %v, want %v`, test.err, string(result), string(test.want))
		}
	}
}

func TestUnauthorized(t *testing.T) {
	var tests = []struct {
		err  error
		want string
	}{
		{
			fmt.Errorf(`Unauthorized`),
			`Unauthorized
`,
		},
	}

	for _, test := range tests {
		writer := httptest.NewRecorder()
		unauthorized(writer, test.err)

		if result := writer.Result().StatusCode; result != http.StatusUnauthorized {
			t.Errorf(`badRequest(%v) = %v, want %v`, test.err, result, http.StatusUnauthorized)
		}

		if result, _ := readBody(writer.Result().Body); string(result) != string(test.want) {
			t.Errorf(`unauthorized(%v) = %v, want %v`, test.err, string(result), string(test.want))
		}
	}
}

func TestForbidden(t *testing.T) {
	var tests = []struct {
	}{
		{},
	}

	for range tests {
		writer := httptest.NewRecorder()
		forbidden(writer)

		if result := writer.Result().StatusCode; result != http.StatusForbidden {
			t.Errorf(`forbidden() = %v, want %v`, result, http.StatusForbidden)
		}
	}
}
<commit_msg>Adding test for errorHandler<commit_after>package docker

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestBadRequest(t *testing.T) {
	var tests = []struct {
		err  error
		want string
	}{
		{
			fmt.Errorf(`BadRequest`),
			`BadRequest
`,
		},
	}

	for _, test := range tests {
		writer := httptest.NewRecorder()
		badRequest(writer, test.err)

		if result := writer.Result().StatusCode; result != http.StatusBadRequest {
			t.Errorf(`badRequest(%v) = %v, want %v`, test.err, result, http.StatusBadRequest)
		}

		if result, _ := readBody(writer.Result().Body); string(result) != string(test.want) {
			t.Errorf(`badRequest(%v) = %v, want %v`, test.err, string(result), string(test.want))
		}
	}
}

func TestUnauthorized(t *testing.T) {
	var tests = []struct {
		err  error
		want string
	}{
		{
			fmt.Errorf(`Unauthorized`),
			`Unauthorized
`,
		},
	}

	for _, test := range tests {
		writer := httptest.NewRecorder()
		unauthorized(writer, test.err)

		if result := writer.Result().StatusCode; result != http.StatusUnauthorized {
			t.Errorf(`badRequest(%v) = %v, want %v`, test.err, result, http.StatusUnauthorized)
		}

		if result, _ := readBody(writer.Result().Body); string(result) != string(test.want) {
			t.Errorf(`unauthorized(%v) = %v, want %v`, test.err, string(result), string(test.want))
		}
	}
}

func TestForbidden(t *testing.T) {
	var tests = []struct {
	}{
		{},
	}

	for range tests {
		writer := httptest.NewRecorder()
		forbidden(writer)

		if result := writer.Result().StatusCode; result != http.StatusForbidden {
			t.Errorf(`forbidden() = %v, want %v`, result, http.StatusForbidden)
		}
	}
}

func TestErrorHandler(t *testing.T) {
	var tests = []struct {
		err  error
		want string
	}{
		{
			fmt.Errorf(`Internal server error`),
			`Internal server error
`,
		},
	}

	for _, test := range tests {
		writer := httptest.NewRecorder()
		errorHandler(writer, test.err)

		if result := writer.Result().StatusCode; result != http.StatusInternalServerError {
			t.Errorf(`errorHandler(%v) = %v, want %v`, test.err, result, http.StatusInternalServerError)
		}
	}
}
<|endoftext|>


<commit_before>package repository

import (
	"github.com/couchbase/gometa/common"
	fdb "github.com/couchbaselabs/goforestdb"
	"log"
	"sync"
)

/////////////////////////////////////////////////////////////////////////////
// Repository
/////////////////////////////////////////////////////////////////////////////

type Repository struct {
	db    *fdb.Database
	mutex sync.Mutex
}

type RepoIterator struct {
	iter *fdb.Iterator
}

/////////////////////////////////////////////////////////////////////////////
// Repository Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Open a repository
//
func OpenRepository() (*Repository, error) {
	return OpenRepositoryWithName(common.REPOSITORY_NAME)
}

func OpenRepositoryWithName(name string) (*Repository, error) {

	config := fdb.DefaultConfig()
	config.SetBufferCacheSize(1024*1024)
	db, err := fdb.Open(name, config)
	if err != nil {
		return nil, err
	}

	repo := &Repository{db: db}
	return repo, nil
}

//
// Update/Insert into the repository
//
func (r *Repository) Set(key string, content []byte) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	log.Printf("Repo.Set(): key %s, len(content) %d", key, len(content))

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	// set value
	err = r.db.SetKV(k, content)
	if err != nil {
		return err
	}

	return r.db.Commit(fdb.COMMIT_NORMAL)
}

//
// Retrieve from repository
//
func (r *Repository) Get(key string) ([]byte, error) {

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return nil, err
	}

	value, err := r.db.GetKV(k)
	log.Printf("Repo.Get(): key %s, found=%s", key, err == nil)
	return value, err
}

//
// Delete from repository
//
func (r *Repository) Delete(key string) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	err = r.db.DeleteKV(k)
	if err != nil {
		return err
	}

	return r.db.Commit(fdb.COMMIT_NORMAL)
}

//
// Close repository.
//
func (r *Repository) Close() {
	// TODO: Does it need mutex?
	if r.db != nil {
		r.db.Close()
		r.db = nil
	}
}

/////////////////////////////////////////////////////////////////////////////
// RepoIterator Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new iterator.  EndKey is inclusive.
//
func (r *Repository) NewIterator(startKey, endKey string) (*RepoIterator, error) {
	// TODO: Check if fdb is closed.

	k1, err := CollateString(startKey)
	if err != nil {
		return nil, err
	}

	k2, err := CollateString(endKey)
	if err != nil {
		return nil, err
	}

	iter, err := r.db.IteratorInit(k1, k2, fdb.ITR_NONE)
	if err != nil {
		return nil, err
	}
	result := &RepoIterator{iter: iter}
	return result, nil
}

// Get value from iterator
func (i *RepoIterator) Next() (key string, content []byte, err error) {

	// TODO: Check if fdb and iterator is closed
	doc, err := i.iter.Next()
	if err != nil {
		return "", nil, err
	}

	key = DecodeString(doc.Key())
	body := doc.Body()

	return key, body, nil
}

// close iterator
func (i *RepoIterator) Close() {
	// TODO: Check if fdb iterator is closed
	i.iter.Close()
}

// This only support ascii.
func CollateString(key string) ([]byte, error) {
	if key == "" {
		return nil, nil
	}

	return ([]byte)(key), nil
}

func DecodeString(data []byte) string {
	return string(data)
}
<commit_msg>Run gofmt on repository/repo.go.<commit_after>package repository

import (
	"github.com/couchbase/gometa/common"
	fdb "github.com/couchbaselabs/goforestdb"
	"log"
	"sync"
)

/////////////////////////////////////////////////////////////////////////////
// Repository
/////////////////////////////////////////////////////////////////////////////

type Repository struct {
	db    *fdb.Database
	mutex sync.Mutex
}

type RepoIterator struct {
	iter *fdb.Iterator
}

/////////////////////////////////////////////////////////////////////////////
// Repository Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Open a repository
//
func OpenRepository() (*Repository, error) {
	return OpenRepositoryWithName(common.REPOSITORY_NAME)
}

func OpenRepositoryWithName(name string) (*Repository, error) {

	config := fdb.DefaultConfig()
	config.SetBufferCacheSize(1024 * 1024)
	db, err := fdb.Open(name, config)
	if err != nil {
		return nil, err
	}

	repo := &Repository{db: db}
	return repo, nil
}

//
// Update/Insert into the repository
//
func (r *Repository) Set(key string, content []byte) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	log.Printf("Repo.Set(): key %s, len(content) %d", key, len(content))

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	// set value
	err = r.db.SetKV(k, content)
	if err != nil {
		return err
	}

	return r.db.Commit(fdb.COMMIT_NORMAL)
}

//
// Retrieve from repository
//
func (r *Repository) Get(key string) ([]byte, error) {

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return nil, err
	}

	value, err := r.db.GetKV(k)
	log.Printf("Repo.Get(): key %s, found=%s", key, err == nil)
	return value, err
}

//
// Delete from repository
//
func (r *Repository) Delete(key string) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	err = r.db.DeleteKV(k)
	if err != nil {
		return err
	}

	return r.db.Commit(fdb.COMMIT_NORMAL)
}

//
// Close repository.
//
func (r *Repository) Close() {
	// TODO: Does it need mutex?
	if r.db != nil {
		r.db.Close()
		r.db = nil
	}
}

/////////////////////////////////////////////////////////////////////////////
// RepoIterator Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new iterator.  EndKey is inclusive.
//
func (r *Repository) NewIterator(startKey, endKey string) (*RepoIterator, error) {
	// TODO: Check if fdb is closed.

	k1, err := CollateString(startKey)
	if err != nil {
		return nil, err
	}

	k2, err := CollateString(endKey)
	if err != nil {
		return nil, err
	}

	iter, err := r.db.IteratorInit(k1, k2, fdb.ITR_NONE)
	if err != nil {
		return nil, err
	}
	result := &RepoIterator{iter: iter}
	return result, nil
}

// Get value from iterator
func (i *RepoIterator) Next() (key string, content []byte, err error) {

	// TODO: Check if fdb and iterator is closed
	doc, err := i.iter.Next()
	if err != nil {
		return "", nil, err
	}

	key = DecodeString(doc.Key())
	body := doc.Body()

	return key, body, nil
}

// close iterator
func (i *RepoIterator) Close() {
	// TODO: Check if fdb iterator is closed
	i.iter.Close()
}

// This only support ascii.
func CollateString(key string) ([]byte, error) {
	if key == "" {
		return nil, nil
	}

	return ([]byte)(key), nil
}

func DecodeString(data []byte) string {
	return string(data)
}
<|endoftext|>


<commit_before>package dragon

import (
	"errors"
	"os"

	"golang.org/x/sync/errgroup"
)

func Imports() error {
	if !existGoImports() {
		return errors.New("goimports command isn't installed.")
	}

	libChan := make(chan lib, 1000)
	done := make(chan error)
	go func() {
		done <- updateZstdlib(libChan)
	}()

	eg := &errgroup.Group{}
	eg.Go(func() error {
		return stdLibs(libChan)
	})
	eg.Go(func() error {
		return gopathLibs(libChan)
	})
	err := eg.Wait()
	if err != nil {
		return err
	}
	close(libChan)

	err = <-done
	if err != nil {
		return err
	}
	return install()
}

type lib struct {
	pkg    string
	object string
	path   string
}

func existGoImports() bool {
	for _, path := range [...]string{outPath(), cmdPath()} {
		if _, err := os.Stat(path); os.IsNotExist(err) {
			return false
		}
	}
	return true
}
<commit_msg>golint.<commit_after>package dragon

import (
	"errors"
	"os"

	"golang.org/x/sync/errgroup"
)

// Imports generate zstdlib.go from api files and libs in GOPATH.
func Imports() error {
	if !existGoImports() {
		return errors.New("goimports command isn't installed")
	}

	libChan := make(chan lib, 1000)
	done := make(chan error)
	go func() {
		done <- updateZstdlib(libChan)
	}()

	eg := &errgroup.Group{}
	eg.Go(func() error {
		return stdLibs(libChan)
	})
	eg.Go(func() error {
		return gopathLibs(libChan)
	})
	err := eg.Wait()
	if err != nil {
		return err
	}
	close(libChan)

	err = <-done
	if err != nil {
		return err
	}
	return install()
}

type lib struct {
	pkg    string
	object string
	path   string
}

func existGoImports() bool {
	for _, path := range [...]string{outPath(), cmdPath()} {
		if _, err := os.Stat(path); os.IsNotExist(err) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/dustin/go-jsonpointer"
)

func main() {
	d, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		log.Fatalf("Error reading json from stdin: %v", err)
	}
	if len(os.Args) == 1 {
		l, err := jsonpointer.ListPointers(d)
		if err != nil {
			log.Fatalf("Error listing pointers: %v", err)
		}
		for _, p := range l {
			fmt.Println(p)
		}
	} else {
		m, err := jsonpointer.FindMany(d, os.Args[1:])
		if err != nil {
			log.Fatalf("Error finding pointers: %v", err)
		}
		for k, v := range m {
			b := &bytes.Buffer{}
			json.Indent(b, v, "", "  ")
			fmt.Printf("%v\n%s\n\n", k, b)
		}
	}
}
<commit_msg>Separate the functions for listing and selecting pointers<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/dustin/go-jsonpointer"
)

func listPointers(d []byte) {
	l, err := jsonpointer.ListPointers(d)
	if err != nil {
		log.Fatalf("Error listing pointers: %v", err)
	}
	for _, p := range l {
		fmt.Println(p)
	}

}

func selectItems(d []byte, pointers []string) {
	m, err := jsonpointer.FindMany(d, pointers)
	if err != nil {
		log.Fatalf("Error finding pointers: %v", err)
	}
	for k, v := range m {
		b := &bytes.Buffer{}
		json.Indent(b, v, "", "  ")
		fmt.Printf("%v\n%s\n\n", k, b)
	}
}

func main() {
	d, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		log.Fatalf("Error reading json from stdin: %v", err)
	}
	if len(os.Args) == 1 {
		listPointers(d)
	} else {
		selectItems(d, os.Args[1:])
	}
}
<|endoftext|>


<commit_before>// Request package provides the request structure
// The package provides access to Headers, Cookies
// Query Params, Post Body and Upload Files
package request

import (
	"net/http"
	"mime/multipart"
	"io"
	"net/url"
	"strings"
	"encoding/json"
	cookie "github.com/DronRathore/goexpress/cookie"
)

type Url struct{
	Username string
	Password string
	Url string
	Path string
	Fragment string
}

// Contains the reader to read the buffer content of
// uploading file
type File struct{
	Name string
	FormName string
	Reader *multipart.Part
}

// Request Structure
type Request struct{
	ref *http.Request
	fileReader *multipart.Reader
	Header map[string]string
	Method string
	URL string
	_url *url.URL
	Params map[string]string // a map to be filled by router
	Query map[string][]string
	Body map[string][]string
	Cookies *cookie.Cookie
	JSON *json.Decoder
	props *map[string]interface{}
}

func (req *Request) Init(request *http.Request, props *map[string]interface{}) *Request{
	req.Header = make(map[string]string)
	req.Body = make(map[string][]string)
	req.Body = request.Form
	req.ref = request
	req.Cookies = &cookie.Cookie{}
	req.Cookies.InitReadOnly(request)
	req.Query = make(map[string][]string)
	req.Query = request.URL.Query()
	req.Method = strings.ToLower(request.Method)
	req.URL = request.URL.Path
	req.Params = make(map[string]string)
	req._url = request.URL
	req.props = props
	req.fileReader = nil
	for key, value := range request.Header {
		// lowercase the header key names
		req.Header[strings.ToLower(key)] = strings.Join(value, ",")
	}

	if req.Header["Content-Type"] == "application/json" {
		req.JSON = json.NewDecoder(request.Body)
	} else {
		request.ParseForm()
	}
	for key, value := range request.PostForm {
		req.Body[key] = value
	}
	return req
}

// todo: Parser for Array and interface
// func (req *Request) parseQuery(){
// 	req._url.RawQuery

// Returns the URL structure
func(req *Request) GetUrl() *url.URL {
	return req._url
}

// Helper that returns original raw http.Request object
func (req *Request) GetRaw() *http.Request{
	return req.ref
}

// In case of file upload request, this function returns a file struct to read
func (req *Request) GetFile() *File {
	if req.fileReader == nil {
		reader, err := req.ref.MultipartReader()
		if err != nil {
			panic("Couldn't get the reader attached")
		}
		req.fileReader = reader
	}
	part, err := req.fileReader.NextPart()
	if err == io.EOF {
		return nil
	}
	var file = &File{}
	file.Name = part.FileName()
	file.FormName = part.FormName()
	file.Reader = part
	return file
}
<commit_msg>[major] Fixes a bug when form is posted without name<commit_after>// Request package provides the request structure
// The package provides access to Headers, Cookies
// Query Params, Post Body and Upload Files
package request

import (
	"net/http"
	"mime/multipart"
	"io"
	"net/url"
	"strings"
	"encoding/json"
	cookie "github.com/DronRathore/goexpress/cookie"
)

type Url struct{
	Username string
	Password string
	Url string
	Path string
	Fragment string
}

// Contains the reader to read the buffer content of
// uploading file
type File struct{
	Name string
	FormName string
	Reader *multipart.Part
}

// Request Structure
type Request struct{
	ref *http.Request
	fileReader *multipart.Reader
	Header map[string]string
	Method string
	URL string
	_url *url.URL
	Params map[string]string // a map to be filled by router
	Query map[string][]string
	Body map[string][]string
	Cookies *cookie.Cookie
	JSON *json.Decoder
	props *map[string]interface{}
}

func (req *Request) Init(request *http.Request, props *map[string]interface{}) *Request{
	req.Header = make(map[string]string)
	req.Body = make(map[string][]string)
	req.Body = request.Form
	req.ref = request
	req.Cookies = &cookie.Cookie{}
	req.Cookies.InitReadOnly(request)
	req.Query = make(map[string][]string)
	req.Query = request.URL.Query()
	req.Method = strings.ToLower(request.Method)
	req.URL = request.URL.Path
	req.Params = make(map[string]string)
	req._url = request.URL
	req.props = props
	req.fileReader = nil
	for key, value := range request.Header {
		// lowercase the header key names
		req.Header[strings.ToLower(key)] = strings.Join(value, ",")
	}

	if req.Header["Content-Type"] == "application/json" {
		req.JSON = json.NewDecoder(request.Body)
	} else {
		request.ParseForm()
	}
	// check if we have an anonymous form posted
	if len(request.PostForm) > 0 && len(req.Body) == 0 {
		req.Body = make(map[string][]string)
	}
	for key, value := range request.PostForm {
		req.Body[key] = value
	}
	return req
}

// todo: Parser for Array and interface
// func (req *Request) parseQuery(){
// 	req._url.RawQuery

// Returns the URL structure
func(req *Request) GetUrl() *url.URL {
	return req._url
}

// Helper that returns original raw http.Request object
func (req *Request) GetRaw() *http.Request{
	return req.ref
}

// In case of file upload request, this function returns a file struct to read
func (req *Request) GetFile() *File {
	if req.fileReader == nil {
		reader, err := req.ref.MultipartReader()
		if err != nil {
			panic("Couldn't get the reader attached")
		}
		req.fileReader = reader
	}
	part, err := req.fileReader.NextPart()
	if err == io.EOF {
		return nil
	}
	var file = &File{}
	file.Name = part.FileName()
	file.FormName = part.FormName()
	file.Reader = part
	return file
}
<|endoftext|>


<commit_before>package script

import (
	"errors"
	"os/exec"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/robfig/cron"

	"github.com/qiniu/log"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/reader"
	. "github.com/qiniu/logkit/utils/models"
)

func init() {
	reader.RegisterConstructor(reader.ModeScript, NewReader)
}

type Reader struct {
	realpath   string // 处理文件路径
	originpath string
	scripttype string

	Cron *cron.Cron //定时任务

	readChan chan []byte
	errChan  chan error

	meta *reader.Meta

	status  int32
	mux     sync.Mutex
	started bool

	execOnStart  bool
	loop         bool
	loopDuration time.Duration

	stats     StatsInfo
	statsLock sync.RWMutex
}

func NewReader(meta *reader.Meta, conf conf.MapConf) (sr reader.Reader, err error) {
	path, _ := conf.GetStringOr(reader.KeyLogPath, "")
	originPath := path

	for {
		path, err = checkPath(meta, path)
		if err != nil {
			continue
		}
		break
	}

	cronSchedule, _ := conf.GetStringOr(reader.KeyScriptCron, "")
	execOnStart, _ := conf.GetBoolOr(reader.KeyScriptExecOnStart, true)
	scriptType, _ := conf.GetStringOr(reader.KeyExecInterpreter, "bash")
	ssr := &Reader{
		originpath:  originPath,
		realpath:    path,
		scripttype:  scriptType,
		Cron:        cron.New(),
		readChan:    make(chan []byte),
		errChan:     make(chan error),
		meta:        meta,
		status:      reader.StatusInit,
		mux:         sync.Mutex{},
		started:     false,
		execOnStart: execOnStart,
		statsLock:   sync.RWMutex{},
	}

	//schedule    string     //定时任务配置串
	if len(cronSchedule) > 0 {
		cronSchedule = strings.ToLower(cronSchedule)
		if strings.HasPrefix(cronSchedule, reader.Loop) {
			ssr.loop = true
			ssr.loopDuration, err = reader.ParseLoopDuration(cronSchedule)
			if err != nil {
				log.Errorf("Runner[%v] %v %v", ssr.meta.RunnerName, ssr.Name(), err)
				err = nil
			}
		} else {
			err = ssr.Cron.AddFunc(cronSchedule, ssr.run)
			if err != nil {
				return
			}
			log.Infof("Runner[%v] %v Cron job added with schedule <%v>", ssr.meta.RunnerName, ssr.Name(), cronSchedule)
		}
	}
	return ssr, nil
}

func (sr *Reader) ReadLine() (data string, err error) {
	if !sr.started {
		sr.Start()
	}
	timer := time.NewTimer(time.Second)
	select {
	case dat := <-sr.readChan:
		data = string(dat)
	case err = <-sr.errChan:
	case <-timer.C:
	}
	timer.Stop()
	return
}

func (s *Reader) sendError(err error) {
	if err == nil {
		return
	}
	defer func() {
		if rec := recover(); rec != nil {
			log.Errorf("Reader %s panic, recovered from %v", s.Name(), rec)
		}
	}()
	s.errChan <- err
}

//Start 仅调用一次，借用ReadLine启动，不能在new实例的时候启动，会有并发问题
func (sr *Reader) Start() {
	sr.mux.Lock()
	defer sr.mux.Unlock()
	if sr.started {
		return
	}
	if sr.loop {
		go sr.LoopRun()
	} else {
		sr.Cron.Start()
		if sr.execOnStart {
			go sr.run()
		}
	}
	sr.started = true
	log.Infof("Runner[%v] %v pull data deamon started", sr.meta.RunnerName, sr.Name())
}

func (sr *Reader) Name() string {
	return "ScriptFile:" + sr.originpath
}

func (sr *Reader) Source() string {
	return sr.originpath
}

func (sr *Reader) SetMode(mode string, v interface{}) error {
	return errors.New("ScriptReader not support readmode")
}

func (sr *Reader) SyncMeta() {}

func (sr *Reader) Close() (err error) {
	sr.Cron.Stop()
	if atomic.CompareAndSwapInt32(&sr.status, reader.StatusRunning, reader.StatusStopping) {
		log.Infof("Runner[%v] %v stopping", sr.meta.RunnerName, sr.Name())
	} else {
		close(sr.readChan)
		close(sr.errChan)
	}
	return
}

func (sr *Reader) LoopRun() {
	for {
		if atomic.LoadInt32(&sr.status) == reader.StatusStopped {
			return
		}
		//run 函数里面处理stopping的逻辑
		sr.run()
		time.Sleep(sr.loopDuration)
	}
}

func (sr *Reader) run() {
	var err error
	// 防止并发run
	for {
		if atomic.LoadInt32(&sr.status) == reader.StatusStopped {
			return
		}
		if atomic.CompareAndSwapInt32(&sr.status, reader.StatusInit, reader.StatusRunning) {
			break
		}
	}

	// running时退出 状态改为Init，以便 cron 调度下次运行
	// stopping时推出改为 stopped，不再运行
	defer func() {
		atomic.CompareAndSwapInt32(&sr.status, reader.StatusRunning, reader.StatusInit)
		if atomic.CompareAndSwapInt32(&sr.status, reader.StatusStopping, reader.StatusStopped) {
			close(sr.readChan)
			close(sr.errChan)
		}
		if err == nil {
			log.Infof("Runner[%v] %v successfully finished", sr.meta.RunnerName, sr.Name())
		}
	}()

	// 开始work逻辑
	for {
		if atomic.LoadInt32(&sr.status) == reader.StatusStopping {
			log.Warnf("Runner[%v] %v stopped from running", sr.meta.RunnerName, sr.Name())
			return
		}
		err = sr.exec()
		if err == nil {
			log.Infof("Runner[%v] %v successfully exec", sr.meta.RunnerName, sr.Name())
			return
		}
		log.Errorf("Runner[%v] %v execute script error [%v]", sr.meta.RunnerName, sr.Name(), err)
		sr.setStatsError(err.Error())
		sr.sendError(err)
		time.Sleep(3 * time.Second)
	}
}

func (sr *Reader) exec() (err error) {
	sr.mux.Lock()
	defer sr.mux.Unlock()
	command := exec.Command(sr.scripttype, sr.realpath) //初始化Cmd

	res, err := command.Output()
	if err != nil {
		return err
	}
	sr.readChan <- res
	return nil
}

func (sr *Reader) setStatsError(err string) {
	sr.statsLock.Lock()
	defer sr.statsLock.Unlock()
	sr.stats.LastError = err
}

func checkPath(meta *reader.Meta, path string) (string, error) {
	for {
		realPath, fileInfo, err := GetRealPath(path)
		if err != nil || fileInfo == nil {
			log.Warnf("Runner[%v] %s - utils.GetRealPath failed, err:%v", meta.RunnerName, path, err)
			time.Sleep(time.Minute)
		}

		fileMode := fileInfo.Mode()
		if !fileMode.IsRegular() {
			log.Warnf("Runner[%v] %s - file failed, err: file is not regular", meta.RunnerName, path)
			time.Sleep(time.Minute)
			continue
		}
		CheckFileMode(realPath, fileMode)
		return realPath, nil
	}
}
<commit_msg>script no sunch file bug fix<commit_after>package script

import (
	"errors"
	"fmt"
	"os/exec"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/robfig/cron"

	"github.com/qiniu/log"
	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/reader"
	. "github.com/qiniu/logkit/utils/models"
)

func init() {
	reader.RegisterConstructor(reader.ModeScript, NewReader)
}

type Reader struct {
	realpath   string // 处理文件路径
	originpath string
	scripttype string

	Cron *cron.Cron //定时任务

	readChan chan []byte
	errChan  chan error

	meta *reader.Meta

	status  int32
	mux     sync.Mutex
	started bool

	execOnStart  bool
	loop         bool
	loopDuration time.Duration

	stats     StatsInfo
	statsLock sync.RWMutex
}

func NewReader(meta *reader.Meta, conf conf.MapConf) (sr reader.Reader, err error) {
	path, _ := conf.GetStringOr(reader.KeyLogPath, "")
	originPath := path

	path, err = checkPath(meta, path)
	if err != nil {
		return nil, err
	}

	cronSchedule, _ := conf.GetStringOr(reader.KeyScriptCron, "")
	execOnStart, _ := conf.GetBoolOr(reader.KeyScriptExecOnStart, true)
	scriptType, _ := conf.GetStringOr(reader.KeyExecInterpreter, "bash")
	ssr := &Reader{
		originpath:  originPath,
		realpath:    path,
		scripttype:  scriptType,
		Cron:        cron.New(),
		readChan:    make(chan []byte),
		errChan:     make(chan error),
		meta:        meta,
		status:      reader.StatusInit,
		mux:         sync.Mutex{},
		started:     false,
		execOnStart: execOnStart,
		statsLock:   sync.RWMutex{},
	}

	//schedule    string     //定时任务配置串
	if len(cronSchedule) > 0 {
		cronSchedule = strings.ToLower(cronSchedule)
		if strings.HasPrefix(cronSchedule, reader.Loop) {
			ssr.loop = true
			ssr.loopDuration, err = reader.ParseLoopDuration(cronSchedule)
			if err != nil {
				log.Errorf("Runner[%v] %v %v", ssr.meta.RunnerName, ssr.Name(), err)
				err = nil
			}
		} else {
			err = ssr.Cron.AddFunc(cronSchedule, ssr.run)
			if err != nil {
				return nil, err
			}
			log.Infof("Runner[%v] %v Cron job added with schedule <%v>", ssr.meta.RunnerName, ssr.Name(), cronSchedule)
		}
	}
	return ssr, nil
}

func (sr *Reader) ReadLine() (data string, err error) {
	if !sr.started {
		sr.Start()
	}
	timer := time.NewTimer(time.Second)
	select {
	case dat := <-sr.readChan:
		data = string(dat)
	case err = <-sr.errChan:
	case <-timer.C:
	}
	timer.Stop()
	return
}

func (s *Reader) sendError(err error) {
	if err == nil {
		return
	}
	defer func() {
		if rec := recover(); rec != nil {
			log.Errorf("Reader %s panic, recovered from %v", s.Name(), rec)
		}
	}()
	s.errChan <- err
}

//Start 仅调用一次，借用ReadLine启动，不能在new实例的时候启动，会有并发问题
func (sr *Reader) Start() {
	sr.mux.Lock()
	defer sr.mux.Unlock()
	if sr.started {
		return
	}
	if sr.loop {
		go sr.LoopRun()
	} else {
		sr.Cron.Start()
		if sr.execOnStart {
			go sr.run()
		}
	}
	sr.started = true
	log.Infof("Runner[%v] %v pull data deamon started", sr.meta.RunnerName, sr.Name())
}

func (sr *Reader) Name() string {
	return "ScriptFile:" + sr.originpath
}

func (sr *Reader) Source() string {
	return sr.originpath
}

func (sr *Reader) SetMode(mode string, v interface{}) error {
	return errors.New("ScriptReader not support readmode")
}

func (sr *Reader) SyncMeta() {}

func (sr *Reader) Close() (err error) {
	sr.Cron.Stop()
	if atomic.CompareAndSwapInt32(&sr.status, reader.StatusRunning, reader.StatusStopping) {
		log.Infof("Runner[%v] %v stopping", sr.meta.RunnerName, sr.Name())
	} else {
		close(sr.readChan)
		close(sr.errChan)
	}
	return
}

func (sr *Reader) LoopRun() {
	for {
		if atomic.LoadInt32(&sr.status) == reader.StatusStopped {
			return
		}
		//run 函数里面处理stopping的逻辑
		sr.run()
		time.Sleep(sr.loopDuration)
	}
}

func (sr *Reader) run() {
	var err error
	// 防止并发run
	for {
		if atomic.LoadInt32(&sr.status) == reader.StatusStopped {
			return
		}
		if atomic.CompareAndSwapInt32(&sr.status, reader.StatusInit, reader.StatusRunning) {
			break
		}
	}

	// running时退出 状态改为Init，以便 cron 调度下次运行
	// stopping时推出改为 stopped，不再运行
	defer func() {
		atomic.CompareAndSwapInt32(&sr.status, reader.StatusRunning, reader.StatusInit)
		if atomic.CompareAndSwapInt32(&sr.status, reader.StatusStopping, reader.StatusStopped) {
			close(sr.readChan)
			close(sr.errChan)
		}
		if err == nil {
			log.Infof("Runner[%v] %v successfully finished", sr.meta.RunnerName, sr.Name())
		}
	}()

	// 开始work逻辑
	for {
		if atomic.LoadInt32(&sr.status) == reader.StatusStopping {
			log.Warnf("Runner[%v] %v stopped from running", sr.meta.RunnerName, sr.Name())
			return
		}
		err = sr.exec()
		if err == nil {
			log.Infof("Runner[%v] %v successfully exec", sr.meta.RunnerName, sr.Name())
			return
		}
		log.Errorf("Runner[%v] %v execute script error [%v]", sr.meta.RunnerName, sr.Name(), err)
		sr.setStatsError(err.Error())
		sr.sendError(err)
		time.Sleep(3 * time.Second)
	}
}

func (sr *Reader) exec() (err error) {
	sr.mux.Lock()
	defer sr.mux.Unlock()
	command := exec.Command(sr.scripttype, sr.realpath) //初始化Cmd

	res, err := command.Output()
	if err != nil {
		return err
	}
	sr.readChan <- res
	return nil
}

func (sr *Reader) setStatsError(err string) {
	sr.statsLock.Lock()
	defer sr.statsLock.Unlock()
	sr.stats.LastError = err
}

func checkPath(meta *reader.Meta, path string) (string, error) {
	for {
		realPath, fileInfo, err := GetRealPath(path)
		if err != nil {
			log.Warnf("Runner[%v] %s - utils.GetRealPath failed, err:%v", meta.RunnerName, path, err)
			time.Sleep(1 * time.Minute)
			continue
		}

		if fileInfo == nil {
			log.Warnf("Runner[%v] %s - utils.GetRealPath file info nil ", meta.RunnerName, path)
			time.Sleep(1 * time.Minute)
			continue
		}

		fileMode := fileInfo.Mode()
		if !fileMode.IsRegular() {
			err = fmt.Errorf("Runner[%v] %s - file failed, err: file is not regular ", meta.RunnerName, path)
			return "", err
		}

		err = CheckFileMode(realPath, fileMode)
		if err != nil {
			err = fmt.Errorf("Runner[%v] %s - file failed, err: %v ", meta.RunnerName, path, err)
			return "", err
		}

		return realPath, nil
	}
}
<|endoftext|>


<commit_before>package weavebox

import (
	"net/http"
	"testing"
)

func BenchmarkGetWithValues(b *testing.B) {
	app := New()
	app.EnableLog = false
	app.Get("/hello/:name", func(ctx *Context) error { return nil })

	for i := 0; i < b.N; i++ {
		r, err := http.NewRequest("GET", "/hello/anthony", nil)
		if err != nil {
			panic(err)
		}
		app.ServeHTTP(nil, r)
	}
}

func BenchmarkSubrouterGetWithValues(b *testing.B) {
	app := New()
	app.EnableLog = false
	admin := app.Subrouter("/admin")
	admin.Get("/:name", func(ctx *Context) error { return nil })

	for i := 0; i < b.N; i++ {
		r, err := http.NewRequest("GET", "/admin/anthony", nil)
		if err != nil {
			panic(err)
		}
		app.ServeHTTP(nil, r)
	}
}
<commit_msg>added bench with logging enabled<commit_after>package weavebox

import (
	"net/http"
	"testing"
)

func BenchmarkGetWithValues(b *testing.B) {
	app := New()
	app.EnableLog = false
	app.Get("/hello/:name", func(ctx *Context) error { return nil })

	for i := 0; i < b.N; i++ {
		r, err := http.NewRequest("GET", "/hello/anthony", nil)
		if err != nil {
			panic(err)
		}
		app.ServeHTTP(nil, r)
	}
}

func BenchmarkSubrouterGetWithValues(b *testing.B) {
	app := New()
	app.EnableLog = false
	admin := app.Subrouter("/admin")
	admin.Get("/:name", func(ctx *Context) error { return nil })

	for i := 0; i < b.N; i++ {
		r, err := http.NewRequest("GET", "/admin/anthony", nil)
		if err != nil {
			panic(err)
		}
		app.ServeHTTP(nil, r)
	}
}

func BenchmarkWithLoggingEnabled(b *testing.B) {
	app := New()
	app.Get("/:name", func(ctx *Context) error { return nil })

	for i := 0; i < b.N; i++ {
		r, err := http.NewRequest("GET", "/anthony", nil)
		if err != nil {
			panic(err)
		}
		app.ServeHTTP(nil, r)
	}
}
<|endoftext|>


<commit_before>package msgpack

import (
	"sync"
)

const DEFAULT_ARR_SIZE = 15

var arrpool = sync.Pool{
	New: func() interface{} {
		return make([]interface{}, DEFAULT_ARR_SIZE)
	},
}

func getNewArray(length int) []interface{} {
	if length <= 15 {
		return arrpool.Get().([]interface{})[:length]
	} else {
		return make([]interface{}, length)
	}
}

/** Each of these functions should take 3 arguments: the buffer to add into, an
 * offset into that buffer (where our writes start), and the value to encode.
 * After encoding the value and placing the byte sequence in the buffer
 * starting at the given offset, the function should return the next available
 * place to place bytes (the next offset to use)
 */

func encodeNil(buf []byte, offset int) int {
	buf[offset] = byte(0xc0)
	return offset + 1
}

func encodeBool(buf []byte, offset int, val bool) int {
	if val {
		buf[offset] = byte(0xc3)
	} else {
		buf[offset] = byte(0xc2)
	}
	return offset + 1
}

func encodeInt(buf []byte, offset, val int) int {
	if val < 128 && val >= 0 {
		buf[offset] = byte(0x7f & val)
		offset += 1
	} else if val < 0 && val > -32 {
		buf[offset] = byte(0xe0 | (0x1f & -val))
		offset += 1
	} else { // go to int64
		// find the smallest mask we can use
		switch val {
		case val & 0xff: // maybe 8 bit
			offset = encodeInt8(buf, offset, int64(val))
		case val & 0xffff: // maybe 16 bit
			offset = encodeInt16(buf, offset, int64(val))
		case val & 0xffffffff: // maybe 32 bit
			offset = encodeInt32(buf, offset, int64(val))
		default:
			offset = encodeInt64(buf, offset, int64(val))
		}
	}
	return offset
}

func encodeInt8(buf []byte, offset int, val int64) int {
	if val > 0x7f {
		return encodeInt16(buf, offset, val)
	}
	buf[offset] = byte(0xd0)
	buf[offset+1] = byte(val)
	return offset + 2
}

func encodeInt16(buf []byte, offset int, val int64) int {
	if val > 0x7fff {
		return encodeInt32(buf, offset, val)
	}
	buf[offset] = byte(0xd1)
	buf[offset+1] = byte(val >> 8)
	buf[offset+2] = byte(val & 0xff)
	return offset + 3
}

func encodeInt32(buf []byte, offset int, val int64) int {
	if val > 0x7fffffff {
		return encodeInt64(buf, offset, val)
	}
	buf[offset] = byte(0xd2)
	buf[offset+1] = byte(val >> 24)
	buf[offset+2] = byte(val >> 16)
	buf[offset+3] = byte(val >> 8)
	buf[offset+4] = byte(val & 0xff)
	return offset + 5
}

func encodeInt64(buf []byte, offset int, val int64) int {
	buf[offset] = byte(0xd3)
	buf[offset+1] = byte(val >> 56)
	buf[offset+2] = byte(val >> 48)
	buf[offset+3] = byte(val >> 40)
	buf[offset+4] = byte(val >> 32)
	buf[offset+5] = byte(val >> 24)
	buf[offset+6] = byte(val >> 16)
	buf[offset+7] = byte(val >> 8)
	buf[offset+8] = byte(val & 0xff)
	return offset + 9
}

func encodeUint(buf []byte, offset int, val uint) int {
	switch val {
	case val & 0xff: // uint8
		buf[offset] = byte(0xcc)
		buf[offset+1] = byte(val)
		offset += 2
	case val & 0xffff: // uint16
		buf[offset] = byte(0xcd)
		buf[offset+1] = byte(val >> 8)
		buf[offset+2] = byte(val & 0xff)
		offset += 3
	case val & 0xffffffff: // uint32
		buf[offset] = byte(0xce)
		buf[offset+1] = byte(val >> 24)
		buf[offset+2] = byte(val >> 16)
		buf[offset+3] = byte(val >> 8)
		buf[offset+4] = byte(val & 0xff)
		offset += 5
	default: // uint64
		buf[offset] = byte(0xcf)
		buf[offset+1] = byte(val >> 56)
		buf[offset+2] = byte(val >> 48)
		buf[offset+3] = byte(val >> 40)
		buf[offset+4] = byte(val >> 32)
		buf[offset+5] = byte(val >> 24)
		buf[offset+6] = byte(val >> 16)
		buf[offset+7] = byte(val >> 8)
		buf[offset+8] = byte(val & 0xff)
		offset += 9
	}
	return offset
}

// Encodes @val as a bigendian unsigned integer in buffer @buf
// starting at offset @offset. Attempts to make it fit in @length
// bytes, and will truncate if it cannot
func encodeLength(buf []byte, offset int, val uint, length int) int {
	switch {
	case length == 1: // uint8
		buf[offset] = byte(val)
		offset += 1
	case length == 2: // uint16
		buf[offset] = byte(val >> 8)
		buf[offset+1] = byte(val & 0xff)
		offset += 2
	case length == 4: // uint32
		buf[offset] = byte(val >> 24)
		buf[offset+1] = byte(val >> 16)
		buf[offset+2] = byte(val >> 8)
		buf[offset+3] = byte(val & 0xff)
		offset += 4
	default: // uint64
		buf[offset] = byte(val >> 56)
		buf[offset+1] = byte(val >> 48)
		buf[offset+2] = byte(val >> 40)
		buf[offset+3] = byte(val >> 32)
		buf[offset+4] = byte(val >> 24)
		buf[offset+5] = byte(val >> 16)
		buf[offset+6] = byte(val >> 8)
		buf[offset+7] = byte(val & 0xff)
		offset += 8
	}
	return offset
}

func encodeString(buf []byte, offset int, val string) int {
	l := len(val)
	switch {
	case l <= 31: // fixstr
		buf[offset] = byte(0xa0 | l)
		offset += 1
	case l <= 255: // str8
		buf[offset] = byte(0xd9)
		buf[offset+1] = byte(l)
		offset += 2
	case l <= 65535: // str16
		buf[offset] = byte(0xda)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 2)
	default: // str32
		buf[offset] = byte(0xdb)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 4)
	}
	for i := 0; i < l; i++ { // TODO fewer copies, e.g. not 1 byte at a time
		buf[offset+i] = val[i]
	}
	offset += l
	return offset
}

func encodeArray(buf []byte, offset int, val []interface{}) int {
	l := len(val)
	switch {
	case l <= 15:
		buf[offset] = byte(0x90 | l)
		offset += 1
	case l <= 65535: // (2^16 - 1)
		buf[offset] = byte(0xdc)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 2)
	default: // up to 4294967295 (2^32 - 1)
		buf[offset] = byte(0xdd)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 4)
	}
	for i := 0; i < l; i++ {
		offset = doEncode(val[i], &buf, offset)
	}
	return offset
}

func encodeMap(buf []byte, offset int, val map[string]interface{}) int {
	l := len(val)
	switch {
	case l <= 15:
		buf[offset] = byte(0x80 | l)
		offset += 1
	case l <= 65535: // 2^16 - 1
		buf[offset] = byte(0xde)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 2)
	default: // up to 4294967295 (2^32 - 1)
		buf[offset] = byte(0xdf)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 4)
	}
	for k, v := range val {
		offset = doEncode(k, &buf, offset)
		offset = doEncode(v, &buf, offset)
	}
	return offset
}

func doEncode(input interface{}, ret *[]byte, offset int) int {
	switch input.(type) {
	case int:
		offset = encodeInt(*ret, offset, input.(int))
	case uint:
		offset = encodeUint(*ret, offset, input.(uint))
	case int64:
		offset = encodeInt64(*ret, offset, input.(int64))
	case uint64:
		offset = encodeUint(*ret, offset, uint(input.(uint64)))
	case string:
		offset = encodeString(*ret, offset, input.(string))
	case map[string]interface{}:
		offset = encodeMap(*ret, offset, input.(map[string]interface{}))
	case []interface{}:
		offset = encodeArray(*ret, offset, input.([]interface{}))
	case bool:
		offset = encodeBool(*ret, offset, input.(bool))
	case nil:
		offset = encodeNil(*ret, offset)
	default:
	}
	return offset
}

// Encodes the input as a msgpack byte array, which is provided
// by the user. This allows the user to control how many allocations
// are done. Returns the length of the encoded message, but does
// not adjust the length of the input array.
func Encode(input interface{}, ret *[]byte) int {
	return doEncode(input, ret, 0)
}
<commit_msg>add some o' dat sweet doc<commit_after>// This MsgPack encoding/decoding library was born out of a partial dissatisfaction with
// the current implementation of (and lack of documentation of) MsgPack encoders/decoders
// for Go. I also wanted some practice writing optimized Go code that attempted to avoid
// allocations and "got out of the way" of libraries using it.
//
// This package is not finished yet, but it is close, and I've made an effort to do things
// in a standard way so that what is actually being done underneath the covers is easy
// to figure out. The focus of this package is not to provide the most fully featured
// MsgPack implementation out there, but rather support the main data types that I
// use in my work (most of the basic types, also []interface{} and map[string]interface{}).
// You'll notice that I don't use the `reflect` package at all, and that's because I
// wanted to see what code looked like when you just did a switch on types instead of
// using runtime reflection to figure out the types. This of course comes at the cost
// of not being able to extend this MsgPack implementation to arbitrary data types.
// Implementing a new type on your own is not difficult, though.
//
// Coming up next are some more convenience methods for doing decoding and encoding
// through writer/reader/buffer interfaces
package msgpack

import (
	"sync"
)

const DEFAULT_ARR_SIZE = 15

var arrpool = sync.Pool{
	New: func() interface{} {
		return make([]interface{}, DEFAULT_ARR_SIZE)
	},
}

func getNewArray(length int) []interface{} {
	if length <= 15 {
		return arrpool.Get().([]interface{})[:length]
	} else {
		return make([]interface{}, length)
	}
}

/** Each of these functions should take 3 arguments: the buffer to add into, an
 * offset into that buffer (where our writes start), and the value to encode.
 * After encoding the value and placing the byte sequence in the buffer
 * starting at the given offset, the function should return the next available
 * place to place bytes (the next offset to use)
 */

func encodeNil(buf []byte, offset int) int {
	buf[offset] = byte(0xc0)
	return offset + 1
}

func encodeBool(buf []byte, offset int, val bool) int {
	if val {
		buf[offset] = byte(0xc3)
	} else {
		buf[offset] = byte(0xc2)
	}
	return offset + 1
}

func encodeInt(buf []byte, offset, val int) int {
	if val < 128 && val >= 0 {
		buf[offset] = byte(0x7f & val)
		offset += 1
	} else if val < 0 && val > -32 {
		buf[offset] = byte(0xe0 | (0x1f & -val))
		offset += 1
	} else { // go to int64
		// find the smallest mask we can use
		switch val {
		case val & 0xff: // maybe 8 bit
			offset = encodeInt8(buf, offset, int64(val))
		case val & 0xffff: // maybe 16 bit
			offset = encodeInt16(buf, offset, int64(val))
		case val & 0xffffffff: // maybe 32 bit
			offset = encodeInt32(buf, offset, int64(val))
		default:
			offset = encodeInt64(buf, offset, int64(val))
		}
	}
	return offset
}

func encodeInt8(buf []byte, offset int, val int64) int {
	if val > 0x7f {
		return encodeInt16(buf, offset, val)
	}
	buf[offset] = byte(0xd0)
	buf[offset+1] = byte(val)
	return offset + 2
}

func encodeInt16(buf []byte, offset int, val int64) int {
	if val > 0x7fff {
		return encodeInt32(buf, offset, val)
	}
	buf[offset] = byte(0xd1)
	buf[offset+1] = byte(val >> 8)
	buf[offset+2] = byte(val & 0xff)
	return offset + 3
}

func encodeInt32(buf []byte, offset int, val int64) int {
	if val > 0x7fffffff {
		return encodeInt64(buf, offset, val)
	}
	buf[offset] = byte(0xd2)
	buf[offset+1] = byte(val >> 24)
	buf[offset+2] = byte(val >> 16)
	buf[offset+3] = byte(val >> 8)
	buf[offset+4] = byte(val & 0xff)
	return offset + 5
}

func encodeInt64(buf []byte, offset int, val int64) int {
	buf[offset] = byte(0xd3)
	buf[offset+1] = byte(val >> 56)
	buf[offset+2] = byte(val >> 48)
	buf[offset+3] = byte(val >> 40)
	buf[offset+4] = byte(val >> 32)
	buf[offset+5] = byte(val >> 24)
	buf[offset+6] = byte(val >> 16)
	buf[offset+7] = byte(val >> 8)
	buf[offset+8] = byte(val & 0xff)
	return offset + 9
}

func encodeUint(buf []byte, offset int, val uint) int {
	switch val {
	case val & 0xff: // uint8
		buf[offset] = byte(0xcc)
		buf[offset+1] = byte(val)
		offset += 2
	case val & 0xffff: // uint16
		buf[offset] = byte(0xcd)
		buf[offset+1] = byte(val >> 8)
		buf[offset+2] = byte(val & 0xff)
		offset += 3
	case val & 0xffffffff: // uint32
		buf[offset] = byte(0xce)
		buf[offset+1] = byte(val >> 24)
		buf[offset+2] = byte(val >> 16)
		buf[offset+3] = byte(val >> 8)
		buf[offset+4] = byte(val & 0xff)
		offset += 5
	default: // uint64
		buf[offset] = byte(0xcf)
		buf[offset+1] = byte(val >> 56)
		buf[offset+2] = byte(val >> 48)
		buf[offset+3] = byte(val >> 40)
		buf[offset+4] = byte(val >> 32)
		buf[offset+5] = byte(val >> 24)
		buf[offset+6] = byte(val >> 16)
		buf[offset+7] = byte(val >> 8)
		buf[offset+8] = byte(val & 0xff)
		offset += 9
	}
	return offset
}

// Encodes @val as a bigendian unsigned integer in buffer @buf
// starting at offset @offset. Attempts to make it fit in @length
// bytes, and will truncate if it cannot
func encodeLength(buf []byte, offset int, val uint, length int) int {
	switch {
	case length == 1: // uint8
		buf[offset] = byte(val)
		offset += 1
	case length == 2: // uint16
		buf[offset] = byte(val >> 8)
		buf[offset+1] = byte(val & 0xff)
		offset += 2
	case length == 4: // uint32
		buf[offset] = byte(val >> 24)
		buf[offset+1] = byte(val >> 16)
		buf[offset+2] = byte(val >> 8)
		buf[offset+3] = byte(val & 0xff)
		offset += 4
	default: // uint64
		buf[offset] = byte(val >> 56)
		buf[offset+1] = byte(val >> 48)
		buf[offset+2] = byte(val >> 40)
		buf[offset+3] = byte(val >> 32)
		buf[offset+4] = byte(val >> 24)
		buf[offset+5] = byte(val >> 16)
		buf[offset+6] = byte(val >> 8)
		buf[offset+7] = byte(val & 0xff)
		offset += 8
	}
	return offset
}

func encodeString(buf []byte, offset int, val string) int {
	l := len(val)
	switch {
	case l <= 31: // fixstr
		buf[offset] = byte(0xa0 | l)
		offset += 1
	case l <= 255: // str8
		buf[offset] = byte(0xd9)
		buf[offset+1] = byte(l)
		offset += 2
	case l <= 65535: // str16
		buf[offset] = byte(0xda)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 2)
	default: // str32
		buf[offset] = byte(0xdb)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 4)
	}
	for i := 0; i < l; i++ { // TODO fewer copies, e.g. not 1 byte at a time
		buf[offset+i] = val[i]
	}
	offset += l
	return offset
}

func encodeArray(buf []byte, offset int, val []interface{}) int {
	l := len(val)
	switch {
	case l <= 15:
		buf[offset] = byte(0x90 | l)
		offset += 1
	case l <= 65535: // (2^16 - 1)
		buf[offset] = byte(0xdc)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 2)
	default: // up to 4294967295 (2^32 - 1)
		buf[offset] = byte(0xdd)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 4)
	}
	for i := 0; i < l; i++ {
		offset = doEncode(val[i], &buf, offset)
	}
	return offset
}

func encodeMap(buf []byte, offset int, val map[string]interface{}) int {
	l := len(val)
	switch {
	case l <= 15:
		buf[offset] = byte(0x80 | l)
		offset += 1
	case l <= 65535: // 2^16 - 1
		buf[offset] = byte(0xde)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 2)
	default: // up to 4294967295 (2^32 - 1)
		buf[offset] = byte(0xdf)
		offset += 1
		offset = encodeLength(buf, offset, uint(l), 4)
	}
	for k, v := range val {
		offset = doEncode(k, &buf, offset)
		offset = doEncode(v, &buf, offset)
	}
	return offset
}

func doEncode(input interface{}, ret *[]byte, offset int) int {
	switch input.(type) {
	case int:
		offset = encodeInt(*ret, offset, input.(int))
	case uint:
		offset = encodeUint(*ret, offset, input.(uint))
	case int64:
		offset = encodeInt64(*ret, offset, input.(int64))
	case uint64:
		offset = encodeUint(*ret, offset, uint(input.(uint64)))
	case string:
		offset = encodeString(*ret, offset, input.(string))
	case map[string]interface{}:
		offset = encodeMap(*ret, offset, input.(map[string]interface{}))
	case []interface{}:
		offset = encodeArray(*ret, offset, input.([]interface{}))
	case bool:
		offset = encodeBool(*ret, offset, input.(bool))
	case nil:
		offset = encodeNil(*ret, offset)
	default:
	}
	return offset
}

// Encodes the input as a msgpack byte array, which is provided
// by the user. This allows the user to control how many allocations
// are done. Returns the length of the encoded message, but does
// not adjust the length of the input array.
func Encode(input interface{}, ret *[]byte) int {
	return doEncode(input, ret, 0)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"sync"
	"syscall"

	"github.com/coduno/app/models"
	"github.com/coduno/app/util"
)

var (
	fileNames = map[string]string{
		"python": "app.py",
		"c":      "app.c",
		"cpp":    "app.cpp",
		"java":   "Application.java",
	}
)

const configFileName string = "coduno.yaml"
const volumePattern string = "coduno-volume"

func startSimpleRun(w http.ResponseWriter, r *http.Request) {
	if !util.CheckMethod(w, r, "POST") {
		return
	}

	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		http.Error(w, "Error reading: "+err.Error(), http.StatusInternalServerError)
		return
	}

	var codeData models.CodeData
	err = json.Unmarshal(body, &codeData)

	if err != nil {
		http.Error(w, "Cannot unmarshal: "+err.Error(), http.StatusInternalServerError)
		return
	}

	for availableLanguage := range fileNames {
		if codeData.Language == availableLanguage {
			goto LANGUAGE_AVAILABLE
		}
	}
	http.Error(w, "Language not available.", http.StatusBadRequest)

LANGUAGE_AVAILABLE:
	tempDir, err := prepareFilesForDockerRun(&codeData)

	if err != nil {
		http.Error(w, "File preparation error: "+err.Error(), http.StatusInternalServerError)
		return
	}

	prepareAndSimpleRun(w, r, tempDir, &codeData)
}

func prepareFilesForDockerRun(codeData *models.CodeData) (tempDir string, err error) {
	tempDir, err = volumeDir()
	if err != nil {
		return
	}
	err = createConfigurationFile(tempDir, codeData.Language)
	if err != nil {
		return
	}
	err = createExecFile(tempDir, codeData)
	if err != nil {
		return
	}
	return tempDir, nil
}

func prepareAndSimpleRun(w http.ResponseWriter, r *http.Request, tempDir string, codeData *models.CodeData) {
	key, build := LogBuildStart("challengeId", codeData.CodeBase, "user")

	volume, err := dockerize(tempDir)

	if err != nil {
		log.Fatal(err)
	}

	cmdUser := exec.Command(
		"docker",
		"run",
		"--rm",
		"-v",
		volume+":/run",
		"coduno_all")

	outUser, err := cmdUser.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	errUser, err := cmdUser.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	_, err = cmdUser.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}

	var runOutput, runErr bytes.Buffer
	cmdUser.Start()
	var wg sync.WaitGroup
	wg.Add(2)

	go PipeOutput(&wg, outUser, os.Stdout, &runOutput)
	go PipeOutput(&wg, errUser, os.Stdout, &runErr)

	exitErr := cmdUser.Wait()
	wg.Wait()
	prepLog, err := ioutil.ReadFile(tempDir + "/prepare.log")
	if err != nil {
		log.Fatal(err)
	}

	var stats syscall.Rusage
	statsData, err := ioutil.ReadFile(tempDir + "/stats.log")
	if err != nil {
		log.Print(err)
	} else {
		err = json.Unmarshal(statsData, &stats)
		if err != nil {
			log.Fatal(err)
		}
	}

	LogRunComplete(key, build, "", runOutput.String(), "", exitErr, string(prepLog), stats)

	var toSend = make(map[string]string)
	toSend["run"] = runOutput.String()
	toSend["err"] = runErr.String()

	json, err := json.Marshal(toSend)
	if err != nil {
		http.Error(w, "Json marshal err: "+err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(json)
}

func createExecFile(tmpDir string, codeData *models.CodeData) (err error) {
	f, err := os.Create(path.Join(tmpDir, fileNames[codeData.Language]))
	if err != nil {
		return
	}
	f.WriteString(codeData.CodeBase)
	f.Close()
	return
}

func createConfigurationFile(tempDir, lang string) error {
	src := path.Join(".", "run_config", lang, "coduno.yaml")
	return copyFileContents(tempDir, src, configFileName)
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(dst, src, fileName string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	dst = dst + "/" + fileName
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	return
}

func main() {
	http.HandleFunc("/api/run/start/simple", startSimpleRun)
	http.ListenAndServe(":8081", nil)
}
<commit_msg>Migrate to dedicated containers<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"sync"
	"syscall"

	"github.com/coduno/app/models"
	"github.com/coduno/app/util"
)

var (
	fileNames = map[string]string{
		"python": "app.py",
		"c":      "app.c",
		"cpp":    "app.cpp",
		"java":   "Application.java",
	}
)

const configFileName string = "coduno.yaml"
const volumePattern string = "coduno-volume"

func startSimpleRun(w http.ResponseWriter, r *http.Request) {
	if !util.CheckMethod(w, r, "POST") {
		return
	}

	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		http.Error(w, "Error reading: "+err.Error(), http.StatusInternalServerError)
		return
	}

	var codeData models.CodeData
	err = json.Unmarshal(body, &codeData)

	if err != nil {
		http.Error(w, "Cannot unmarshal: "+err.Error(), http.StatusInternalServerError)
		return
	}

	for availableLanguage := range fileNames {
		if codeData.Language == availableLanguage {
			goto LANGUAGE_AVAILABLE
		}
	}
	http.Error(w, "Language not available.", http.StatusBadRequest)

LANGUAGE_AVAILABLE:
	tempDir, err := prepareFilesForDockerRun(&codeData)

	if err != nil {
		http.Error(w, "File preparation error: "+err.Error(), http.StatusInternalServerError)
		return
	}

	prepareAndSimpleRun(w, r, tempDir, &codeData)
}

func prepareFilesForDockerRun(codeData *models.CodeData) (tempDir string, err error) {
	tempDir, err = volumeDir()
	if err != nil {
		return
	}
	err = createExecFile(tempDir, codeData)
	if err != nil {
		return
	}
	return tempDir, nil
}

func prepareAndSimpleRun(w http.ResponseWriter, r *http.Request, tempDir string, codeData *models.CodeData) {
	key, build := LogBuildStart("challengeId", codeData.CodeBase, "user")

	volume, err := dockerize(tempDir)

	if err != nil {
		log.Fatal(err)
	}

	cmdUser := exec.Command(
		"docker",
		"run",
		"--rm",
		"-v",
		volume+":/run",
		"coduno/fingerprint-"+codeData.Language)

	outUser, err := cmdUser.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	errUser, err := cmdUser.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	_, err = cmdUser.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}

	var runOutput, runErr bytes.Buffer
	cmdUser.Start()
	var wg sync.WaitGroup
	wg.Add(2)

	go PipeOutput(&wg, outUser, os.Stdout, &runOutput)
	go PipeOutput(&wg, errUser, os.Stdout, &runErr)

	exitErr := cmdUser.Wait()
	wg.Wait()
	prepLog, err := ioutil.ReadFile(tempDir + "/prepare.log")
	if err != nil {
		log.Fatal(err)
	}

	var stats syscall.Rusage
	statsData, err := ioutil.ReadFile(tempDir + "/stats.log")
	if err != nil {
		log.Print(err)
	} else {
		err = json.Unmarshal(statsData, &stats)
		if err != nil {
			log.Fatal(err)
		}
	}

	LogRunComplete(key, build, "", runOutput.String(), "", exitErr, string(prepLog), stats)

	var toSend = make(map[string]string)
	toSend["run"] = runOutput.String()
	toSend["err"] = runErr.String()

	json, err := json.Marshal(toSend)
	if err != nil {
		http.Error(w, "Json marshal err: "+err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(json)
}

func createExecFile(tmpDir string, codeData *models.CodeData) (err error) {
	f, err := os.Create(path.Join(tmpDir, fileNames[codeData.Language]))
	if err != nil {
		return
	}
	f.WriteString(codeData.CodeBase)
	f.Close()
	return
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(dst, src, fileName string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	dst = dst + "/" + fileName
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	return
}

func main() {
	http.HandleFunc("/api/run/start/simple", startSimpleRun)
	http.ListenAndServe(":8081", nil)
}
<|endoftext|>