<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package rest

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"testing"

	"github.com/gorilla/mux"

	"github.com/couchbaselabs/cbgt"
)

func TestMustEncode(t *testing.T) {
	defer func() {
		r := recover()
		if r == nil {
			t.Errorf("expected must encode panic and recover")
		}
	}()
	MustEncode(&bytes.Buffer{}, func() {})
	t.Errorf("expected must encode panic")
}

// Implements ManagerEventHandlers interface.
type TestMEH struct {
	lastPIndex *cbgt.PIndex
	lastCall   string
	ch         chan bool
}

func (meh *TestMEH) OnRegisterPIndex(pindex *cbgt.PIndex) {
	meh.lastPIndex = pindex
	meh.lastCall = "OnRegisterPIndex"
	if meh.ch != nil {
		meh.ch <- true
	}
}

func (meh *TestMEH) OnUnregisterPIndex(pindex *cbgt.PIndex) {
	meh.lastPIndex = pindex
	meh.lastCall = "OnUnregisterPIndex"
	if meh.ch != nil {
		meh.ch <- true
	}
}

func TestNewRESTRouter(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	ring, err := cbgt.NewMsgRing(nil, 1)

	cfg := cbgt.NewCfgMem()
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000",
		emptyDir, "some-datasource", nil)
	r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring,
		AssetDir, Asset)
	if r == nil || meta == nil || err != nil {
		t.Errorf("expected no errors")
	}

	mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		[]string{"queryer", "anotherTag"},
		"", 1, "", ":1000", emptyDir, "some-datasource", nil)
	r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring,
		AssetDir, Asset)
	if r == nil || meta == nil || err != nil {
		t.Errorf("expected no errors")
	}
}

type RESTHandlerTest struct {
	Desc          string
	Path          string
	Method        string
	Params        url.Values
	Body          []byte
	Status        int
	ResponseBody  []byte
	ResponseMatch map[string]bool

	Before func()
	After  func()
}

func (test *RESTHandlerTest) check(t *testing.T,
	record *httptest.ResponseRecorder) {
	desc := test.Desc
	if desc == "" {
		desc = test.Path + " " + test.Method
	}

	if got, want := record.Code, test.Status; got != want {
		t.Errorf("%s: response code = %d, want %d", desc, got, want)
		t.Errorf("%s: response body = %s", desc, record.Body)
	}
	got := bytes.TrimRight(record.Body.Bytes(), "\n")
	if test.ResponseBody != nil {
		if !reflect.DeepEqual(got, test.ResponseBody) {
			t.Errorf("%s: expected: '%s', got: '%s'",
				desc, test.ResponseBody, got)
		}
	}
	for pattern, shouldMatch := range test.ResponseMatch {
		didMatch := bytes.Contains(got, []byte(pattern))
		if didMatch != shouldMatch {
			t.Errorf("%s: expected match %t for pattern %s, got %t",
				desc, shouldMatch, pattern, didMatch)
			t.Errorf("%s: response body was: %s", desc, got)
		}
	}
}

func testRESTHandlers(t *testing.T,
	tests []*RESTHandlerTest, router *mux.Router) {
	for _, test := range tests {
		if test.Before != nil {
			test.Before()
		}
		if test.Method != "NOOP" {
			req := &http.Request{
				Method: test.Method,
				URL:    &url.URL{Path: test.Path},
				Form:   test.Params,
				Body:   ioutil.NopCloser(bytes.NewBuffer(test.Body)),
			}
			record := httptest.NewRecorder()
			router.ServeHTTP(record, req)
			test.check(t, record)
		}
		if test.After != nil {
			test.After()
		}
	}
}

func TestHandlersForRuntimeOps(t *testing.T) {
	emptyDir, err := ioutil.TempDir("./tmp", "test")
	if err != nil {
		t.Errorf("tempdir err: %v", err)
	}
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	err = mgr.Start("wanted")
	if err != nil {
		t.Errorf("expected no start err, got: %v", err)
	}

	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
	mr.Write([]byte("hello"))
	mr.Write([]byte("world"))

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
		AssetDir, Asset)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []*RESTHandlerTest{
		{
			Path:   "/api/runtime",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				"arch":       true,
				"go":         true,
				"GOMAXPROCS": true,
				"GOROOT":     true,
			},
		},
		{
			Path:          "/api/runtime/args",
			Method:        "GET",
			Params:        nil,
			Body:          nil,
			Status:        http.StatusOK,
			ResponseMatch: map[string]bool{
			// Actual production args are different from "go test" context.
			},
		},
		{
			Path:         "/api/runtime/gc",
			Method:       "POST",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(nil),
		},
		{
			Path:   "/api/runtime/statsMem",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				"Alloc":      true,
				"TotalAlloc": true,
			},
		},
		{
			Path:   "/api/runtime/profile/cpu",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				"incorrect or missing secs parameter": true,
			},
		},
	}

	testRESTHandlers(t, tests, router)
}

func TestHandlersForEmptyManager(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	err := mgr.Start("wanted")
	if err != nil {
		t.Errorf("expected start ok")
	}
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
	mr.Write([]byte("hello"))
	mr.Write([]byte("world"))

	mgr.AddEvent([]byte(`"fizz"`))
	mgr.AddEvent([]byte(`"buzz"`))

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
		AssetDir, Asset)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []*RESTHandlerTest{
		{
			Desc:         "log on empty msg ring",
			Path:         "/api/log",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`),
		},
		{
			Desc:   "cfg on empty manaager",
			Path:   "/api/cfg",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:       true,
				`"indexDefs":null`:    true,
				`"nodeDefsKnown":{`:   true,
				`"nodeDefsWanted":{`:  true,
				`"planPIndexes":null`: true,
			},
		},
		{
			Desc:   "cfg refresh on empty, unchanged manager",
			Path:   "/api/cfgRefresh",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager kick on empty, unchanged manager",
			Path:   "/api/managerKick",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager meta",
			Path:   "/api/managerMeta",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:    true,
				`"startSamples":{`: true,
			},
		},
		{
			Desc:   "feed stats when no feeds",
			Path:   "/api/stats",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{`: true,
				`}`: true,
			},
		},
		{
			Desc:         "list empty indexes",
			Path:         "/api/index",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
		},
		{
			Desc:         "try to get a nonexistent index",
			Path:         "/api/index/NOT-AN-INDEX",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       400,
			ResponseBody: []byte(`not an index`),
		},
		{
			Desc:   "try to create a default index with no params",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`rest_create_index: indexType is required`: true,
			},
		},
		{
			Desc:   "try to create a default index with no sourceType",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: url.Values{
				"indexType": []string{"no-care"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`rest_create_index: sourceType is required`: true,
			},
		},
		{
			Desc:   "try to create a default index with bad indexType",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"no-care"},
				"sourceType": []string{"couchbase"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`unknown indexType`: true,
			},
		},
		{
			Desc:   "try to delete a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX",
			Method: "DELETE",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`no indexes`: true,
			},
		},
		{
			Desc:   "try to count a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "try to query a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/query",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "create an index with bogus indexType",
			Path:   "/api/index/idxBogusIndexType",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"not-a-real-index-type"},
				"sourceType": []string{"nil"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`error`: true,
			},
		},
	}

	testRESTHandlers(t, tests, router)
}
<commit_msg>TestInitStaticRouter<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package rest

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"testing"

	"github.com/gorilla/mux"

	"github.com/couchbaselabs/cbgt"
)

func TestInitStaticRouter(t *testing.T) {
	r := mux.NewRouter()

	staticDir := ""
	staticETag := ""
	pagesHandler := RewriteURL("/", http.FileServer(AssetFS()))

	r = InitStaticRouter(r,
		staticDir, staticETag, []string{
			"/indexes",
			"/nodes",
			"/monitor",
			"/manage",
			"/logs",
			"/debug",
		}, pagesHandler)
	if r == nil {
		t.Errorf("expected r")
	}
}

func TestMustEncode(t *testing.T) {
	defer func() {
		r := recover()
		if r == nil {
			t.Errorf("expected must encode panic and recover")
		}
	}()
	MustEncode(&bytes.Buffer{}, func() {})
	t.Errorf("expected must encode panic")
}

// Implements ManagerEventHandlers interface.
type TestMEH struct {
	lastPIndex *cbgt.PIndex
	lastCall   string
	ch         chan bool
}

func (meh *TestMEH) OnRegisterPIndex(pindex *cbgt.PIndex) {
	meh.lastPIndex = pindex
	meh.lastCall = "OnRegisterPIndex"
	if meh.ch != nil {
		meh.ch <- true
	}
}

func (meh *TestMEH) OnUnregisterPIndex(pindex *cbgt.PIndex) {
	meh.lastPIndex = pindex
	meh.lastCall = "OnUnregisterPIndex"
	if meh.ch != nil {
		meh.ch <- true
	}
}

func TestNewRESTRouter(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	ring, err := cbgt.NewMsgRing(nil, 1)

	cfg := cbgt.NewCfgMem()
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000",
		emptyDir, "some-datasource", nil)
	r, meta, err := NewRESTRouter("v0", mgr, emptyDir, "", ring,
		AssetDir, Asset)
	if r == nil || meta == nil || err != nil {
		t.Errorf("expected no errors")
	}

	mgr = cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		[]string{"queryer", "anotherTag"},
		"", 1, "", ":1000", emptyDir, "some-datasource", nil)
	r, meta, err = NewRESTRouter("v0", mgr, emptyDir, "", ring,
		AssetDir, Asset)
	if r == nil || meta == nil || err != nil {
		t.Errorf("expected no errors")
	}
}

type RESTHandlerTest struct {
	Desc          string
	Path          string
	Method        string
	Params        url.Values
	Body          []byte
	Status        int
	ResponseBody  []byte
	ResponseMatch map[string]bool

	Before func()
	After  func()
}

func (test *RESTHandlerTest) check(t *testing.T,
	record *httptest.ResponseRecorder) {
	desc := test.Desc
	if desc == "" {
		desc = test.Path + " " + test.Method
	}

	if got, want := record.Code, test.Status; got != want {
		t.Errorf("%s: response code = %d, want %d", desc, got, want)
		t.Errorf("%s: response body = %s", desc, record.Body)
	}
	got := bytes.TrimRight(record.Body.Bytes(), "\n")
	if test.ResponseBody != nil {
		if !reflect.DeepEqual(got, test.ResponseBody) {
			t.Errorf("%s: expected: '%s', got: '%s'",
				desc, test.ResponseBody, got)
		}
	}
	for pattern, shouldMatch := range test.ResponseMatch {
		didMatch := bytes.Contains(got, []byte(pattern))
		if didMatch != shouldMatch {
			t.Errorf("%s: expected match %t for pattern %s, got %t",
				desc, shouldMatch, pattern, didMatch)
			t.Errorf("%s: response body was: %s", desc, got)
		}
	}
}

func testRESTHandlers(t *testing.T,
	tests []*RESTHandlerTest, router *mux.Router) {
	for _, test := range tests {
		if test.Before != nil {
			test.Before()
		}
		if test.Method != "NOOP" {
			req := &http.Request{
				Method: test.Method,
				URL:    &url.URL{Path: test.Path},
				Form:   test.Params,
				Body:   ioutil.NopCloser(bytes.NewBuffer(test.Body)),
			}
			record := httptest.NewRecorder()
			router.ServeHTTP(record, req)
			test.check(t, record)
		}
		if test.After != nil {
			test.After()
		}
	}
}

func TestHandlersForRuntimeOps(t *testing.T) {
	emptyDir, err := ioutil.TempDir("./tmp", "test")
	if err != nil {
		t.Errorf("tempdir err: %v", err)
	}
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	err = mgr.Start("wanted")
	if err != nil {
		t.Errorf("expected no start err, got: %v", err)
	}

	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
	mr.Write([]byte("hello"))
	mr.Write([]byte("world"))

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
		AssetDir, Asset)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []*RESTHandlerTest{
		{
			Path:   "/api/runtime",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				"arch":       true,
				"go":         true,
				"GOMAXPROCS": true,
				"GOROOT":     true,
			},
		},
		{
			Path:          "/api/runtime/args",
			Method:        "GET",
			Params:        nil,
			Body:          nil,
			Status:        http.StatusOK,
			ResponseMatch: map[string]bool{
			// Actual production args are different from "go test" context.
			},
		},
		{
			Path:         "/api/runtime/gc",
			Method:       "POST",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(nil),
		},
		{
			Path:   "/api/runtime/statsMem",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				"Alloc":      true,
				"TotalAlloc": true,
			},
		},
		{
			Path:   "/api/runtime/profile/cpu",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				"incorrect or missing secs parameter": true,
			},
		},
	}

	testRESTHandlers(t, tests, router)
}

func TestHandlersForEmptyManager(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := cbgt.NewCfgMem()
	meh := &TestMEH{}
	mgr := cbgt.NewManager(cbgt.VERSION, cfg, cbgt.NewUUID(),
		nil, "", 1, "", ":1000", emptyDir, "some-datasource", meh)
	err := mgr.Start("wanted")
	if err != nil {
		t.Errorf("expected start ok")
	}
	mgr.Kick("test-start-kick")

	mr, _ := cbgt.NewMsgRing(os.Stderr, 1000)
	mr.Write([]byte("hello"))
	mr.Write([]byte("world"))

	mgr.AddEvent([]byte(`"fizz"`))
	mgr.AddEvent([]byte(`"buzz"`))

	router, _, err := NewRESTRouter("v0", mgr, "static", "", mr,
		AssetDir, Asset)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []*RESTHandlerTest{
		{
			Desc:         "log on empty msg ring",
			Path:         "/api/log",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"messages":["hello","world"],"events":["fizz","buzz"]}`),
		},
		{
			Desc:   "cfg on empty manaager",
			Path:   "/api/cfg",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:       true,
				`"indexDefs":null`:    true,
				`"nodeDefsKnown":{`:   true,
				`"nodeDefsWanted":{`:  true,
				`"planPIndexes":null`: true,
			},
		},
		{
			Desc:   "cfg refresh on empty, unchanged manager",
			Path:   "/api/cfgRefresh",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager kick on empty, unchanged manager",
			Path:   "/api/managerKick",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager meta",
			Path:   "/api/managerMeta",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:    true,
				`"startSamples":{`: true,
			},
		},
		{
			Desc:   "feed stats when no feeds",
			Path:   "/api/stats",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{`: true,
				`}`: true,
			},
		},
		{
			Desc:         "list empty indexes",
			Path:         "/api/index",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
		},
		{
			Desc:         "try to get a nonexistent index",
			Path:         "/api/index/NOT-AN-INDEX",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       400,
			ResponseBody: []byte(`not an index`),
		},
		{
			Desc:   "try to create a default index with no params",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`rest_create_index: indexType is required`: true,
			},
		},
		{
			Desc:   "try to create a default index with no sourceType",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: url.Values{
				"indexType": []string{"no-care"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`rest_create_index: sourceType is required`: true,
			},
		},
		{
			Desc:   "try to create a default index with bad indexType",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"no-care"},
				"sourceType": []string{"couchbase"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`unknown indexType`: true,
			},
		},
		{
			Desc:   "try to delete a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX",
			Method: "DELETE",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`no indexes`: true,
			},
		},
		{
			Desc:   "try to count a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "try to query a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/query",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "create an index with bogus indexType",
			Path:   "/api/index/idxBogusIndexType",
			Method: "PUT",
			Params: url.Values{
				"indexType":  []string{"not-a-real-index-type"},
				"sourceType": []string{"nil"},
			},
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`error`: true,
			},
		},
	}

	testRESTHandlers(t, tests, router)
}
<|endoftext|>


<commit_before>package packer

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"sync"
)

type UiColor uint

const (
	UiColorRed     UiColor = 31
	UiColorGreen           = 32
	UiColorYellow          = 33
	UiColorBlue            = 34
	UiColorMagenta         = 35
	UiColorCyan            = 36
)

// The Ui interface handles all communication for Packer with the outside
// world. This sort of control allows us to strictly control how output
// is formatted and various levels of output.
type Ui interface {
	Ask(string) (string, error)
	Say(string)
	Message(string)
	Error(string)
}

// ColoredUi is a UI that is colored using terminal colors.
type ColoredUi struct {
	Color      UiColor
	ErrorColor UiColor
	Ui         Ui
}

// PrefixedUi is a UI that wraps another UI implementation and adds a
// prefix to all the messages going out.
type PrefixedUi struct {
	SayPrefix     string
	MessagePrefix string
	Ui            Ui
}

// The ReaderWriterUi is a UI that writes and reads from standard Go
// io.Reader and io.Writer.
type ReaderWriterUi struct {
	Reader      io.Reader
	Writer      io.Writer
	l           sync.Mutex
	interrupted bool
}

func (u *ColoredUi) Ask(query string) (string, error) {
	return u.Ui.Ask(u.colorize(query, u.Color, true))
}

func (u *ColoredUi) Say(message string) {
	u.Ui.Say(u.colorize(message, u.Color, true))
}

func (u *ColoredUi) Message(message string) {
	u.Ui.Message(u.colorize(message, u.Color, false))
}

func (u *ColoredUi) Error(message string) {
	color := u.ErrorColor
	if color == 0 {
		color = UiColorRed
	}

	u.Ui.Error(u.colorize(message, color, true))
}

func (u *ColoredUi) colorize(message string, color UiColor, bold bool) string {
	attr := 0
	if bold {
		attr = 1
	}

	return fmt.Sprintf("\033[%d;%d;40m%s\033[0m", attr, color, message)
}

func (u *PrefixedUi) Ask(query string) (string, error) {
	return u.Ui.Ask(fmt.Sprintf("%s: %s", u.SayPrefix, query))
}

func (u *PrefixedUi) Say(message string) {
	u.Ui.Say(fmt.Sprintf("%s: %s", u.SayPrefix, message))
}

func (u *PrefixedUi) Message(message string) {
	u.Ui.Message(fmt.Sprintf("%s: %s", u.MessagePrefix, message))
}

func (u *PrefixedUi) Error(message string) {
	u.Ui.Error(fmt.Sprintf("%s: %s", u.SayPrefix, message))
}

func (rw *ReaderWriterUi) Ask(query string) (string, error) {
	rw.l.Lock()
	defer rw.l.Unlock()

	if rw.interrupted {
		return "", errors.New("interrupted")
	}

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, os.Interrupt)
	defer signal.Stop(sigCh)

	log.Printf("ui: ask: %s", query)
	if query != "" {
		if _, err := fmt.Fprint(rw.Writer, query+" "); err != nil {
			return "", err
		}
	}

	result := make(chan string, 1)
	go func() {
		var line string
		if _, err := fmt.Fscanln(rw.Reader, &line); err != nil {
			log.Printf("ui: scan err: %s", err)
		}

		result <- line
	}()

	select {
	case line := <-result:
		return line, nil
	case <-sigCh:
		rw.interrupted = true
		return "", errors.New("interrupted")
	}
}

func (rw *ReaderWriterUi) Say(message string) {
	rw.l.Lock()
	defer rw.l.Unlock()

	log.Printf("ui: %s", message)
	_, err := fmt.Fprint(rw.Writer, message+"\n")
	if err != nil {
		panic(err)
	}
}

func (rw *ReaderWriterUi) Message(message string) {
	rw.l.Lock()
	defer rw.l.Unlock()

	log.Printf("ui: %s", message)
	_, err := fmt.Fprintf(rw.Writer, message+"\n")
	if err != nil {
		panic(err)
	}
}

func (rw *ReaderWriterUi) Error(message string) {
	rw.l.Lock()
	defer rw.l.Unlock()

	log.Printf("ui error: %s", message)
	_, err := fmt.Fprint(rw.Writer, message+"\n")
	if err != nil {
		panic(err)
	}
}
<commit_msg>packer: Output a newline when interrupted for UI<commit_after>package packer

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"sync"
)

type UiColor uint

const (
	UiColorRed     UiColor = 31
	UiColorGreen           = 32
	UiColorYellow          = 33
	UiColorBlue            = 34
	UiColorMagenta         = 35
	UiColorCyan            = 36
)

// The Ui interface handles all communication for Packer with the outside
// world. This sort of control allows us to strictly control how output
// is formatted and various levels of output.
type Ui interface {
	Ask(string) (string, error)
	Say(string)
	Message(string)
	Error(string)
}

// ColoredUi is a UI that is colored using terminal colors.
type ColoredUi struct {
	Color      UiColor
	ErrorColor UiColor
	Ui         Ui
}

// PrefixedUi is a UI that wraps another UI implementation and adds a
// prefix to all the messages going out.
type PrefixedUi struct {
	SayPrefix     string
	MessagePrefix string
	Ui            Ui
}

// The ReaderWriterUi is a UI that writes and reads from standard Go
// io.Reader and io.Writer.
type ReaderWriterUi struct {
	Reader      io.Reader
	Writer      io.Writer
	l           sync.Mutex
	interrupted bool
}

func (u *ColoredUi) Ask(query string) (string, error) {
	return u.Ui.Ask(u.colorize(query, u.Color, true))
}

func (u *ColoredUi) Say(message string) {
	u.Ui.Say(u.colorize(message, u.Color, true))
}

func (u *ColoredUi) Message(message string) {
	u.Ui.Message(u.colorize(message, u.Color, false))
}

func (u *ColoredUi) Error(message string) {
	color := u.ErrorColor
	if color == 0 {
		color = UiColorRed
	}

	u.Ui.Error(u.colorize(message, color, true))
}

func (u *ColoredUi) colorize(message string, color UiColor, bold bool) string {
	attr := 0
	if bold {
		attr = 1
	}

	return fmt.Sprintf("\033[%d;%d;40m%s\033[0m", attr, color, message)
}

func (u *PrefixedUi) Ask(query string) (string, error) {
	return u.Ui.Ask(fmt.Sprintf("%s: %s", u.SayPrefix, query))
}

func (u *PrefixedUi) Say(message string) {
	u.Ui.Say(fmt.Sprintf("%s: %s", u.SayPrefix, message))
}

func (u *PrefixedUi) Message(message string) {
	u.Ui.Message(fmt.Sprintf("%s: %s", u.MessagePrefix, message))
}

func (u *PrefixedUi) Error(message string) {
	u.Ui.Error(fmt.Sprintf("%s: %s", u.SayPrefix, message))
}

func (rw *ReaderWriterUi) Ask(query string) (string, error) {
	rw.l.Lock()
	defer rw.l.Unlock()

	if rw.interrupted {
		return "", errors.New("interrupted")
	}

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, os.Interrupt)
	defer signal.Stop(sigCh)

	log.Printf("ui: ask: %s", query)
	if query != "" {
		if _, err := fmt.Fprint(rw.Writer, query+" "); err != nil {
			return "", err
		}
	}

	result := make(chan string, 1)
	go func() {
		var line string
		if _, err := fmt.Fscanln(rw.Reader, &line); err != nil {
			log.Printf("ui: scan err: %s", err)
		}

		result <- line
	}()

	select {
	case line := <-result:
		return line, nil
	case <-sigCh:
		// Print a newline so that any further output starts properly
		// on a new line.
		fmt.Fprintln(rw.Writer)

		// Mark that we were interrupted so future Ask calls fail.
		rw.interrupted = true

		return "", errors.New("interrupted")
	}
}

func (rw *ReaderWriterUi) Say(message string) {
	rw.l.Lock()
	defer rw.l.Unlock()

	log.Printf("ui: %s", message)
	_, err := fmt.Fprint(rw.Writer, message+"\n")
	if err != nil {
		panic(err)
	}
}

func (rw *ReaderWriterUi) Message(message string) {
	rw.l.Lock()
	defer rw.l.Unlock()

	log.Printf("ui: %s", message)
	_, err := fmt.Fprintf(rw.Writer, message+"\n")
	if err != nil {
		panic(err)
	}
}

func (rw *ReaderWriterUi) Error(message string) {
	rw.l.Lock()
	defer rw.l.Unlock()

	log.Printf("ui error: %s", message)
	_, err := fmt.Fprint(rw.Writer, message+"\n")
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"github.com/realglobe-Inc/go-lib-rg/file"
	"github.com/realglobe-Inc/go-lib-rg/rglog/level"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type parameters struct {
	// 画面表示ログ。
	consLv level.Level

	// 追加ログ。
	logType string
	logLv   level.Level

	// ファイルログ。
	logPath string

	// fluentd ログ。
	fluAddr string
	fluTag  string

	// 秘密鍵置き場。
	priKeyContType string

	// ファイルベース秘密鍵置き場。
	priKeyContPath string

	// ソケット。
	socType string

	// UNIX ソケット。
	socPath string

	// TCP ソケット。
	socPort int

	// プロトコル。
	protType string

	// キャッシュの有効期間。
	caExpiDur time.Duration

	// 称する TA の ID。
	taId string

	// 署名に使うハッシュ関数。
	hashName string

	// 有効期限ギリギリのセッションを避けるための遊び。
	sessMargin time.Duration

	// 同一ホスト用の http.Client の保持期間。
	cliExpiDur time.Duration

	// セッションを事前に検査するボディサイズの下限。
	threSize int
}

func parseParameters(args ...string) (param *parameters, err error) {

	const label = "access-proxy"

	flags := util.NewFlagSet("edo-"+label+" parameters", flag.ExitOnError)
	flags.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage:")
		fmt.Fprintln(os.Stderr, "  "+args[0]+" [{FLAG}...]")
		fmt.Fprintln(os.Stderr, "FLAG:")
		flags.PrintDefaults()
	}

	param = &parameters{}

	flags.Var(level.Var(&param.consLv, level.INFO), "consLv", "Console log level.")
	flags.StringVar(&param.logType, "logType", "", "Extra log type.")
	flags.Var(level.Var(&param.logLv, level.ALL), "logLv", "Extra log level.")
	flags.StringVar(&param.logPath, "logPath", filepath.Join(os.TempDir(), "edo", label+".log"), "File log path.")
	flags.StringVar(&param.fluAddr, "fluAddr", "localhost:24224", "fluentd address.")
	flags.StringVar(&param.fluTag, "fluTag", "edo."+label, "fluentd tag.")

	flags.StringVar(&param.priKeyContType, "priKeyContType", "file", "Private key container type.")
	flags.StringVar(&param.priKeyContPath, "priKeyContPath", filepath.Join("sandbox", "private-key"), "Private key container directory.")

	flags.StringVar(&param.socType, "socType", "tcp", "Socket type.")
	flags.StringVar(&param.socPath, "socPath", filepath.Join(os.TempDir(), "edo", label), "UNIX socket path.")
	flags.IntVar(&param.socPort, "socPort", 16051, "TCP socket port.")
	flags.StringVar(&param.protType, "protType", "http", "Protocol type.")

	flags.DurationVar(&param.caExpiDur, "caExpiDur", 10*time.Minute, "Cache expiration duration.")

	flags.StringVar(&param.taId, "taId", "", "TA ID.")
	flags.StringVar(&param.hashName, "hashName", "sha256", "Sign hash type.")

	flags.DurationVar(&param.sessMargin, "sessMargin", time.Minute, "Margin for session expiration duration.")
	flags.DurationVar(&param.cliExpiDur, "cliExpiDur", 10*time.Minute, "Client expiration duration.")
	flags.IntVar(&param.threSize, "threSize", 8192, "Maximum byte size of request body for no session check.")

	var config string
	flags.StringVar(&config, "f", "", "Config file path.")

	// 実行引数を読んで、設定ファイルを指定させてから、
	// 設定ファイルを読んで、また実行引数を読む。
	flags.Parse(args[1:])
	if config != "" {
		if exist, err := file.IsExist(config); err != nil {
			return nil, erro.Wrap(err)
		} else if !exist {
			log.Warn("Config file " + config + " is not exist.")
		} else {
			buff, err := ioutil.ReadFile(config)
			if err != nil {
				return nil, erro.Wrap(err)
			}
			flags.CompleteParse(strings.Fields(string(buff)))
		}
	}
	flags.Parse(args[1:])

	if l := len(flags.Args()); l > 0 {
		log.Warn("Ignore extra parameters ", flags.Args(), ".")
	}

	return param, nil
}
<commit_msg>使用するファイルが1つのディレクトリ以下に収まるようにデフォルト値を変更<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"github.com/realglobe-Inc/go-lib-rg/file"
	"github.com/realglobe-Inc/go-lib-rg/rglog/level"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type parameters struct {
	// 画面表示ログ。
	consLv level.Level

	// 追加ログ。
	logType string
	logLv   level.Level

	// ファイルログ。
	logPath string

	// fluentd ログ。
	fluAddr string
	fluTag  string

	// 秘密鍵置き場。
	priKeyContType string

	// ファイルベース秘密鍵置き場。
	priKeyContPath string

	// ソケット。
	socType string

	// UNIX ソケット。
	socPath string

	// TCP ソケット。
	socPort int

	// プロトコル。
	protType string

	// キャッシュの有効期間。
	caExpiDur time.Duration

	// 称する TA の ID。
	taId string

	// 署名に使うハッシュ関数。
	hashName string

	// 有効期限ギリギリのセッションを避けるための遊び。
	sessMargin time.Duration

	// 同一ホスト用の http.Client の保持期間。
	cliExpiDur time.Duration

	// セッションを事前に検査するボディサイズの下限。
	threSize int
}

func parseParameters(args ...string) (param *parameters, err error) {

	const label = "edo-access-proxy"

	flags := util.NewFlagSet(label+" parameters", flag.ExitOnError)
	flags.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage:")
		fmt.Fprintln(os.Stderr, "  "+args[0]+" [{FLAG}...]")
		fmt.Fprintln(os.Stderr, "FLAG:")
		flags.PrintDefaults()
	}

	param = &parameters{}

	flags.Var(level.Var(&param.consLv, level.INFO), "consLv", "Console log level.")
	flags.StringVar(&param.logType, "logType", "", "Extra log type.")
	flags.Var(level.Var(&param.logLv, level.ALL), "logLv", "Extra log level.")
	flags.StringVar(&param.logPath, "logPath", filepath.Join(filepath.Dir(os.Args[0]), "log", label+".log"), "File log path.")
	flags.StringVar(&param.fluAddr, "fluAddr", "localhost:24224", "fluentd address.")
	flags.StringVar(&param.fluTag, "fluTag", "edo."+label, "fluentd tag.")

	flags.StringVar(&param.priKeyContType, "priKeyContType", "file", "Private key container type.")
	flags.StringVar(&param.priKeyContPath, "priKeyContPath", filepath.Join(filepath.Dir(os.Args[0]), "private_keys"), "Private key container directory.")

	flags.StringVar(&param.socType, "socType", "tcp", "Socket type.")
	flags.StringVar(&param.socPath, "socPath", filepath.Join(filepath.Dir(os.Args[0]), "run", label+".soc"), "UNIX socket path.")
	flags.IntVar(&param.socPort, "socPort", 16051, "TCP socket port.")
	flags.StringVar(&param.protType, "protType", "http", "Protocol type.")

	flags.DurationVar(&param.caExpiDur, "caExpiDur", 10*time.Minute, "Cache expiration duration.")

	flags.StringVar(&param.taId, "taId", "", "Default TA ID.")
	flags.StringVar(&param.hashName, "hashName", "sha256", "Sign hash type.")

	flags.DurationVar(&param.sessMargin, "sessMargin", time.Minute, "Margin for session expiration duration.")
	flags.DurationVar(&param.cliExpiDur, "cliExpiDur", 10*time.Minute, "Client expiration duration.")
	flags.IntVar(&param.threSize, "threSize", 8192, "Maximum byte size of request body for skipping session check.")

	var config string
	flags.StringVar(&config, "f", "", "Config file path.")

	// 実行引数を読んで、設定ファイルを指定させてから、
	// 設定ファイルを読んで、また実行引数を読む。
	flags.Parse(args[1:])
	if config != "" {
		if exist, err := file.IsExist(config); err != nil {
			return nil, erro.Wrap(err)
		} else if !exist {
			log.Warn("Config file " + config + " is not exist.")
		} else {
			buff, err := ioutil.ReadFile(config)
			if err != nil {
				return nil, erro.Wrap(err)
			}
			flags.CompleteParse(strings.Fields(string(buff)))
		}
	}
	flags.Parse(args[1:])

	if l := len(flags.Args()); l > 0 {
		log.Warn("Ignore extra parameters ", flags.Args(), ".")
	}

	return param, nil
}
<|endoftext|>


<commit_before>package goka

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lovoo/goka/kafka"
	"github.com/lovoo/goka/logger"
	"github.com/lovoo/goka/storage"

	"github.com/Shopify/sarama"
)

const (
	defaultPartitionChannelSize = 10
	stallPeriod                 = 30 * time.Second
	stalledTimeout              = 2 * time.Minute
)

type partition struct {
	log   logger.Logger
	topic string

	ch      chan kafka.Event
	st      *storageProxy
	proxy   kafkaProxy
	process processCallback

	dying    chan bool
	done     chan bool
	stopFlag int64

	recoveredFlag int32
	hwm           int64
	offset        int64

	recoveredOnce sync.Once

	stats         *PartitionStats
	lastStats     *PartitionStats
	requestStats  chan bool
	responseStats chan *PartitionStats
}

type kafkaProxy interface {
	Add(string, int64)
	Remove(string)
	AddGroup()
	Stop()
}

type processCallback func(msg *message, st storage.Storage, wg *sync.WaitGroup, pstats *PartitionStats) (int, error)

func newPartition(log logger.Logger, topic string, cb processCallback, st *storageProxy, proxy kafkaProxy, channelSize int) *partition {
	return &partition{
		log:   log,
		topic: topic,

		ch:    make(chan kafka.Event, channelSize),
		dying: make(chan bool),
		done:  make(chan bool),

		st:            st,
		recoveredOnce: sync.Once{},
		proxy:         proxy,
		process:       cb,

		stats:         newPartitionStats(),
		lastStats:     newPartitionStats(),
		requestStats:  make(chan bool),
		responseStats: make(chan *PartitionStats, 1),
	}
}

func (p *partition) start() error {
	defer close(p.done)
	defer p.proxy.Stop()
	p.stats.Table.StartTime = time.Now()

	if !p.st.Stateless() {
		err := p.st.Open()
		if err != nil {
			return err
		}
		defer p.st.Close()

		if err := p.recover(); err != nil {
			return err
		}
	} else {
		p.markRecovered()
	}

	// if stopped, just return
	if atomic.LoadInt64(&p.stopFlag) == 1 {
		return nil
	}
	return p.run()
}

func (p *partition) startCatchup() error {
	defer close(p.done)
	defer p.proxy.Stop()
	p.stats.Table.StartTime = time.Now()

	err := p.st.Open()
	if err != nil {
		return err
	}
	defer p.st.Close()

	return p.catchup()
}

func (p *partition) stop() {
	atomic.StoreInt64(&p.stopFlag, 1)
	close(p.dying)
	<-p.done
	close(p.ch)
}

///////////////////////////////////////////////////////////////////////////////
// processing
///////////////////////////////////////////////////////////////////////////////
func newMessage(ev *kafka.Message) *message {
	return &message{
		Topic:     string(ev.Topic),
		Partition: int32(ev.Partition),
		Offset:    int64(ev.Offset),
		Timestamp: ev.Timestamp,
		Data:      ev.Value,
		Key:       string(ev.Key),
	}
}

func (p *partition) run() error {
	var wg sync.WaitGroup
	p.proxy.AddGroup()
	defer wg.Wait()

	for {
		select {
		case ev, isOpen := <-p.ch:
			// channel already closed, ev will be nil
			if !isOpen {
				return nil
			}
			switch ev := ev.(type) {
			case *kafka.Message:
				if ev.Topic == p.topic {
					return fmt.Errorf("received message from group table topic after recovery: %s", p.topic)
				}

				updates, err := p.process(newMessage(ev), p.st, &wg, p.stats)
				if err != nil {
					return fmt.Errorf("error processing message: %v", err)
				}
				p.offset += int64(updates)
				p.hwm = p.offset + 1

				// metrics
				s := p.stats.Input[ev.Topic]
				s.Count++
				s.Bytes += len(ev.Value)
				if !ev.Timestamp.IsZero() {
					s.Delay = time.Since(ev.Timestamp)
				}
				p.stats.Input[ev.Topic] = s

			case *kafka.NOP:
				// don't do anything but also don't log.
			case *kafka.EOF:
				if ev.Topic != p.topic {
					return fmt.Errorf("received EOF of topic that is not ours. This should not happend (ours=%s, received=%s)", p.topic, ev.Topic)
				}
			default:
				return fmt.Errorf("load: cannot handle %T = %v", ev, ev)
			}

		case <-p.requestStats:
			p.lastStats = newPartitionStats().init(p.stats, p.offset, p.hwm)
			select {
			case p.responseStats <- p.lastStats:
			case <-p.dying:
				return nil
			}

		case <-p.dying:
			return nil
		}

	}
}

///////////////////////////////////////////////////////////////////////////////
// loading storage
///////////////////////////////////////////////////////////////////////////////

func (p *partition) catchup() error {
	return p.load(true)
}

func (p *partition) recover() error {
	return p.load(false)
}

func (p *partition) recovered() bool {
	return atomic.LoadInt32(&p.recoveredFlag) == 1
}

func (p *partition) load(catchup bool) error {
	// fetch local offset
	local, err := p.st.GetOffset(sarama.OffsetOldest)
	if err != nil {
		return fmt.Errorf("Error reading local offset: %v", err)
	}
	p.proxy.Add(p.topic, local)
	defer p.proxy.Remove(p.topic)

	stallTicker := time.NewTicker(stallPeriod)
	defer stallTicker.Stop()

	// reset stats after load
	defer p.stats.reset()

	var lastMessage time.Time
	for {
		select {
		case ev, isOpen := <-p.ch:

			// channel already closed, ev will be nil
			if !isOpen {
				return nil
			}

			switch ev := ev.(type) {
			case *kafka.BOF:
				p.hwm = ev.Hwm

				if ev.Offset == ev.Hwm {
					// nothing to recover
					if err := p.markRecovered(); err != nil {
						return fmt.Errorf("error setting recovered: %v", err)
					}
				}

			case *kafka.EOF:
				p.offset = ev.Hwm - 1
				p.hwm = ev.Hwm

				if err := p.markRecovered(); err != nil {
					return fmt.Errorf("error setting recovered: %v", err)
				}

				if catchup {
					continue
				}
				return nil

			case *kafka.Message:
				lastMessage = time.Now()
				if ev.Topic != p.topic {
					return fmt.Errorf("load: wrong topic = %s", ev.Topic)
				}
				err := p.storeEvent(ev)
				if err != nil {
					return fmt.Errorf("load: error updating storage: %v", err)
				}
				p.offset = ev.Offset
				if p.offset >= p.hwm-1 {
					if err := p.markRecovered(); err != nil {
						return fmt.Errorf("error setting recovered: %v", err)
					}
				}

				// update metrics
				s := p.stats.Input[ev.Topic]
				s.Count++
				s.Bytes += len(ev.Value)
				if !ev.Timestamp.IsZero() {
					s.Delay = time.Since(ev.Timestamp)
				}
				p.stats.Input[ev.Topic] = s
				p.stats.Table.Stalled = false

			case *kafka.NOP:
				// don't do anything

			default:
				return fmt.Errorf("load: cannot handle %T = %v", ev, ev)
			}

		case now := <-stallTicker.C:
			// only set to stalled, if the last message was earlier
			// than the stalled timeout
			if now.Sub(lastMessage) > stalledTimeout {
				p.stats.Table.Stalled = true
			}

		case <-p.requestStats:
			p.lastStats = newPartitionStats().init(p.stats, p.offset, p.hwm)
			select {
			case p.responseStats <- p.lastStats:
			case <-p.dying:
				return nil
			}

		case <-p.dying:
			return nil
		}
	}
}

func (p *partition) storeEvent(msg *kafka.Message) error {
	err := p.st.Update(msg.Key, msg.Value)
	if err != nil {
		return fmt.Errorf("Error from the update callback while recovering from the log: %v", err)
	}
	err = p.st.SetOffset(int64(msg.Offset))
	if err != nil {
		return fmt.Errorf("Error updating offset in local storage while recovering from the log: %v", err)
	}
	return nil
}

// mark storage as recovered
func (p *partition) markRecovered() (err error) {
	p.recoveredOnce.Do(func() {
		p.lastStats = newPartitionStats().init(p.stats, p.offset, p.hwm)
		p.lastStats.Table.Status = PartitionPreparing

		// before marking partition as recovered, stop reading from topic
		p.proxy.Remove(p.topic)

		// drain events channel
		done := make(chan bool)
		go func() {
			for {
				select {
				case <-p.ch:
				case <-done:
					return

				}
			}
		}()
		defer close(done)

		// while storage is being marked as recovered, we drop all messages
		// from topic since this may take long
		if err = p.st.MarkRecovered(); err != nil {
			return
		}

		// start reading from topic again
		p.proxy.Add(p.topic, p.hwm)

		// update stats
		p.stats.Table.Status = PartitionRunning
		p.stats.Table.RecoveryTime = time.Now()

		atomic.StoreInt32(&p.recoveredFlag, 1)
	})
	return
}

func (p *partition) fetchStats() *PartitionStats {
	timer := time.NewTimer(100 * time.Millisecond)
	defer timer.Stop()

	select {
	case p.requestStats <- true:
	case <-p.dying:
		// if closing, return empty stats
		return newPartitionStats()
	case <-timer.C:
		return p.lastStats
	}

	select {
	case s := <-p.responseStats:
		return s
	case <-p.dying:
		// if closing, return empty stats
		return newPartitionStats()
	}
}
<commit_msg>use catchup flag in markRecovered<commit_after>package goka

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lovoo/goka/kafka"
	"github.com/lovoo/goka/logger"
	"github.com/lovoo/goka/storage"

	"github.com/Shopify/sarama"
)

const (
	defaultPartitionChannelSize = 10
	stallPeriod                 = 30 * time.Second
	stalledTimeout              = 2 * time.Minute
)

type partition struct {
	log   logger.Logger
	topic string

	ch      chan kafka.Event
	st      *storageProxy
	proxy   kafkaProxy
	process processCallback

	dying    chan bool
	done     chan bool
	stopFlag int64

	recoveredFlag int32
	hwm           int64
	offset        int64

	recoveredOnce sync.Once

	stats         *PartitionStats
	lastStats     *PartitionStats
	requestStats  chan bool
	responseStats chan *PartitionStats
}

type kafkaProxy interface {
	Add(string, int64)
	Remove(string)
	AddGroup()
	Stop()
}

type processCallback func(msg *message, st storage.Storage, wg *sync.WaitGroup, pstats *PartitionStats) (int, error)

func newPartition(log logger.Logger, topic string, cb processCallback, st *storageProxy, proxy kafkaProxy, channelSize int) *partition {
	return &partition{
		log:   log,
		topic: topic,

		ch:    make(chan kafka.Event, channelSize),
		dying: make(chan bool),
		done:  make(chan bool),

		st:            st,
		recoveredOnce: sync.Once{},
		proxy:         proxy,
		process:       cb,

		stats:         newPartitionStats(),
		lastStats:     newPartitionStats(),
		requestStats:  make(chan bool),
		responseStats: make(chan *PartitionStats, 1),
	}
}

func (p *partition) start() error {
	defer close(p.done)
	defer p.proxy.Stop()
	p.stats.Table.StartTime = time.Now()

	if !p.st.Stateless() {
		err := p.st.Open()
		if err != nil {
			return err
		}
		defer p.st.Close()

		if err := p.recover(); err != nil {
			return err
		}
	} else {
		p.markRecovered(false)
	}

	// if stopped, just return
	if atomic.LoadInt64(&p.stopFlag) == 1 {
		return nil
	}
	return p.run()
}

func (p *partition) startCatchup() error {
	defer close(p.done)
	defer p.proxy.Stop()
	p.stats.Table.StartTime = time.Now()

	err := p.st.Open()
	if err != nil {
		return err
	}
	defer p.st.Close()

	return p.catchup()
}

func (p *partition) stop() {
	atomic.StoreInt64(&p.stopFlag, 1)
	close(p.dying)
	<-p.done
	close(p.ch)
}

///////////////////////////////////////////////////////////////////////////////
// processing
///////////////////////////////////////////////////////////////////////////////
func newMessage(ev *kafka.Message) *message {
	return &message{
		Topic:     string(ev.Topic),
		Partition: int32(ev.Partition),
		Offset:    int64(ev.Offset),
		Timestamp: ev.Timestamp,
		Data:      ev.Value,
		Key:       string(ev.Key),
	}
}

func (p *partition) run() error {
	var wg sync.WaitGroup
	p.proxy.AddGroup()
	defer wg.Wait()

	for {
		select {
		case ev, isOpen := <-p.ch:
			// channel already closed, ev will be nil
			if !isOpen {
				return nil
			}
			switch ev := ev.(type) {
			case *kafka.Message:
				if ev.Topic == p.topic {
					return fmt.Errorf("received message from group table topic after recovery: %s", p.topic)
				}

				updates, err := p.process(newMessage(ev), p.st, &wg, p.stats)
				if err != nil {
					return fmt.Errorf("error processing message: %v", err)
				}
				p.offset += int64(updates)
				p.hwm = p.offset + 1

				// metrics
				s := p.stats.Input[ev.Topic]
				s.Count++
				s.Bytes += len(ev.Value)
				if !ev.Timestamp.IsZero() {
					s.Delay = time.Since(ev.Timestamp)
				}
				p.stats.Input[ev.Topic] = s

			case *kafka.NOP:
				// don't do anything but also don't log.
			case *kafka.EOF:
				if ev.Topic != p.topic {
					return fmt.Errorf("received EOF of topic that is not ours. This should not happend (ours=%s, received=%s)", p.topic, ev.Topic)
				}
			default:
				return fmt.Errorf("load: cannot handle %T = %v", ev, ev)
			}

		case <-p.requestStats:
			p.lastStats = newPartitionStats().init(p.stats, p.offset, p.hwm)
			select {
			case p.responseStats <- p.lastStats:
			case <-p.dying:
				return nil
			}

		case <-p.dying:
			return nil
		}

	}
}

///////////////////////////////////////////////////////////////////////////////
// loading storage
///////////////////////////////////////////////////////////////////////////////

func (p *partition) catchup() error {
	return p.load(true)
}

func (p *partition) recover() error {
	return p.load(false)
}

func (p *partition) recovered() bool {
	return atomic.LoadInt32(&p.recoveredFlag) == 1
}

func (p *partition) load(catchup bool) error {
	// fetch local offset
	local, err := p.st.GetOffset(sarama.OffsetOldest)
	if err != nil {
		return fmt.Errorf("Error reading local offset: %v", err)
	}
	p.proxy.Add(p.topic, local)
	defer p.proxy.Remove(p.topic)

	stallTicker := time.NewTicker(stallPeriod)
	defer stallTicker.Stop()

	// reset stats after load
	defer p.stats.reset()

	var lastMessage time.Time
	for {
		select {
		case ev, isOpen := <-p.ch:

			// channel already closed, ev will be nil
			if !isOpen {
				return nil
			}

			switch ev := ev.(type) {
			case *kafka.BOF:
				p.hwm = ev.Hwm

				if ev.Offset == ev.Hwm {
					// nothing to recover
					if err := p.markRecovered(false); err != nil {
						return fmt.Errorf("error setting recovered: %v", err)
					}
				}

			case *kafka.EOF:
				p.offset = ev.Hwm - 1
				p.hwm = ev.Hwm

				if err := p.markRecovered(catchup); err != nil {
					return fmt.Errorf("error setting recovered: %v", err)
				}

				if catchup {
					continue
				}
				return nil

			case *kafka.Message:
				lastMessage = time.Now()
				if ev.Topic != p.topic {
					return fmt.Errorf("load: wrong topic = %s", ev.Topic)
				}
				err := p.storeEvent(ev)
				if err != nil {
					return fmt.Errorf("load: error updating storage: %v", err)
				}
				p.offset = ev.Offset
				if p.offset >= p.hwm-1 {
					if err := p.markRecovered(catchup); err != nil {
						return fmt.Errorf("error setting recovered: %v", err)
					}
				}

				// update metrics
				s := p.stats.Input[ev.Topic]
				s.Count++
				s.Bytes += len(ev.Value)
				if !ev.Timestamp.IsZero() {
					s.Delay = time.Since(ev.Timestamp)
				}
				p.stats.Input[ev.Topic] = s
				p.stats.Table.Stalled = false

			case *kafka.NOP:
				// don't do anything

			default:
				return fmt.Errorf("load: cannot handle %T = %v", ev, ev)
			}

		case now := <-stallTicker.C:
			// only set to stalled, if the last message was earlier
			// than the stalled timeout
			if now.Sub(lastMessage) > stalledTimeout {
				p.stats.Table.Stalled = true
			}

		case <-p.requestStats:
			p.lastStats = newPartitionStats().init(p.stats, p.offset, p.hwm)
			select {
			case p.responseStats <- p.lastStats:
			case <-p.dying:
				return nil
			}

		case <-p.dying:
			return nil
		}
	}
}

func (p *partition) storeEvent(msg *kafka.Message) error {
	err := p.st.Update(msg.Key, msg.Value)
	if err != nil {
		return fmt.Errorf("Error from the update callback while recovering from the log: %v", err)
	}
	err = p.st.SetOffset(int64(msg.Offset))
	if err != nil {
		return fmt.Errorf("Error updating offset in local storage while recovering from the log: %v", err)
	}
	return nil
}

// mark storage as recovered
func (p *partition) markRecovered(catchup bool) (err error) {
	p.recoveredOnce.Do(func() {
		p.lastStats = newPartitionStats().init(p.stats, p.offset, p.hwm)
		p.lastStats.Table.Status = PartitionPreparing

		if catchup {
			// if catching up (views), stop reading from topic before marking
			// partition as recovered to avoid blocking other partitions when
			// p.ch gets full
			p.proxy.Remove(p.topic)

			// drain events channel -- we'll fetch them again later
			done := make(chan bool)
			go func() {
				for {
					select {
					case <-p.ch:
					case <-done:
						return

					}
				}
			}()
			defer close(done)
		}

		// mark storage as recovered -- this may take long
		if err = p.st.MarkRecovered(); err != nil {
			return
		}

		if catchup {
			// start reading from topic again if in catchup mode
			p.proxy.Add(p.topic, p.hwm)
		}

		// update stats
		p.stats.Table.Status = PartitionRunning
		p.stats.Table.RecoveryTime = time.Now()

		atomic.StoreInt32(&p.recoveredFlag, 1)
	})
	return
}

func (p *partition) fetchStats() *PartitionStats {
	timer := time.NewTimer(100 * time.Millisecond)
	defer timer.Stop()

	select {
	case p.requestStats <- true:
	case <-p.dying:
		// if closing, return empty stats
		return newPartitionStats()
	case <-timer.C:
		return p.lastStats
	}

	select {
	case s := <-p.responseStats:
		return s
	case <-p.dying:
		// if closing, return empty stats
		return newPartitionStats()
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The goyy Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package client

import (
	"gopkg.in/goyy/goyy.v0/data/result"
	"gopkg.in/goyy/goyy.v0/util/errors"
	"gopkg.in/goyy/goyy.v0/util/strings"
	"io/ioutil"
	"net/http"
	"net/url"
)

type Client struct {
	URL         string
	Params      url.Values
	Header      http.Header
	Cookies     []*http.Cookie
	Timeout     int
	OnTimeout   func()
	OnError     func(error)
	OnCompleted func(*result.Client)
}

func (me *Client) DoGet() {
	me.do("GET")
}

func (me *Client) DoPost() {
	me.do("POST")
}

func (me *Client) GoGet() {
	go me.do("GET")
}

func (me *Client) GoPost() {
	go me.do("POST")
}

func (me *Client) QueueGet() {
	go me.do("GET")
}

func (me *Client) QueuePost() {
	go me.do("POST")
}

func (me *Client) onError(err error) {
	if me.OnError != nil {
		me.OnError(err)
	} else {
		panic(err.Error())
	}
}

func (me *Client) do(method string) {
	if strings.IsBlank(me.URL) {
		me.onError(errors.NewNotBlank("URL"))
		return
	}
	client := &http.Client{}
	req, err := me.getRequest(method)
	if err != nil {
		me.onError(err)
		return
	}
	resp, err := client.Do(req)
	if err != nil {
		logger.Debug(err.Error())
		me.onError(err)
		return
	}
	if resp.StatusCode >= 400 {
		me.onError(errors.New(resp.Status))
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.Debug(err.Error())
		me.onError(err)
		return
	}
	if me.OnCompleted != nil {
		r := &result.Client{
			Body:       body,
			Status:     resp.Status,
			StatusCode: resp.StatusCode,
			Header:     resp.Header,
			Cookies:    resp.Cookies(),
		}
		me.OnCompleted(r)
	}
}

func (me *Client) getRequest(method string) (*http.Request, error) {
	if method == "GET" {
		url := me.URL
		if me.Params != nil {
			if strings.Contains(me.URL, "?") {
				url = me.URL + "&" + me.Params.Encode()
			} else {
				url = me.URL + "?" + me.Params.Encode()
			}
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			logger.Debug(err.Error())
			return req, err
		}
		req.Header = me.Header
		for _, c := range me.Cookies {
			req.AddCookie(c)
		}
		return req, err
	} else {
		req, err := http.NewRequest("POST", me.URL, strings.NewReader(me.Params.Encode()))
		if err != nil {
			logger.Debug(err.Error())
			return req, err
		}
		req.Header = me.Header
		if strings.IsBlank(req.Header.Get("Content-Type")) {
			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
		}
		for _, c := range me.Cookies {
			req.AddCookie(c)
		}
		return req, err
	}
}
<commit_msg>Add error log output<commit_after>// Copyright 2014 The goyy Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package client

import (
	"gopkg.in/goyy/goyy.v0/data/result"
	"gopkg.in/goyy/goyy.v0/util/errors"
	"gopkg.in/goyy/goyy.v0/util/strings"
	"io/ioutil"
	"net/http"
	"net/url"
)

type Client struct {
	URL         string
	Params      url.Values
	Header      http.Header
	Cookies     []*http.Cookie
	Timeout     int
	OnTimeout   func()
	OnError     func(error)
	OnCompleted func(*result.Client)
}

func (me *Client) DoGet() {
	me.do("GET")
}

func (me *Client) DoPost() {
	me.do("POST")
}

func (me *Client) GoGet() {
	go me.do("GET")
}

func (me *Client) GoPost() {
	go me.do("POST")
}

func (me *Client) QueueGet() {
	go me.do("GET")
}

func (me *Client) QueuePost() {
	go me.do("POST")
}

func (me *Client) onError(err error) {
	logger.Error(err.Error())
	if me.OnError != nil {
		me.OnError(err)
	}
}

func (me *Client) do(method string) {
	if strings.IsBlank(me.URL) {
		me.onError(errors.NewNotBlank("URL"))
		return
	}
	client := &http.Client{}
	req, err := me.getRequest(method)
	if err != nil {
		me.onError(err)
		return
	}
	resp, err := client.Do(req)
	if err != nil {
		logger.Debug(err.Error())
		me.onError(err)
		return
	}
	if resp.StatusCode >= 400 {
		me.onError(errors.New(resp.Status))
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.Debug(err.Error())
		me.onError(err)
		return
	}
	if me.OnCompleted != nil {
		r := &result.Client{
			Body:       body,
			Status:     resp.Status,
			StatusCode: resp.StatusCode,
			Header:     resp.Header,
			Cookies:    resp.Cookies(),
		}
		me.OnCompleted(r)
	}
}

func (me *Client) getRequest(method string) (*http.Request, error) {
	if method == "GET" {
		url := me.URL
		if me.Params != nil {
			if strings.Contains(me.URL, "?") {
				url = me.URL + "&" + me.Params.Encode()
			} else {
				url = me.URL + "?" + me.Params.Encode()
			}
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			logger.Debug(err.Error())
			return req, err
		}
		req.Header = me.Header
		for _, c := range me.Cookies {
			req.AddCookie(c)
		}
		return req, err
	} else {
		req, err := http.NewRequest("POST", me.URL, strings.NewReader(me.Params.Encode()))
		if err != nil {
			logger.Debug(err.Error())
			return req, err
		}
		req.Header = me.Header
		if strings.IsBlank(req.Header.Get("Content-Type")) {
			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
		}
		for _, c := range me.Cookies {
			req.AddCookie(c)
		}
		return req, err
	}
}
<|endoftext|>


<commit_before>package reporting

import (
	"bytes"
	"encoding/json"
	"github.com/smartystreets/goconvey/printing"
)

func (self *jsonReporter) BeginStory(story *StoryReport) {
	top := newScopeResult(story.Name, self.depth, story.File, story.Line)
	self.scopes = append(self.scopes, top)
	self.stack = append(self.stack, top)
}

func (self *jsonReporter) Enter(scope *ScopeReport) {
	self.depth++
	if _, found := self.titlesById[scope.ID]; !found {
		self.registerScope(scope)
	}
}
func (self *jsonReporter) registerScope(scope *ScopeReport) {
	self.titlesById[scope.ID] = scope.ID
	next := newScopeResult(scope.Title, self.depth, scope.File, scope.Line)
	self.scopes = append(self.scopes, next)
	self.stack = append(self.stack, next)
}

func (self *jsonReporter) Report(report *AssertionReport) {
	current := self.stack[len(self.stack)-1]
	current.Assertions = append(current.Assertions, newAssertionResult(report))
}

func (self *jsonReporter) Exit() {
	self.depth--
	if len(self.stack) > 0 {
		self.stack = self.stack[:len(self.stack)-1]
	}
}

func (self *jsonReporter) EndStory() {
	self.report()
	self.reset()
}
func (self *jsonReporter) report() {
	serialized, _ := json.Marshal(self.scopes)
	var buffer bytes.Buffer
	json.Indent(&buffer, serialized, "", "  ")
	self.out.Print(buffer.String() + ",\n")
}
func (self *jsonReporter) reset() {
	self.titlesById = make(map[string]string)
	self.scopes = []*ScopeResult{}
	self.stack = []*ScopeResult{}
	self.depth = 0
}

func NewJsonReporter(out *printing.Printer) *jsonReporter {
	self := &jsonReporter{}
	self.out = out
	self.reset()
	return self
}

type jsonReporter struct {
	out        *printing.Printer
	titlesById map[string]string
	scopes     []*ScopeResult
	stack      []*ScopeResult
	depth      int
}

type ScopeResult struct {
	Title      string
	File       string
	Line       int
	Depth      int
	Assertions []AssertionResult
}

func newScopeResult(title string, depth int, file string, line int) *ScopeResult {
	self := &ScopeResult{}
	self.Title = title
	self.Depth = depth
	self.File = file
	self.Line = line
	self.Assertions = []AssertionResult{}
	return self
}

type AssertionResult struct {
	File    string
	Line    int
	Failure string
	Error   interface{}
	Skipped bool

	// TODO: I'm going to have to parse this turn it into a structure that
	// can accomodate turning the file paths into urls when templated...
	StackTrace string
}

func newAssertionResult(report *AssertionReport) AssertionResult {
	self := AssertionResult{}
	self.File = report.File
	self.Line = report.Line
	self.Failure = report.Failure
	self.Error = report.Error
	self.StackTrace = report.stackTrace
	self.Skipped = report.Skipped
	return self
}
<commit_msg>Because we're going to parse verbose go test output we don't need to keep a scope just for the gotest function.<commit_after>package reporting

import (
	"bytes"
	"encoding/json"
	"github.com/smartystreets/goconvey/printing"
)

func (self *jsonReporter) BeginStory(story *StoryReport) {}

func (self *jsonReporter) Enter(scope *ScopeReport) {
	if _, found := self.titlesById[scope.ID]; !found {
		self.registerScope(scope)
	}
	self.depth++
}
func (self *jsonReporter) registerScope(scope *ScopeReport) {
	self.titlesById[scope.ID] = scope.ID
	next := newScopeResult(scope.Title, self.depth, scope.File, scope.Line)
	self.scopes = append(self.scopes, next)
	self.stack = append(self.stack, next)
}

func (self *jsonReporter) Report(report *AssertionReport) {
	current := self.stack[len(self.stack)-1]
	current.Assertions = append(current.Assertions, newAssertionResult(report))
}

func (self *jsonReporter) Exit() {
	self.depth--
	if len(self.stack) > 0 {
		self.stack = self.stack[:len(self.stack)-1]
	}
}

func (self *jsonReporter) EndStory() {
	self.report()
	self.reset()
}
func (self *jsonReporter) report() {
	serialized, _ := json.Marshal(self.scopes)
	var buffer bytes.Buffer
	json.Indent(&buffer, serialized, "", "  ")
	self.out.Print(buffer.String() + ",\n")
}
func (self *jsonReporter) reset() {
	self.titlesById = make(map[string]string)
	self.scopes = []*ScopeResult{}
	self.stack = []*ScopeResult{}
	self.depth = 0
}

func NewJsonReporter(out *printing.Printer) *jsonReporter {
	self := &jsonReporter{}
	self.out = out
	self.reset()
	return self
}

type jsonReporter struct {
	out        *printing.Printer
	titlesById map[string]string
	scopes     []*ScopeResult
	stack      []*ScopeResult
	depth      int
}

type ScopeResult struct {
	Title      string
	File       string
	Line       int
	Depth      int
	Assertions []AssertionResult
}

func newScopeResult(title string, depth int, file string, line int) *ScopeResult {
	self := &ScopeResult{}
	self.Title = title
	self.Depth = depth
	self.File = file
	self.Line = line
	self.Assertions = []AssertionResult{}
	return self
}

type AssertionResult struct {
	File    string
	Line    int
	Failure string
	Error   interface{}
	Skipped bool

	// TODO: I'm going to have to parse this turn it into a structure that
	// can accomodate turning the file paths into urls when templated...
	StackTrace string
}

func newAssertionResult(report *AssertionReport) AssertionResult {
	self := AssertionResult{}
	self.File = report.File
	self.Line = report.Line
	self.Failure = report.Failure
	self.Error = report.Error
	self.StackTrace = report.stackTrace
	self.Skipped = report.Skipped
	return self
}
<|endoftext|>


<commit_before>/*
 * MumbleDJ
 * By Matthieu Grieger
 * songqueue.go
 * Copyright (c) 2014 Matthieu Grieger (MIT License)
 */

 package main

 type SongQueue struct {
 	songs []Song
 }

 func NewSongQueue() *SongQueue {
 	return &SongQueue{}
 }<commit_msg>Add some unfinished methods to SongQueue<commit_after>/*
 * MumbleDJ
 * By Matthieu Grieger
 * songqueue.go
 * Copyright (c) 2014 Matthieu Grieger (MIT License)
 */

package main

type SongQueue struct {
	queue Queue
}

func NewSongQueue() *SongQueue {
	return &SongQueue{}
}

func (q *SongQueue) AddSong(s *Song) bool {
	return false
}

func (q *SongQueue) NextSong() bool {
	return false
}

//func (q *SongQueue) CurrentSong() Song {
//	return false
//}
<|endoftext|>


<commit_before>package controllers

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/nicomo/abacaxi/logger"
	"github.com/nicomo/abacaxi/models"
)

const (
	kbartNumFields = 25
)

func fileIO(pp parseparams, report *models.Report) ([]models.Record, error) {
	// slice will hold successfully parsed records
	var records []models.Record

	// retrieve target service (e.g. ebook package) for this file
	myTS, err := models.GetTargetService(pp.tsname)
	if err != nil {
		return records, err
	}

	// open file
	f, err := os.Open(pp.fpath)
	if err != nil {
		return nil, errors.New("cannot open the source file")
	}
	defer f.Close()

	reader := csv.NewReader(f)

	// target service csv has n fields, separator is ;
	if pp.filetype == "publishercsv" {
		reader.FieldsPerRecord = len(pp.csvconf)
	} else {
		reader.FieldsPerRecord = kbartNumFields // kbart is a const: always 25 fields
	}
	reader.Comma = ';' //TODO: detect separator, tab or comma, rather than force comma

	// counters to keep track of records parsed, for logging
	line := 1
	var rejectedLines []int

	for {
		// read a row
		r, err := reader.Read()

		// if at EOF, break out of loop
		if err != nil {
			if err == io.EOF {
				break
			}
			rejectedLines = append(rejectedLines, line)
			line++
			continue
		}

		// validate we don't have an unicode replacement char. in the string
		// if we do abort: source file isn't proper utf8
		for _, v := range r {
			if strings.ContainsRune(v, '\uFFFD') {
				err := errors.New("parsing failed: non utf-8 character in file")
				return records, err
			}
		}

		// parse each line into a struct
		record, err := fileParseRow(r, pp.csvconf)
		if err != nil {
			logger.Error.Println(err, r)
			continue
		}

		// add TS to record
		record.TargetServices = append(record.TargetServices, myTS)

		// add record to slice
		records = append(records, record)

		line++
	}

	// log number of records successfully parsed
	report.Text = append(report.Text, fmt.Sprintf("successfully parsed %d lines from %s",
		len(records), pp.fpath))

	// log the lines we rejected, if any
	if len(rejectedLines) > 0 {
		report.Text = append(report.Text, fmt.Sprintf("lines rejected in source file: %v", rejectedLines))
	}

	// no records parsed
	if len(records) == 0 {
		err := errors.New("couldn't parse a single line: check your input file")
		return records, err

	}

	return records, nil
}

func fileParseRow(row []string, csvConf map[string]int) (models.Record, error) {

	var record models.Record

	if csvConf == nil { // the csv Configuration is nil, we default to kbart values

		record.PublicationTitle = row[0]

		// ISBNs : validate & cleanup, convert isbn 10 <-> isbn13
		// Identifiers Print ID
		err := getIsbnIdentifiers(row[1], &record, models.IDTypePrint)
		if err != nil && row[1] != "" { // doesn't look like an isbn, might be issn, cleanup and add as is
			idCleaned := strings.Trim(strings.Replace(row[1], "-", "", -1), " ")
			record.Identifiers = append(record.Identifiers, models.Identifier{Identifier: idCleaned, IDType: models.IDTypePrint})
		}
		// Identifiers Online ID
		err = getIsbnIdentifiers(row[2], &record, models.IDTypeOnline)
		if err != nil && row[2] != "" { // doesn't look like an isbn, might be issn, cleanup and add as is
			idCleaned := strings.Trim(strings.Replace(row[2], "-", "", -1), " ")
			record.Identifiers = append(record.Identifiers, models.Identifier{Identifier: idCleaned, IDType: models.IDTypeOnline})
		}

		record.DateFirstIssueOnline = row[3]
		record.NumFirstVolOnline = row[4]
		record.NumFirstIssueOnline = row[5]
		record.DateLastIssueOnline = row[6]
		record.NumLastVolOnline = row[7]
		record.NumLastIssueOnline = row[8]
		record.TitleURL = row[9]
		record.FirstAuthor = row[10]
		record.TitleID = row[11]
		record.EmbargoInfo = row[12]
		record.CoverageDepth = row[13]
		record.Notes = row[14]
		record.PublisherName = row[15]
		record.PublicationType = row[16]
		record.DateMonographPublishedPrint = row[17]
		record.DateMonographPublishedOnline = row[18]
		record.MonographVolume = row[19]
		record.MonographEdition = row[20]
		record.FirstEditor = row[21]
		record.ParentPublicationTitleID = row[22]
		record.PrecedingPublicationTitleID = row[23]
		record.AccessType = row[24]
	} else { // we do have a csv configuration
		if i, ok := csvConf["publicationtitle"]; ok {
			record.PublicationTitle = row[i-1]
		}
		if i, ok := csvConf["identifierprint"]; ok {
			err := getIsbnIdentifiers(row[i-1], &record, models.IDTypePrint)
			if err != nil && row[i-1] != "" { // doesn't look like an isbn, might be issn, clean up and add as is
				idCleaned := strings.Trim(strings.Replace(row[i-1], "-", "", -1), " ")
				record.Identifiers = append(record.Identifiers, models.Identifier{Identifier: idCleaned, IDType: models.IDTypePrint})
			}
		}
		if i, ok := csvConf["identifieronline"]; ok {
			err := getIsbnIdentifiers(row[i-1], &record, models.IDTypeOnline)
			if err != nil && row[i-1] != "" { // doesn't look like an isbn, might be issn, clean up and add as is
				idCleaned := strings.Trim(strings.Replace(row[i-1], "-", "", -1), " ")
				record.Identifiers = append(record.Identifiers, models.Identifier{Identifier: idCleaned, IDType: models.IDTypeOnline})
			}
		}

		if i, ok := csvConf["datefirstissueonline"]; ok {
			record.DateFirstIssueOnline = row[i-1]
		}
		if i, ok := csvConf["numfirstvolonline"]; ok {
			record.NumFirstVolOnline = row[i-1]
		}
		if i, ok := csvConf["numfirstissueonline"]; ok {
			record.NumFirstIssueOnline = row[i-1]
		}
		if i, ok := csvConf["datelastissueonline"]; ok {
			record.DateLastIssueOnline = row[i-1]
		}
		if i, ok := csvConf["numlastvolonline"]; ok {
			record.NumLastVolOnline = row[i-1]
		}
		if i, ok := csvConf["numlastissueonline"]; ok {
			record.NumLastIssueOnline = row[i-1]
		}
		if i, ok := csvConf["titleurl"]; ok {
			record.TitleURL = row[i-1]
		}
		if i, ok := csvConf["firstauthor"]; ok {
			record.FirstAuthor = row[i-1]
		}
		if i, ok := csvConf["titleid"]; ok {
			record.TitleID = row[i-1]
		}
		if i, ok := csvConf["embargoinfo"]; ok {
			record.EmbargoInfo = row[i-1]
		}
		if i, ok := csvConf["coveragedepth"]; ok {
			record.CoverageDepth = row[i-1]
		}
		if i, ok := csvConf["notes"]; ok {
			record.Notes = row[i-1]
		}
		if i, ok := csvConf["publishername"]; ok {
			record.PublisherName = row[i-1]
		}
		if i, ok := csvConf["publicationtype"]; ok {
			record.PublicationType = row[i-1]
		}
		if i, ok := csvConf["datemonographpublishedprint"]; ok {
			record.DateMonographPublishedPrint = row[i-1]
		}
		if i, ok := csvConf["datemonographpublishedonline"]; ok {
			record.DateMonographPublishedOnline = row[i-1]
		}
		if i, ok := csvConf["monographvolume"]; ok {
			record.MonographVolume = row[i-1]
		}
		if i, ok := csvConf["monographedition"]; ok {
			record.MonographEdition = row[i-1]
		}
		if i, ok := csvConf["firsteditor"]; ok {
			record.FirstEditor = row[i-1]
		}
		if i, ok := csvConf["parentpublicationtitleid"]; ok {
			record.ParentPublicationTitleID = row[i-1]
		}
		if i, ok := csvConf["precedingpublicationtitleid"]; ok {
			record.PrecedingPublicationTitleID = row[i-1]
		}
		if i, ok := csvConf["accesstype"]; ok {
			record.AccessType = row[i-1]
		}
	}

	if !validateRecord(record) {
		recordNotValid := errors.New("record not valid")
		return record, recordNotValid
	}

	return record, nil
}

func validateRecord(record models.Record) bool {
	if len(record.Identifiers) == 0 || record.PublicationTitle == "" {
		return false
	}
	return true
}
<commit_msg>fixes #46<commit_after>package controllers

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/nicomo/abacaxi/logger"
	"github.com/nicomo/abacaxi/models"
)

const (
	kbartNumFields = 25
)

func fileIO(pp parseparams, report *models.Report) ([]models.Record, error) {
	// slice will hold successfully parsed records
	var records []models.Record

	// retrieve target service (e.g. ebook package) for this file
	myTS, err := models.GetTargetService(pp.tsname)
	if err != nil {
		return records, err
	}

	// open file
	f, err := os.Open(pp.fpath)
	if err != nil {
		return nil, errors.New("cannot open the source file")
	}
	defer f.Close()

	reader := csv.NewReader(f)

	// target service csv has n fields, separator is ;
	if pp.filetype == "publishercsv" {
		reader.FieldsPerRecord = len(pp.csvconf)
	} else {
		reader.FieldsPerRecord = kbartNumFields // kbart is a const: always 25 fields
	}
	reader.Comma = ';' //TODO: detect separator, tab or comma, rather than force comma

	// counters to keep track of records parsed, for logging
	line := 1
	var rejectedLines []int

	for {
		// read a row
		r, err := reader.Read()

		// if at EOF, break out of loop
		if err != nil {
			if err == io.EOF {
				break
			}
			rejectedLines = append(rejectedLines, line)
			line++
			continue
		}

		// validate we don't have an unicode replacement char. in the string
		// if we do abort: source file isn't proper utf8
		for _, v := range r {
			if strings.ContainsRune(v, '\uFFFD') {
				err := errors.New("parsing failed: non utf-8 character in file")
				return records, err
			}
		}

		// parse each line into a struct
		record, err := fileParseRow(r, pp.csvconf)
		if err != nil {
			logger.Error.Println(err, r)
			continue
		}

		// add TS to record
		record.TargetServices = append(record.TargetServices, myTS)
		if myTS.Active {
			record.Active = true
		}

		// add record to slice
		records = append(records, record)

		line++
	}

	// log number of records successfully parsed
	report.Text = append(report.Text, fmt.Sprintf("successfully parsed %d lines from %s",
		len(records), pp.fpath))

	// log the lines we rejected, if any
	if len(rejectedLines) > 0 {
		report.Text = append(report.Text, fmt.Sprintf("lines rejected in source file: %v", rejectedLines))
	}

	// no records parsed
	if len(records) == 0 {
		err := errors.New("couldn't parse a single line: check your input file")
		return records, err

	}

	return records, nil
}

func fileParseRow(row []string, csvConf map[string]int) (models.Record, error) {

	var record models.Record

	if csvConf == nil { // the csv Configuration is nil, we default to kbart values

		record.PublicationTitle = row[0]

		// ISBNs : validate & cleanup, convert isbn 10 <-> isbn13
		// Identifiers Print ID
		err := getIsbnIdentifiers(row[1], &record, models.IDTypePrint)
		if err != nil && row[1] != "" { // doesn't look like an isbn, might be issn, cleanup and add as is
			idCleaned := strings.Trim(strings.Replace(row[1], "-", "", -1), " ")
			record.Identifiers = append(record.Identifiers, models.Identifier{Identifier: idCleaned, IDType: models.IDTypePrint})
		}
		// Identifiers Online ID
		err = getIsbnIdentifiers(row[2], &record, models.IDTypeOnline)
		if err != nil && row[2] != "" { // doesn't look like an isbn, might be issn, cleanup and add as is
			idCleaned := strings.Trim(strings.Replace(row[2], "-", "", -1), " ")
			record.Identifiers = append(record.Identifiers, models.Identifier{Identifier: idCleaned, IDType: models.IDTypeOnline})
		}

		record.DateFirstIssueOnline = row[3]
		record.NumFirstVolOnline = row[4]
		record.NumFirstIssueOnline = row[5]
		record.DateLastIssueOnline = row[6]
		record.NumLastVolOnline = row[7]
		record.NumLastIssueOnline = row[8]
		record.TitleURL = row[9]
		record.FirstAuthor = row[10]
		record.TitleID = row[11]
		record.EmbargoInfo = row[12]
		record.CoverageDepth = row[13]
		record.Notes = row[14]
		record.PublisherName = row[15]
		record.PublicationType = row[16]
		record.DateMonographPublishedPrint = row[17]
		record.DateMonographPublishedOnline = row[18]
		record.MonographVolume = row[19]
		record.MonographEdition = row[20]
		record.FirstEditor = row[21]
		record.ParentPublicationTitleID = row[22]
		record.PrecedingPublicationTitleID = row[23]
		record.AccessType = row[24]
	} else { // we do have a csv configuration
		if i, ok := csvConf["publicationtitle"]; ok {
			record.PublicationTitle = row[i-1]
		}
		if i, ok := csvConf["identifierprint"]; ok {
			err := getIsbnIdentifiers(row[i-1], &record, models.IDTypePrint)
			if err != nil && row[i-1] != "" { // doesn't look like an isbn, might be issn, clean up and add as is
				idCleaned := strings.Trim(strings.Replace(row[i-1], "-", "", -1), " ")
				record.Identifiers = append(record.Identifiers, models.Identifier{Identifier: idCleaned, IDType: models.IDTypePrint})
			}
		}
		if i, ok := csvConf["identifieronline"]; ok {
			err := getIsbnIdentifiers(row[i-1], &record, models.IDTypeOnline)
			if err != nil && row[i-1] != "" { // doesn't look like an isbn, might be issn, clean up and add as is
				idCleaned := strings.Trim(strings.Replace(row[i-1], "-", "", -1), " ")
				record.Identifiers = append(record.Identifiers, models.Identifier{Identifier: idCleaned, IDType: models.IDTypeOnline})
			}
		}

		if i, ok := csvConf["datefirstissueonline"]; ok {
			record.DateFirstIssueOnline = row[i-1]
		}
		if i, ok := csvConf["numfirstvolonline"]; ok {
			record.NumFirstVolOnline = row[i-1]
		}
		if i, ok := csvConf["numfirstissueonline"]; ok {
			record.NumFirstIssueOnline = row[i-1]
		}
		if i, ok := csvConf["datelastissueonline"]; ok {
			record.DateLastIssueOnline = row[i-1]
		}
		if i, ok := csvConf["numlastvolonline"]; ok {
			record.NumLastVolOnline = row[i-1]
		}
		if i, ok := csvConf["numlastissueonline"]; ok {
			record.NumLastIssueOnline = row[i-1]
		}
		if i, ok := csvConf["titleurl"]; ok {
			record.TitleURL = row[i-1]
		}
		if i, ok := csvConf["firstauthor"]; ok {
			record.FirstAuthor = row[i-1]
		}
		if i, ok := csvConf["titleid"]; ok {
			record.TitleID = row[i-1]
		}
		if i, ok := csvConf["embargoinfo"]; ok {
			record.EmbargoInfo = row[i-1]
		}
		if i, ok := csvConf["coveragedepth"]; ok {
			record.CoverageDepth = row[i-1]
		}
		if i, ok := csvConf["notes"]; ok {
			record.Notes = row[i-1]
		}
		if i, ok := csvConf["publishername"]; ok {
			record.PublisherName = row[i-1]
		}
		if i, ok := csvConf["publicationtype"]; ok {
			record.PublicationType = row[i-1]
		}
		if i, ok := csvConf["datemonographpublishedprint"]; ok {
			record.DateMonographPublishedPrint = row[i-1]
		}
		if i, ok := csvConf["datemonographpublishedonline"]; ok {
			record.DateMonographPublishedOnline = row[i-1]
		}
		if i, ok := csvConf["monographvolume"]; ok {
			record.MonographVolume = row[i-1]
		}
		if i, ok := csvConf["monographedition"]; ok {
			record.MonographEdition = row[i-1]
		}
		if i, ok := csvConf["firsteditor"]; ok {
			record.FirstEditor = row[i-1]
		}
		if i, ok := csvConf["parentpublicationtitleid"]; ok {
			record.ParentPublicationTitleID = row[i-1]
		}
		if i, ok := csvConf["precedingpublicationtitleid"]; ok {
			record.PrecedingPublicationTitleID = row[i-1]
		}
		if i, ok := csvConf["accesstype"]; ok {
			record.AccessType = row[i-1]
		}
	}

	if !validateRecord(record) {
		recordNotValid := errors.New("record not valid")
		return record, recordNotValid
	}

	return record, nil
}

func validateRecord(record models.Record) bool {
	if len(record.Identifiers) == 0 || record.PublicationTitle == "" {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package hoverfly_test

import (
	"bytes"
	"github.com/dghubble/sling"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"io/ioutil"
)

var _ = Describe("Using Hoverfly to return responses by request templates", func() {

	Context("With a request template loaded for matching on URL + headers", func() {

		var (
			jsonRequestResponsePair *bytes.Buffer
		)

		BeforeEach(func() {
			jsonRequestResponsePair = bytes.NewBufferString(`{"data":[{"requestTemplate": {"path": "/path1", "method": "GET", "destination": "www.virtual.com"}, "response": {"status": 201, "encodedBody": false, "body": "body1", "headers": {"Header": ["value1"]}}}, {"requestTemplate": {"path": "/path2", "method": "GET", "destination": "www.virtual.com", "headers": {"Header": ["value2"]}}, "response": {"status": 202, "body": "body2", "headers": {"Header": ["value2"]}}}]}`)

		})

		Context("When running in proxy mode", func() {

			BeforeEach(func() {
				hoverflyCmd = startHoverfly(adminPort, proxyPort)
				SetHoverflyMode("simulate")
				ImportHoverflyTemplates(jsonRequestResponsePair)
			})

			AfterEach(func() {
				stopHoverfly()
			})

			It("Should find a match", func() {
				resp := DoRequestThroughProxy(sling.New().Get("http://www.virtual.com/path2").Add("Header", "value2"))
				body, err := ioutil.ReadAll(resp.Body)
				Expect(err).To(BeNil())
				Expect(resp.StatusCode).To(Equal(202))
				Expect(string(body)).To(Equal("body2"))
			})
		})

		Context("When running in webserver mode", func() {

			BeforeEach(func() {
				hoverflyCmd = startHoverflyWebServer(adminPort, proxyPort)
				ImportHoverflyTemplates(jsonRequestResponsePair)
			})

			It("Should find a match", func() {
				request := sling.New().Get("http://localhost:"+proxyPortAsString+"/path2").Add("Header", "value2")

				resp := DoRequest(request)
				body, err := ioutil.ReadAll(resp.Body)
				Expect(err).To(BeNil())
				Expect(resp.StatusCode).To(Equal(202))
				Expect(string(body)).To(Equal("body2"))
			})

			AfterEach(func() {
				stopHoverfly()
			})
		})

	})

})
<commit_msg>Updated a functional test to use the /api/records endpoint over the /api/templates endpoint<commit_after>package hoverfly_test

import (
	"bytes"
	"github.com/dghubble/sling"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"io/ioutil"
)

var _ = Describe("Using Hoverfly to return responses by request templates", func() {

	Context("With a request template loaded for matching on URL + headers", func() {

		var (
			jsonRequestResponsePair *bytes.Buffer
		)

		BeforeEach(func() {
			jsonRequestResponsePair = bytes.NewBufferString(`
			{
				"data": [{
					"request": {
						"requestType": "template",
						"path": "/path1",
						"method": "GET",
						"destination": "www.virtual.com"
					},
					"response": {
						"status": 201,
						"encodedBody": false,
						"body": "body1",
						"headers": {
							"Header": ["value1"]
						}
					}
				}, {
					"request": {
						"requestType": "template",
						"path": "/path2",
						"method": "GET",
						"destination": "www.virtual.com",
						"headers": {
							"Header": ["value2"]
						}
					},
					"response": {
						"status": 202,
						"body": "body2",
						"headers": {
							"Header": ["value2"]
						}
					}
				}]
			}
			`)
		})

		Context("When running in proxy mode", func() {

			BeforeEach(func() {
				hoverflyCmd = startHoverfly(adminPort, proxyPort)
				SetHoverflyMode("simulate")
				ImportHoverflyRecords(jsonRequestResponsePair)
			})

			AfterEach(func() {
				stopHoverfly()
			})

			It("Should find a match", func() {
				resp := DoRequestThroughProxy(sling.New().Get("http://www.virtual.com/path2").Add("Header", "value2"))
				body, err := ioutil.ReadAll(resp.Body)
				Expect(err).To(BeNil())
				Expect(resp.StatusCode).To(Equal(202))
				Expect(string(body)).To(Equal("body2"))
			})
		})

		Context("When running in webserver mode", func() {

			BeforeEach(func() {
				hoverflyCmd = startHoverflyWebServer(adminPort, proxyPort)
				ImportHoverflyRecords(jsonRequestResponsePair)
			})

			It("Should find a match", func() {
				request := sling.New().Get("http://localhost:"+proxyPortAsString+"/path2").Add("Header", "value2")

				resp := DoRequest(request)
				body, err := ioutil.ReadAll(resp.Body)
				Expect(err).To(BeNil())
				Expect(resp.StatusCode).To(Equal(202))
				Expect(string(body)).To(Equal("body2"))
			})

			AfterEach(func() {
				stopHoverfly()
			})
		})

	})

})
<|endoftext|>