<commit_before>// Automatically generated by MockGen. DO NOT EDIT!
// Source: github.com/google/trillian (interfaces: TrillianLogClient)

package mockclient

import (
	gomock "github.com/golang/mock/gomock"
	trillian "github.com/google/trillian"
	context "golang.org/x/net/context"
	grpc "google.golang.org/grpc"
)

// MockTrillianLogClient is a mock of TrillianLogClient interface
type MockTrillianLogClient struct {
	ctrl     *gomock.Controller
	recorder *MockTrillianLogClientMockRecorder
}

// MockTrillianLogClientMockRecorder is the mock recorder for MockTrillianLogClient
type MockTrillianLogClientMockRecorder struct {
	mock *MockTrillianLogClient
}

// NewMockTrillianLogClient creates a new mock instance
func NewMockTrillianLogClient(ctrl *gomock.Controller) *MockTrillianLogClient {
	mock := &MockTrillianLogClient{ctrl: ctrl}
	mock.recorder = &MockTrillianLogClientMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (_m *MockTrillianLogClient) EXPECT() *MockTrillianLogClientMockRecorder {
	return _m.recorder
}

// GetConsistencyProof mocks base method
func (_m *MockTrillianLogClient) GetConsistencyProof(_param0 context.Context, _param1 *trillian.GetConsistencyProofRequest, _param2 ...grpc.CallOption) (*trillian.GetConsistencyProofResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetConsistencyProof", _s...)
	ret0, _ := ret[0].(*trillian.GetConsistencyProofResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetConsistencyProof indicates an expected call of GetConsistencyProof
func (_mr *MockTrillianLogClientMockRecorder) GetConsistencyProof(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetConsistencyProof", _s...)
}

// GetEntryAndProof mocks base method
func (_m *MockTrillianLogClient) GetEntryAndProof(_param0 context.Context, _param1 *trillian.GetEntryAndProofRequest, _param2 ...grpc.CallOption) (*trillian.GetEntryAndProofResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetEntryAndProof", _s...)
	ret0, _ := ret[0].(*trillian.GetEntryAndProofResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetEntryAndProof indicates an expected call of GetEntryAndProof
func (_mr *MockTrillianLogClientMockRecorder) GetEntryAndProof(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetEntryAndProof", _s...)
}

// GetInclusionProof mocks base method
func (_m *MockTrillianLogClient) GetInclusionProof(_param0 context.Context, _param1 *trillian.GetInclusionProofRequest, _param2 ...grpc.CallOption) (*trillian.GetInclusionProofResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetInclusionProof", _s...)
	ret0, _ := ret[0].(*trillian.GetInclusionProofResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetInclusionProof indicates an expected call of GetInclusionProof
func (_mr *MockTrillianLogClientMockRecorder) GetInclusionProof(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetInclusionProof", _s...)
}

// GetInclusionProofByHash mocks base method
func (_m *MockTrillianLogClient) GetInclusionProofByHash(_param0 context.Context, _param1 *trillian.GetInclusionProofByHashRequest, _param2 ...grpc.CallOption) (*trillian.GetInclusionProofByHashResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetInclusionProofByHash", _s...)
	ret0, _ := ret[0].(*trillian.GetInclusionProofByHashResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetInclusionProofByHash indicates an expected call of GetInclusionProofByHash
func (_mr *MockTrillianLogClientMockRecorder) GetInclusionProofByHash(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetInclusionProofByHash", _s...)
}

// GetLatestSignedLogRoot mocks base method
func (_m *MockTrillianLogClient) GetLatestSignedLogRoot(_param0 context.Context, _param1 *trillian.GetLatestSignedLogRootRequest, _param2 ...grpc.CallOption) (*trillian.GetLatestSignedLogRootResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetLatestSignedLogRoot", _s...)
	ret0, _ := ret[0].(*trillian.GetLatestSignedLogRootResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetLatestSignedLogRoot indicates an expected call of GetLatestSignedLogRoot
func (_mr *MockTrillianLogClientMockRecorder) GetLatestSignedLogRoot(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetLatestSignedLogRoot", _s...)
}

// GetLeavesByHash mocks base method
func (_m *MockTrillianLogClient) GetLeavesByHash(_param0 context.Context, _param1 *trillian.GetLeavesByHashRequest, _param2 ...grpc.CallOption) (*trillian.GetLeavesByHashResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetLeavesByHash", _s...)
	ret0, _ := ret[0].(*trillian.GetLeavesByHashResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetLeavesByHash indicates an expected call of GetLeavesByHash
func (_mr *MockTrillianLogClientMockRecorder) GetLeavesByHash(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetLeavesByHash", _s...)
}

// GetLeavesByIndex mocks base method
func (_m *MockTrillianLogClient) GetLeavesByIndex(_param0 context.Context, _param1 *trillian.GetLeavesByIndexRequest, _param2 ...grpc.CallOption) (*trillian.GetLeavesByIndexResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetLeavesByIndex", _s...)
	ret0, _ := ret[0].(*trillian.GetLeavesByIndexResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetLeavesByIndex indicates an expected call of GetLeavesByIndex
func (_mr *MockTrillianLogClientMockRecorder) GetLeavesByIndex(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetLeavesByIndex", _s...)
}

// GetSequencedLeafCount mocks base method
func (_m *MockTrillianLogClient) GetSequencedLeafCount(_param0 context.Context, _param1 *trillian.GetSequencedLeafCountRequest, _param2 ...grpc.CallOption) (*trillian.GetSequencedLeafCountResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetSequencedLeafCount", _s...)
	ret0, _ := ret[0].(*trillian.GetSequencedLeafCountResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetSequencedLeafCount indicates an expected call of GetSequencedLeafCount
func (_mr *MockTrillianLogClientMockRecorder) GetSequencedLeafCount(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetSequencedLeafCount", _s...)
}

// QueueLeaf mocks base method
func (_m *MockTrillianLogClient) QueueLeaf(_param0 context.Context, _param1 *trillian.QueueLeafRequest, _param2 ...grpc.CallOption) (*trillian.QueueLeafResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "QueueLeaf", _s...)
	ret0, _ := ret[0].(*trillian.QueueLeafResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// QueueLeaf indicates an expected call of QueueLeaf
func (_mr *MockTrillianLogClientMockRecorder) QueueLeaf(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "QueueLeaf", _s...)
}

// QueueLeaves mocks base method
func (_m *MockTrillianLogClient) QueueLeaves(_param0 context.Context, _param1 *trillian.QueueLeavesRequest, _param2 ...grpc.CallOption) (*trillian.QueueLeavesResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "QueueLeaves", _s...)
	ret0, _ := ret[0].(*trillian.QueueLeavesResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// QueueLeaves indicates an expected call of QueueLeaves
func (_mr *MockTrillianLogClientMockRecorder) QueueLeaves(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "QueueLeaves", _s...)
}
<commit_msg>trillian: update mock for latest mockgen (#29)<commit_after>// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/google/trillian (interfaces: TrillianLogClient)

package mockclient

import (
	gomock "github.com/golang/mock/gomock"
	trillian "github.com/google/trillian"
	context "golang.org/x/net/context"
	grpc "google.golang.org/grpc"
)

// MockTrillianLogClient is a mock of TrillianLogClient interface
type MockTrillianLogClient struct {
	ctrl     *gomock.Controller
	recorder *MockTrillianLogClientMockRecorder
}

// MockTrillianLogClientMockRecorder is the mock recorder for MockTrillianLogClient
type MockTrillianLogClientMockRecorder struct {
	mock *MockTrillianLogClient
}

// NewMockTrillianLogClient creates a new mock instance
func NewMockTrillianLogClient(ctrl *gomock.Controller) *MockTrillianLogClient {
	mock := &MockTrillianLogClient{ctrl: ctrl}
	mock.recorder = &MockTrillianLogClientMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (_m *MockTrillianLogClient) EXPECT() *MockTrillianLogClientMockRecorder {
	return _m.recorder
}

// GetConsistencyProof mocks base method
func (_m *MockTrillianLogClient) GetConsistencyProof(_param0 context.Context, _param1 *trillian.GetConsistencyProofRequest, _param2 ...grpc.CallOption) (*trillian.GetConsistencyProofResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetConsistencyProof", _s...)
	ret0, _ := ret[0].(*trillian.GetConsistencyProofResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetConsistencyProof indicates an expected call of GetConsistencyProof
func (_mr *MockTrillianLogClientMockRecorder) GetConsistencyProof(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetConsistencyProof", _s...)
}

// GetEntryAndProof mocks base method
func (_m *MockTrillianLogClient) GetEntryAndProof(_param0 context.Context, _param1 *trillian.GetEntryAndProofRequest, _param2 ...grpc.CallOption) (*trillian.GetEntryAndProofResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetEntryAndProof", _s...)
	ret0, _ := ret[0].(*trillian.GetEntryAndProofResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetEntryAndProof indicates an expected call of GetEntryAndProof
func (_mr *MockTrillianLogClientMockRecorder) GetEntryAndProof(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetEntryAndProof", _s...)
}

// GetInclusionProof mocks base method
func (_m *MockTrillianLogClient) GetInclusionProof(_param0 context.Context, _param1 *trillian.GetInclusionProofRequest, _param2 ...grpc.CallOption) (*trillian.GetInclusionProofResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetInclusionProof", _s...)
	ret0, _ := ret[0].(*trillian.GetInclusionProofResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetInclusionProof indicates an expected call of GetInclusionProof
func (_mr *MockTrillianLogClientMockRecorder) GetInclusionProof(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetInclusionProof", _s...)
}

// GetInclusionProofByHash mocks base method
func (_m *MockTrillianLogClient) GetInclusionProofByHash(_param0 context.Context, _param1 *trillian.GetInclusionProofByHashRequest, _param2 ...grpc.CallOption) (*trillian.GetInclusionProofByHashResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetInclusionProofByHash", _s...)
	ret0, _ := ret[0].(*trillian.GetInclusionProofByHashResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetInclusionProofByHash indicates an expected call of GetInclusionProofByHash
func (_mr *MockTrillianLogClientMockRecorder) GetInclusionProofByHash(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetInclusionProofByHash", _s...)
}

// GetLatestSignedLogRoot mocks base method
func (_m *MockTrillianLogClient) GetLatestSignedLogRoot(_param0 context.Context, _param1 *trillian.GetLatestSignedLogRootRequest, _param2 ...grpc.CallOption) (*trillian.GetLatestSignedLogRootResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetLatestSignedLogRoot", _s...)
	ret0, _ := ret[0].(*trillian.GetLatestSignedLogRootResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetLatestSignedLogRoot indicates an expected call of GetLatestSignedLogRoot
func (_mr *MockTrillianLogClientMockRecorder) GetLatestSignedLogRoot(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetLatestSignedLogRoot", _s...)
}

// GetLeavesByHash mocks base method
func (_m *MockTrillianLogClient) GetLeavesByHash(_param0 context.Context, _param1 *trillian.GetLeavesByHashRequest, _param2 ...grpc.CallOption) (*trillian.GetLeavesByHashResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetLeavesByHash", _s...)
	ret0, _ := ret[0].(*trillian.GetLeavesByHashResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetLeavesByHash indicates an expected call of GetLeavesByHash
func (_mr *MockTrillianLogClientMockRecorder) GetLeavesByHash(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetLeavesByHash", _s...)
}

// GetLeavesByIndex mocks base method
func (_m *MockTrillianLogClient) GetLeavesByIndex(_param0 context.Context, _param1 *trillian.GetLeavesByIndexRequest, _param2 ...grpc.CallOption) (*trillian.GetLeavesByIndexResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetLeavesByIndex", _s...)
	ret0, _ := ret[0].(*trillian.GetLeavesByIndexResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetLeavesByIndex indicates an expected call of GetLeavesByIndex
func (_mr *MockTrillianLogClientMockRecorder) GetLeavesByIndex(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetLeavesByIndex", _s...)
}

// GetSequencedLeafCount mocks base method
func (_m *MockTrillianLogClient) GetSequencedLeafCount(_param0 context.Context, _param1 *trillian.GetSequencedLeafCountRequest, _param2 ...grpc.CallOption) (*trillian.GetSequencedLeafCountResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "GetSequencedLeafCount", _s...)
	ret0, _ := ret[0].(*trillian.GetSequencedLeafCountResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// GetSequencedLeafCount indicates an expected call of GetSequencedLeafCount
func (_mr *MockTrillianLogClientMockRecorder) GetSequencedLeafCount(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetSequencedLeafCount", _s...)
}

// QueueLeaf mocks base method
func (_m *MockTrillianLogClient) QueueLeaf(_param0 context.Context, _param1 *trillian.QueueLeafRequest, _param2 ...grpc.CallOption) (*trillian.QueueLeafResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "QueueLeaf", _s...)
	ret0, _ := ret[0].(*trillian.QueueLeafResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// QueueLeaf indicates an expected call of QueueLeaf
func (_mr *MockTrillianLogClientMockRecorder) QueueLeaf(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "QueueLeaf", _s...)
}

// QueueLeaves mocks base method
func (_m *MockTrillianLogClient) QueueLeaves(_param0 context.Context, _param1 *trillian.QueueLeavesRequest, _param2 ...grpc.CallOption) (*trillian.QueueLeavesResponse, error) {
	_s := []interface{}{_param0, _param1}
	for _, _x := range _param2 {
		_s = append(_s, _x)
	}
	ret := _m.ctrl.Call(_m, "QueueLeaves", _s...)
	ret0, _ := ret[0].(*trillian.QueueLeavesResponse)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// QueueLeaves indicates an expected call of QueueLeaves
func (_mr *MockTrillianLogClientMockRecorder) QueueLeaves(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
	_s := append([]interface{}{arg0, arg1}, arg2...)
	return _mr.mock.ctrl.RecordCall(_mr.mock, "QueueLeaves", _s...)
}
<|endoftext|>


<commit_before>package scrape

import (
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/slotix/dataflowkit/extract"
	"github.com/slotix/dataflowkit/paginate"
	"github.com/temoto/robotstxt"
)

type Extractor struct {
	Type   string      `json:"type"`
	Params interface{} `json:"params"`
}

type field struct {
	Name     string `json:"name" validate:"required"`
	Selector string `json:"selector" validate:"required"`
	//Count     int       `json:"count"`
	Extractor Extractor `json:"extractor"`
	Details   *Payload  `json:"details"`
}

type paginator struct {
	Selector  string `json:"selector"`
	Attribute string `json:"attr"`
	MaxPages  int    `json:"maxPages"`
}

type Payload struct {
	Name string `json:"name" validate:"required"`
	//Request             splash.Request `json:"request"`
	Request interface{} `json:"request"`
	Fields  []field     `json:"fields"`
	//PayloadMD5 encodes payload content to MD5. It is used for generating file name to be stored.
	PayloadMD5          []byte        `json:"payloadMD5"`
	Format              string        `json:"format"`
	Paginator           *paginator    `json:"paginator"`
	PaginateResults     *bool         `json:"paginateResults"`
	FetchDelay          time.Duration `json:"fetchDelay"`
	RandomizeFetchDelay *bool         `json:"randomizeFetchDelay"`
	RetryTimes          int           `json:"retryTimes"`
}

// The DividePageFunc type is used to extract a page's blocks during a scrape.
// For more information, please see the documentation on the ScrapeConfig type.
type DividePageFunc func(*goquery.Selection) []*goquery.Selection

// A Part represents a given chunk of data that is to be extracted from every
// block in each page of a scrape.
type Part struct {
	// The name of this part.  Required, and will be used to aggregate results.
	Name string

	// A sub-selector within the given block to process.  Pass in "." to use
	// the root block's selector with no modification.
	Selector string
	// TODO(andrew-d): Consider making this an interface too.

	// Extractor contains the logic on how to extract some results from the
	// selector that is provided to this Piece.
	Extractor extract.Extractor
	Details   *Scraper
}

//Scraper struct consolidates settings for scraping task.
type Scraper struct {
	// Paginator is the Paginator to use for this current scrape.
	//
	// If Paginator is nil, then no pagination is performed and it is assumed that
	// the initial URL is the only page.
	Paginator paginate.Paginator

	// DividePage splits a page into individual 'blocks'.  When scraping, we treat
	// each page as if it contains some number of 'blocks', each of which can be
	// further subdivided into what actually needs to be extracted.
	//
	// If the DividePage function is nil, then no division is performed and the
	// page is assumed to contain a single block containing the entire <body>
	// tag.
	DividePage DividePageFunc

	// Parts contains the list of data that is extracted for each block.  For
	// every block that is the result of the DividePage function (above), all of
	// the Parts entries receives the selector representing the block, and can
	// return a result.  If the returned result is nil, then the Part is
	// considered not to exist in this block, and is not included.
	//
	// Note: if a Part's Extractor returns an error, it results in the scrape
	// being aborted - this can be useful if you need to ensure that a given Part
	// is required, for example.
	Parts []Part
	//Opts contains options that are used during the progress of a
	// scrape.
	Opts ScrapeOptions
}

// Results describes the results of a scrape.  It contains a list of all
// pages (URLs) visited during the process, along with all results generated
// from each Part in each page.
type Results struct {
	// Visited contain a map[url]error during this scrape.
	// Always contains at least one element - the initial URL.
	Visited map[string]error

	// Output represents combined results after parsing from each Part of each page.  Essentially, the top-level array
	// is for each page, the second-level array is for each block in a page, and
	// the final map[string]interface{} is the mapping of Part.Name to results.
	Output [][]map[string]interface{}
}

type Session struct {
	Robots  *robotstxt.RobotsData
	//Cookies string
}
type Task struct {
	ID      string
	Scraper *Scraper
	Session
	//	Err     []error
	Status string
	Results
}
<commit_msg>scrape structure monor changes<commit_after>package scrape

import (
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/slotix/dataflowkit/extract"
	"github.com/slotix/dataflowkit/paginate"
	"github.com/temoto/robotstxt"
)

type Extractor struct {
	Type   string      `json:"type"`
	Params interface{} `json:"params"`
}

type field struct {
	Name     string `json:"name"`
	Selector string `json:"selector"`
	//Count     int       `json:"count"`
	Extractor Extractor `json:"extractor"`
	Details   *Payload  `json:"details"`
}

type paginator struct {
	Selector  string `json:"selector"`
	Attribute string `json:"attr"`
	MaxPages  int    `json:"maxPages"`
}

type Payload struct {
	Name string `json:"name"`
	//Request             splash.Request `json:"request"`
	Request interface{} `json:"request"`
	Fields  []field     `json:"fields"`
	//PayloadMD5 encodes payload content to MD5. It is used for generating file name to be stored.
	PayloadMD5          []byte        `json:"payloadMD5"`
	Format              string        `json:"format"`
	Paginator           *paginator    `json:"paginator"`
	PaginateResults     *bool         `json:"paginateResults"`
	FetchDelay          time.Duration `json:"fetchDelay"`
	RandomizeFetchDelay *bool         `json:"randomizeFetchDelay"`
	RetryTimes          int           `json:"retryTimes"`
}

// The DividePageFunc type is used to extract a page's blocks during a scrape.
// For more information, please see the documentation on the ScrapeConfig type.
type DividePageFunc func(*goquery.Selection) []*goquery.Selection

// A Part represents a given chunk of data that is to be extracted from every
// block in each page of a scrape.
type Part struct {
	// The name of this part.  Required, and will be used to aggregate results.
	Name string

	// A sub-selector within the given block to process.  Pass in "." to use
	// the root block's selector with no modification.
	Selector string
	// TODO(andrew-d): Consider making this an interface too.

	// Extractor contains the logic on how to extract some results from the
	// selector that is provided to this Piece.
	Extractor extract.Extractor
	Details   *Scraper
}

//Scraper struct consolidates settings for scraping task.
type Scraper struct {
	// Paginator is the Paginator to use for this current scrape.
	//
	// If Paginator is nil, then no pagination is performed and it is assumed that
	// the initial URL is the only page.
	Paginator paginate.Paginator

	// DividePage splits a page into individual 'blocks'.  When scraping, we treat
	// each page as if it contains some number of 'blocks', each of which can be
	// further subdivided into what actually needs to be extracted.
	//
	// If the DividePage function is nil, then no division is performed and the
	// page is assumed to contain a single block containing the entire <body>
	// tag.
	DividePage DividePageFunc

	// Parts contains the list of data that is extracted for each block.  For
	// every block that is the result of the DividePage function (above), all of
	// the Parts entries receives the selector representing the block, and can
	// return a result.  If the returned result is nil, then the Part is
	// considered not to exist in this block, and is not included.
	//
	// Note: if a Part's Extractor returns an error, it results in the scrape
	// being aborted - this can be useful if you need to ensure that a given Part
	// is required, for example.
	Parts []Part
	//Opts contains options that are used during the progress of a
	// scrape.
	Opts ScrapeOptions
}

// Results describes the results of a scrape.  It contains a list of all
// pages (URLs) visited during the process, along with all results generated
// from each Part in each page.
type Results struct {
	// Visited contain a map[url]error during this scrape.
	// Always contains at least one element - the initial URL.
	Visited map[string]error

	// Output represents combined results after parsing from each Part of each page.  Essentially, the top-level array
	// is for each page, the second-level array is for each block in a page, and
	// the final map[string]interface{} is the mapping of Part.Name to results.
	Output [][]map[string]interface{}
}

type Session struct {
	Robots  *robotstxt.RobotsData
	//Cookies string
}
type Task struct {
	ID      string
	Scraper *Scraper
	Session
	Status string
	Results
}
<|endoftext|>


<commit_before>package matchers

import (
	"testing"
)

func TestEqualTo(t *testing.T) {
	m := EqualTo{true}
	if !m.Match(true) {
		t.Error("true is not true")
	}
	if m.Match(false) {
		t.Error("true is false")
	}
}

func TestAssertThat(t *testing.T) {
	f := new(testing.T)
	AssertThat(Protect{f}, true, EqualTo{true})
	if f.Failed() {
		t.Error("true is not true")
	}
	AssertThat(Protect{f}, true, EqualTo{false})
	if !f.Failed() {
		t.Error("true is false")
	}
}

func TestIs(t *testing.T) {
	AssertThat(t, true, Is{true})
	AssertThat(t, true, Is{EqualTo{true}})
	AssertThat(t, Expect{true, Is{false}}, Fails{})
}

func TestNot(t *testing.T) {
	AssertThat(t, true, Not{false})
	AssertThat(t, Expect{true, Not{true}}, Fails{})
}

func TestAllOf(t *testing.T) {
	AssertThat(t, true, AllOf{Is{true}, Not{false}})
	AssertThat(t, Expect{true, AllOf{Is{false}, Not{true}}}, Fails{})
	AssertThat(t, Expect{true, AllOf{Is{true}, Not{true}}}, Fails{})
	AssertThat(t, Expect{true, AllOf{Is{false}, Not{true}}}, Fails{})
}

func TestAnyOf(t *testing.T) {
	AssertThat(t, true, AnyOf{Is{true}, Not{true}})
	AssertThat(t, true, AnyOf{Is{false}, Not{false}})
	AssertThat(t, true, AnyOf{Is{true}, Not{false}})
	AssertThat(t, Expect{true, AnyOf{Is{false}, Not{true}}}, Fails{})
}

func TestElementsAre(t *testing.T) {
	AssertThat(t, []int{1, 2, 3, 4, 5}, ElementsAre{5, 1, 4, 2, 3})
	AssertThat(t, []int{1, 2}, Not{ElementsAre{1, 2, 2}})
	AssertThat(t, []int{1, 2}, Not{ElementsAre{1, 3}})
}

func TestTypeOf(t *testing.T) {
	AssertThat(t, "zzzzz", TypeOf{string("")})
	AssertThat(t, 1, Not{TypeOf{string("")}})
}

func TestFails(t *testing.T) {
	AssertThat(t, Expect{Expect{true, Is{true}}, Fails{}}, Fails{})
}

func TestFailsPanic(t *testing.T) {
	defer func() {
		e := recover()
		AssertThat(t, e, Is{Not{nil}})
	}()
	AssertThat(t, true, Fails{})
}
<commit_msg>Expectation instead of negotiation<commit_after>package matchers

import (
	"testing"
)

func TestEqualTo(t *testing.T) {
	m := EqualTo{true}
	if !m.Match(true) {
		t.Error("true is not true")
	}
	if m.Match(false) {
		t.Error("true is false")
	}
}

func TestAssertThat(t *testing.T) {
	f := new(testing.T)
	AssertThat(Protect{f}, true, EqualTo{true})
	if f.Failed() {
		t.Error("true is not true")
	}
	AssertThat(Protect{f}, true, EqualTo{false})
	if !f.Failed() {
		t.Error("true is false")
	}
}

func TestIs(t *testing.T) {
	AssertThat(t, true, Is{true})
	AssertThat(t, true, Is{EqualTo{true}})
	AssertThat(t, Expect{true, Is{false}}, Fails{})
}

func TestNot(t *testing.T) {
	AssertThat(t, true, Not{false})
	AssertThat(t, Expect{true, Not{true}}, Fails{})
}

func TestAllOf(t *testing.T) {
	AssertThat(t, true, AllOf{Is{true}, Not{false}})
	AssertThat(t, Expect{true, AllOf{Is{false}, Not{true}}}, Fails{})
	AssertThat(t, Expect{true, AllOf{Is{true}, Not{true}}}, Fails{})
	AssertThat(t, Expect{true, AllOf{Is{false}, Not{true}}}, Fails{})
}

func TestAnyOf(t *testing.T) {
	AssertThat(t, true, AnyOf{Is{true}, Not{true}})
	AssertThat(t, true, AnyOf{Is{false}, Not{false}})
	AssertThat(t, true, AnyOf{Is{true}, Not{false}})
	AssertThat(t, Expect{true, AnyOf{Is{false}, Not{true}}}, Fails{})
}

func TestElementsAre(t *testing.T) {
	AssertThat(t, []int{1, 2, 3, 4, 5}, ElementsAre{5, 1, 4, 2, 3})
	AssertThat(t, Expect{[]int{1, 2}, ElementsAre{1, 2, 2}}, Fails{})
	AssertThat(t, Expect{[]int{1, 2}, ElementsAre{1, 3}}, Fails{})
}

func TestTypeOf(t *testing.T) {
	AssertThat(t, "zzzzz", TypeOf{string("")})
	AssertThat(t, Expect{1, TypeOf{string("")}}, Fails{})
}

func TestFails(t *testing.T) {
	AssertThat(t, Expect{Expect{true, Is{true}}, Fails{}}, Fails{})
}

func TestFailsPanic(t *testing.T) {
	defer func() {
		e := recover()
		AssertThat(t, e, Is{Not{nil}})
	}()
	AssertThat(t, true, Fails{})
}
<|endoftext|>


<commit_before>//
// Copyright 2014 Cristian Maglie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package serial

/*

// MSDN article on Serial Communications:
// http://msdn.microsoft.com/en-us/library/ff802693.aspx

// Arduino Playground article on serial communication with Windows API:
// http://playground.arduino.cc/Interfacing/CPPWindows

*/

import (
	"syscall"
	"time"
)

// opaque type that implements SerialPort interface for Windows
type SerialPort struct {
	handle syscall.Handle
}

//sys RegEnumValue(key syscall.Handle, index uint32, name *uint16, nameLen *uint32, reserved *uint32, class *uint16, value *uint16, valueLen *uint32) (regerrno error) = advapi32.RegEnumValueW

func GetPortsList() ([]string, error) {
	subKey, err := syscall.UTF16PtrFromString("HARDWARE\\DEVICEMAP\\SERIALCOMM\\")
	if err != nil {
		return nil, &SerialPortError{code: ERROR_ENUMERATING_PORTS}
	}

	var h syscall.Handle
	if syscall.RegOpenKeyEx(syscall.HKEY_LOCAL_MACHINE, subKey, 0, syscall.KEY_READ, &h) != nil {
		return nil, &SerialPortError{code: ERROR_ENUMERATING_PORTS}
	}
	defer syscall.RegCloseKey(h)

	var valuesCount uint32
	if syscall.RegQueryInfoKey(h, nil, nil, nil, nil, nil, nil, &valuesCount, nil, nil, nil, nil) != nil {
		return nil, &SerialPortError{code: ERROR_ENUMERATING_PORTS}
	}

	list := make([]string, valuesCount)
	for i := range list {
		var data [1024]uint16
		dataSize := uint32(len(data))
		var name [1024]uint16
		nameSize := uint32(len(name))
		if RegEnumValue(h, uint32(i), &name[0], &nameSize, nil, nil, &data[0], &dataSize) != nil {
			return nil, &SerialPortError{code: ERROR_ENUMERATING_PORTS}
		}
		list[i] = syscall.UTF16ToString(data[:])
	}
	return list, nil
}

func (port *SerialPort) Close() error {
	return syscall.CloseHandle(port.handle)
}

func (port *SerialPort) Read(p []byte) (int, error) {
	var readed uint32
	params := &DCB{}
	for {
		if err := syscall.ReadFile(port.handle, p, &readed, nil); err != nil {
			return int(readed), err
		}
		if readed > 0 {
			return int(readed), nil
		}

		// At the moment it seems that the only reliable way to check if
		// a serial port is alive in Windows is to check if the SetCommState
		// function fails.

		GetCommState(port.handle, params)
		if err := SetCommState(port.handle, params); err != nil {
			port.Close()
			return 0, err
		}
	}
}

func (port *SerialPort) Write(p []byte) (int, error) {
	var writed uint32
	err := syscall.WriteFile(port.handle, p, &writed, nil)
	return int(writed), err
}

/// Set a BREAK
func (port *SerialPort) SendBreak(breakTime int) error {
	// Set the Break
	if SetCommBreak(port.handle) != nil {
		port.Close()
		return &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}

	// Sleep for a period
	time.Sleep(time.Duration(breakTime) * time.Millisecond)

	// Reset the BREAK
	if ClearCommBreak(port.handle) != nil {
		port.Close()
		return &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}
	return nil
}

const (
	DCB_BINARY                   = 0x00000001
	DCB_PARITY                   = 0x00000002
	DCB_OUT_X_CTS_FLOW           = 0x00000004
	DCB_OUT_X_DSR_FLOW           = 0x00000008
	DCB_DTR_CONTROL_DISABLE_MASK = ^0x00000030
	DCB_DTR_CONTROL_ENABLE       = 0x00000010
	DCB_DTR_CONTROL_HANDSHAKE    = 0x00000020
	DCB_DSR_SENSITIVITY          = 0x00000040
	DCB_TX_CONTINUE_ON_XOFF      = 0x00000080
	DCB_OUT_X                    = 0x00000100
	DCB_IN_X                     = 0x00000200
	DCB_ERROR_CHAR               = 0x00000400
	DCB_NULL                     = 0x00000800
	DCB_RTS_CONTROL_DISABLE_MASK = ^0x00003000
	DCB_RTS_CONTROL_ENABLE       = 0x00001000
	DCB_RTS_CONTROL_HANDSHAKE    = 0x00002000
	DCB_RTS_CONTROL_TOGGLE       = 0x00003000
	DCB_ABORT_ON_ERROR           = 0x00004000
)

type DCB struct {
	DCBlength uint32
	BaudRate  uint32

	// Flags field is a bitfield
	//  fBinary            :1
	//  fParity            :1
	//  fOutxCtsFlow       :1
	//  fOutxDsrFlow       :1
	//  fDtrControl        :2
	//  fDsrSensitivity    :1
	//  fTXContinueOnXoff  :1
	//  fOutX              :1
	//  fInX               :1
	//  fErrorChar         :1
	//  fNull              :1
	//  fRtsControl        :2
	//  fAbortOnError      :1
	//  fDummy2            :17
	Flags uint32

	wReserved  uint16
	XonLim     uint16
	XoffLim    uint16
	ByteSize   byte
	Parity     byte
	StopBits   byte
	XonChar    byte
	XoffChar   byte
	ErrorChar  byte
	EofChar    byte
	EvtChar    byte
	wReserved1 uint16
}

type COMMTIMEOUTS struct {
	ReadIntervalTimeout         uint32
	ReadTotalTimeoutMultiplier  uint32
	ReadTotalTimeoutConstant    uint32
	WriteTotalTimeoutMultiplier uint32
	WriteTotalTimeoutConstant   uint32
}

//sys GetCommState(handle syscall.Handle, dcb *DCB) (err error)
//sys SetCommState(handle syscall.Handle, dcb *DCB) (err error)
//sys SetCommTimeouts(handle syscall.Handle, timeouts *COMMTIMEOUTS) (err error)

const (
	NOPARITY    = 0 // Default
	ODDPARITY   = 1
	EVENPARITY  = 2
	MARKPARITY  = 3
	SPACEPARITY = 4
)

const (
	ONESTOPBIT   = 0 // Default
	ONE5STOPBITS = 1
	TWOSTOPBITS  = 2
)

/// Set the Baud rate, data bits, stop bit and Parity
/// Default is 9600 8N1
func (port *SerialPort) SetMode(mode *Mode) error {
	params := DCB{}
	if GetCommState(port.handle, &params) != nil {
		port.Close()
		return &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}
	if mode.BaudRate == 0 {
		params.BaudRate = 9600 // Default to 9600
	} else {
		params.BaudRate = uint32(mode.BaudRate)
	}
	if mode.DataBits == 0 {
		params.ByteSize = 8 // Default to 8 bits
	} else {
		params.ByteSize = byte(mode.DataBits)
	}
	params.StopBits = byte(mode.StopBits)
	params.Parity = byte(mode.Parity)
	if SetCommState(port.handle, &params) != nil {
		port.Close()
		return &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}
	return nil
}

func OpenPort(portName string, mode *Mode) (*SerialPort, error) {
	portName = "\\\\.\\" + portName
	path, err := syscall.UTF16PtrFromString(portName)
	if err != nil {
		return nil, err
	}
	handle, err := syscall.CreateFile(
		path,
		syscall.GENERIC_READ|syscall.GENERIC_WRITE,
		0, nil,
		syscall.OPEN_EXISTING,
		0, //syscall.FILE_FLAG_OVERLAPPED,
		0)
	if err != nil {
		switch err {
		case syscall.ERROR_ACCESS_DENIED:
			return nil, &SerialPortError{code: ERROR_PORT_BUSY}
		case syscall.ERROR_FILE_NOT_FOUND:
			return nil, &SerialPortError{code: ERROR_PORT_NOT_FOUND}
		}
		return nil, err
	}
	// Create the serial port
	port := &SerialPort{
		handle: handle,
	}

	// Set port parameters
	if port.SetMode(mode) != nil {
		port.Close()
		return nil, &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}

	params := &DCB{}
	if GetCommState(port.handle, params) != nil {
		port.Close()
		return nil, &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}
	params.Flags |= DCB_RTS_CONTROL_ENABLE | DCB_DTR_CONTROL_ENABLE
	params.Flags &= ^uint32(DCB_OUT_X_CTS_FLOW)
	params.Flags &= ^uint32(DCB_OUT_X_DSR_FLOW)
	params.Flags &= ^uint32(DCB_DSR_SENSITIVITY)
	params.Flags |= DCB_TX_CONTINUE_ON_XOFF
	params.Flags &= ^uint32(DCB_IN_X | DCB_OUT_X)
	params.Flags &= ^uint32(DCB_ERROR_CHAR)
	params.Flags &= ^uint32(DCB_NULL)
	params.Flags &= ^uint32(DCB_ABORT_ON_ERROR)
	params.XonLim = 2048
	params.XoffLim = 512
	params.XonChar = 17  // DC1
	params.XoffChar = 19 // C3
	if SetCommState(port.handle, params) != nil {
		port.Close()
		return nil, &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}

	// Set timeouts to 1 second
	timeouts := &COMMTIMEOUTS{
		ReadIntervalTimeout:         0xFFFFFFFF,
		ReadTotalTimeoutMultiplier:  0xFFFFFFFF,
		ReadTotalTimeoutConstant:    1000, // 1 sec
		WriteTotalTimeoutConstant:   0,
		WriteTotalTimeoutMultiplier: 0,
	}
	if SetCommTimeouts(port.handle, timeouts) != nil {
		port.Close()
		return nil, &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}

	return port, nil
}

// vi:ts=2
<commit_msg>remove comments<commit_after>//
// Copyright 2014 Cristian Maglie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package serial

/*

// MSDN article on Serial Communications:
// http://msdn.microsoft.com/en-us/library/ff802693.aspx

// Arduino Playground article on serial communication with Windows API:
// http://playground.arduino.cc/Interfacing/CPPWindows

*/

import (
	"syscall"
	"time"
)

// opaque type that implements SerialPort interface for Windows
type SerialPort struct {
	handle syscall.Handle
}

//sys RegEnumValue(key syscall.Handle, index uint32, name *uint16, nameLen *uint32, reserved *uint32, class *uint16, value *uint16, valueLen *uint32) (regerrno error) = advapi32.RegEnumValueW

func GetPortsList() ([]string, error) {
	subKey, err := syscall.UTF16PtrFromString("HARDWARE\\DEVICEMAP\\SERIALCOMM\\")
	if err != nil {
		return nil, &SerialPortError{code: ERROR_ENUMERATING_PORTS}
	}

	var h syscall.Handle
	if syscall.RegOpenKeyEx(syscall.HKEY_LOCAL_MACHINE, subKey, 0, syscall.KEY_READ, &h) != nil {
		return nil, &SerialPortError{code: ERROR_ENUMERATING_PORTS}
	}
	defer syscall.RegCloseKey(h)

	var valuesCount uint32
	if syscall.RegQueryInfoKey(h, nil, nil, nil, nil, nil, nil, &valuesCount, nil, nil, nil, nil) != nil {
		return nil, &SerialPortError{code: ERROR_ENUMERATING_PORTS}
	}

	list := make([]string, valuesCount)
	for i := range list {
		var data [1024]uint16
		dataSize := uint32(len(data))
		var name [1024]uint16
		nameSize := uint32(len(name))
		if RegEnumValue(h, uint32(i), &name[0], &nameSize, nil, nil, &data[0], &dataSize) != nil {
			return nil, &SerialPortError{code: ERROR_ENUMERATING_PORTS}
		}
		list[i] = syscall.UTF16ToString(data[:])
	}
	return list, nil
}

func (port *SerialPort) Close() error {
	return syscall.CloseHandle(port.handle)
}

func (port *SerialPort) Read(p []byte) (int, error) {
	var readed uint32
	params := &DCB{}
	for {
		if err := syscall.ReadFile(port.handle, p, &readed, nil); err != nil {
			return int(readed), err
		}
		if readed > 0 {
			return int(readed), nil
		}

		// At the moment it seems that the only reliable way to check if
		// a serial port is alive in Windows is to check if the SetCommState
		// function fails.

		GetCommState(port.handle, params)
		if err := SetCommState(port.handle, params); err != nil {
			port.Close()
			return 0, err
		}
	}
}

func (port *SerialPort) Write(p []byte) (int, error) {
	var writed uint32
	err := syscall.WriteFile(port.handle, p, &writed, nil)
	return int(writed), err
}

func (port *SerialPort) SendBreak(breakTime int) error {
	// Set the Break
	if SetCommBreak(port.handle) != nil {
		port.Close()
		return &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}

	// Sleep for a period
	time.Sleep(time.Duration(breakTime) * time.Millisecond)

	// Reset the BREAK
	if ClearCommBreak(port.handle) != nil {
		port.Close()
		return &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}
	return nil
}

const (
	DCB_BINARY                   = 0x00000001
	DCB_PARITY                   = 0x00000002
	DCB_OUT_X_CTS_FLOW           = 0x00000004
	DCB_OUT_X_DSR_FLOW           = 0x00000008
	DCB_DTR_CONTROL_DISABLE_MASK = ^0x00000030
	DCB_DTR_CONTROL_ENABLE       = 0x00000010
	DCB_DTR_CONTROL_HANDSHAKE    = 0x00000020
	DCB_DSR_SENSITIVITY          = 0x00000040
	DCB_TX_CONTINUE_ON_XOFF      = 0x00000080
	DCB_OUT_X                    = 0x00000100
	DCB_IN_X                     = 0x00000200
	DCB_ERROR_CHAR               = 0x00000400
	DCB_NULL                     = 0x00000800
	DCB_RTS_CONTROL_DISABLE_MASK = ^0x00003000
	DCB_RTS_CONTROL_ENABLE       = 0x00001000
	DCB_RTS_CONTROL_HANDSHAKE    = 0x00002000
	DCB_RTS_CONTROL_TOGGLE       = 0x00003000
	DCB_ABORT_ON_ERROR           = 0x00004000
)

type DCB struct {
	DCBlength uint32
	BaudRate  uint32

	// Flags field is a bitfield
	//  fBinary            :1
	//  fParity            :1
	//  fOutxCtsFlow       :1
	//  fOutxDsrFlow       :1
	//  fDtrControl        :2
	//  fDsrSensitivity    :1
	//  fTXContinueOnXoff  :1
	//  fOutX              :1
	//  fInX               :1
	//  fErrorChar         :1
	//  fNull              :1
	//  fRtsControl        :2
	//  fAbortOnError      :1
	//  fDummy2            :17
	Flags uint32

	wReserved  uint16
	XonLim     uint16
	XoffLim    uint16
	ByteSize   byte
	Parity     byte
	StopBits   byte
	XonChar    byte
	XoffChar   byte
	ErrorChar  byte
	EofChar    byte
	EvtChar    byte
	wReserved1 uint16
}

type COMMTIMEOUTS struct {
	ReadIntervalTimeout         uint32
	ReadTotalTimeoutMultiplier  uint32
	ReadTotalTimeoutConstant    uint32
	WriteTotalTimeoutMultiplier uint32
	WriteTotalTimeoutConstant   uint32
}

//sys GetCommState(handle syscall.Handle, dcb *DCB) (err error)
//sys SetCommState(handle syscall.Handle, dcb *DCB) (err error)
//sys SetCommTimeouts(handle syscall.Handle, timeouts *COMMTIMEOUTS) (err error)

const (
	NOPARITY    = 0 // Default
	ODDPARITY   = 1
	EVENPARITY  = 2
	MARKPARITY  = 3
	SPACEPARITY = 4
)

const (
	ONESTOPBIT   = 0 // Default
	ONE5STOPBITS = 1
	TWOSTOPBITS  = 2
)

/// Set the Baud rate, data bits, stop bit and Parity
/// Default is 9600 8N1
func (port *SerialPort) SetMode(mode *Mode) error {
	params := DCB{}
	if GetCommState(port.handle, &params) != nil {
		port.Close()
		return &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}
	if mode.BaudRate == 0 {
		params.BaudRate = 9600 // Default to 9600
	} else {
		params.BaudRate = uint32(mode.BaudRate)
	}
	if mode.DataBits == 0 {
		params.ByteSize = 8 // Default to 8 bits
	} else {
		params.ByteSize = byte(mode.DataBits)
	}
	params.StopBits = byte(mode.StopBits)
	params.Parity = byte(mode.Parity)
	if SetCommState(port.handle, &params) != nil {
		port.Close()
		return &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}
	return nil
}

func OpenPort(portName string, mode *Mode) (*SerialPort, error) {
	portName = "\\\\.\\" + portName
	path, err := syscall.UTF16PtrFromString(portName)
	if err != nil {
		return nil, err
	}
	handle, err := syscall.CreateFile(
		path,
		syscall.GENERIC_READ|syscall.GENERIC_WRITE,
		0, nil,
		syscall.OPEN_EXISTING,
		0, //syscall.FILE_FLAG_OVERLAPPED,
		0)
	if err != nil {
		switch err {
		case syscall.ERROR_ACCESS_DENIED:
			return nil, &SerialPortError{code: ERROR_PORT_BUSY}
		case syscall.ERROR_FILE_NOT_FOUND:
			return nil, &SerialPortError{code: ERROR_PORT_NOT_FOUND}
		}
		return nil, err
	}
	// Create the serial port
	port := &SerialPort{
		handle: handle,
	}

	// Set port parameters
	if port.SetMode(mode) != nil {
		port.Close()
		return nil, &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}

	params := &DCB{}
	if GetCommState(port.handle, params) != nil {
		port.Close()
		return nil, &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}
	params.Flags |= DCB_RTS_CONTROL_ENABLE | DCB_DTR_CONTROL_ENABLE
	params.Flags &= ^uint32(DCB_OUT_X_CTS_FLOW)
	params.Flags &= ^uint32(DCB_OUT_X_DSR_FLOW)
	params.Flags &= ^uint32(DCB_DSR_SENSITIVITY)
	params.Flags |= DCB_TX_CONTINUE_ON_XOFF
	params.Flags &= ^uint32(DCB_IN_X | DCB_OUT_X)
	params.Flags &= ^uint32(DCB_ERROR_CHAR)
	params.Flags &= ^uint32(DCB_NULL)
	params.Flags &= ^uint32(DCB_ABORT_ON_ERROR)
	params.XonLim = 2048
	params.XoffLim = 512
	params.XonChar = 17  // DC1
	params.XoffChar = 19 // C3
	if SetCommState(port.handle, params) != nil {
		port.Close()
		return nil, &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}

	// Set timeouts to 1 second
	timeouts := &COMMTIMEOUTS{
		ReadIntervalTimeout:         0xFFFFFFFF,
		ReadTotalTimeoutMultiplier:  0xFFFFFFFF,
		ReadTotalTimeoutConstant:    1000, // 1 sec
		WriteTotalTimeoutConstant:   0,
		WriteTotalTimeoutMultiplier: 0,
	}
	if SetCommTimeouts(port.handle, timeouts) != nil {
		port.Close()
		return nil, &SerialPortError{code: ERROR_INVALID_SERIAL_PORT}
	}

	return port, nil
}

// vi:ts=2
<|endoftext|>


<commit_before>// Copyright (c) 2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"math/rand"
	"os/exec"
	"time"

	log "github.com/Sirupsen/logrus"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"k8s.io/client-go/kubernetes"

	"github.com/projectcalico/felix/k8sfv/leastsquares"
)

var _ = Describe("testing the test code", func() {

	It("should get non-nil value from getMac", func() {
		m := getMac()
		log.WithField("mac", m).Info("Generated MAC address")
		Expect(m).ToNot(BeNil())
	})
})

var _ = Context("with a k8s clientset", func() {

	var (
		clientset *kubernetes.Clientset
		nsPrefix  string
		d         deployment
	)

	BeforeEach(func() {
		log.Info(">>> BeforeEach <<<")
		clientset = initialize(k8sServerEndpoint)
		nsPrefix = getNamespacePrefix()
	})

	AfterEach(func() {
		log.Info(">>> AfterEach <<<")
		time.Sleep(10 * time.Second)
		cleanupAll(clientset, nsPrefix)
	})

	Context("with 1 remote node", func() {

		BeforeEach(func() {
			log.Info(">>> BeforeEach <<<")
			d = NewDeployment(clientset, 1, false)
		})

		It("should create 10k endpoints", func() {
			addNamespaces(clientset, nsPrefix)
			addEndpoints(clientset, nsPrefix, d, 10000)
		})

		It("should not leak memory", func() {
			const (
				cycles = 20
				ignore = 12
			)
			iiAverage := 0.5 * (ignore + cycles - 1)
			addNamespaces(clientset, nsPrefix)
			heapInUseMeasurements := []leastsquares.Point{}
			heapAllocMeasurements := []leastsquares.Point{}
			for ii := 0; ii < cycles; ii++ {
				// Add 10,000 endpoints.
				addEndpoints(clientset, nsPrefix, d, 10000)

				// Allow a little time for Felix to finish digesting those.
				time.Sleep(10 * time.Second)

				// Get Felix to GC and dump heap memory profile.
				exec.Command("pkill", "-USR1", "calico-felix").Run()
				time.Sleep(2 * time.Second)

				// Get current occupancy.
				heapInUse := getFelixFloatMetric("go_memstats_heap_inuse_bytes")
				heapAlloc := getFelixFloatMetric("go_memstats_heap_alloc_bytes")
				log.WithFields(log.Fields{
					"iteration": ii,
					"heapInUse": heapInUse,
					"heapAlloc": heapAlloc,
				}).Info("Bytes in use now")

				gaugeVecHeapAllocBytes.WithLabelValues(
					"felix",
					testName,
					fmt.Sprintf("iteration%d", ii),
					codeLevel,
				).Set(
					heapAlloc,
				)

				// Discard the first occupancy measurements since the first runs
				// have the advantage of running in a clean, unfragmented heap.
				if ii >= ignore {
					heapInUseMeasurements = append(
						heapInUseMeasurements,
						leastsquares.Point{float64(ii) - iiAverage, heapInUse},
					)
					heapAllocMeasurements = append(
						heapAllocMeasurements,
						leastsquares.Point{float64(ii) - iiAverage, heapAlloc},
					)
				}

				// Delete endpoints, then pause before continuing to the next cycle.
				cleanupAllPods(clientset, nsPrefix)
				time.Sleep(10 * time.Second)
			}

			gradient, constant := leastsquares.LeastSquaresMethod(heapInUseMeasurements)
			log.WithFields(log.Fields{
				"gradient": gradient,
				"constant": constant,
			}).Info("Least squares fit for inuse")
			gradient, constant = leastsquares.LeastSquaresMethod(heapAllocMeasurements)
			log.WithFields(log.Fields{
				"gradient": gradient,
				"constant": constant,
			}).Info("Least squares fit for alloc")

			// Initial strawman is that we don't expect to see any increase in memory
			// over the long term.  Given just 10 iterations, let's say that we require
			// the average gradient, per iteration, to be less than 2% of the average
			// occupancy.
			log.WithField("bytes", constant).Info("Average occupancy")
			increase := gradient * 100 / constant
			log.WithField("%", increase).Info("Increase per iteration")

			gaugeVecOccupancyMeanBytes.WithLabelValues(
				"felix", testName, codeLevel).Set(constant)
			gaugeVecOccupancyIncreasePercent.WithLabelValues(
				"felix", testName, codeLevel).Set(increase)

			Expect(increase).To(BeNumerically("<", 2))
		})
	})

	Context("with 1 local node", func() {

		BeforeEach(func() {
			log.Info(">>> BeforeEach <<<")
			d = NewDeployment(clientset, 0, true)
		})

		It("should handle a local endpoint", func() {
			createNamespace(clientset, nsPrefix+"test", nil)
			createPod(clientset, d, nsPrefix+"test", podSpec{})
			time.Sleep(10 * time.Second)
		})

		It("should handle 10 local endpoints", func() {
			createNamespace(clientset, nsPrefix+"test", nil)
			for ii := 0; ii < 10; ii++ {
				createPod(clientset, d, nsPrefix+"test", podSpec{})
			}
			time.Sleep(10 * time.Second)
		})

		It("should handle 100 local endpoints", func() {
			createNamespace(clientset, nsPrefix+"test", nil)
			for ii := 0; ii < 100; ii++ {
				createPod(clientset, d, nsPrefix+"test", podSpec{})
			}
			time.Sleep(10 * time.Second)
		})

		It("should create two isolated namespaces", func() {
			createIsolatedNamespace(clientset, nsPrefix+"test1", nil)
			createIsolatedNamespace(clientset, nsPrefix+"test2", nil)
			createNetworkPolicy(clientset, nsPrefix+"test1")
			createNetworkPolicy(clientset, nsPrefix+"test2")
			createPod(clientset, d, nsPrefix+"test1", podSpec{})
			createPod(clientset, d, nsPrefix+"test1", podSpec{})
			createPod(clientset, d, nsPrefix+"test1", podSpec{})
			createPod(clientset, d, nsPrefix+"test2", podSpec{})
			createPod(clientset, d, nsPrefix+"test2", podSpec{})
			createPod(clientset, d, nsPrefix+"test2", podSpec{})
		})

	})

	Context("with 1 local and 9 remote nodes", func() {

		BeforeEach(func() {
			log.Info(">>> BeforeEach <<<")
			d = NewDeployment(clientset, 9, true)
		})

		It("should add and remove 1000 pods, of which about 100 on local node", func() {
			createNamespace(clientset, nsPrefix+"scale", nil)
			for cycle := 0; cycle < 10; cycle++ {
				for ii := 0; ii < 1000; ii++ {
					createPod(clientset, d, nsPrefix+"scale", podSpec{})
					time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
				}
				time.Sleep(5 * time.Second)
				cleanupAllPods(clientset, nsPrefix)
				time.Sleep(1 * time.Second)
			}
			time.Sleep(20 * time.Second)
		})
	})
})
<commit_msg>k8sfv: utility function triggerFelixGCAndMemoryDump<commit_after>// Copyright (c) 2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"math/rand"
	"os/exec"
	"time"

	log "github.com/Sirupsen/logrus"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"k8s.io/client-go/kubernetes"

	"github.com/projectcalico/felix/k8sfv/leastsquares"
)

var _ = Describe("testing the test code", func() {

	It("should get non-nil value from getMac", func() {
		m := getMac()
		log.WithField("mac", m).Info("Generated MAC address")
		Expect(m).ToNot(BeNil())
	})
})

var _ = Context("with a k8s clientset", func() {

	var (
		clientset *kubernetes.Clientset
		nsPrefix  string
		d         deployment
	)

	BeforeEach(func() {
		log.Info(">>> BeforeEach <<<")
		clientset = initialize(k8sServerEndpoint)
		nsPrefix = getNamespacePrefix()
	})

	AfterEach(func() {
		log.Info(">>> AfterEach <<<")
		time.Sleep(10 * time.Second)
		cleanupAll(clientset, nsPrefix)
	})

	Context("with 1 remote node", func() {

		BeforeEach(func() {
			log.Info(">>> BeforeEach <<<")
			d = NewDeployment(clientset, 1, false)
		})

		It("should create 10k endpoints", func() {
			addNamespaces(clientset, nsPrefix)
			addEndpoints(clientset, nsPrefix, d, 10000)
		})

		It("should not leak memory", func() {
			const (
				cycles = 20
				ignore = 12
			)
			iiAverage := 0.5 * (ignore + cycles - 1)
			addNamespaces(clientset, nsPrefix)
			heapInUseMeasurements := []leastsquares.Point{}
			heapAllocMeasurements := []leastsquares.Point{}
			for ii := 0; ii < cycles; ii++ {
				// Add 10,000 endpoints.
				addEndpoints(clientset, nsPrefix, d, 10000)

				// Allow a little time for Felix to finish digesting those.
				time.Sleep(10 * time.Second)

				// Get Felix to GC and dump heap memory profile.
				triggerFelixGCAndMemoryDump()

				// Get current occupancy.
				heapInUse := getFelixFloatMetric("go_memstats_heap_inuse_bytes")
				heapAlloc := getFelixFloatMetric("go_memstats_heap_alloc_bytes")
				log.WithFields(log.Fields{
					"iteration": ii,
					"heapInUse": heapInUse,
					"heapAlloc": heapAlloc,
				}).Info("Bytes in use now")

				gaugeVecHeapAllocBytes.WithLabelValues(
					"felix",
					testName,
					fmt.Sprintf("iteration%d", ii),
					codeLevel,
				).Set(
					heapAlloc,
				)

				// Discard the first occupancy measurements since the first runs
				// have the advantage of running in a clean, unfragmented heap.
				if ii >= ignore {
					heapInUseMeasurements = append(
						heapInUseMeasurements,
						leastsquares.Point{float64(ii) - iiAverage, heapInUse},
					)
					heapAllocMeasurements = append(
						heapAllocMeasurements,
						leastsquares.Point{float64(ii) - iiAverage, heapAlloc},
					)
				}

				// Delete endpoints, then pause before continuing to the next cycle.
				cleanupAllPods(clientset, nsPrefix)
				time.Sleep(10 * time.Second)
			}

			gradient, constant := leastsquares.LeastSquaresMethod(heapInUseMeasurements)
			log.WithFields(log.Fields{
				"gradient": gradient,
				"constant": constant,
			}).Info("Least squares fit for inuse")
			gradient, constant = leastsquares.LeastSquaresMethod(heapAllocMeasurements)
			log.WithFields(log.Fields{
				"gradient": gradient,
				"constant": constant,
			}).Info("Least squares fit for alloc")

			// Initial strawman is that we don't expect to see any increase in memory
			// over the long term.  Given just 10 iterations, let's say that we require
			// the average gradient, per iteration, to be less than 2% of the average
			// occupancy.
			log.WithField("bytes", constant).Info("Average occupancy")
			increase := gradient * 100 / constant
			log.WithField("%", increase).Info("Increase per iteration")

			gaugeVecOccupancyMeanBytes.WithLabelValues(
				"felix", testName, codeLevel).Set(constant)
			gaugeVecOccupancyIncreasePercent.WithLabelValues(
				"felix", testName, codeLevel).Set(increase)

			Expect(increase).To(BeNumerically("<", 2))
		})
	})

	Context("with 1 local node", func() {

		BeforeEach(func() {
			log.Info(">>> BeforeEach <<<")
			d = NewDeployment(clientset, 0, true)
		})

		It("should handle a local endpoint", func() {
			createNamespace(clientset, nsPrefix+"test", nil)
			createPod(clientset, d, nsPrefix+"test", podSpec{})
			time.Sleep(10 * time.Second)
		})

		It("should handle 10 local endpoints", func() {
			createNamespace(clientset, nsPrefix+"test", nil)
			for ii := 0; ii < 10; ii++ {
				createPod(clientset, d, nsPrefix+"test", podSpec{})
			}
			time.Sleep(10 * time.Second)
		})

		It("should handle 100 local endpoints", func() {
			createNamespace(clientset, nsPrefix+"test", nil)
			for ii := 0; ii < 100; ii++ {
				createPod(clientset, d, nsPrefix+"test", podSpec{})
			}
			time.Sleep(10 * time.Second)
		})

		It("should create two isolated namespaces", func() {
			createIsolatedNamespace(clientset, nsPrefix+"test1", nil)
			createIsolatedNamespace(clientset, nsPrefix+"test2", nil)
			createNetworkPolicy(clientset, nsPrefix+"test1")
			createNetworkPolicy(clientset, nsPrefix+"test2")
			createPod(clientset, d, nsPrefix+"test1", podSpec{})
			createPod(clientset, d, nsPrefix+"test1", podSpec{})
			createPod(clientset, d, nsPrefix+"test1", podSpec{})
			createPod(clientset, d, nsPrefix+"test2", podSpec{})
			createPod(clientset, d, nsPrefix+"test2", podSpec{})
			createPod(clientset, d, nsPrefix+"test2", podSpec{})
		})

	})

	Context("with 1 local and 9 remote nodes", func() {

		BeforeEach(func() {
			log.Info(">>> BeforeEach <<<")
			d = NewDeployment(clientset, 9, true)
		})

		It("should add and remove 1000 pods, of which about 100 on local node", func() {
			createNamespace(clientset, nsPrefix+"scale", nil)
			for cycle := 0; cycle < 10; cycle++ {
				for ii := 0; ii < 1000; ii++ {
					createPod(clientset, d, nsPrefix+"scale", podSpec{})
					time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
				}
				time.Sleep(5 * time.Second)
				cleanupAllPods(clientset, nsPrefix)
				time.Sleep(1 * time.Second)
			}
			time.Sleep(20 * time.Second)
		})
	})
})

func triggerFelixGCAndMemoryDump() {
	exec.Command("pkill", "-USR1", "calico-felix").Run()
	time.Sleep(2 * time.Second)
}
<|endoftext|>


<commit_before>package scepserver

import (
	"context"
	"crypto"
	"crypto/rsa"
	"crypto/sha1"
	"crypto/x509"
	"encoding/asn1"
	"errors"
	"math/big"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/micromdm/scep/challenge"
	"github.com/micromdm/scep/csrverifier"
	"github.com/micromdm/scep/depot"
	"github.com/micromdm/scep/scep"
)

// Service is the interface for all supported SCEP server operations.
type Service interface {
	// GetCACaps returns a list of options
	// which are supported by the server.
	GetCACaps(ctx context.Context) ([]byte, error)

	// GetCACert returns CA certificate or
	// a CA certificate chain with intermediates
	// in a PKCS#7 Degenerate Certificates format
	GetCACert(ctx context.Context) ([]byte, int, error)

	// PKIOperation handles incoming SCEP messages such as PKCSReq and
	// sends back a CertRep PKIMessag.
	PKIOperation(ctx context.Context, msg []byte) ([]byte, error)

	// GetNextCACert returns a replacement certificate or certificate chain
	// when the old one expires. The response format is a PKCS#7 Degenerate
	// Certificates type.
	GetNextCACert(ctx context.Context) ([]byte, error)
}

type service struct {
	depot                   depot.Depot
	ca                      []*x509.Certificate // CA cert or chain
	caKey                   *rsa.PrivateKey
	caKeyPassword           []byte
	csrTemplate             *x509.Certificate
	challengePassword       string
	supportDynamciChallenge bool
	dynamicChallengeStore   challenge.Store
	csrVerifier             csrverifier.CSRVerifier
	allowRenewal            int // days before expiry, 0 to disable
	clientValidity          int // client cert validity in days

	/// info logging is implemented in the service middleware layer.
	debugLogger log.Logger
}

// SCEPChallenge returns a brand new, random dynamic challenge.
func (svc *service) SCEPChallenge() (string, error) {
	if !svc.supportDynamciChallenge {
		return svc.challengePassword, nil
	}

	return svc.dynamicChallengeStore.SCEPChallenge()
}

func (svc *service) GetCACaps(ctx context.Context) ([]byte, error) {
	defaultCaps := []byte("Renewal\nSHA-1\nSHA-256\nAES\nDES3\nSCEPStandard\nPOSTPKIOperation")
	return defaultCaps, nil
}

func (svc *service) GetCACert(ctx context.Context) ([]byte, int, error) {
	if len(svc.ca) == 0 {
		return nil, 0, errors.New("missing CA Cert")
	}
	if len(svc.ca) == 1 {
		return svc.ca[0].Raw, 1, nil
	}
	data, err := scep.DegenerateCertificates(svc.ca)
	return data, len(svc.ca), err
}

func (svc *service) PKIOperation(ctx context.Context, data []byte) ([]byte, error) {
	msg, err := scep.ParsePKIMessage(data, scep.WithLogger(svc.debugLogger))
	if err != nil {
		return nil, err
	}
	ca := svc.ca[0]
	if err := msg.DecryptPKIEnvelope(svc.ca[0], svc.caKey); err != nil {
		return nil, err
	}

	// validate challenge passwords
	if msg.MessageType == scep.PKCSReq {
		CSRIsValid := false

		if svc.csrVerifier != nil {
			result, err := svc.csrVerifier.Verify(msg.CSRReqMessage.RawDecrypted)
			if err != nil {
				return nil, err
			}
			CSRIsValid = result
			if !CSRIsValid {
				svc.debugLogger.Log("err", "CSR is not valid")
			}
		} else {
			CSRIsValid = svc.challengePasswordMatch(msg.CSRReqMessage.ChallengePassword)
			if !CSRIsValid {
				svc.debugLogger.Log("err", "scep challenge password does not match")
			}
		}

		if !CSRIsValid {
			certRep, err := msg.Fail(ca, svc.caKey, scep.BadRequest)
			if err != nil {
				return nil, err
			}
			return certRep.Raw, nil
		}
	}

	csr := msg.CSRReqMessage.CSR
	id, err := generateSubjectKeyID(csr.PublicKey)
	if err != nil {
		return nil, err
	}

	serial, err := svc.depot.Serial()
	if err != nil {
		return nil, err
	}

	duration := svc.clientValidity

	// create cert template
	tmpl := &x509.Certificate{
		SerialNumber: serial,
		Subject:      csr.Subject,
		NotBefore:    time.Now().Add(-600).UTC(),
		NotAfter:     time.Now().AddDate(0, 0, duration).UTC(),
		SubjectKeyId: id,
		KeyUsage:     x509.KeyUsageDigitalSignature,
		ExtKeyUsage: []x509.ExtKeyUsage{
			x509.ExtKeyUsageClientAuth,
		},
		SignatureAlgorithm: csr.SignatureAlgorithm,
		EmailAddresses:     csr.EmailAddresses,
	}

	certRep, err := msg.SignCSR(ca, svc.caKey, tmpl)
	if err != nil {
		return nil, err
	}

	crt := certRep.CertRepMessage.Certificate
	name := certName(crt)

	// Test if this certificate is already in the CADB, revoke if needed
	// revocation is done if the validity of the existing certificate is
	// less than allowRenewal (14 days by default)
	_, err = svc.depot.HasCN(name, svc.allowRenewal, crt, false)
	if err != nil {
		return nil, err
	}

	if err := svc.depot.Put(name, crt); err != nil {
		return nil, err
	}

	return certRep.Raw, nil
}

func certName(crt *x509.Certificate) string {
	if crt.Subject.CommonName != "" {
		return crt.Subject.CommonName
	}
	return string(crt.Signature)
}

func (svc *service) GetNextCACert(ctx context.Context) ([]byte, error) {
	panic("not implemented")
}

func (svc *service) challengePasswordMatch(pw string) bool {
	if svc.challengePassword == "" && !svc.supportDynamciChallenge {
		// empty password, don't validate
		return true
	}
	if !svc.supportDynamciChallenge && svc.challengePassword == pw {
		return true
	}

	if svc.supportDynamciChallenge {
		valid, err := svc.dynamicChallengeStore.HasChallenge(pw)
		if err != nil {
			svc.debugLogger.Log(err)
			return false
		}
		return valid
	}

	return false
}

// ServiceOption is a server configuration option
type ServiceOption func(*service) error

// WithCSRVerifier is an option argument to NewService
// which allows setting a CSR verifier.
func WithCSRVerifier(csrVerifier csrverifier.CSRVerifier) ServiceOption {
	return func(s *service) error {
		s.csrVerifier = csrVerifier
		return nil
	}
}

// ChallengePassword is an optional argument to NewService
// which allows setting a preshared key for SCEP.
func ChallengePassword(pw string) ServiceOption {
	return func(s *service) error {
		s.challengePassword = pw
		return nil
	}
}

// CAKeyPassword is an optional argument to NewService for
// specifying the CA private key password.
func CAKeyPassword(pw []byte) ServiceOption {
	return func(s *service) error {
		s.caKeyPassword = pw
		return nil
	}
}

// allowRenewal sets the days before expiry which we are allowed to renew (optional)
func AllowRenewal(duration int) ServiceOption {
	return func(s *service) error {
		s.allowRenewal = duration
		return nil
	}
}

// ClientValidity sets the validity of signed client certs in days (optional parameter)
func ClientValidity(duration int) ServiceOption {
	return func(s *service) error {
		s.clientValidity = duration
		return nil
	}
}

// WithLogger configures a logger for the SCEP Service.
// By default, a no-op logger is used.
func WithLogger(logger log.Logger) ServiceOption {
	return func(s *service) error {
		s.debugLogger = logger
		return nil
	}
}

func WithDynamicChallenges(cache challenge.Store) ServiceOption {
	return func(s *service) error {
		s.supportDynamciChallenge = true
		s.dynamicChallengeStore = cache
		return nil
	}
}

// NewService creates a new scep service
func NewService(depot depot.Depot, opts ...ServiceOption) (Service, error) {
	s := &service{
		depot:       depot,
		debugLogger: log.NewNopLogger(),
	}
	for _, opt := range opts {
		if err := opt(s); err != nil {
			return nil, err
		}
	}

	var err error
	s.ca, s.caKey, err = depot.CA(s.caKeyPassword)
	if err != nil {
		return nil, err
	}
	return s, nil
}

// rsaPublicKey reflects the ASN.1 structure of a PKCS#1 public key.
type rsaPublicKey struct {
	N *big.Int
	E int
}

// GenerateSubjectKeyID generates SubjectKeyId used in Certificate
// ID is 160-bit SHA-1 hash of the value of the BIT STRING subjectPublicKey
func generateSubjectKeyID(pub crypto.PublicKey) ([]byte, error) {
	var pubBytes []byte
	var err error
	switch pub := pub.(type) {
	case *rsa.PublicKey:
		pubBytes, err = asn1.Marshal(rsaPublicKey{
			N: pub.N,
			E: pub.E,
		})
		if err != nil {
			return nil, err
		}
	default:
		return nil, errors.New("only RSA public key is supported")
	}

	hash := sha1.Sum(pubBytes)

	return hash[:], nil
}
<commit_msg>copy more alternative names<commit_after>package scepserver

import (
	"context"
	"crypto"
	"crypto/rsa"
	"crypto/sha1"
	"crypto/x509"
	"encoding/asn1"
	"errors"
	"math/big"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/micromdm/scep/challenge"
	"github.com/micromdm/scep/csrverifier"
	"github.com/micromdm/scep/depot"
	"github.com/micromdm/scep/scep"
)

// Service is the interface for all supported SCEP server operations.
type Service interface {
	// GetCACaps returns a list of options
	// which are supported by the server.
	GetCACaps(ctx context.Context) ([]byte, error)

	// GetCACert returns CA certificate or
	// a CA certificate chain with intermediates
	// in a PKCS#7 Degenerate Certificates format
	GetCACert(ctx context.Context) ([]byte, int, error)

	// PKIOperation handles incoming SCEP messages such as PKCSReq and
	// sends back a CertRep PKIMessag.
	PKIOperation(ctx context.Context, msg []byte) ([]byte, error)

	// GetNextCACert returns a replacement certificate or certificate chain
	// when the old one expires. The response format is a PKCS#7 Degenerate
	// Certificates type.
	GetNextCACert(ctx context.Context) ([]byte, error)
}

type service struct {
	depot                   depot.Depot
	ca                      []*x509.Certificate // CA cert or chain
	caKey                   *rsa.PrivateKey
	caKeyPassword           []byte
	csrTemplate             *x509.Certificate
	challengePassword       string
	supportDynamciChallenge bool
	dynamicChallengeStore   challenge.Store
	csrVerifier             csrverifier.CSRVerifier
	allowRenewal            int // days before expiry, 0 to disable
	clientValidity          int // client cert validity in days

	/// info logging is implemented in the service middleware layer.
	debugLogger log.Logger
}

// SCEPChallenge returns a brand new, random dynamic challenge.
func (svc *service) SCEPChallenge() (string, error) {
	if !svc.supportDynamciChallenge {
		return svc.challengePassword, nil
	}

	return svc.dynamicChallengeStore.SCEPChallenge()
}

func (svc *service) GetCACaps(ctx context.Context) ([]byte, error) {
	defaultCaps := []byte("Renewal\nSHA-1\nSHA-256\nAES\nDES3\nSCEPStandard\nPOSTPKIOperation")
	return defaultCaps, nil
}

func (svc *service) GetCACert(ctx context.Context) ([]byte, int, error) {
	if len(svc.ca) == 0 {
		return nil, 0, errors.New("missing CA Cert")
	}
	if len(svc.ca) == 1 {
		return svc.ca[0].Raw, 1, nil
	}
	data, err := scep.DegenerateCertificates(svc.ca)
	return data, len(svc.ca), err
}

func (svc *service) PKIOperation(ctx context.Context, data []byte) ([]byte, error) {
	msg, err := scep.ParsePKIMessage(data, scep.WithLogger(svc.debugLogger))
	if err != nil {
		return nil, err
	}
	ca := svc.ca[0]
	if err := msg.DecryptPKIEnvelope(svc.ca[0], svc.caKey); err != nil {
		return nil, err
	}

	// validate challenge passwords
	if msg.MessageType == scep.PKCSReq {
		CSRIsValid := false

		if svc.csrVerifier != nil {
			result, err := svc.csrVerifier.Verify(msg.CSRReqMessage.RawDecrypted)
			if err != nil {
				return nil, err
			}
			CSRIsValid = result
			if !CSRIsValid {
				svc.debugLogger.Log("err", "CSR is not valid")
			}
		} else {
			CSRIsValid = svc.challengePasswordMatch(msg.CSRReqMessage.ChallengePassword)
			if !CSRIsValid {
				svc.debugLogger.Log("err", "scep challenge password does not match")
			}
		}

		if !CSRIsValid {
			certRep, err := msg.Fail(ca, svc.caKey, scep.BadRequest)
			if err != nil {
				return nil, err
			}
			return certRep.Raw, nil
		}
	}

	csr := msg.CSRReqMessage.CSR
	id, err := generateSubjectKeyID(csr.PublicKey)
	if err != nil {
		return nil, err
	}

	serial, err := svc.depot.Serial()
	if err != nil {
		return nil, err
	}

	duration := svc.clientValidity

	// create cert template
	tmpl := &x509.Certificate{
		SerialNumber: serial,
		Subject:      csr.Subject,
		NotBefore:    time.Now().Add(-600).UTC(),
		NotAfter:     time.Now().AddDate(0, 0, duration).UTC(),
		SubjectKeyId: id,
		KeyUsage:     x509.KeyUsageDigitalSignature,
		ExtKeyUsage: []x509.ExtKeyUsage{
			x509.ExtKeyUsageClientAuth,
		},
		SignatureAlgorithm: csr.SignatureAlgorithm,
		DNSNames:           csr.DNSNames,
		EmailAddresses:     csr.EmailAddresses,
		IPAddresses:        csr.IPAddresses,
		URIs:               csr.URIs,
	}

	certRep, err := msg.SignCSR(ca, svc.caKey, tmpl)
	if err != nil {
		return nil, err
	}

	crt := certRep.CertRepMessage.Certificate
	name := certName(crt)

	// Test if this certificate is already in the CADB, revoke if needed
	// revocation is done if the validity of the existing certificate is
	// less than allowRenewal (14 days by default)
	_, err = svc.depot.HasCN(name, svc.allowRenewal, crt, false)
	if err != nil {
		return nil, err
	}

	if err := svc.depot.Put(name, crt); err != nil {
		return nil, err
	}

	return certRep.Raw, nil
}

func certName(crt *x509.Certificate) string {
	if crt.Subject.CommonName != "" {
		return crt.Subject.CommonName
	}
	return string(crt.Signature)
}

func (svc *service) GetNextCACert(ctx context.Context) ([]byte, error) {
	panic("not implemented")
}

func (svc *service) challengePasswordMatch(pw string) bool {
	if svc.challengePassword == "" && !svc.supportDynamciChallenge {
		// empty password, don't validate
		return true
	}
	if !svc.supportDynamciChallenge && svc.challengePassword == pw {
		return true
	}

	if svc.supportDynamciChallenge {
		valid, err := svc.dynamicChallengeStore.HasChallenge(pw)
		if err != nil {
			svc.debugLogger.Log(err)
			return false
		}
		return valid
	}

	return false
}

// ServiceOption is a server configuration option
type ServiceOption func(*service) error

// WithCSRVerifier is an option argument to NewService
// which allows setting a CSR verifier.
func WithCSRVerifier(csrVerifier csrverifier.CSRVerifier) ServiceOption {
	return func(s *service) error {
		s.csrVerifier = csrVerifier
		return nil
	}
}

// ChallengePassword is an optional argument to NewService
// which allows setting a preshared key for SCEP.
func ChallengePassword(pw string) ServiceOption {
	return func(s *service) error {
		s.challengePassword = pw
		return nil
	}
}

// CAKeyPassword is an optional argument to NewService for
// specifying the CA private key password.
func CAKeyPassword(pw []byte) ServiceOption {
	return func(s *service) error {
		s.caKeyPassword = pw
		return nil
	}
}

// allowRenewal sets the days before expiry which we are allowed to renew (optional)
func AllowRenewal(duration int) ServiceOption {
	return func(s *service) error {
		s.allowRenewal = duration
		return nil
	}
}

// ClientValidity sets the validity of signed client certs in days (optional parameter)
func ClientValidity(duration int) ServiceOption {
	return func(s *service) error {
		s.clientValidity = duration
		return nil
	}
}

// WithLogger configures a logger for the SCEP Service.
// By default, a no-op logger is used.
func WithLogger(logger log.Logger) ServiceOption {
	return func(s *service) error {
		s.debugLogger = logger
		return nil
	}
}

func WithDynamicChallenges(cache challenge.Store) ServiceOption {
	return func(s *service) error {
		s.supportDynamciChallenge = true
		s.dynamicChallengeStore = cache
		return nil
	}
}

// NewService creates a new scep service
func NewService(depot depot.Depot, opts ...ServiceOption) (Service, error) {
	s := &service{
		depot:       depot,
		debugLogger: log.NewNopLogger(),
	}
	for _, opt := range opts {
		if err := opt(s); err != nil {
			return nil, err
		}
	}

	var err error
	s.ca, s.caKey, err = depot.CA(s.caKeyPassword)
	if err != nil {
		return nil, err
	}
	return s, nil
}

// rsaPublicKey reflects the ASN.1 structure of a PKCS#1 public key.
type rsaPublicKey struct {
	N *big.Int
	E int
}

// GenerateSubjectKeyID generates SubjectKeyId used in Certificate
// ID is 160-bit SHA-1 hash of the value of the BIT STRING subjectPublicKey
func generateSubjectKeyID(pub crypto.PublicKey) ([]byte, error) {
	var pubBytes []byte
	var err error
	switch pub := pub.(type) {
	case *rsa.PublicKey:
		pubBytes, err = asn1.Marshal(rsaPublicKey{
			N: pub.N,
			E: pub.E,
		})
		if err != nil {
			return nil, err
		}
	default:
		return nil, errors.New("only RSA public key is supported")
	}

	hash := sha1.Sum(pubBytes)

	return hash[:], nil
}
<|endoftext|>


<commit_before>package search

import (
	"container/heap"
	"math"

	gr "github.com/gonum/graph"
	"github.com/gonum/graph/concrete"
)

type searchFuncs struct {
	successors, predecessors, neighbors    func(gr.Node) []gr.Node
	isSuccessor, isPredecessor, isNeighbor func(gr.Node, gr.Node) bool
	cost                                   gr.CostFunc
	heuristicCost                          gr.HeuristicCostFunc
}

func genIsSuccessor(graph gr.DirectedGraph) func(gr.Node, gr.Node) bool {
	return func(node, succ gr.Node) bool {
		return graph.EdgeTo(node, succ) != nil
	}
}

func genIsPredecessor(graph gr.DirectedGraph) func(gr.Node, gr.Node) bool {
	return func(node, succ gr.Node) bool {
		return graph.EdgeTo(succ, node) != nil
	}
}

func genIsNeighbor(graph gr.Graph) func(gr.Node, gr.Node) bool {
	return func(node, succ gr.Node) bool {
		return graph.EdgeBetween(succ, node) != nil
	}
}

// Sets up the cost functions and successor functions so I don't have to do a type switch every
// time. This almost always does more work than is necessary, but since it's only executed once
// per function, and graph functions are rather costly, the "extra work" should be negligible.
func setupFuncs(graph gr.Graph, cost gr.CostFunc, heuristicCost gr.HeuristicCostFunc) searchFuncs {

	sf := searchFuncs{}

	switch g := graph.(type) {
	case gr.DirectedGraph:
		sf.successors = g.Successors
		sf.predecessors = g.Predecessors
		sf.neighbors = g.Neighbors
		sf.isSuccessor = genIsSuccessor(g)
		sf.isPredecessor = genIsPredecessor(g)
		sf.isNeighbor = genIsNeighbor(g)
	default:
		sf.successors = g.Neighbors
		sf.predecessors = g.Neighbors
		sf.neighbors = g.Neighbors
		sf.isSuccessor = genIsNeighbor(g)
		sf.isPredecessor = genIsNeighbor(g)
		sf.isNeighbor = genIsNeighbor(g)
	}

	if heuristicCost != nil {
		sf.heuristicCost = heuristicCost
	} else {
		if g, ok := graph.(gr.HeuristicCoster); ok {
			sf.heuristicCost = g.HeuristicCost
		} else {
			sf.heuristicCost = NullHeuristic
		}
	}

	if cost != nil {
		sf.cost = cost
	} else {
		if g, ok := graph.(gr.Coster); ok {
			sf.cost = g.Cost
		} else {
			sf.cost = UniformCost
		}
	}

	return sf
}

func NullHeuristic(node1, node2 gr.Node) float64 {
	return 0.0
}

func UniformCost(e gr.Edge) float64 {
	if e == nil {
		return math.Inf(1)
	}

	return 1.0
}

/** Sorts a list of edges by weight, agnostic to repeated edges as well as direction **/

type edgeSorter []concrete.WeightedEdge

func (el edgeSorter) Len() int {
	return len(el)
}

func (el edgeSorter) Less(i, j int) bool {
	return el[i].Cost < el[j].Cost
}

func (el edgeSorter) Swap(i, j int) {
	el[i], el[j] = el[j], el[i]
}

/** Keeps track of a node's scores so they can be used in a priority queue for A* **/

type internalNode struct {
	gr.Node
	gscore, fscore float64
}

/* A* stuff */
type aStarPriorityQueue struct {
	indexList map[int]int
	nodes     []internalNode
}

func (pq *aStarPriorityQueue) Less(i, j int) bool {
	// As the heap documentation says, a priority queue is listed if the actual values
	// are treated as if they were negative
	return pq.nodes[i].fscore < pq.nodes[j].fscore
}

func (pq *aStarPriorityQueue) Swap(i, j int) {
	pq.indexList[pq.nodes[i].ID()] = j
	pq.indexList[pq.nodes[j].ID()] = i

	pq.nodes[i], pq.nodes[j] = pq.nodes[j], pq.nodes[i]
}

func (pq *aStarPriorityQueue) Len() int {
	return len(pq.nodes)
}

func (pq *aStarPriorityQueue) Push(x interface{}) {
	node := x.(internalNode)
	pq.nodes = append(pq.nodes, node)
	pq.indexList[node.ID()] = len(pq.nodes) - 1
}

func (pq *aStarPriorityQueue) Pop() interface{} {
	x := pq.nodes[len(pq.nodes)-1]
	pq.nodes = pq.nodes[:len(pq.nodes)-1]
	delete(pq.indexList, x.ID())

	return x
}

func (pq *aStarPriorityQueue) Fix(id int, newGScore, newFScore float64) {
	if i, ok := pq.indexList[id]; ok {
		pq.nodes[i].gscore = newGScore
		pq.nodes[i].fscore = newFScore
		heap.Fix(pq, i)
	}
}

func (pq *aStarPriorityQueue) Find(id int) (internalNode, bool) {
	loc, ok := pq.indexList[id]
	if ok {
		return pq.nodes[loc], true
	} else {
		return internalNode{}, false
	}

}

func (pq *aStarPriorityQueue) Exists(id int) bool {
	_, ok := pq.indexList[id]
	return ok
}

type denseNodeSorter []gr.Node

func (dns denseNodeSorter) Less(i, j int) bool {
	return dns[i].ID() < dns[j].ID()
}

func (dns denseNodeSorter) Swap(i, j int) {
	dns[i], dns[j] = dns[j], dns[i]
}

func (dns denseNodeSorter) Len() int {
	return len(dns)
}

// General utility funcs

// Rebuilds a path backwards from the goal.
func rebuildPath(predecessors map[int]gr.Node, goal gr.Node) []gr.Node {
	if n, ok := goal.(internalNode); ok {
		goal = n.Node
	}
	path := []gr.Node{goal}
	curr := goal
	for prev, ok := predecessors[curr.ID()]; ok; prev, ok = predecessors[curr.ID()] {
		if n, ok := prev.(internalNode); ok {
			prev = n.Node
		}
		path = append(path, prev)
		curr = prev
	}

	// Reverse the path since it was built backwards
	for i, j := 0, len(path)-1; i < j; i, j = i+1, j-1 {
		path[i], path[j] = path[j], path[i]
	}

	return path
}
<commit_msg>Fixed redundant declaration<commit_after>package search

import (
	"container/heap"

	gr "github.com/gonum/graph"
	"github.com/gonum/graph/concrete"
)

type searchFuncs struct {
	successors, predecessors, neighbors    func(gr.Node) []gr.Node
	isSuccessor, isPredecessor, isNeighbor func(gr.Node, gr.Node) bool
	cost                                   gr.CostFunc
	heuristicCost                          gr.HeuristicCostFunc
}

func genIsSuccessor(graph gr.DirectedGraph) func(gr.Node, gr.Node) bool {
	return func(node, succ gr.Node) bool {
		return graph.EdgeTo(node, succ) != nil
	}
}

func genIsPredecessor(graph gr.DirectedGraph) func(gr.Node, gr.Node) bool {
	return func(node, succ gr.Node) bool {
		return graph.EdgeTo(succ, node) != nil
	}
}

func genIsNeighbor(graph gr.Graph) func(gr.Node, gr.Node) bool {
	return func(node, succ gr.Node) bool {
		return graph.EdgeBetween(succ, node) != nil
	}
}

// Sets up the cost functions and successor functions so I don't have to do a type switch every
// time. This almost always does more work than is necessary, but since it's only executed once
// per function, and graph functions are rather costly, the "extra work" should be negligible.
func setupFuncs(graph gr.Graph, cost gr.CostFunc, heuristicCost gr.HeuristicCostFunc) searchFuncs {

	sf := searchFuncs{}

	switch g := graph.(type) {
	case gr.DirectedGraph:
		sf.successors = g.Successors
		sf.predecessors = g.Predecessors
		sf.neighbors = g.Neighbors
		sf.isSuccessor = genIsSuccessor(g)
		sf.isPredecessor = genIsPredecessor(g)
		sf.isNeighbor = genIsNeighbor(g)
	default:
		sf.successors = g.Neighbors
		sf.predecessors = g.Neighbors
		sf.neighbors = g.Neighbors
		sf.isSuccessor = genIsNeighbor(g)
		sf.isPredecessor = genIsNeighbor(g)
		sf.isNeighbor = genIsNeighbor(g)
	}

	if heuristicCost != nil {
		sf.heuristicCost = heuristicCost
	} else {
		if g, ok := graph.(gr.HeuristicCoster); ok {
			sf.heuristicCost = g.HeuristicCost
		} else {
			sf.heuristicCost = NullHeuristic
		}
	}

	if cost != nil {
		sf.cost = cost
	} else {
		if g, ok := graph.(gr.Coster); ok {
			sf.cost = g.Cost
		} else {
			sf.cost = UniformCost
		}
	}

	return sf
}

/** Sorts a list of edges by weight, agnostic to repeated edges as well as direction **/

type edgeSorter []concrete.WeightedEdge

func (el edgeSorter) Len() int {
	return len(el)
}

func (el edgeSorter) Less(i, j int) bool {
	return el[i].Cost < el[j].Cost
}

func (el edgeSorter) Swap(i, j int) {
	el[i], el[j] = el[j], el[i]
}

/** Keeps track of a node's scores so they can be used in a priority queue for A* **/

type internalNode struct {
	gr.Node
	gscore, fscore float64
}

/* A* stuff */
type aStarPriorityQueue struct {
	indexList map[int]int
	nodes     []internalNode
}

func (pq *aStarPriorityQueue) Less(i, j int) bool {
	// As the heap documentation says, a priority queue is listed if the actual values
	// are treated as if they were negative
	return pq.nodes[i].fscore < pq.nodes[j].fscore
}

func (pq *aStarPriorityQueue) Swap(i, j int) {
	pq.indexList[pq.nodes[i].ID()] = j
	pq.indexList[pq.nodes[j].ID()] = i

	pq.nodes[i], pq.nodes[j] = pq.nodes[j], pq.nodes[i]
}

func (pq *aStarPriorityQueue) Len() int {
	return len(pq.nodes)
}

func (pq *aStarPriorityQueue) Push(x interface{}) {
	node := x.(internalNode)
	pq.nodes = append(pq.nodes, node)
	pq.indexList[node.ID()] = len(pq.nodes) - 1
}

func (pq *aStarPriorityQueue) Pop() interface{} {
	x := pq.nodes[len(pq.nodes)-1]
	pq.nodes = pq.nodes[:len(pq.nodes)-1]
	delete(pq.indexList, x.ID())

	return x
}

func (pq *aStarPriorityQueue) Fix(id int, newGScore, newFScore float64) {
	if i, ok := pq.indexList[id]; ok {
		pq.nodes[i].gscore = newGScore
		pq.nodes[i].fscore = newFScore
		heap.Fix(pq, i)
	}
}

func (pq *aStarPriorityQueue) Find(id int) (internalNode, bool) {
	loc, ok := pq.indexList[id]
	if ok {
		return pq.nodes[loc], true
	} else {
		return internalNode{}, false
	}

}

func (pq *aStarPriorityQueue) Exists(id int) bool {
	_, ok := pq.indexList[id]
	return ok
}

type denseNodeSorter []gr.Node

func (dns denseNodeSorter) Less(i, j int) bool {
	return dns[i].ID() < dns[j].ID()
}

func (dns denseNodeSorter) Swap(i, j int) {
	dns[i], dns[j] = dns[j], dns[i]
}

func (dns denseNodeSorter) Len() int {
	return len(dns)
}

// General utility funcs

// Rebuilds a path backwards from the goal.
func rebuildPath(predecessors map[int]gr.Node, goal gr.Node) []gr.Node {
	if n, ok := goal.(internalNode); ok {
		goal = n.Node
	}
	path := []gr.Node{goal}
	curr := goal
	for prev, ok := predecessors[curr.ID()]; ok; prev, ok = predecessors[curr.ID()] {
		if n, ok := prev.(internalNode); ok {
			prev = n.Node
		}
		path = append(path, prev)
		curr = prev
	}

	// Reverse the path since it was built backwards
	for i, j := 0, len(path)-1; i < j; i, j = i+1, j-1 {
		path[i], path[j] = path[j], path[i]
	}

	return path
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/base64"
	"encoding/json"
	"log"
	"net/http"
	"time"

	"github.com/ant0ine/go-json-rest/rest"
	"github.com/joshheinrichs/geosource/server/transactions"
	"github.com/pborman/uuid"
)

func GetPosts(w rest.ResponseWriter, req *rest.Request) {
	userId, err := GetUserId(req)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	posts, err := transactions.GetPosts(userId)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(posts)
	w.WriteHeader(http.StatusOK)
}

func GetPost(w rest.ResponseWriter, req *rest.Request) {
	userId, err := GetUserId(req)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	post, err := transactions.GetPost(userId, req.PathParam("pid"))
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(post)
	w.WriteHeader(http.StatusOK)
}

func AddPost(w rest.ResponseWriter, req *rest.Request) {
	userId, err := GetUserId(req)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	var jsonBody json.RawMessage
	err = req.DecodeJsonPayload(&jsonBody)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	submissionChannel := struct {
		Channel string `json:"channel"`
	}{}
	err = json.Unmarshal(jsonBody, &submissionChannel)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	channel, err := transactions.GetChannel(userId, submissionChannel.Channel)
	if err != nil {
		log.Println("could not find channel ", submissionChannel.Channel)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	post, err := channel.UnmarshalSubmissionToPost(jsonBody)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	post.CreatorId = userId
	post.Id = base64.RawURLEncoding.EncodeToString(uuid.NewRandom())
	post.time = time.Now().UTC()

	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	err = transactions.AddPost(userId, post)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
}

func RemovePost(w rest.ResponseWriter, req *rest.Request) {}
<commit_msg>Fix broken field name<commit_after>package api

import (
	"encoding/base64"
	"encoding/json"
	"log"
	"net/http"
	"time"

	"github.com/ant0ine/go-json-rest/rest"
	"github.com/joshheinrichs/geosource/server/transactions"
	"github.com/pborman/uuid"
)

func GetPosts(w rest.ResponseWriter, req *rest.Request) {
	userId, err := GetUserId(req)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	posts, err := transactions.GetPosts(userId)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(posts)
	w.WriteHeader(http.StatusOK)
}

func GetPost(w rest.ResponseWriter, req *rest.Request) {
	userId, err := GetUserId(req)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	post, err := transactions.GetPost(userId, req.PathParam("pid"))
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteJson(post)
	w.WriteHeader(http.StatusOK)
}

func AddPost(w rest.ResponseWriter, req *rest.Request) {
	userId, err := GetUserId(req)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	var jsonBody json.RawMessage
	err = req.DecodeJsonPayload(&jsonBody)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	submissionChannel := struct {
		Channel string `json:"channel"`
	}{}
	err = json.Unmarshal(jsonBody, &submissionChannel)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	channel, err := transactions.GetChannel(userId, submissionChannel.Channel)
	if err != nil {
		log.Println("could not find channel ", submissionChannel.Channel)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	post, err := channel.UnmarshalSubmissionToPost(jsonBody)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	post.CreatorId = userId
	post.Id = base64.RawURLEncoding.EncodeToString(uuid.NewRandom())
	post.Time = time.Now().UTC()

	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	err = transactions.AddPost(userId, post)
	if err != nil {
		log.Println(err)
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
}

func RemovePost(w rest.ResponseWriter, req *rest.Request) {}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
    fmt.Printf("hello, world\n")
}<commit_msg>basic go server app<commit_after>package main

import (
    "fmt"
    "html"
    "log"
    "net/http"
)

func main() {
    http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
        fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path))
    })

    log.Fatal(http.ListenAndServe(":8080", nil))

}<|endoftext|>


<commit_before>package serverrpc

import (
	"encoding/json"
	"github.com/golang/glog"
	"github.com/hyperhq/hyperd/types"
	"golang.org/x/net/context"
	"io"
)

func (s *ServerRPC) ExecCreate(ctx context.Context, req *types.ExecCreateRequest) (*types.ExecCreateResponse, error) {
	glog.V(3).Infof("create exec %v", req.String())

	cmd, err := json.Marshal(req.Command)
	if err != nil {
		return nil, err
	}

	execId, err := s.daemon.CreateExec(req.ContainerID, string(cmd), req.Tty)
	if err != nil {
		glog.Errorf("ExecCreate error: %v", err)
		return nil, err
	}

	return &types.ExecCreateResponse{
		ExecID: execId,
	}, nil
}

func (s *ServerRPC) ExecStart(stream types.PublicAPI_ExecStartServer) error {
	req, err := stream.Recv()
	if err != nil {
		return err
	}

	inReader, inWriter := io.Pipe()
	outReader, outWriter := io.Pipe()
	go func() {
		defer outReader.Close()
		buf := make([]byte, 32)
		for {
			nr, err := outReader.Read(buf)
			if nr > 0 {
				if err := stream.Send(&types.ExecStartResponse{buf[:nr]}); err != nil {
					glog.Errorf("Send to stream error: %v", err)
					return
				}
			}
			if err == io.EOF {
				break
			}
			if err != nil {
				glog.Errorf("Read from pipe error: %v", err)
				return
			}
		}
	}()
	go func() {
		defer inWriter.Close()
		for {
			req, err := stream.Recv()
			if err != nil && err != io.EOF {
				glog.Errorf("Receive from stream error: %v", err)
				return
			}
			if req != nil && req.Stdin != nil {
				nw, ew := inWriter.Write(req.Stdin)
				if ew != nil {
					glog.Errorf("Write pipe error: %v", ew)
					return
				}
				if nw != len(req.Stdin) {
					glog.Errorf("Write data length is not enougt, write: %d success: %d", len(req.Stdin), nw)
					return
				}
			}
			if err == io.EOF {
				break
			}
		}
	}()

	err = s.daemon.StartExec(inReader, outWriter, req.ContainerID, req.ExecID)
	if err != nil {
		return err
	}
	return nil
}

// Wait gets exitcode by container and processId
func (s *ServerRPC) Wait(c context.Context, req *types.WaitRequest) (*types.WaitResponse, error) {
	glog.V(3).Infof("Wait with request %v", req.String())

	//FIXME need update if param NoHang is enabled
	code, err := s.daemon.ExitCode(req.Container, req.ProcessId)
	if err != nil {
		glog.Errorf("Wait error: %v", err)
		return nil, err
	}

	return &types.WaitResponse{
		ExitCode: int32(code),
	}, nil
}

// ExecSignal sends a singal to specified exec of specified container
func (s *ServerRPC) ExecSignal(ctx context.Context, req *types.ExecSignalRequest) (*types.ExecSignalResponse, error) {
	glog.V(3).Infof("ExecSignal with request %v", req.String())

	err := s.daemon.KillExec(req.ContainerID, req.ExecID, req.Signal)
	if err != nil {
		glog.Errorf("Kill Process %s of container %s with signal %d failed: %v", req.ExecID, req.ContainerID, req.Signal, err)
		return nil, err
	}

	return &types.ExecSignalResponse{}, nil
}
<commit_msg>Add ExecVM to ServerRPC<commit_after>package serverrpc

import (
	"encoding/json"
	"io"

	"github.com/golang/glog"
	"github.com/hyperhq/hyperd/types"
	"golang.org/x/net/context"
)

func (s *ServerRPC) ExecCreate(ctx context.Context, req *types.ExecCreateRequest) (*types.ExecCreateResponse, error) {
	glog.V(3).Infof("create exec %v", req.String())

	cmd, err := json.Marshal(req.Command)
	if err != nil {
		return nil, err
	}

	execId, err := s.daemon.CreateExec(req.ContainerID, string(cmd), req.Tty)
	if err != nil {
		glog.Errorf("ExecCreate error: %v", err)
		return nil, err
	}

	return &types.ExecCreateResponse{
		ExecID: execId,
	}, nil
}

func (s *ServerRPC) ExecStart(stream types.PublicAPI_ExecStartServer) error {
	req, err := stream.Recv()
	if err != nil {
		return err
	}

	inReader, inWriter := io.Pipe()
	outReader, outWriter := io.Pipe()
	go func() {
		defer outReader.Close()
		buf := make([]byte, 32)
		for {
			nr, err := outReader.Read(buf)
			if nr > 0 {
				if err := stream.Send(&types.ExecStartResponse{buf[:nr]}); err != nil {
					glog.Errorf("Send to stream error: %v", err)
					return
				}
			}
			if err == io.EOF {
				break
			}
			if err != nil {
				glog.Errorf("Read from pipe error: %v", err)
				return
			}
		}
	}()
	go func() {
		defer inWriter.Close()
		for {
			req, err := stream.Recv()
			if err != nil && err != io.EOF {
				glog.Errorf("Receive from stream error: %v", err)
				return
			}
			if req != nil && req.Stdin != nil {
				nw, ew := inWriter.Write(req.Stdin)
				if ew != nil {
					glog.Errorf("Write pipe error: %v", ew)
					return
				}
				if nw != len(req.Stdin) {
					glog.Errorf("Write data length is not enougt, write: %d success: %d", len(req.Stdin), nw)
					return
				}
			}
			if err == io.EOF {
				break
			}
		}
	}()

	err = s.daemon.StartExec(inReader, outWriter, req.ContainerID, req.ExecID)
	if err != nil {
		return err
	}
	return nil
}

// Wait gets exitcode by container and processId
func (s *ServerRPC) Wait(c context.Context, req *types.WaitRequest) (*types.WaitResponse, error) {
	glog.V(3).Infof("Wait with request %v", req.String())

	//FIXME need update if param NoHang is enabled
	code, err := s.daemon.ExitCode(req.Container, req.ProcessId)
	if err != nil {
		glog.Errorf("Wait error: %v", err)
		return nil, err
	}

	return &types.WaitResponse{
		ExitCode: int32(code),
	}, nil
}

// ExecSignal sends a singal to specified exec of specified container
func (s *ServerRPC) ExecSignal(ctx context.Context, req *types.ExecSignalRequest) (*types.ExecSignalResponse, error) {
	glog.V(3).Infof("ExecSignal with request %v", req.String())

	err := s.daemon.KillExec(req.ContainerID, req.ExecID, req.Signal)
	if err != nil {
		glog.Errorf("Kill Process %s of container %s with signal %d failed: %v", req.ExecID, req.ContainerID, req.Signal, err)
		return nil, err
	}

	return &types.ExecSignalResponse{}, nil
}

func (s *ServerRPC) ExecVM(stream types.PublicAPI_ExecVMServer) error {
	req, err := stream.Recv()
	if err != nil {
		return err
	}

	cmd, err := json.Marshal(req.Command)
	if err != nil {
		return err
	}

	inReader, inWriter := io.Pipe()
	outReader, outWriter := io.Pipe()
	go func() {
		defer outReader.Close()
		buf := make([]byte, 32)
		for {
			nr, err := outReader.Read(buf)
			if nr > 0 {
				if err := stream.Send(&types.ExecVMResponse{
					Stdout: buf[:nr],
				}); err != nil {
					glog.Errorf("Send to stream error: %v", err)
					return
				}
			}
			if err == io.EOF {
				break
			}
			if err != nil {
				glog.Errorf("Read from pipe error: %v", err)
				return
			}
		}
	}()

	go func() {
		defer inWriter.Close()
		for {
			recv, err := stream.Recv()
			if err != nil && err != io.EOF {
				glog.Errorf("Receive from stream error: %v", err)
				return
			}
			if recv != nil && recv.Stdin != nil {
				nw, ew := inWriter.Write(recv.Stdin)
				if ew != nil {
					glog.Errorf("Write pipe error: %v", ew)
					return
				}
				if nw != len(recv.Stdin) {
					glog.Errorf("Write data length is not enougt, write: %d success: %d", len(recv.Stdin), nw)
					return
				}
			}
			if err == io.EOF {
				break
			}
		}
	}()

	code, err := s.daemon.ExecVM(req.PodID, string(cmd), inReader, outWriter, outWriter)
	if err != nil {
		return err
	}
	if err := stream.Send(&types.ExecVMResponse{
		ExitCode: int32(code),
	}); err != nil {
		glog.Errorf("Send to stream error: %v", err)
		return err
	}

	return nil
}
<|endoftext|>