<commit_before>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"errors"
	"fmt"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/gomit"
	"github.com/pborman/uuid"

	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/scheduler_event"
	"github.com/intelsdi-x/snap/pkg/schedule"
	"github.com/intelsdi-x/snap/scheduler/wmap"
)

const (
	// DefaultDeadlineDuration - The default timeout is 5 second
	DefaultDeadlineDuration = time.Second * 5
	// DefaultStopOnFailure - The default stopping a failure is after three tries
	DefaultStopOnFailure = 3
)

var (
	taskLogger = schedulerLogger.WithField("_module", "scheduler-task")

	// ErrTaskNotFound - The error message for task not found
	ErrTaskNotFound = errors.New("Task not found")
	// ErrTaskNotStopped - The error message for task must be stopped
	ErrTaskNotStopped = errors.New("Task must be stopped")
	// ErrTaskHasAlreadyBeenAdded - The error message for task has already been added
	ErrTaskHasAlreadyBeenAdded = errors.New("Task has already been added")
	// ErrTaskDisabledOnFailures - The error message for task disabled due to consecutive failures
	ErrTaskDisabledOnFailures = errors.New("Task disabled due to consecutive failures")
	// ErrTaskNotDisabled - The error message for task must be disabled
	ErrTaskNotDisabled = errors.New("Task must be disabled")
)

type task struct {
	sync.Mutex //protects state

	id                 string
	name               string
	schResponseChan    chan schedule.Response
	killChan           chan struct{}
	schedule           schedule.Schedule
	workflow           *schedulerWorkflow
	state              core.TaskState
	creationTime       time.Time
	lastFireTime       time.Time
	manager            managesWork
	metricsManager     managesMetrics
	deadlineDuration   time.Duration
	hitCount           uint
	missedIntervals    uint
	failureMutex       sync.Mutex
	failedRuns         uint
	lastFailureMessage string
	lastFailureTime    time.Time
	stopOnFailure      uint
	eventEmitter       gomit.Emitter
}

//NewTask creates a Task
func newTask(s schedule.Schedule, wf *schedulerWorkflow, m *workManager, mm managesMetrics, emitter gomit.Emitter, opts ...core.TaskOption) *task {

	//Task would always be given a default name.
	//However if a user want to change this name, she can pass optional arguments, in form of core.TaskOption
	//The new name then get over written.

	taskID := uuid.New()
	name := fmt.Sprintf("Task-%s", taskID)
	wf.eventEmitter = emitter
	task := &task{
		id:               taskID,
		name:             name,
		schResponseChan:  make(chan schedule.Response),
		schedule:         s,
		state:            core.TaskStopped,
		creationTime:     time.Now(),
		workflow:         wf,
		manager:          m,
		metricsManager:   mm,
		deadlineDuration: DefaultDeadlineDuration,
		stopOnFailure:    DefaultStopOnFailure,
		eventEmitter:     emitter,
	}
	//set options
	for _, opt := range opts {
		opt(task)
	}
	return task
}

// Option sets the options specified.
// Returns an option to optionally restore the last arg's previous value.
func (t *task) Option(opts ...core.TaskOption) core.TaskOption {
	var previous core.TaskOption
	for _, opt := range opts {
		previous = opt(t)
	}
	return previous
}

//Returns the name of the task
func (t *task) GetName() string {
	return t.name
}

func (t *task) SetName(name string) {
	t.name = name
}

// CreateTime returns the time the task was created.
func (t *task) CreationTime() *time.Time {
	return &t.creationTime
}

func (t *task) DeadlineDuration() time.Duration {
	return t.deadlineDuration
}

func (t *task) SetDeadlineDuration(d time.Duration) {
	t.deadlineDuration = d
}

func (t *task) SetTaskID(id string) {
	t.id = id
}

// HitCount returns the number of times the task has fired.
func (t *task) HitCount() uint {
	return t.hitCount
}

// Id returns the tasks Id.
func (t *task) ID() string {
	return t.id
}

// LastRunTime returns the time of the tasks last run.
func (t *task) LastRunTime() *time.Time {
	return &t.lastFireTime
}

// MissedCount returns the number of intervals missed.
func (t *task) MissedCount() uint {
	return t.missedIntervals
}

// FailedRuns returns the number of intervals missed.
func (t *task) FailedCount() uint {
	return t.failedRuns
}

// LastFailureMessage returns the last error from a task run
func (t *task) LastFailureMessage() string {
	return t.lastFailureMessage
}

// State returns state of the task.
func (t *task) State() core.TaskState {
	return t.state
}

// Status returns the state of the workflow.
func (t *task) Status() WorkflowState {
	return t.workflow.State()
}

func (t *task) SetStopOnFailure(v uint) {
	t.stopOnFailure = v
}

func (t *task) SetID(id string) {
	t.id = id
}

func (t *task) GetStopOnFailure() uint {
	return t.stopOnFailure
}

// Spin will start a task spinning in its own routine while it waits for its
// schedule.
func (t *task) Spin() {
	// We need to lock long enough to change state
	t.Lock()
	defer t.Unlock()
	if t.state == core.TaskStopped {
		t.state = core.TaskSpinning
		t.killChan = make(chan struct{})
		// spin in a goroutine
		go t.spin()
	}
}

func (t *task) Stop() {
	t.Lock()
	defer t.Unlock()
	if t.state == core.TaskFiring || t.state == core.TaskSpinning {
		t.state = core.TaskStopping
		close(t.killChan)
	}
}

//Enable changes the state from Disabled to Stopped
func (t *task) Enable() error {
	t.Lock()
	defer t.Unlock()

	if t.state != core.TaskDisabled {
		return ErrTaskNotDisabled
	}
	t.state = core.TaskStopped

	return nil
}

func (t *task) Kill() {
	t.Lock()
	defer t.Unlock()
	if t.state == core.TaskFiring || t.state == core.TaskSpinning {
		close(t.killChan)
		t.state = core.TaskDisabled
	}
}

func (t *task) WMap() *wmap.WorkflowMap {
	return t.workflow.workflowMap
}

func (t *task) Schedule() schedule.Schedule {
	return t.schedule
}

func (t *task) spin() {
	var consecutiveFailures uint
	for {
		taskLogger.Debug("task spin loop")
		// Start go routine to wait on schedule
		go t.waitForSchedule()
		// wait here on
		//  schResponseChan - response from schedule
		//  killChan - signals task needs to be stopped
		select {
		case sr := <-t.schResponseChan:
			switch sr.State() {
			// If response show this schedule is stil active we fire
			case schedule.Active:
				t.missedIntervals += sr.Missed()
				t.lastFireTime = time.Now()
				t.hitCount++
				t.fire()
				if t.lastFailureTime == t.lastFireTime {
					consecutiveFailures++
					taskLogger.WithFields(log.Fields{
						"_block":                    "spin",
						"task-id":                   t.id,
						"task-name":                 t.name,
						"consecutive failures":      consecutiveFailures,
						"consecutive failure limit": t.stopOnFailure,
						"error":                     t.lastFailureMessage,
					}).Warn("Task failed")
				} else {
					consecutiveFailures = 0
				}
				if consecutiveFailures >= t.stopOnFailure {
					taskLogger.WithFields(log.Fields{
						"_block":               "spin",
						"task-id":              t.id,
						"task-name":            t.name,
						"consecutive failures": consecutiveFailures,
						"error":                t.lastFailureMessage,
					}).Error(ErrTaskDisabledOnFailures)
					// You must lock on state change for tasks
					t.Lock()
					t.state = core.TaskDisabled
					t.Unlock()
					// Send task disabled event
					event := new(scheduler_event.TaskDisabledEvent)
					event.TaskID = t.id
					event.Why = fmt.Sprintf("Task disabled with error: %s", t.lastFailureMessage)
					defer t.eventEmitter.Emit(event)
					return
				}
			// Schedule has ended
			case schedule.Ended:
				// You must lock task to change state
				t.Lock()
				t.state = core.TaskEnded
				t.Unlock()
				return //spin

			// Schedule has errored
			case schedule.Error:
				// You must lock task to change state
				t.Lock()
				t.state = core.TaskDisabled
				t.Unlock()
				return //spin

			}
		case <-t.killChan:
			// Only here can it truly be stopped
			t.Lock()
			t.state = core.TaskStopped
			t.lastFireTime = time.Time{}
			t.Unlock()
			return
		}
	}
}

func (t *task) fire() {
	t.Lock()
	defer t.Unlock()

	t.state = core.TaskFiring
	t.workflow.Start(t)
	t.state = core.TaskSpinning
}

func (t *task) waitForSchedule() {
	select {
	case <-t.killChan:
		return
	case t.schResponseChan <- t.schedule.Wait(t.lastFireTime):
	}
}

// RecordFailure updates the failed runs and last failure properties
func (t *task) RecordFailure(e []error) {
	// We synchronize this update to ensure it is atomic
	t.failureMutex.Lock()
	defer t.failureMutex.Unlock()
	t.failedRuns++
	t.lastFailureTime = t.lastFireTime
	t.lastFailureMessage = e[len(e)-1].Error()
}

type taskCollection struct {
	*sync.Mutex

	table map[string]*task
}

func newTaskCollection() *taskCollection {
	return &taskCollection{
		Mutex: &sync.Mutex{},

		table: make(map[string]*task),
	}
}

// Get given a task id returns a Task or nil if not found
func (t *taskCollection) Get(id string) *task {
	t.Lock()
	defer t.Unlock()

	if t, ok := t.table[id]; ok {
		return t
	}
	return nil
}

// Add given a reference to a task adds it to the collection of tasks.  An
// error is returned if the task alredy exists in the collection.
func (t *taskCollection) add(task *task) error {
	t.Lock()
	defer t.Unlock()

	if _, ok := t.table[task.id]; !ok {
		//If we don't already have this task in the collection save it
		t.table[task.id] = task
	} else {
		taskLogger.WithFields(log.Fields{
			"_module": "scheduler-taskCollection",
			"_block":  "add",
			"task id": task.id,
		}).Error(ErrTaskHasAlreadyBeenAdded.Error())
		return ErrTaskHasAlreadyBeenAdded
	}

	return nil
}

// remove will remove a given task from tasks.  The task must be stopped.
// Can return errors ErrTaskNotFound and ErrTaskNotStopped.
func (t *taskCollection) remove(task *task) error {
	t.Lock()
	defer t.Unlock()
	if _, ok := t.table[task.id]; ok {
		if task.state != core.TaskStopped {
			taskLogger.WithFields(log.Fields{
				"_block":  "remove",
				"task id": task.id,
			}).Error(ErrTaskNotStopped)
			return ErrTaskNotStopped
		}
		delete(t.table, task.id)
	} else {
		taskLogger.WithFields(log.Fields{
			"_block":  "remove",
			"task id": task.id,
		}).Error(ErrTaskNotFound)
		return ErrTaskNotFound
	}
	return nil
}

// Table returns a copy of the taskCollection
func (t *taskCollection) Table() map[string]*task {
	t.Lock()
	defer t.Unlock()
	tasks := make(map[string]*task)
	for id, t := range t.table {
		tasks[id] = t
	}
	return tasks
}
<commit_msg>Fixes #807 - Resets the lastFireTime on a task.Spin()<commit_after>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"errors"
	"fmt"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/gomit"
	"github.com/pborman/uuid"

	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/scheduler_event"
	"github.com/intelsdi-x/snap/pkg/schedule"
	"github.com/intelsdi-x/snap/scheduler/wmap"
)

const (
	// DefaultDeadlineDuration - The default timeout is 5 second
	DefaultDeadlineDuration = time.Second * 5
	// DefaultStopOnFailure - The default stopping a failure is after three tries
	DefaultStopOnFailure = 3
)

var (
	taskLogger = schedulerLogger.WithField("_module", "scheduler-task")

	// ErrTaskNotFound - The error message for task not found
	ErrTaskNotFound = errors.New("Task not found")
	// ErrTaskNotStopped - The error message for task must be stopped
	ErrTaskNotStopped = errors.New("Task must be stopped")
	// ErrTaskHasAlreadyBeenAdded - The error message for task has already been added
	ErrTaskHasAlreadyBeenAdded = errors.New("Task has already been added")
	// ErrTaskDisabledOnFailures - The error message for task disabled due to consecutive failures
	ErrTaskDisabledOnFailures = errors.New("Task disabled due to consecutive failures")
	// ErrTaskNotDisabled - The error message for task must be disabled
	ErrTaskNotDisabled = errors.New("Task must be disabled")
)

type task struct {
	sync.Mutex //protects state

	id                 string
	name               string
	schResponseChan    chan schedule.Response
	killChan           chan struct{}
	schedule           schedule.Schedule
	workflow           *schedulerWorkflow
	state              core.TaskState
	creationTime       time.Time
	lastFireTime       time.Time
	manager            managesWork
	metricsManager     managesMetrics
	deadlineDuration   time.Duration
	hitCount           uint
	missedIntervals    uint
	failureMutex       sync.Mutex
	failedRuns         uint
	lastFailureMessage string
	lastFailureTime    time.Time
	stopOnFailure      uint
	eventEmitter       gomit.Emitter
}

//NewTask creates a Task
func newTask(s schedule.Schedule, wf *schedulerWorkflow, m *workManager, mm managesMetrics, emitter gomit.Emitter, opts ...core.TaskOption) *task {

	//Task would always be given a default name.
	//However if a user want to change this name, she can pass optional arguments, in form of core.TaskOption
	//The new name then get over written.

	taskID := uuid.New()
	name := fmt.Sprintf("Task-%s", taskID)
	wf.eventEmitter = emitter
	task := &task{
		id:               taskID,
		name:             name,
		schResponseChan:  make(chan schedule.Response),
		schedule:         s,
		state:            core.TaskStopped,
		creationTime:     time.Now(),
		workflow:         wf,
		manager:          m,
		metricsManager:   mm,
		deadlineDuration: DefaultDeadlineDuration,
		stopOnFailure:    DefaultStopOnFailure,
		eventEmitter:     emitter,
	}
	//set options
	for _, opt := range opts {
		opt(task)
	}
	return task
}

// Option sets the options specified.
// Returns an option to optionally restore the last arg's previous value.
func (t *task) Option(opts ...core.TaskOption) core.TaskOption {
	var previous core.TaskOption
	for _, opt := range opts {
		previous = opt(t)
	}
	return previous
}

//Returns the name of the task
func (t *task) GetName() string {
	return t.name
}

func (t *task) SetName(name string) {
	t.name = name
}

// CreateTime returns the time the task was created.
func (t *task) CreationTime() *time.Time {
	return &t.creationTime
}

func (t *task) DeadlineDuration() time.Duration {
	return t.deadlineDuration
}

func (t *task) SetDeadlineDuration(d time.Duration) {
	t.deadlineDuration = d
}

func (t *task) SetTaskID(id string) {
	t.id = id
}

// HitCount returns the number of times the task has fired.
func (t *task) HitCount() uint {
	return t.hitCount
}

// Id returns the tasks Id.
func (t *task) ID() string {
	return t.id
}

// LastRunTime returns the time of the tasks last run.
func (t *task) LastRunTime() *time.Time {
	return &t.lastFireTime
}

// MissedCount returns the number of intervals missed.
func (t *task) MissedCount() uint {
	return t.missedIntervals
}

// FailedRuns returns the number of intervals missed.
func (t *task) FailedCount() uint {
	return t.failedRuns
}

// LastFailureMessage returns the last error from a task run
func (t *task) LastFailureMessage() string {
	return t.lastFailureMessage
}

// State returns state of the task.
func (t *task) State() core.TaskState {
	return t.state
}

// Status returns the state of the workflow.
func (t *task) Status() WorkflowState {
	return t.workflow.State()
}

func (t *task) SetStopOnFailure(v uint) {
	t.stopOnFailure = v
}

func (t *task) SetID(id string) {
	t.id = id
}

func (t *task) GetStopOnFailure() uint {
	return t.stopOnFailure
}

// Spin will start a task spinning in its own routine while it waits for its
// schedule.
func (t *task) Spin() {
	// We need to lock long enough to change state
	t.Lock()
	defer t.Unlock()
	// Reset the lastFireTime at each Spin.
	// This ensures misses are tracked only forward of the point
	// in time that a task starts spinning. E.g. stopping a task,
	// waiting a period of time, and starting the task won't show
	// misses for the interval while stopped.
	t.lastFireTime = time.Now()
	if t.state == core.TaskStopped {
		t.state = core.TaskSpinning
		t.killChan = make(chan struct{})
		// spin in a goroutine
		go t.spin()
	}
}

func (t *task) Stop() {
	t.Lock()
	defer t.Unlock()
	if t.state == core.TaskFiring || t.state == core.TaskSpinning {
		t.state = core.TaskStopping
		close(t.killChan)
	}
}

//Enable changes the state from Disabled to Stopped
func (t *task) Enable() error {
	t.Lock()
	defer t.Unlock()

	if t.state != core.TaskDisabled {
		return ErrTaskNotDisabled
	}
	t.state = core.TaskStopped

	return nil
}

func (t *task) Kill() {
	t.Lock()
	defer t.Unlock()
	if t.state == core.TaskFiring || t.state == core.TaskSpinning {
		close(t.killChan)
		t.state = core.TaskDisabled
	}
}

func (t *task) WMap() *wmap.WorkflowMap {
	return t.workflow.workflowMap
}

func (t *task) Schedule() schedule.Schedule {
	return t.schedule
}

func (t *task) spin() {
	var consecutiveFailures uint
	for {
		taskLogger.Debug("task spin loop")
		// Start go routine to wait on schedule
		go t.waitForSchedule()
		// wait here on
		//  schResponseChan - response from schedule
		//  killChan - signals task needs to be stopped
		select {
		case sr := <-t.schResponseChan:
			switch sr.State() {
			// If response show this schedule is stil active we fire
			case schedule.Active:
				t.missedIntervals += sr.Missed()
				t.lastFireTime = time.Now()
				t.hitCount++
				t.fire()
				if t.lastFailureTime == t.lastFireTime {
					consecutiveFailures++
					taskLogger.WithFields(log.Fields{
						"_block":                    "spin",
						"task-id":                   t.id,
						"task-name":                 t.name,
						"consecutive failures":      consecutiveFailures,
						"consecutive failure limit": t.stopOnFailure,
						"error":                     t.lastFailureMessage,
					}).Warn("Task failed")
				} else {
					consecutiveFailures = 0
				}
				if consecutiveFailures >= t.stopOnFailure {
					taskLogger.WithFields(log.Fields{
						"_block":               "spin",
						"task-id":              t.id,
						"task-name":            t.name,
						"consecutive failures": consecutiveFailures,
						"error":                t.lastFailureMessage,
					}).Error(ErrTaskDisabledOnFailures)
					// You must lock on state change for tasks
					t.Lock()
					t.state = core.TaskDisabled
					t.Unlock()
					// Send task disabled event
					event := new(scheduler_event.TaskDisabledEvent)
					event.TaskID = t.id
					event.Why = fmt.Sprintf("Task disabled with error: %s", t.lastFailureMessage)
					defer t.eventEmitter.Emit(event)
					return
				}
			// Schedule has ended
			case schedule.Ended:
				// You must lock task to change state
				t.Lock()
				t.state = core.TaskEnded
				t.Unlock()
				return //spin

			// Schedule has errored
			case schedule.Error:
				// You must lock task to change state
				t.Lock()
				t.state = core.TaskDisabled
				t.Unlock()
				return //spin

			}
		case <-t.killChan:
			// Only here can it truly be stopped
			t.Lock()
			t.state = core.TaskStopped
			t.lastFireTime = time.Time{}
			t.Unlock()
			return
		}
	}
}

func (t *task) fire() {
	t.Lock()
	defer t.Unlock()

	t.state = core.TaskFiring
	t.workflow.Start(t)
	t.state = core.TaskSpinning
}

func (t *task) waitForSchedule() {
	select {
	case <-t.killChan:
		return
	case t.schResponseChan <- t.schedule.Wait(t.lastFireTime):
	}
}

// RecordFailure updates the failed runs and last failure properties
func (t *task) RecordFailure(e []error) {
	// We synchronize this update to ensure it is atomic
	t.failureMutex.Lock()
	defer t.failureMutex.Unlock()
	t.failedRuns++
	t.lastFailureTime = t.lastFireTime
	t.lastFailureMessage = e[len(e)-1].Error()
}

type taskCollection struct {
	*sync.Mutex

	table map[string]*task
}

func newTaskCollection() *taskCollection {
	return &taskCollection{
		Mutex: &sync.Mutex{},

		table: make(map[string]*task),
	}
}

// Get given a task id returns a Task or nil if not found
func (t *taskCollection) Get(id string) *task {
	t.Lock()
	defer t.Unlock()

	if t, ok := t.table[id]; ok {
		return t
	}
	return nil
}

// Add given a reference to a task adds it to the collection of tasks.  An
// error is returned if the task alredy exists in the collection.
func (t *taskCollection) add(task *task) error {
	t.Lock()
	defer t.Unlock()

	if _, ok := t.table[task.id]; !ok {
		//If we don't already have this task in the collection save it
		t.table[task.id] = task
	} else {
		taskLogger.WithFields(log.Fields{
			"_module": "scheduler-taskCollection",
			"_block":  "add",
			"task id": task.id,
		}).Error(ErrTaskHasAlreadyBeenAdded.Error())
		return ErrTaskHasAlreadyBeenAdded
	}

	return nil
}

// remove will remove a given task from tasks.  The task must be stopped.
// Can return errors ErrTaskNotFound and ErrTaskNotStopped.
func (t *taskCollection) remove(task *task) error {
	t.Lock()
	defer t.Unlock()
	if _, ok := t.table[task.id]; ok {
		if task.state != core.TaskStopped {
			taskLogger.WithFields(log.Fields{
				"_block":  "remove",
				"task id": task.id,
			}).Error(ErrTaskNotStopped)
			return ErrTaskNotStopped
		}
		delete(t.table, task.id)
	} else {
		taskLogger.WithFields(log.Fields{
			"_block":  "remove",
			"task id": task.id,
		}).Error(ErrTaskNotFound)
		return ErrTaskNotFound
	}
	return nil
}

// Table returns a copy of the taskCollection
func (t *taskCollection) Table() map[string]*task {
	t.Lock()
	defer t.Unlock()
	tasks := make(map[string]*task)
	for id, t := range t.table {
		tasks[id] = t
	}
	return tasks
}
<|endoftext|>


<commit_before>package schema

import (
	"reflect"
	"fmt"

	"github.com/xitongsys/parquet-go/parquet"
	"github.com/xitongsys/parquet-go/types"
)

// Get object type from schema by reflect
func (self *SchemaHandler) GetTypes() []reflect.Type {
	ln := int32(len(self.SchemaElements))
	elements := make([][]int32, ln)
	for i := 0; i<int(ln); i++ {
		elements[i] = []int32{}
	}

	elementTypes := make([]reflect.Type, ln)

	var pos int32 = 0
	stack := make([][2]int32, 0) //stack item[0]: index of schemas; item[1]: numChildren
	for pos < ln || len(stack) > 0 {
		if len(stack) == 0 || stack[len(stack)-1][1] > 0 {
			if len(stack) > 0 {
				stack[len(stack)-1][1]--
				idx := stack[len(stack)-1][0]
				elements[idx] = append(elements[idx], pos)
			}
			item := [2]int32{pos, self.SchemaElements[pos].GetNumChildren()}
			stack = append(stack, item)
			pos++

		} else {
			curlen := len(stack) - 1
			idx := stack[curlen][0]
			nc := self.SchemaElements[idx].GetNumChildren()
			pT, cT := self.SchemaElements[idx].Type, self.SchemaElements[idx].ConvertedType
			rT := self.SchemaElements[idx].RepetitionType

			if nc == 0 {
				if *rT != parquet.FieldRepetitionType_REPEATED {
					elementTypes[idx] = types.ParquetTypeToGoReflectType(pT, rT)

				} else {
					elementTypes[idx] = reflect.SliceOf(types.ParquetTypeToGoReflectType(pT, nil))
				}

			} else {
				if cT != nil && *cT == parquet.ConvertedType_LIST &&
					len(elements[idx]) == 1 &&
					self.GetInName(int(elements[idx][0])) == "List" &&
					len(elements[elements[idx][0]]) == 1 &&
					self.GetInName(int(elements[elements[idx][0]][0])) == "Element" {
						cidx := elements[elements[idx][0]][0]
						elementTypes [idx] = reflect.SliceOf(elementTypes[cidx])

				} else if cT != nil && *cT == parquet.ConvertedType_MAP &&
					len(elements[idx]) == 1 &&
					self.GetInName(int(elements[idx][0])) == "Key_value" &&
					len(elements[elements[idx][0]]) == 2 &&
					self.GetInName(int(elements[elements[idx][0]][0])) == "Key" &&
					self.GetInName(int(elements[elements[idx][0]][1])) == "Value"{
						kIdx, vIdx := elements[elements[idx][0]][0], elements[elements[idx][0]][1]
						kT, vT := elementTypes[kIdx], elementTypes[vIdx]
						elementTypes[idx] = reflect.MapOf(kT, vT)

				}else {
					fields := []reflect.StructField{}
					for _, ci := range elements[idx] {
						fields = append(fields, reflect.StructField{
							Name: self.Infos[ci].InName,
							Type: elementTypes[ci],
						})
					}

					structType := reflect.StructOf(fields)

					if *rT == parquet.FieldRepetitionType_REQUIRED {
						elementTypes[idx] = structType

					} else if *rT == parquet.FieldRepetitionType_OPTIONAL {
						elementTypes[idx] = reflect.New(structType).Type()

					} else if *rT == parquet.FieldRepetitionType_REPEATED {
						elementTypes[idx] = reflect.SliceOf(structType)
					}
				}
			}

			stack = stack[:curlen]
		}
	}

	return elementTypes
}

func (self *SchemaHandler) GetType(prefixPath string) (reflect.Type, error) {
	prefixPath, err := self.ConvertToInPathStr(prefixPath)
	if err != nil {
		return nil, err
	}

	ts := self.GetTypes()
	if idx, ok := self.MapIndex[prefixPath]; !ok {
		return nil, fmt.Errorf("[GetType] Can't find %v", prefixPath)
	} else {
		return ts[idx], nil
	}
}
<commit_msg>fix rt nil bug<commit_after>package schema

import (
	"reflect"
	"fmt"

	"github.com/xitongsys/parquet-go/parquet"
	"github.com/xitongsys/parquet-go/types"
)

// Get object type from schema by reflect
func (self *SchemaHandler) GetTypes() []reflect.Type {
	ln := int32(len(self.SchemaElements))
	elements := make([][]int32, ln)
	for i := 0; i<int(ln); i++ {
		elements[i] = []int32{}
	}

	elementTypes := make([]reflect.Type, ln)

	var pos int32 = 0
	stack := make([][2]int32, 0) //stack item[0]: index of schemas; item[1]: numChildren
	for pos < ln || len(stack) > 0 {
		if len(stack) == 0 || stack[len(stack)-1][1] > 0 {
			if len(stack) > 0 {
				stack[len(stack)-1][1]--
				idx := stack[len(stack)-1][0]
				elements[idx] = append(elements[idx], pos)
			}
			item := [2]int32{pos, self.SchemaElements[pos].GetNumChildren()}
			stack = append(stack, item)
			pos++

		} else {
			curlen := len(stack) - 1
			idx := stack[curlen][0]
			nc := self.SchemaElements[idx].GetNumChildren()
			pT, cT := self.SchemaElements[idx].Type, self.SchemaElements[idx].ConvertedType
			rT := self.SchemaElements[idx].RepetitionType

			if nc == 0 {
				if *rT != parquet.FieldRepetitionType_REPEATED {
					elementTypes[idx] = types.ParquetTypeToGoReflectType(pT, rT)

				} else {
					elementTypes[idx] = reflect.SliceOf(types.ParquetTypeToGoReflectType(pT, nil))
				}

			} else {
				if cT != nil && *cT == parquet.ConvertedType_LIST &&
					len(elements[idx]) == 1 &&
					self.GetInName(int(elements[idx][0])) == "List" &&
					len(elements[elements[idx][0]]) == 1 &&
					self.GetInName(int(elements[elements[idx][0]][0])) == "Element" {
						cidx := elements[elements[idx][0]][0]
						elementTypes [idx] = reflect.SliceOf(elementTypes[cidx])

				} else if cT != nil && *cT == parquet.ConvertedType_MAP &&
					len(elements[idx]) == 1 &&
					self.GetInName(int(elements[idx][0])) == "Key_value" &&
					len(elements[elements[idx][0]]) == 2 &&
					self.GetInName(int(elements[elements[idx][0]][0])) == "Key" &&
					self.GetInName(int(elements[elements[idx][0]][1])) == "Value"{
						kIdx, vIdx := elements[elements[idx][0]][0], elements[elements[idx][0]][1]
						kT, vT := elementTypes[kIdx], elementTypes[vIdx]
						elementTypes[idx] = reflect.MapOf(kT, vT)

				}else {
					fields := []reflect.StructField{}
					for _, ci := range elements[idx] {
						fields = append(fields, reflect.StructField{
							Name: self.Infos[ci].InName,
							Type: elementTypes[ci],
						})
					}

					structType := reflect.StructOf(fields)

					if rT == nil || *rT == parquet.FieldRepetitionType_REQUIRED {
						elementTypes[idx] = structType

					} else if *rT == parquet.FieldRepetitionType_OPTIONAL {
						elementTypes[idx] = reflect.New(structType).Type()

					} else if *rT == parquet.FieldRepetitionType_REPEATED {
						elementTypes[idx] = reflect.SliceOf(structType)
					}
				}
			}

			stack = stack[:curlen]
		}
	}

	return elementTypes
}

func (self *SchemaHandler) GetType(prefixPath string) (reflect.Type, error) {
	prefixPath, err := self.ConvertToInPathStr(prefixPath)
	if err != nil {
		return nil, err
	}

	ts := self.GetTypes()
	if idx, ok := self.MapIndex[prefixPath]; !ok {
		return nil, fmt.Errorf("[GetType] Can't find %v", prefixPath)
	} else {
		return ts[idx], nil
	}
}
<|endoftext|>


<commit_before>package cuckoo

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math/rand"
	"strconv"
	"testing"
)

const ItemSize = uint64(64)

// Test identical state after operations
// Test contains after insert/remove sequence
// Test insert same value twice

func GetBytes(val string) []byte {
	buf := make([]byte, ItemSize)
	copy(buf, bytes.NewBufferString(val).Bytes())
	return buf
}

func randBucket(numBuckets uint64) uint64 {
	result := rand.Uint64() % numBuckets
	return result
}

func TestGetCapacity(t *testing.T) {
	fmt.Printf("TestGetCapacity: ...\n")

	table := NewTable("t", 10, 2, ItemSize, nil, 0)
	if table.GetCapacity() != 20 {
		t.Fatalf("table returned wrong value for GetCapacity=%v. Expecting 20\n", table.GetCapacity())
	}

	table = NewTable("t", 1, 1, ItemSize, nil, 0)
	if table.GetCapacity() != 1 {
		t.Fatalf("table returned wrong value for GetCapacity=%v. Expecting 1\n", table.GetCapacity())
	}

	table = NewTable("t", 0, 0, ItemSize, nil, 0)
	if table.GetCapacity() != 0 {
		t.Fatalf("table returned wrong value for GetCapacity=%v. Expecting 0\n", table.GetCapacity())
	}

	fmt.Printf("... done \n")
}

func TestInvalidConstruction(t *testing.T) {
	fmt.Printf("TestInvalidConstruction: ...\n")
	data := make([]byte, 7)
	table := NewTable("t", 2, 3, 4, data, 0)
	if table != nil {
		t.Fatalf("created a improperly sized table\n")
	}
	fmt.Printf("... done \n")
}

func TestBasic(t *testing.T) {
	table := NewTable("t", 10, 2, ItemSize, nil, 0)

	fmt.Printf("TestBasic: check empty...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("TestBasic: Check contains non-existent value...\n")
	if table.Contains(&Item{0, GetBytes(""), 0, 1}) {
		t.Fatalf("empty table returned true for Contains()\n")
	}

	fmt.Printf("TestBasic: remove non-existent value ...\n")
	if table.Remove(&Item{1, GetBytes("value1"), 0, 1}) {
		t.Fatalf("empty table returned true for Remove()\n")
	}

	fmt.Printf("TestBasic: check empty...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("TestBasic: Insert improperly sized value ...\n")
	ok, itm := table.Insert(&Item{1, []byte{0, 0}, 0, 1})
	if itm != nil || ok {
		t.Fatalf("should have failed inserting a malformed item\n")
	}

	fmt.Printf("TestBasic: Insert value ...\n")
	ok, itm = table.Insert(&Item{1, GetBytes("value1"), 0, 1})
	if itm != nil || !ok {
		t.Fatalf("error inserting into table (0, 1, value1)\n")
	}

	fmt.Printf("TestBasic: Check inserted value...\n")
	if !table.Contains(&Item{1, GetBytes("value1"), 0, 1}) {
		t.Fatalf("cannot find recently inserted value\n")
	}

	fmt.Printf("TestBasic: Check inserted value w/o full reference...\n")
	if !table.Contains(&Item{1, GetBytes(""), 0, 1}) {
		t.Fatalf("cannot find recently inserted value\n")
	}

	fmt.Printf("TestBasic: Check non-existent value...\n")
	if table.Contains(&Item{2, GetBytes("value2"), 0, 1}) {
		t.Fatalf("contains a non-existent value\n")
	}

	fmt.Printf("TestBasic: check 1 element...\n")
	if table.GetNumElements() != 1 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("TestBasic: remove existing value ...\n")
	if !table.Remove(&Item{1, GetBytes("value1"), 0, 1}) {
		t.Fatalf("error removing existing value (0, 1, value1)\n")
	}

	fmt.Printf("TestBasic: check 0 element...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("TestBasic: remove recently removed value ...\n")
	if table.Remove(&Item{1, GetBytes("value1"), 0, 1}) {
		t.Fatalf("empty table returned true for Remove()\n")
	}

	fmt.Printf("TestBasic: check 0 element...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("... done \n")
}

func TestBucket(t *testing.T) {
	fmt.Printf("TestBucket ...\n")
	table := NewTable("t", 10, 2, ItemSize, nil, 0)

	items := []*Item{
		{1, GetBytes("value1"), 5, 5},
		{2, GetBytes("value2"), 5, 5},
		{3, GetBytes("value3"), 5, 6},
	}
	for _, v := range items {
		ok, _ := table.Insert(v)
		if !ok {
			t.Fatalf("Failed to insert item %v", v)
		}
	}

	bucket, err := table.Bucket(items[0])
	if bucket != 5 || err != nil {
		t.Fatalf("Table should report expected item position %v", items[0])
	}
	bucket, err = table.Bucket(items[1])
	if bucket != 5 || err != nil {
		t.Fatalf("Table should report expected item position %v", items[1])
	}
	bucket, err = table.Bucket(items[2])
	if bucket != 6 || err != nil {
		t.Fatalf("Table should report expected item position %v", items[2])
	}

	nonItem := &Item{4, GetBytes("value4"), 1, 1}
	_, err = table.Bucket(nonItem)
	if err == nil {
		t.Fatalf("Table should report expected item position %v", nonItem)
	}

	fmt.Printf("... done \n")
}

func TestOutOfBounds(t *testing.T) {
	table := NewTable("t", 10, 2, ItemSize, nil, 0)

	fmt.Printf("TestOutOfBounds: Insert() out of bounds...\n")
	ok, itm := table.Insert(&Item{1, GetBytes("value1"), 100, 100})
	if ok {
		t.Fatalf("Insert returned true with out of bound buckets\n")
	}
	if itm != nil {
		t.Fatalf("Insert returned wrong values\n")
	}

	fmt.Printf("TestOutOfBounds: Contains() out of bounds...\n")
	if table.Contains(&Item{1, GetBytes("value1"), 100, 100}) {
		t.Fatalf("Contains returned true with out of bound buckets\n")
	}

	fmt.Printf("TestOutOfBounds: Remove() out of bounds...\n")
	if table.Remove(&Item{1, GetBytes("value1"), 100, 100}) {
		t.Fatalf("Remove() returned true with out of bound buckets\n")
	}

	fmt.Printf("... done \n")
}

func TestFullTable(t *testing.T) {
	numBuckets := uint64(100)
	depth := uint64(4)

	capacity := numBuckets * depth
	entries := make([]Item, 0, capacity)
	table := NewTable("t", numBuckets, depth, ItemSize, nil, 0)
	ok := true
	count := uint64(0)
	var evic *Item
	var b1, b2 uint64

	// Insert random values until we've reached a limit
	fmt.Printf("TestFullTable: Insert until reach capacity...\n")
	for ok {
		b1 = randBucket(numBuckets)
		b2 = randBucket(numBuckets)
		id := rand.Uint64()
		val := GetBytes(strconv.Itoa(rand.Int()))
		entries = append(entries, Item{id, nil, b1, b2})
		ok, evic = table.Insert(&Item{id, val, b1, b2})

		if ok {
			count++
			if !table.Contains(&Item{id, nil, b1, b2}) {
				t.Fatalf("Insert() succeeded, but Contains failed\n")
			}
			if count != table.GetNumElements() {
				t.Fatalf("Number of successful Inserts(), %v, does not match GetNumElements(), %v \n",
					count,
					table.GetNumElements())
			}
		}
	}

	// Middle count check
	fmt.Printf("TestFullTable: Fully Loaded check...\n")
	if count != table.GetNumElements() {
		t.Fatalf("Number of successful Inserts(), %v, does not match GetNumElements(), %v \n",
			count,
			table.GetNumElements())
	}
	maxCount := count

	// Remove elements one by one
	fmt.Printf("TestFullTable: Remove each element...\n")
	for _, entry := range entries {
		if !entry.Equals(evic) {
			ok = table.Remove(&entry)
			if !ok {
				t.Fatalf("Cannot Remove() an element believed to be in the table. item %d of %d",
					count,
					maxCount)
			} else {
				count--
				if count != table.GetNumElements() {
					t.Fatalf("GetNumElements()=%v returned a value that didn't match what was expected=%v \n",
						table.GetNumElements(),
						count)
				}
			}
		}
	}

	// Final count check
	fmt.Printf("TestFullTable: Final count check...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("GetNumElements() returns %v when table should be empty \n", table.GetNumElements())
	}

	fmt.Printf("... done\n")
}

func TestDuplicateValues(t *testing.T) {
	fmt.Printf("TestDuplicateValues: ...\n")
	table := NewTable("t", 10, 2, ItemSize, nil, 0)

	ok, itm := table.Insert(&Item{1, GetBytes("v"), 0, 1})
	if itm != nil || !ok {
		t.Fatalf("Error inserting value \n")
	}

	ok, itm = table.Insert(&Item{2, GetBytes("v"), 0, 1})
	if itm != nil || !ok {
		t.Fatalf("Error inserting value again \n")
	}

	ok, itm = table.Insert(&Item{3, GetBytes("v"), 1, 2})
	if itm != nil || !ok {
		t.Fatalf("Error inserting value in shifted buckets\n")
	}

	if !table.Remove(&Item{1, GetBytes("v"), 0, 1}) {
		t.Fatalf("Error removing value 1st time\n")
	}

	if !table.Remove(&Item{2, GetBytes("v"), 0, 1}) {
		t.Fatalf("Error removing value 2nd time\n")
	}

	if !table.Remove(&Item{3, GetBytes("v"), 1, 2}) {
		t.Fatalf("Error removing value 3rd time\n")
	}

	fmt.Printf("... done\n")
}

func TestLoadFactor(t *testing.T) {
	fmt.Printf("TestLoadFactor: ...\n")
	numBuckets := uint64(1000)
	var table *Table

	for depth := uint64(1); depth < uint64(32); depth *= 2 {
		count := -1
		ok := true
		table = NewTable("t", numBuckets, depth, ItemSize, nil, int64(depth))
		for ok {
			count++
			val := GetBytes(strconv.Itoa(rand.Int()))
			ok, _ = table.Insert(&Item{rand.Uint64(), val, randBucket(numBuckets), randBucket(numBuckets)})
		}

		if table.GetNumElements() != uint64(count) {
			t.Fatalf("Mismatch count=%v with GetNumElements()=%v \n", count, table.GetNumElements())
		}
		fmt.Printf("count=%v, depth=%v, capacity=%v, loadfactor=%v \n",
			count,
			depth,
			table.GetCapacity(),
			(float64(count) / float64(table.GetCapacity())))
	}

	fmt.Printf("... done\n")
}

func BenchmarkBigTable(b *testing.B) {
	//numMessages := uint64(1073741824) //2^30
	numMessages := uint64(268435456) //2^28
	dataSize := uint64(8)
	depth := uint64(4)
	numBuckets := numMessages / depth
	data := make([]byte, dataSize)
	table := NewTable("t", numBuckets, depth, dataSize, nil, 0)
	end := uint64(float64(numMessages) * 0.90)
	var ok bool
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		for x := uint64(0); x < end; x++ {
			binary.PutUvarint(data, x)
			item := &Item{
				ID:      x,
				Data:    data,
				Bucket1: rand.Uint64() % numBuckets,
				Bucket2: rand.Uint64() % numBuckets,
			}
			ok, _ = table.Insert(item)
			if !ok {
				b.Fatalf("error inserting into table after %v inserts", x)
			}
		}
	}
}
<commit_msg>rename benchmark<commit_after>package cuckoo

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math/rand"
	"strconv"
	"testing"
)

const ItemSize = uint64(64)

// Test identical state after operations
// Test contains after insert/remove sequence
// Test insert same value twice

func GetBytes(val string) []byte {
	buf := make([]byte, ItemSize)
	copy(buf, bytes.NewBufferString(val).Bytes())
	return buf
}

func randBucket(numBuckets uint64) uint64 {
	result := rand.Uint64() % numBuckets
	return result
}

func TestGetCapacity(t *testing.T) {
	fmt.Printf("TestGetCapacity: ...\n")

	table := NewTable("t", 10, 2, ItemSize, nil, 0)
	if table.GetCapacity() != 20 {
		t.Fatalf("table returned wrong value for GetCapacity=%v. Expecting 20\n", table.GetCapacity())
	}

	table = NewTable("t", 1, 1, ItemSize, nil, 0)
	if table.GetCapacity() != 1 {
		t.Fatalf("table returned wrong value for GetCapacity=%v. Expecting 1\n", table.GetCapacity())
	}

	table = NewTable("t", 0, 0, ItemSize, nil, 0)
	if table.GetCapacity() != 0 {
		t.Fatalf("table returned wrong value for GetCapacity=%v. Expecting 0\n", table.GetCapacity())
	}

	fmt.Printf("... done \n")
}

func TestInvalidConstruction(t *testing.T) {
	fmt.Printf("TestInvalidConstruction: ...\n")
	data := make([]byte, 7)
	table := NewTable("t", 2, 3, 4, data, 0)
	if table != nil {
		t.Fatalf("created a improperly sized table\n")
	}
	fmt.Printf("... done \n")
}

func TestBasic(t *testing.T) {
	table := NewTable("t", 10, 2, ItemSize, nil, 0)

	fmt.Printf("TestBasic: check empty...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("TestBasic: Check contains non-existent value...\n")
	if table.Contains(&Item{0, GetBytes(""), 0, 1}) {
		t.Fatalf("empty table returned true for Contains()\n")
	}

	fmt.Printf("TestBasic: remove non-existent value ...\n")
	if table.Remove(&Item{1, GetBytes("value1"), 0, 1}) {
		t.Fatalf("empty table returned true for Remove()\n")
	}

	fmt.Printf("TestBasic: check empty...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("TestBasic: Insert improperly sized value ...\n")
	ok, itm := table.Insert(&Item{1, []byte{0, 0}, 0, 1})
	if itm != nil || ok {
		t.Fatalf("should have failed inserting a malformed item\n")
	}

	fmt.Printf("TestBasic: Insert value ...\n")
	ok, itm = table.Insert(&Item{1, GetBytes("value1"), 0, 1})
	if itm != nil || !ok {
		t.Fatalf("error inserting into table (0, 1, value1)\n")
	}

	fmt.Printf("TestBasic: Check inserted value...\n")
	if !table.Contains(&Item{1, GetBytes("value1"), 0, 1}) {
		t.Fatalf("cannot find recently inserted value\n")
	}

	fmt.Printf("TestBasic: Check inserted value w/o full reference...\n")
	if !table.Contains(&Item{1, GetBytes(""), 0, 1}) {
		t.Fatalf("cannot find recently inserted value\n")
	}

	fmt.Printf("TestBasic: Check non-existent value...\n")
	if table.Contains(&Item{2, GetBytes("value2"), 0, 1}) {
		t.Fatalf("contains a non-existent value\n")
	}

	fmt.Printf("TestBasic: check 1 element...\n")
	if table.GetNumElements() != 1 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("TestBasic: remove existing value ...\n")
	if !table.Remove(&Item{1, GetBytes("value1"), 0, 1}) {
		t.Fatalf("error removing existing value (0, 1, value1)\n")
	}

	fmt.Printf("TestBasic: check 0 element...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("TestBasic: remove recently removed value ...\n")
	if table.Remove(&Item{1, GetBytes("value1"), 0, 1}) {
		t.Fatalf("empty table returned true for Remove()\n")
	}

	fmt.Printf("TestBasic: check 0 element...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("empty table returned %v for GetNumElements()\n", table.GetNumElements())
	}

	fmt.Printf("... done \n")
}

func TestBucket(t *testing.T) {
	fmt.Printf("TestBucket ...\n")
	table := NewTable("t", 10, 2, ItemSize, nil, 0)

	items := []*Item{
		{1, GetBytes("value1"), 5, 5},
		{2, GetBytes("value2"), 5, 5},
		{3, GetBytes("value3"), 5, 6},
	}
	for _, v := range items {
		ok, _ := table.Insert(v)
		if !ok {
			t.Fatalf("Failed to insert item %v", v)
		}
	}

	bucket, err := table.Bucket(items[0])
	if bucket != 5 || err != nil {
		t.Fatalf("Table should report expected item position %v", items[0])
	}
	bucket, err = table.Bucket(items[1])
	if bucket != 5 || err != nil {
		t.Fatalf("Table should report expected item position %v", items[1])
	}
	bucket, err = table.Bucket(items[2])
	if bucket != 6 || err != nil {
		t.Fatalf("Table should report expected item position %v", items[2])
	}

	nonItem := &Item{4, GetBytes("value4"), 1, 1}
	_, err = table.Bucket(nonItem)
	if err == nil {
		t.Fatalf("Table should report expected item position %v", nonItem)
	}

	fmt.Printf("... done \n")
}

func TestOutOfBounds(t *testing.T) {
	table := NewTable("t", 10, 2, ItemSize, nil, 0)

	fmt.Printf("TestOutOfBounds: Insert() out of bounds...\n")
	ok, itm := table.Insert(&Item{1, GetBytes("value1"), 100, 100})
	if ok {
		t.Fatalf("Insert returned true with out of bound buckets\n")
	}
	if itm != nil {
		t.Fatalf("Insert returned wrong values\n")
	}

	fmt.Printf("TestOutOfBounds: Contains() out of bounds...\n")
	if table.Contains(&Item{1, GetBytes("value1"), 100, 100}) {
		t.Fatalf("Contains returned true with out of bound buckets\n")
	}

	fmt.Printf("TestOutOfBounds: Remove() out of bounds...\n")
	if table.Remove(&Item{1, GetBytes("value1"), 100, 100}) {
		t.Fatalf("Remove() returned true with out of bound buckets\n")
	}

	fmt.Printf("... done \n")
}

func TestFullTable(t *testing.T) {
	numBuckets := uint64(100)
	depth := uint64(4)

	capacity := numBuckets * depth
	entries := make([]Item, 0, capacity)
	table := NewTable("t", numBuckets, depth, ItemSize, nil, 0)
	ok := true
	count := uint64(0)
	var evic *Item
	var b1, b2 uint64

	// Insert random values until we've reached a limit
	fmt.Printf("TestFullTable: Insert until reach capacity...\n")
	for ok {
		b1 = randBucket(numBuckets)
		b2 = randBucket(numBuckets)
		id := rand.Uint64()
		val := GetBytes(strconv.Itoa(rand.Int()))
		entries = append(entries, Item{id, nil, b1, b2})
		ok, evic = table.Insert(&Item{id, val, b1, b2})

		if ok {
			count++
			if !table.Contains(&Item{id, nil, b1, b2}) {
				t.Fatalf("Insert() succeeded, but Contains failed\n")
			}
			if count != table.GetNumElements() {
				t.Fatalf("Number of successful Inserts(), %v, does not match GetNumElements(), %v \n",
					count,
					table.GetNumElements())
			}
		}
	}

	// Middle count check
	fmt.Printf("TestFullTable: Fully Loaded check...\n")
	if count != table.GetNumElements() {
		t.Fatalf("Number of successful Inserts(), %v, does not match GetNumElements(), %v \n",
			count,
			table.GetNumElements())
	}
	maxCount := count

	// Remove elements one by one
	fmt.Printf("TestFullTable: Remove each element...\n")
	for _, entry := range entries {
		if !entry.Equals(evic) {
			ok = table.Remove(&entry)
			if !ok {
				t.Fatalf("Cannot Remove() an element believed to be in the table. item %d of %d",
					count,
					maxCount)
			} else {
				count--
				if count != table.GetNumElements() {
					t.Fatalf("GetNumElements()=%v returned a value that didn't match what was expected=%v \n",
						table.GetNumElements(),
						count)
				}
			}
		}
	}

	// Final count check
	fmt.Printf("TestFullTable: Final count check...\n")
	if table.GetNumElements() != 0 {
		t.Fatalf("GetNumElements() returns %v when table should be empty \n", table.GetNumElements())
	}

	fmt.Printf("... done\n")
}

func TestDuplicateValues(t *testing.T) {
	fmt.Printf("TestDuplicateValues: ...\n")
	table := NewTable("t", 10, 2, ItemSize, nil, 0)

	ok, itm := table.Insert(&Item{1, GetBytes("v"), 0, 1})
	if itm != nil || !ok {
		t.Fatalf("Error inserting value \n")
	}

	ok, itm = table.Insert(&Item{2, GetBytes("v"), 0, 1})
	if itm != nil || !ok {
		t.Fatalf("Error inserting value again \n")
	}

	ok, itm = table.Insert(&Item{3, GetBytes("v"), 1, 2})
	if itm != nil || !ok {
		t.Fatalf("Error inserting value in shifted buckets\n")
	}

	if !table.Remove(&Item{1, GetBytes("v"), 0, 1}) {
		t.Fatalf("Error removing value 1st time\n")
	}

	if !table.Remove(&Item{2, GetBytes("v"), 0, 1}) {
		t.Fatalf("Error removing value 2nd time\n")
	}

	if !table.Remove(&Item{3, GetBytes("v"), 1, 2}) {
		t.Fatalf("Error removing value 3rd time\n")
	}

	fmt.Printf("... done\n")
}

func TestLoadFactor(t *testing.T) {
	fmt.Printf("TestLoadFactor: ...\n")
	numBuckets := uint64(1000)
	var table *Table

	for depth := uint64(1); depth < uint64(32); depth *= 2 {
		count := -1
		ok := true
		table = NewTable("t", numBuckets, depth, ItemSize, nil, int64(depth))
		for ok {
			count++
			val := GetBytes(strconv.Itoa(rand.Int()))
			ok, _ = table.Insert(&Item{rand.Uint64(), val, randBucket(numBuckets), randBucket(numBuckets)})
		}

		if table.GetNumElements() != uint64(count) {
			t.Fatalf("Mismatch count=%v with GetNumElements()=%v \n", count, table.GetNumElements())
		}
		fmt.Printf("count=%v, depth=%v, capacity=%v, loadfactor=%v \n",
			count,
			depth,
			table.GetCapacity(),
			(float64(count) / float64(table.GetCapacity())))
	}

	fmt.Printf("... done\n")
}

func BenchmarkInserts(b *testing.B) {
	//numMessages := uint64(1073741824) //2^30
	numMessages := uint64(268435456) //2^28
	dataSize := uint64(8)
	depth := uint64(4)
	numBuckets := numMessages / depth
	data := make([]byte, dataSize)
	table := NewTable("t", numBuckets, depth, dataSize, nil, 0)
	end := uint64(float64(numMessages) * 0.90)
	var ok bool
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		// Insert up to capacity
		for x := uint64(0); x < end; x++ {
			binary.PutUvarint(data, x)
			item := &Item{
				ID:      x,
				Data:    data,
				Bucket1: rand.Uint64() % numBuckets,
				Bucket2: rand.Uint64() % numBuckets,
			}
			ok, _ = table.Insert(item)
			if !ok {
				b.Fatalf("error inserting into table after %v inserts", x)
			}
		}
	}
}
<|endoftext|>


<commit_before>package r

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var r *rand.Rand
var rootDir string

type Session struct {
	Key     string
	Dir     string
	Output  string
	Command string
}

type RCall struct {
	Package   string
	Function  string
	Arguments string
}

// Installs the given packages on the server. packages is a space
// separated list with the packages.
func installPackages(packages string) error {
	cmd := exec.Command("R", "-q", "-e", "install.packages(c("+packages+"),  repos='http://cran.us.r-project.org'), dependencies=TRUE")

	var out bytes.Buffer
	cmd.Stdout = &out

	fmt.Println(out.String())
	err := cmd.Run()
	if err != nil {
		fmt.Println("Could not install packages:", out.String())
		return err
	}
	return nil
}

// Installs the package found at src. Should be a tgz ball.
func InstallPackageFromSource(src string) (string, error) {

	cmd := exec.Command("R", "CMD", "INSTALL", src)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()
	if err != nil {
		fmt.Println("Could not install local package:", out.String())
		return "", err
	}
	return out.String(), nil

}

func newCall() (string, string, error) {
	h := md5.New()
	k := h.Sum([]byte(strconv.Itoa(r.Int())))

	hash := hex.EncodeToString(k)
	key := ".s" + hash

	wd := rootDir + "/" + key
	err := os.MkdirAll(wd, 0755)
	if err != nil {
		return "", "", err
	}

	err = os.Chdir(wd)

	if err != nil {
		return "", "", err
	}

	return key, wd, nil

}

// Executes function. Returns tmp key for use in Get
func Call(pkg, fun, args string) (*Session, error) {

	key, wd, err := newCall()
	if err != nil {
		return nil, err
	}

	// Replace argument names with real argument names using the
	// keys from previous calls. Also load data from previous
	// calls before running cmd

	argList := strings.Split(args, ",")
	loadArgs := []string{}

	if argList[0] != "" {
		finalArgs := []string{}

		for _, arg := range argList {

			// special case when we use a column vector as argument value
			// e,g. d = c(2,1,3,4) only d=c(2, is added. we need to
			// append the rest to get the full vector.
			if len(strings.Split(arg, "=")) == 1 {
				finalArgs[len(finalArgs)-1] = finalArgs[len(finalArgs)-1] + "," + arg
				continue
			}

			argName := strings.Split(arg, "=")[0]
			argVal := strings.Split(arg, "=")[1]
			if strings.HasPrefix(argVal, ".s") {
				loadArgs = append(loadArgs, "load('"+rootDir+"/"+argVal+"/.RData');")

				argVal = strings.TrimPrefix(argVal, ".")

			}
			finalArgs = append(finalArgs, argName+"="+argVal)
		}

		args = strings.Join(finalArgs, ",")
	}

	varName := strings.TrimPrefix(key, ".")
	command := varName + "=" + pkg + "::" + fun + "(" + args + "); " + varName

	if len(loadArgs) > 0 {
		loadString := strings.Join(loadArgs, "")
		command = loadString + command
	}

	cmd := exec.Command("R", "--save", "-q", "-e", command)
	cmd.Dir = wd

	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err = cmd.Run()

	var output string
	if err != nil {
		output = stderr.String()
		// remove kvik tmp key
		output = strings.Replace(output, varName+"=", "", -1)
	} else {
		output = stdout.String()
	}

	return &Session{key, wd, output, command}, err
}

// get results for a func call
func Get(key, format string) ([]byte, error) {

	dir := rootDir + "/" + key
	err := os.Chdir(dir)
	if err != nil {
		return nil, err
	}

	varName := strings.TrimPrefix(key, ".")

	extension := "." + format
	_, err = os.Stat("output" + extension)
	if err == nil {
		return ioutil.ReadFile(dir + "/output" + extension)
	}

	var command string
	if format == "csv" {
		command = "write.table(" + varName + ", sep=',', row.names=FALSE, file='output" + extension + "')"
	} else if format == "json" {
		command = "js=jsonlite::toJSON(" + varName + "); write(js, file='output" + extension + "')"
	} else if format == "pdf" {
		return ioutil.ReadFile(dir + "/Rplots.pdf")
	} else if format == "png" {
		cmd := exec.Command("pdftoppm", "-png", dir+"/Rplots.pdf", "plot")

		var out bytes.Buffer
		cmd.Stdout = &out

		err := cmd.Run()
		if err != nil {
			fmt.Println("Could not convert rplot to png:", out.String())
			return nil, err
		}

		return ioutil.ReadFile(dir + "/plot-1.png")
	} else {
		return nil, errors.New("Unknown format")
	}

	cmd := exec.Command("R", "--save", "-q", "-e", command)
	cmd.Dir = dir

	var out bytes.Buffer
	cmd.Stdout = &out

	err = cmd.Run()

	if err != nil {
		fmt.Println("ERROR", out.String())
		return nil, err
	}

	return ioutil.ReadFile(dir + "/output" + extension)

}

func ScriptHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("script handler")
	_, wd, err := newCall()
	if err != nil {
		fmt.Println("new call fail:", err)
		w.Write([]byte(err.Error()))
		return
	}

	file, _, err := r.FormFile("file")
	defer file.Close()

	if err != nil {
		fmt.Fprintln(w, err)
		return
	}

	fmt.Println("workdir", wd)

	fileLoc := wd + "/script.R"

	out, err := os.Create(fileLoc)
	if err != nil {
		fmt.Fprintf(w, "Failed to open the file for writing")
		return
	}
	defer out.Close()
	_, err = io.Copy(out, file)
	if err != nil {
		fmt.Fprintln(w, err)
		return
	}

	output, err := executeScript(wd)
	if err != nil {
		w.Write([]byte("ERROR, could not execute script. Output:\n" + output))
		return
	}

	cmd := exec.Command("zip", "-r", "-q", "output.zip", ".")
	cmd.Dir = wd

	var outbuffer bytes.Buffer
	cmd.Stdout = &outbuffer

	err = cmd.Run()

	if err != nil {
		fmt.Println("ERROR", outbuffer.String())
		w.Write([]byte("Could not zip output file"))
		return
	}

	http.ServeFile(w, r, "output.zip")

	return

}

func executeScript(wd string) (string, error) {

	filename := wd + "/script.R"

	cmd := exec.Command("R", "--save", "-f", filename)
	cmd.Dir = wd

	fmt.Println(cmd)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()

	if err != nil {

		errMessage := out.String()
		fmt.Println("ERROR", errMessage)
		return errMessage, err
	}

	return out.String(), nil
}

func InstalledPackages() ([]byte, error) {

	pkg := "utils"
	fun := "installed.packages"
	args := ""

	s, err := Call(pkg, fun, args)
	if err != nil {
		fmt.Println("could not get installed packages")
		return nil, err
	}

	pkg = "base"
	fun = "as.data.frame"
	args = "x=" + s.Key

	s, err = Call(pkg, fun, args)
	if err != nil {
		fmt.Println("could not get installed packages")
		return nil, err
	}

	return Get(s.Key, "json")
}

type PackageInfo struct {
	Package               string
	LibPath               string
	Version               string
	Priority              string
	Depends               string
	Imports               string
	LinkingTo             string
	Suggests              string
	Enhances              string
	Licence               string
	License_is_FOSS       string
	Licence_restricts_use string
	OS_type               string
	MD5sum                string
	NeedsCompilation      string
	Built                 string
}
<commit_msg>duplicate installed pkgs bug<commit_after>package r

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var r *rand.Rand
var rootDir string

type Session struct {
	Key     string
	Dir     string
	Output  string
	Command string
}

type RCall struct {
	Package   string
	Function  string
	Arguments string
}

// Installs the given packages on the server. packages is a space
// separated list with the packages.
func installPackages(packages string) error {
	cmd := exec.Command("R", "-q", "-e", "install.packages(c("+packages+"),  repos='http://cran.us.r-project.org'), dependencies=TRUE")

	var out bytes.Buffer
	cmd.Stdout = &out

	fmt.Println(out.String())
	err := cmd.Run()
	if err != nil {
		fmt.Println("Could not install packages:", out.String())
		return err
	}
	return nil
}

// Installs the package found at src. Should be a tgz ball.
func InstallPackageFromSource(src string) (string, error) {

	cmd := exec.Command("R", "CMD", "INSTALL", src)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()
	if err != nil {
		fmt.Println("Could not install local package:", out.String())
		return "", err
	}
	return out.String(), nil

}

func newCall() (string, string, error) {
	h := md5.New()
	k := h.Sum([]byte(strconv.Itoa(r.Int())))

	hash := hex.EncodeToString(k)
	key := ".s" + hash

	wd := rootDir + "/" + key
	err := os.MkdirAll(wd, 0755)
	if err != nil {
		return "", "", err
	}

	err = os.Chdir(wd)

	if err != nil {
		return "", "", err
	}

	return key, wd, nil

}

// Executes function. Returns tmp key for use in Get
func Call(pkg, fun, args string) (*Session, error) {

	key, wd, err := newCall()
	if err != nil {
		return nil, err
	}

	// Replace argument names with real argument names using the
	// keys from previous calls. Also load data from previous
	// calls before running cmd

	argList := strings.Split(args, ",")
	loadArgs := []string{}

	if argList[0] != "" {
		finalArgs := []string{}

		for _, arg := range argList {

			// special case when we use a column vector as argument value
			// e,g. d = c(2,1,3,4) only d=c(2, is added. we need to
			// append the rest to get the full vector.
			if len(strings.Split(arg, "=")) == 1 {
				finalArgs[len(finalArgs)-1] = finalArgs[len(finalArgs)-1] + "," + arg
				continue
			}

			argName := strings.Split(arg, "=")[0]
			argVal := strings.Split(arg, "=")[1]
			if strings.HasPrefix(argVal, ".s") {
				loadArgs = append(loadArgs, "load('"+rootDir+"/"+argVal+"/.RData');")

				argVal = strings.TrimPrefix(argVal, ".")

			}
			finalArgs = append(finalArgs, argName+"="+argVal)
		}

		args = strings.Join(finalArgs, ",")
	}

	varName := strings.TrimPrefix(key, ".")
	command := varName + "=" + pkg + "::" + fun + "(" + args + "); " + varName

	if len(loadArgs) > 0 {
		loadString := strings.Join(loadArgs, "")
		command = loadString + command
	}

	cmd := exec.Command("R", "--save", "-q", "-e", command)
	cmd.Dir = wd

	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err = cmd.Run()

	var output string
	if err != nil {
		output = stderr.String()
		// remove kvik tmp key
		output = strings.Replace(output, varName+"=", "", -1)
	} else {
		output = stdout.String()
	}

	return &Session{key, wd, output, command}, err
}

// get results for a func call
func Get(key, format string) ([]byte, error) {

	dir := rootDir + "/" + key
	err := os.Chdir(dir)
	if err != nil {
		return nil, err
	}

	varName := strings.TrimPrefix(key, ".")

	extension := "." + format
	_, err = os.Stat("output" + extension)
	if err == nil {
		return ioutil.ReadFile(dir + "/output" + extension)
	}

	var command string
	if format == "csv" {
		command = "write.table(" + varName + ", sep=',', row.names=FALSE, file='output" + extension + "')"
	} else if format == "json" {
		command = "js=jsonlite::toJSON(" + varName + "); write(js, file='output" + extension + "')"
	} else if format == "pdf" {
		return ioutil.ReadFile(dir + "/Rplots.pdf")
	} else if format == "png" {
		cmd := exec.Command("pdftoppm", "-png", dir+"/Rplots.pdf", "plot")

		var out bytes.Buffer
		cmd.Stdout = &out

		err := cmd.Run()
		if err != nil {
			fmt.Println("Could not convert rplot to png:", out.String())
			return nil, err
		}

		return ioutil.ReadFile(dir + "/plot-1.png")
	} else {
		return nil, errors.New("Unknown format")
	}

	cmd := exec.Command("R", "--save", "-q", "-e", command)
	cmd.Dir = dir

	var out bytes.Buffer
	cmd.Stdout = &out

	err = cmd.Run()

	if err != nil {
		fmt.Println("ERROR", out.String())
		return nil, err
	}

	return ioutil.ReadFile(dir + "/output" + extension)

}

func ScriptHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("script handler")
	_, wd, err := newCall()
	if err != nil {
		fmt.Println("new call fail:", err)
		w.Write([]byte(err.Error()))
		return
	}

	file, _, err := r.FormFile("file")
	defer file.Close()

	if err != nil {
		fmt.Fprintln(w, err)
		return
	}

	fmt.Println("workdir", wd)

	fileLoc := wd + "/script.R"

	out, err := os.Create(fileLoc)
	if err != nil {
		fmt.Fprintf(w, "Failed to open the file for writing")
		return
	}
	defer out.Close()
	_, err = io.Copy(out, file)
	if err != nil {
		fmt.Fprintln(w, err)
		return
	}

	output, err := executeScript(wd)
	if err != nil {
		w.Write([]byte("ERROR, could not execute script. Output:\n" + output))
		return
	}

	cmd := exec.Command("zip", "-r", "-q", "output.zip", ".")
	cmd.Dir = wd

	var outbuffer bytes.Buffer
	cmd.Stdout = &outbuffer

	err = cmd.Run()

	if err != nil {
		fmt.Println("ERROR", outbuffer.String())
		w.Write([]byte("Could not zip output file"))
		return
	}

	http.ServeFile(w, r, "output.zip")

	return

}

func executeScript(wd string) (string, error) {

	filename := wd + "/script.R"

	cmd := exec.Command("R", "--save", "-f", filename)
	cmd.Dir = wd

	fmt.Println(cmd)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()

	if err != nil {

		errMessage := out.String()
		fmt.Println("ERROR", errMessage)
		return errMessage, err
	}

	return out.String(), nil
}

func InstalledPackages() ([]byte, error) {

	pkg := "utils"
	fun := "installed.packages"
	args := ""

	s, err := Call(pkg, fun, args)
	if err != nil {
		fmt.Println("Could not get installed packages")
		fmt.Println(s.Output)
		return nil, err
	}

	pkg = "base"
	fun = "as.data.frame"
	args = "x=" + s.Key + ",row.names=make.names(installed.packages(), unique=TRUE)"

	s, err = Call(pkg, fun, args)
	if err != nil {
		fmt.Println("Could not get installed packages")
		fmt.Println(s.Output)
		return nil, err
	}

	return Get(s.Key, "json")
}

type PackageInfo struct {
	Package               string
	LibPath               string
	Version               string
	Priority              string
	Depends               string
	Imports               string
	LinkingTo             string
	Suggests              string
	Enhances              string
	Licence               string
	License_is_FOSS       string
	Licence_restricts_use string
	OS_type               string
	MD5sum                string
	NeedsCompilation      string
	Built                 string
}
<|endoftext|>


<commit_before>package search

import (
	"fmt"

	"github.com/gernest/ngorm/engine"
	"github.com/gernest/ngorm/model"
	"github.com/gernest/ngorm/regexes"
	"github.com/gernest/ngorm/util"
)

func Where(e *engine.Engine, query interface{}, values ...interface{}) *engine.Engine {
	e.Search.WhereConditions = append(e.Search.WhereConditions, map[string]interface{}{"query": query, "args": values})
	return e
}

func Not(e *engine.Engine, query interface{}, values ...interface{}) *engine.Engine {
	e.Search.NotConditions = append(e.Search.NotConditions, map[string]interface{}{"query": query, "args": values})
	return e
}

func Or(e *engine.Engine, query interface{}, values ...interface{}) *engine.Engine {
	e.Search.OrConditions = append(e.Search.OrConditions, map[string]interface{}{"query": query, "args": values})
	return e
}

func Attr(e *engine.Engine, attrs ...interface{}) *engine.Engine {
	e.Search.InitAttrs = append(e.Search.InitAttrs, toSearchableMap(attrs...))
	return e
}

func toSearchableMap(attrs ...interface{}) (result interface{}) {
	if len(attrs) > 1 {
		if str, ok := attrs[0].(string); ok {
			result = map[string]interface{}{str: attrs[1]}
		}
	} else if len(attrs) == 1 {
		if attr, ok := attrs[0].(map[string]interface{}); ok {
			result = attr
		}

		if attr, ok := attrs[0].(interface{}); ok {
			result = attr
		}
	}
	return
}

func Assign(e *engine.Engine, attrs ...interface{}) *engine.Engine {
	e.Search.AssignAttrs = append(e.Search.AssignAttrs, toSearchableMap(attrs...))
	return e
}

func Order(e *engine.Engine, value interface{}, reorder ...bool) *engine.Engine {
	if len(reorder) > 0 && reorder[0] {
		e.Search.Orders = []interface{}{}
	}

	if value != nil {
		e.Search.Orders = append(e.Search.Orders, value)
	}
	return e
}

func Select(e *engine.Engine, query interface{}, values ...interface{}) *engine.Engine {
	if regexes.DistinctSQL.MatchString(fmt.Sprint(query)) {
		e.Search.IgnoreOrderQuery = true
	}
	e.Search.Selects = map[string]interface{}{"query": query, "args": values}
	return e
}

func Omit(e *engine.Engine, columns ...string) *engine.Engine {
	e.Search.Omits = columns
	return e
}

func Limit(e *engine.Engine, limit interface{}) *engine.Engine {
	e.Search.Limit = limit
	return e
}

func Offset(e *engine.Engine, offset interface{}) *engine.Engine {
	e.Search.Offset = offset
	return e
}

func Group(e *engine.Engine, query interface{}) *engine.Engine {
	s, err := util.GetInterfaceAsSQL(query)
	if err != nil {
		e.Error = err
	} else {
		e.Search.Group = s
	}
	return e
}

func Having(e *engine.Engine, query interface{}, values ...interface{}) *engine.Engine {
	e.Search.HavingConditions = append(e.Search.HavingConditions, map[string]interface{}{"query": query, "args": values})
	return e
}

func Join(e *engine.Engine, query interface{}, values ...interface{}) *engine.Engine {
	e.Search.JoinConditions = append(e.Search.JoinConditions, map[string]interface{}{"query": query, "args": values})
	return e
}

func Preload(e *engine.Engine, schema string, values ...interface{}) *engine.Engine {
	var preloads []model.SearchPreload
	for _, preload := range e.Search.Preload {
		if preload.Schema != schema {
			preloads = append(preloads, preload)
		}
	}
	preloads = append(preloads, model.SearchPreload{schema, values})
	e.Search.Preload = preloads
	return e
}

func Raw(e *engine.Engine, b bool) *engine.Engine {
	e.Search.Raw = b
	return e
}

func Unscoped(e *engine.Engine, b bool) *engine.Engine {
	e.Search.Unscoped = b
	return e
}

func Table(e *engine.Engine, name string) *engine.Engine {
	e.Search.TableName = name
	return e
}
<commit_msg>[search] Fix lint errors<commit_after>package search

import (
	"fmt"

	"github.com/gernest/ngorm/engine"
	"github.com/gernest/ngorm/model"
	"github.com/gernest/ngorm/regexes"
	"github.com/gernest/ngorm/util"
)

//Where adds WHERE search condition.
func Where(e *engine.Engine, query interface{}, values ...interface{}) {
	e.Search.WhereConditions = append(e.Search.WhereConditions, map[string]interface{}{"query": query, "args": values})
}

//Not adds NOT search condition
func Not(e *engine.Engine, query interface{}, values ...interface{}) {
	e.Search.NotConditions = append(e.Search.NotConditions, map[string]interface{}{"query": query, "args": values})
}

//Or add OR search condition
func Or(e *engine.Engine, query interface{}, values ...interface{}) {
	e.Search.OrConditions = append(e.Search.OrConditions, map[string]interface{}{"query": query, "args": values})
}

//Attr add attributes
func Attr(e *engine.Engine, attrs ...interface{}) {
	e.Search.InitAttrs = append(e.Search.InitAttrs, toSearchableMap(attrs...))
}

func toSearchableMap(attrs ...interface{}) (result interface{}) {
	if len(attrs) > 1 {
		if str, ok := attrs[0].(string); ok {
			result = map[string]interface{}{str: attrs[1]}
		}
	} else if len(attrs) == 1 {
		if attr, ok := attrs[0].(map[string]interface{}); ok {
			result = attr
		}

		if attr, ok := attrs[0].(interface{}); ok {
			result = attr
		}
	}
	return
}

//Assign assigns attrs
func Assign(e *engine.Engine, attrs ...interface{}) {
	e.Search.AssignAttrs = append(e.Search.AssignAttrs, toSearchableMap(attrs...))
}

//Order add ORDER BY search condition
func Order(e *engine.Engine, value interface{}, reorder ...bool) {
	if len(reorder) > 0 && reorder[0] {
		e.Search.Orders = []interface{}{}
	}

	if value != nil {
		e.Search.Orders = append(e.Search.Orders, value)
	}
}

//Select add SELECT querry
func Select(e *engine.Engine, query interface{}, values ...interface{}) {
	if regexes.DistinctSQL.MatchString(fmt.Sprint(query)) {
		e.Search.IgnoreOrderQuery = true
	}
	e.Search.Selects = map[string]interface{}{"query": query, "args": values}
}

//Omit ommits seacrh condition
func Omit(e *engine.Engine, columns ...string) {
	e.Search.Omits = columns
}

//Limit add search LIMIT
func Limit(e *engine.Engine, limit interface{}) {
	e.Search.Limit = limit
}

//Offset add search OFFSET
func Offset(e *engine.Engine, offset interface{}) {
	e.Search.Offset = offset
}

//Group  add GROUP BY search condition.
func Group(e *engine.Engine, query interface{}) {
	s, err := util.GetInterfaceAsSQL(query)
	if err != nil {
		e.Error = err
	} else {
		e.Search.Group = s
	}
}

//Having add HAVING condition
func Having(e *engine.Engine, query interface{}, values ...interface{}) {
	e.Search.HavingConditions = append(e.Search.HavingConditions, map[string]interface{}{"query": query, "args": values})
}

//Join add JOIN condition
func Join(e *engine.Engine, query interface{}, values ...interface{}) {
	e.Search.JoinConditions = append(e.Search.JoinConditions, map[string]interface{}{"query": query, "args": values})
}

//Preload add preloading condition
func Preload(e *engine.Engine, schema string, values ...interface{}) {
	var preloads []model.SearchPreload
	for _, preload := range e.Search.Preload {
		if preload.Schema != schema {
			preloads = append(preloads, preload)
		}
	}
	preloads = append(preloads, model.SearchPreload{Schema: schema, Conditions: values})
	e.Search.Preload = preloads
}

//Raw set the seacrh querry to RAw
func Raw(e *engine.Engine, b bool) {
	e.Search.Raw = b
}

//Unscoped set the search scope status
func Unscoped(e *engine.Engine, b bool) {
	e.Search.Unscoped = b
}

//Table set the search table name.
func Table(e *engine.Engine, name string) {
	e.Search.TableName = name
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mingrammer/go-codelab/models"
)

// worker struct has necessary information to run sensor data producing job
type worker struct {
	ticker      *time.Ticker
	sensor      models.Sensor
	sensorError float64
	serverPort  int
}

// counter is for counting the number of sending request
type counter struct {
	mutex sync.Mutex
	n     int
}

func (c *counter) count() {
	c.mutex.Lock()
	c.n++
	c.mutex.Unlock()
}

func (c *counter) value() int {
	return c.n
}

// sensorWorker will be ran in goroutine to produce (random) sensor data and sends it to server
func sensorWorker(done <-chan struct{}, w worker, c *counter) {
	for {
		select {
		case <-done:
			return
		case <-w.ticker.C:
			sensorData := w.sensor.GenerateSensorData(w.sensorError)
			url := getRequestServerURL(w.serverPort)

			fmt.Println(sensorData.SendingOutputString())

			sendJSONSensorData(url, sensorData)

			c.count()
		}
	}
}

func main() {
	numCPUs := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPUs)

	var sendCounter counter
	var wg sync.WaitGroup
	const numWorkers = 3

	done := make(chan struct{})

	wg.Add(numWorkers)

	workerList := [numWorkers]worker{
		worker{
			ticker:      time.NewTicker(500 * time.Millisecond),
			sensor:      models.GyroSensor{},
			sensorError: 4.0,
			serverPort:  8001,
		},
		worker{
			ticker:      time.NewTicker(500 * time.Millisecond),
			sensor:      models.AccelSensor{},
			sensorError: 12.0,
			serverPort:  8002,
		},
		worker{
			ticker:      time.NewTicker(2 * time.Second),
			sensor:      models.TempSensor{},
			sensorError: 1.5,
			serverPort:  8003,
		},
	}

	for _, w := range workerList {
		go func(w worker) {
			sensorWorker(done, w, &sendCounter)
			wg.Done()
		}(w)
	}

	go func() {
		for {
			if sendCounter.value() > 100 {
				close(done)
				return
			}
		}
	}()

	wg.Wait()

	fmt.Printf("\n[Count : %d] Sending is stopped.\n", sendCounter.value())
}

// getRequestServerURL just concatenate the server url and port number to get complete server url
func getRequestServerURL(port int) string {
	urlComponents := []string{"http://127.0.0.1", strconv.Itoa(port)}

	return strings.Join(urlComponents, ":")
}

// sendJSONSensorData sends the sensor data as JSON type to server
func sendJSONSensorData(url string, sensorValues interface{}) {
	jsonBytes, err := json.Marshal(sensorValues)
	if err != nil {
		log.Fatal("Error occurs when mashaling the gyro sensor values")
	}

	buff := bytes.NewBuffer(jsonBytes)

	resp, err := http.Post(url, "application/json", buff)
	if err != nil || resp.StatusCode != 200 {
		log.Fatal("Error occurs when request the post data")
	}
}
<commit_msg>Use models.Sensor instead of interface{} as argument<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mingrammer/go-codelab/models"
)

// worker struct has necessary information to run sensor data producing job
type worker struct {
	ticker      *time.Ticker
	sensor      models.Sensor
	sensorError float64
	serverPort  int
}

// counter is for counting the number of sending request
type counter struct {
	mutex sync.Mutex
	n     int
}

func (c *counter) count() {
	c.mutex.Lock()
	c.n++
	c.mutex.Unlock()
}

func (c *counter) value() int {
	return c.n
}

// sensorWorker will be ran in goroutine to produce (random) sensor data and sends it to server
func sensorWorker(done <-chan struct{}, w worker, c *counter) {
	for {
		select {
		case <-done:
			return
		case <-w.ticker.C:
			sensorData := w.sensor.GenerateSensorData(w.sensorError)
			url := getRequestServerURL(w.serverPort)

			fmt.Println(sensorData.SendingOutputString())

			sendJSONSensorData(url, sensorData)

			c.count()
		}
	}
}

func main() {
	numCPUs := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPUs)

	var sendCounter counter
	var wg sync.WaitGroup
	const numWorkers = 3

	done := make(chan struct{})

	wg.Add(numWorkers)

	workerList := [numWorkers]worker{
		worker{
			ticker:      time.NewTicker(500 * time.Millisecond),
			sensor:      models.GyroSensor{},
			sensorError: 4.0,
			serverPort:  8001,
		},
		worker{
			ticker:      time.NewTicker(500 * time.Millisecond),
			sensor:      models.AccelSensor{},
			sensorError: 12.0,
			serverPort:  8002,
		},
		worker{
			ticker:      time.NewTicker(2 * time.Second),
			sensor:      models.TempSensor{},
			sensorError: 1.5,
			serverPort:  8003,
		},
	}

	for _, w := range workerList {
		go func(w worker) {
			sensorWorker(done, w, &sendCounter)
			wg.Done()
		}(w)
	}

	go func() {
		for {
			if sendCounter.value() > 100 {
				close(done)
				return
			}
		}
	}()

	wg.Wait()

	fmt.Printf("\n[Count : %d] Sending is stopped.\n", sendCounter.value())
}

// getRequestServerURL just concatenate the server url and port number to get complete server url
func getRequestServerURL(port int) string {
	urlComponents := []string{"http://127.0.0.1", strconv.Itoa(port)}

	return strings.Join(urlComponents, ":")
}

// sendJSONSensorData sends the sensor data as JSON type to server
func sendJSONSensorData(url string, sensorValues models.Sensor) {
	jsonBytes, err := json.Marshal(sensorValues)
	if err != nil {
		log.Fatal("Error occurs when mashaling the gyro sensor values")
	}

	buff := bytes.NewBuffer(jsonBytes)

	resp, err := http.Post(url, "application/json", buff)
	if err != nil || resp.StatusCode != 200 {
		log.Fatal("Error occurs when request the post data")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/mingrammer/go-codelab/models"
	"log"
	"net/http"
	"os"
	"sync"
)

type logBuffer struct {
	buffer chan interface{}
	mux    sync.Mutex
}

type TempHandler struct {
	buf *logBuffer
}

type GyroHandler struct {
	buf *logBuffer
}

type AccelHandler struct {
	buf *logBuffer
}

func (m *TempHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.TempSensor

	fileHandle, err := os.OpenFile("./log/Temp.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)

	if err != nil {
		fmt.Println(err)
		log.Fatal("Error Opening Temp.log")
	}

	// logger := log.New(fileHandle, "", log.LstdFlags)
	defer fileHandle.Close()

	decoder := json.NewDecoder(req.Body)
	err = decoder.Decode(&data)
	if err != nil {
		fmt.Println("Somethink wrong")
	}

	defer req.Body.Close()
	// log.Println("Temperature Data Incoming")
	m.buf.mux.Lock()
	defer m.buf.mux.Unlock()
	fmt.Println("Locked at Temp")
	m.buf.buffer <- data
	fmt.Println("Allocated at Temp")
}

func (m *GyroHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.GyroSensor

	fileHandle, err := os.OpenFile("./log/Gyro.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)

	if err != nil {
		fmt.Println(err)
		log.Fatal("Error Opening Gyro.log")
	}

	// logger := log.New(fileHandle, "", log.LstdFlags)
	defer fileHandle.Close()

	decoder := json.NewDecoder(req.Body)
	err = decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}

	defer req.Body.Close()
	// log.Println("Gyro Sensor Data Incoming")
	m.buf.mux.Lock()
	defer m.buf.mux.Unlock()
	fmt.Println("Locked at Gyro")
	m.buf.buffer <- data
	fmt.Println("Allocated at Gyro")
}

func (m *AccelHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.AccelSensor

	fileHandle, err := os.OpenFile("./log/Accel.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)

	if err != nil {
		fmt.Println(err)
		log.Fatal("Error Opening Accel.log")
	}

	// logger := log.New(fileHandle, "", log.LstdFlags)
	defer fileHandle.Close()

	decoder := json.NewDecoder(req.Body)
	err = decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}

	defer req.Body.Close()
	// log.Println("Accelorator Sensor Data Incoming")

	m.buf.mux.Lock()
	defer m.buf.mux.Unlock()
	fmt.Println("Locked at Accel")
	m.buf.buffer <- data
	fmt.Println("Allocated at Accel")

}

func fileLogger(buf *logBuffer) {
	for i := range buf.buffer {
		fmt.Println("Locked at fileLogger")
		switch v := i.(type) {
		case models.GyroSensor:
			fmt.Println("[%s of %s Received]\n%s", v.Name, v.Type, v)
		case models.AccelSensor:
			fmt.Println("[%s of %s Received]\n%s", v.Name, v.Type, v)
		case models.TempSensor:
			fmt.Println("[%s of %s Received]\n%s", v.Name, v.Type, v)
		}
	}
}

func main() {
	var wg sync.WaitGroup

	wg.Add(4)

	logBuf := &logBuffer{buffer: make(chan interface{})}
	gyroHander := &GyroHandler{buf: logBuf}
	accelHandler := &AccelHandler{buf: logBuf}
	tempHandler := &TempHandler{buf: logBuf}

	go http.ListenAndServe(":8001", gyroHander)
	go http.ListenAndServe(":8002", accelHandler)
	go http.ListenAndServe(":8003", tempHandler)
	go fileLogger(logBuf)

	wg.Wait()
}
<commit_msg>Remove newline between line of opening file and line of checking error<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/mingrammer/go-codelab/models"
	"log"
	"net/http"
	"os"
	"sync"
)

type logBuffer struct {
	buffer chan interface{}
	mux    sync.Mutex
}

type TempHandler struct {
	buf *logBuffer
}

type GyroHandler struct {
	buf *logBuffer
}

type AccelHandler struct {
	buf *logBuffer
}

func (m *TempHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.TempSensor

	fileHandle, err := os.OpenFile("./log/Temp.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		fmt.Println(err)
		log.Fatal("Error Opening Temp.log")
	}

	// logger := log.New(fileHandle, "", log.LstdFlags)
	defer fileHandle.Close()

	decoder := json.NewDecoder(req.Body)
	err = decoder.Decode(&data)
	if err != nil {
		fmt.Println("Somethink wrong")
	}

	defer req.Body.Close()
	// log.Println("Temperature Data Incoming")
	m.buf.mux.Lock()
	defer m.buf.mux.Unlock()
	fmt.Println("Locked at Temp")
	m.buf.buffer <- data
	fmt.Println("Allocated at Temp")
}

func (m *GyroHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.GyroSensor

	fileHandle, err := os.OpenFile("./log/Gyro.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		fmt.Println(err)
		log.Fatal("Error Opening Gyro.log")
	}

	// logger := log.New(fileHandle, "", log.LstdFlags)
	defer fileHandle.Close()

	decoder := json.NewDecoder(req.Body)
	err = decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}

	defer req.Body.Close()
	// log.Println("Gyro Sensor Data Incoming")
	m.buf.mux.Lock()
	defer m.buf.mux.Unlock()
	fmt.Println("Locked at Gyro")
	m.buf.buffer <- data
	fmt.Println("Allocated at Gyro")
}

func (m *AccelHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.AccelSensor

	fileHandle, err := os.OpenFile("./log/Accel.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		fmt.Println(err)
		log.Fatal("Error Opening Accel.log")
	}

	// logger := log.New(fileHandle, "", log.LstdFlags)
	defer fileHandle.Close()

	decoder := json.NewDecoder(req.Body)
	err = decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}

	defer req.Body.Close()
	// log.Println("Accelorator Sensor Data Incoming")

	m.buf.mux.Lock()
	defer m.buf.mux.Unlock()
	fmt.Println("Locked at Accel")
	m.buf.buffer <- data
	fmt.Println("Allocated at Accel")

}

func fileLogger(buf *logBuffer) {
	for i := range buf.buffer {
		fmt.Println("Locked at fileLogger")
		switch v := i.(type) {
		case models.GyroSensor:
			fmt.Println("[%s of %s Received]\n%s", v.Name, v.Type, v)
		case models.AccelSensor:
			fmt.Println("[%s of %s Received]\n%s", v.Name, v.Type, v)
		case models.TempSensor:
			fmt.Println("[%s of %s Received]\n%s", v.Name, v.Type, v)
		}
	}
}

func main() {
	var wg sync.WaitGroup

	wg.Add(4)

	logBuf := &logBuffer{buffer: make(chan interface{})}
	gyroHander := &GyroHandler{buf: logBuf}
	accelHandler := &AccelHandler{buf: logBuf}
	tempHandler := &TempHandler{buf: logBuf}

	go http.ListenAndServe(":8001", gyroHander)
	go http.ListenAndServe(":8002", accelHandler)
	go http.ListenAndServe(":8003", tempHandler)
	go fileLogger(logBuf)

	wg.Wait()
}
<|endoftext|>


<commit_before>package authentication

type AuthMiddleware struct {
	AB backends.AuthBackend
}
<commit_msg>getting new auth middleware<commit_after>package authentication

type AuthMiddleware struct {
	AB backends.AuthBackend
}

func GetNewAuthenticationMiddleware(authBackend backends.AuthBackend) *AuthMiddleware {
	return &AuthMiddleware{AB: authBackend}
}
<|endoftext|>


<commit_before>package rep

import (
	"math"
	"testing"
)

func TestAngleToClock(t *testing.T) {
	cases := []struct {
		angle float64
		clock int
	}{
		{0, 3},
		{math.Pi / 2, 12},
		{math.Pi, 9},
		{math.Pi * 3 / 2, 6},

		{0 + math.Pi*6, 3},
		{0 - math.Pi*6, 3},

		{math.Pi/2 + math.Pi/13, 12},
		{math.Pi/2 - math.Pi/13, 12},
	}

	for _, c := range cases {
		if got := angleToClock(c.angle); got != c.clock {
			t.Errorf("Expected: %v, got: %v", c.clock, got)
		}
	}
}

func TestCalcSQ(t *testing.T) {
	cases := []struct {
		unspent, income int64
		sq              int
	}{
		// Test table taken from http://www.teamliquid.net/forum/starcraft-2/266019-do-you-macro-like-a-pro
		{959, 1970, 94},
		{408, 1358, 95},
		{263, 844, 85},
		{447, 1459, 96},
		{585, 1849, 106},
		{262, 901, 88},
		{814, 1734, 89},
		{442, 1486, 98},
		{696, 1682, 92},
		{1641, 2107, 82},
		{670, 1291, 74},
		{2201, 2148, 74},
		{584, 1540, 91},
		{2222, 2085, 70},
		{556, 1486, 90},
		{1580, 2078, 82},
		{547, 1219, 78},
	}

	for _, c := range cases {
		if got := calcSQ(c.unspent, c.income); got != c.sq {
			t.Errorf("Expected: %v, got: %v", c.sq, got)
		}
	}
}
<commit_msg>Added tests for isMainBuilding().<commit_after>package rep

import (
	"math"
	"testing"
)

func TestIsMainBuilding(t *testing.T) {
	cases := []struct {
		name   string
		isMain bool
	}{
		{"Nexus", true},
		{"CommandCenter", true},
		{"Hatchery", true},
		{"", false},
		{"nexus", false},
		{"kitty", false},
	}

	for _, c := range cases {
		if got := isMainBuilding(c.name); got != c.isMain {
			t.Errorf("Expected: %v, got: %v", c.isMain, got)
		}
	}
}

func TestAngleToClock(t *testing.T) {
	cases := []struct {
		angle float64
		clock int
	}{
		{0, 3},
		{math.Pi / 2, 12},
		{math.Pi, 9},
		{math.Pi * 3 / 2, 6},

		{0 + math.Pi*6, 3},
		{0 - math.Pi*6, 3},

		{math.Pi/2 + math.Pi/13, 12},
		{math.Pi/2 - math.Pi/13, 12},
	}

	for _, c := range cases {
		if got := angleToClock(c.angle); got != c.clock {
			t.Errorf("Expected: %v, got: %v", c.clock, got)
		}
	}
}

func TestCalcSQ(t *testing.T) {
	cases := []struct {
		unspent, income int64
		sq              int
	}{
		// Test table taken from http://www.teamliquid.net/forum/starcraft-2/266019-do-you-macro-like-a-pro
		{959, 1970, 94},
		{408, 1358, 95},
		{263, 844, 85},
		{447, 1459, 96},
		{585, 1849, 106},
		{262, 901, 88},
		{814, 1734, 89},
		{442, 1486, 98},
		{696, 1682, 92},
		{1641, 2107, 82},
		{670, 1291, 74},
		{2201, 2148, 74},
		{584, 1540, 91},
		{2222, 2085, 70},
		{556, 1486, 90},
		{1580, 2078, 82},
		{547, 1219, 78},
	}

	for _, c := range cases {
		if got := calcSQ(c.unspent, c.income); got != c.sq {
			t.Errorf("Expected: %v, got: %v", c.sq, got)
		}
	}
}
<|endoftext|>