<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"github.com/paulkramme/toml"
	"log"
	"os"
	"os/signal"
	"syscall"
)

func main() {
	fmt.Println("BTSOOT - Copyright (c) 2017 Paul Kramme All Rights Reserved.")

	ConfigLocation := flag.String("config", "./btsoot.conf", "Specifies configfile location")
	flag.Parse()

	var Config Configuration
	_, err := toml.DecodeFile(*ConfigLocation, &Config)
	if err != nil {
		fmt.Println("Couldn't find or open config file.")
		panic(err)
	}
	db, err := sql.Open("sqlite3", Config.DBFileLocation)
	if err != nil {
		panic(err)
	}
	defer db.Close()
	err = db.Ping()
	if err != nil {
		panic(err)
	}

	ProcessList := CreateMasterProcessList()

	// NOTE: Init standard threads...
	go UpdateProcess(ProcessList[UpdateThreadID])
	go WebServer(ProcessList[WebserverThreadID])
	go ScanningProcess(ProcessList[ScanThreadID])
	signals := make(chan os.Signal, 1)

	log.Println("Startup complete...")

	signal.Notify(signals, syscall.SIGINT)
	<-signals
	fmt.Println("Exiting. Please wait...")

	for k, v := range ProcessList {
		fmt.Printf("Sending stop (%x) to THREADID=%d\n", StopCode, k)
		v.Channel <- StopCode
	}
	for len(ProcessList) != 0 {
		for k, v := range ProcessList {
			select {
			case callback := <-v.Channel:
				if callback == ConfirmCode {
					fmt.Printf("THREADID=%d: Confirmation (%x)\n", k, ConfirmCode)
					delete(ProcessList, k)
				} else if callback == ErrorCode {
					fmt.Println("THREADID=%d: Error (%x)\n", k, ErrorCode)
				}
			default:
				// NOTE: Thread did not respond, wait for the next flyby
			}
		}
	}
}
<commit_msg>Add unblock code<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"github.com/paulkramme/toml"
	"log"
	"os"
	"os/signal"
	"syscall"
)

func main() {
	fmt.Println("BTSOOT - Copyright (c) 2017 Paul Kramme All Rights Reserved.")

	ConfigLocation := flag.String("config", "./btsoot.conf", "Specifies configfile location")
	flag.Parse()

	var Config Configuration
	_, err := toml.DecodeFile(*ConfigLocation, &Config)
	if err != nil {
		fmt.Println("Couldn't find or open config file.")
		panic(err)
	}
	db, err := sql.Open("sqlite3", Config.DBFileLocation)
	if err != nil {
		panic(err)
	}
	defer db.Close()
	err = db.Ping()
	if err != nil {
		panic(err)
	}

	ProcessList := CreateMasterProcessList()

	// NOTE: Init standard threads...
	go UpdateProcess(ProcessList[UpdateThreadID])
	go WebServer(ProcessList[WebserverThreadID])
	go ScanningProcess(ProcessList[ScanThreadID])
	signals := make(chan os.Signal, 1)

	log.Println("Startup complete...")

	// NOTE: Wait for SIGINT
	signal.Notify(signals, syscall.SIGINT)
	<-signals
	fmt.Println("Exiting. Please wait...")

	// NOTE: Copy ProcessList
	ContactList := make(map[int]Process)
	for k, v := range ProcessList {
		ContactList[k] = v
	}

	// NOTE: Sending StopCode to registered threads inside ProcessList
	//ContactList := ProcessList
	for len(ContactList) != 0 {
		fmt.Println("Masterloop")
		for ContactListKey, _ := range ContactList {
			fmt.Println("Shortloop")
			v := ProcessList[ContactListKey]
			select {
			case v.Channel <- StopCode:
				fmt.Printf("Sending stop (%x) to THREADID=%d\n", StopCode, ContactListKey)
				delete(ContactList, ContactListKey)
			default:
				// NOTE: Wait for the next flyby to contact thread
			}
		}
	}

	fmt.Println(ProcessList)
	// NOTE: Receiving reponse code and deleting registration from ProcessList
	for len(ProcessList) != 0 {
		//fmt.Println(len(ProcessList), "threads are registered.")
		for k, v := range ProcessList {
			select {
			case callback := <-v.Channel:
				if callback == ConfirmCode {
					fmt.Printf("THREADID=%d: Confirmation (%x)\n", k, ConfirmCode)
					delete(ProcessList, k)
				} else if callback == ErrorCode {
					fmt.Println("THREADID=%d: Error (%x)\n", k, ErrorCode)
				}
			default:
				// NOTE: Thread did not respond, wait for the next flyby
			}
		}
	}
	fmt.Print(ProcessList)
}
<|endoftext|>


<commit_before>package clw11

/*
#cgo windows linux LDFLAGS: -lOpenCL
#cgo darwin LDFLAGS: -framework OpenCL

#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif
*/
import "C"
import (
	"unsafe"
)

type (
	Memory      C.cl_mem
	MemoryFlags C.cl_mem_flags
	MapFlags    C.cl_map_flags
)

// Bitfield.
const (
	MemoryReadWrite        MemoryFlags = C.CL_MEM_READ_WRITE
	MemoryWriteOnly        MemoryFlags = C.CL_MEM_WRITE_ONLY
	MemoryReadOnly         MemoryFlags = C.CL_MEM_READ_ONLY
	MemoryUseHostPointer   MemoryFlags = C.CL_MEM_USE_HOST_PTR
	MemoryAllocHostPointer MemoryFlags = C.CL_MEM_ALLOC_HOST_PTR
	MemoryCopyHostPointer  MemoryFlags = C.CL_MEM_COPY_HOST_PTR
)

// Bitfield.
const (
	MapRead  MapFlags = C.CL_MAP_READ
	MapWrite MapFlags = C.CL_MAP_WRITE
)

func CreateBuffer(context Context, flags MemoryFlags, size Size, host_ptr []byte) (Memory, error) {
	var host unsafe.Pointer
	if host_ptr != nil {
		host = unsafe.Pointer(&host_ptr[0])
	}
	var err C.cl_int
	memory := C.clCreateBuffer(context, C.cl_mem_flags(flags), C.size_t(size), host, &err)
	return Memory(memory), toError(err)
}

func ReleaseMemObject(memobject Memory) error {
	return toError(C.clReleaseMemObject(memobject))
}

func EnqueueReadBuffer(command_queue CommandQueue, buffer Memory, blocking_read Bool, offset, cb Size, ptr []byte,
	wait_list []Event, event *Event) error {

	return toError(C.clEnqueueReadBuffer(command_queue, buffer, C.cl_bool(blocking_read), C.size_t(offset),
		C.size_t(cb), unsafe.Pointer(&ptr[0]), C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]),
		(*C.cl_event)(event)))
}

func EnqueueWriteBuffer(command_queue CommandQueue, buffer Memory, blocking_read Bool, offset, cb Size, ptr []byte,
	wait_list []Event, event *Event) error {

	return toError(C.clEnqueueWriteBuffer(command_queue, buffer, C.cl_bool(blocking_read), C.size_t(offset),
		C.size_t(cb), unsafe.Pointer(&ptr[0]), C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]),
		(*C.cl_event)(event)))
}

func EnqueueMapBuffer(command_queue CommandQueue, buffer Memory, blocking_map Bool, map_flags MapFlags, offset, cb Size,
	wait_list []Event, event *Event) ([]byte, error) {

	var err C.cl_int
	mapped := C.clEnqueueMapBuffer(command_queue, buffer, C.cl_bool(blocking_map), C.cl_map_flags(map_flags),
		C.size_t(offset), C.size_t(cb), C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]), (*C.cl_event)(event),
		&err)
	return toBytes(mapped, int(cb)), toError(err)
}

func EnqueueUnmapMemObject(command_queue, memobj Memory, mapped_ptr []byte, wait_list []Event, event *Event) error {
	return toError(C.clEnqueueUnmapMemObject(command_queue, memobj, unsafe.Pointer(&mapped_ptr[0]),
		C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]), (*C.cl_event)(event)))
}
<commit_msg>Added copy buffer.<commit_after>package clw11

/*
#cgo windows linux LDFLAGS: -lOpenCL
#cgo darwin LDFLAGS: -framework OpenCL

#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif
*/
import "C"
import (
	"unsafe"
)

type (
	Memory      C.cl_mem
	MemoryFlags C.cl_mem_flags
	MapFlags    C.cl_map_flags
)

// Bitfield.
const (
	MemoryReadWrite        MemoryFlags = C.CL_MEM_READ_WRITE
	MemoryWriteOnly        MemoryFlags = C.CL_MEM_WRITE_ONLY
	MemoryReadOnly         MemoryFlags = C.CL_MEM_READ_ONLY
	MemoryUseHostPointer   MemoryFlags = C.CL_MEM_USE_HOST_PTR
	MemoryAllocHostPointer MemoryFlags = C.CL_MEM_ALLOC_HOST_PTR
	MemoryCopyHostPointer  MemoryFlags = C.CL_MEM_COPY_HOST_PTR
)

// Bitfield.
const (
	MapRead  MapFlags = C.CL_MAP_READ
	MapWrite MapFlags = C.CL_MAP_WRITE
)

func CreateBuffer(context Context, flags MemoryFlags, size Size, host_ptr []byte) (Memory, error) {
	var host unsafe.Pointer
	if host_ptr != nil {
		host = unsafe.Pointer(&host_ptr[0])
	}
	var err C.cl_int
	memory := C.clCreateBuffer(context, C.cl_mem_flags(flags), C.size_t(size), host, &err)
	return Memory(memory), toError(err)
}

func ReleaseMemObject(memobject Memory) error {
	return toError(C.clReleaseMemObject(memobject))
}

func EnqueueReadBuffer(command_queue CommandQueue, buffer Memory, blocking_read Bool, offset, cb Size, ptr []byte,
	wait_list []Event, event *Event) error {

	return toError(C.clEnqueueReadBuffer(command_queue, buffer, C.cl_bool(blocking_read), C.size_t(offset),
		C.size_t(cb), unsafe.Pointer(&ptr[0]), C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]),
		(*C.cl_event)(event)))
}

func EnqueueWriteBuffer(command_queue CommandQueue, buffer Memory, blocking_read Bool, offset, cb Size, ptr []byte,
	wait_list []Event, event *Event) error {

	return toError(C.clEnqueueWriteBuffer(command_queue, buffer, C.cl_bool(blocking_read), C.size_t(offset),
		C.size_t(cb), unsafe.Pointer(&ptr[0]), C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]),
		(*C.cl_event)(event)))
}

func EnqueueCopyBuffer(command_queue CommandQueue, src_buffer, dst_buffer Memory, src_offset, dst_offset, cb Size,
	wait_list []Event, event *Event) error {

	return toError(C.clEnqueueCopyBuffer(command_queue, src_buffer, dst_buffer, C.size_t(src_offset),
		C.size_t(dst_offset), C.size_t(cb), C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]),
		(*C.cl_event)(event)))
}

func EnqueueMapBuffer(command_queue CommandQueue, buffer Memory, blocking_map Bool, map_flags MapFlags, offset, cb Size,
	wait_list []Event, event *Event) ([]byte, error) {

	var err C.cl_int
	mapped := C.clEnqueueMapBuffer(command_queue, buffer, C.cl_bool(blocking_map), C.cl_map_flags(map_flags),
		C.size_t(offset), C.size_t(cb), C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]), (*C.cl_event)(event),
		&err)
	return toBytes(mapped, int(cb)), toError(err)
}

func EnqueueUnmapMemObject(command_queue, memobj Memory, mapped_ptr []byte, wait_list []Event, event *Event) error {
	return toError(C.clEnqueueUnmapMemObject(command_queue, memobj, unsafe.Pointer(&mapped_ptr[0]),
		C.cl_uint(len(wait_list)), (*C.cl_event)(&wait_list[0]), (*C.cl_event)(event)))
}
<|endoftext|>


<commit_before>package stdres

import "fmt"

var colorDisabled bool

func DisableColor() {
	colorDisabled = true
}

func EnableColor() {
	colorDisabled = false
}

type Result uint

// Foreground colors
const (
	RESET   = "\033[00m"
	BLACK   = "\033[30m"
	RED     = "\033[31m"
	GREEN   = "\033[32m"
	YELLOW  = "\033[33m"
	BLUE    = "\033[34m"
	MAGENTA = "\033[35m"
	CYAN    = "\033[36m"
	WHITE   = "\033[37m"
)

const (
	UNKNOWN Result = iota
	PENDING
	FAILURE
	SUCCESS
	INFO
	PLAIN
)

type Buffer struct {
	buffer []*Record
}

// Record struct keeps record of message to print and current result
type Record struct {
	Result
	Message string
}

func (out *Record) String() string {
	return string(out.Message)
}

// Flash iterates through all records in Buffer and flush it to stdout preceded by color matching current status.
// Each record will be followed by color reset if it was preceded by a color.
func (outBuffer *Buffer) Flush() {
	toFlush := outBuffer.buffer
	outBuffer.buffer = make([]*Record, 0, 500)

	// Print with colors disabled
	if colorDisabled {
		for _, out := range toFlush {
			message := out.String()
			fmt.Print(message)
		}

		return
	}

	// Print with colors enabled
	for _, record := range toFlush {
		var color string

		switch record.Result {
		case UNKNOWN:
			color = CYAN
		case PENDING:
			color = YELLOW
		case SUCCESS:
			color = GREEN
		case FAILURE:
			color = RED
		case INFO:
			color = BLUE
		case PLAIN:
			color = WHITE
		}

		fmt.Print(color, record.String(), RESET)
	}
}

func (outBuffer *Buffer) printer(message string, newline bool) *Record {
	if outBuffer.buffer == nil {
		outBuffer.buffer = make([]*Record, 0, 500)
	}

	if newline {
		message = message + "\n"
	}

	out := &Record{UNKNOWN, message}
	outBuffer.buffer = append(outBuffer.buffer, out)

	return out
}

// Println records message to buffer to be printed later.
// Text not printed until Buffer.Flush has been called.
// It returns current status and text string as Record.
func (outBuffer *Buffer) Print(message string) *Record {
	return outBuffer.printer(message, false)
}

// Println records message followed by newline to buffer to be printed later
// Text not printed until Buffer.Flush has been called.
// It returns current status and text string as Record.
func (outBuffer *Buffer) Println(message string) *Record {
	return outBuffer.printer(message, true)
}
<commit_msg>FEATURE: Don't export color constants<commit_after>package stdres

import "fmt"

var colorDisabled bool

func DisableColor() {
	colorDisabled = true
}

func EnableColor() {
	colorDisabled = false
}

type Result uint

// Foreground colors
const (
	reset   = "\033[00m"
	black   = "\033[30m"
	red     = "\033[31m"
	green   = "\033[32m"
	yellow  = "\033[33m"
	blue    = "\033[34m"
	magenta = "\033[35m"
	cyan    = "\033[36m"
	white   = "\033[37m"
)

const (
	UNKNOWN Result = iota
	PENDING
	FAILURE
	SUCCESS
	INFO
	PLAIN
)

type Buffer struct {
	buffer []*Record
}

// Record struct keeps record of message to print and current result
type Record struct {
	Result
	Message string
}

func (out *Record) String() string {
	return string(out.Message)
}

// Flash iterates through all records in Buffer and flush it to stdout preceded by color matching current status.
// Each record will be followed by color reset if it was preceded by a color.
func (outBuffer *Buffer) Flush() {
	toFlush := outBuffer.buffer
	outBuffer.buffer = make([]*Record, 0, 500)

	// Print with colors disabled
	if colorDisabled {
		for _, out := range toFlush {
			message := out.String()
			fmt.Print(message)
		}

		return
	}

	// Print with colors enabled
	for _, record := range toFlush {
		var color string

		switch record.Result {
		case UNKNOWN:
			color = cyan
		case PENDING:
			color = yellow
		case SUCCESS:
			color = green
		case FAILURE:
			color = red
		case INFO:
			color = blue
		case PLAIN:
			color = white
		}

		fmt.Print(color, record.String(), reset)
	}
}

func (outBuffer *Buffer) printer(message string, newline bool) *Record {
	if outBuffer.buffer == nil {
		outBuffer.buffer = make([]*Record, 0, 500)
	}

	if newline {
		message = message + "\n"
	}

	out := &Record{UNKNOWN, message}
	outBuffer.buffer = append(outBuffer.buffer, out)

	return out
}

// Println records message to buffer to be printed later.
// Text not printed until Buffer.Flush has been called.
// It returns current status and text string as Record.
func (outBuffer *Buffer) Print(message string) *Record {
	return outBuffer.printer(message, false)
}

// Println records message followed by newline to buffer to be printed later
// Text not printed until Buffer.Flush has been called.
// It returns current status and text string as Record.
func (outBuffer *Buffer) Println(message string) *Record {
	return outBuffer.printer(message, true)
}
<|endoftext|>


<commit_before>package cl11

import (
	"fmt"
	"reflect"
	"unsafe"

	clw "github.com/rdwilliamson/clw11"
)

// A buffer object stores a one-dimensional collection of elements. Elements of
// a buffer object can be a scalar data type (such as an int, float), vector
// data type, or a user-defined structure.
type Buffer struct {
	id clw.Mem

	// The context the buffer was created on.
	Context *Context

	// The size of the buffer in bytes.
	Size int64

	// Usage information for the buffer from the device's point of view.
	Flags MemFlags

	// The host backed memory for the buffer, if applicable.
	Host interface{}

	// The parent buffer, if applicable.
	Buffer *Buffer

	// The offset in bytes within the parent buffer.
	Origin int64
}

// A source and destination rectangular region.
type Rect struct {

	// The source offset. For a 2D rectangle SrcOrigin[2] should be 0. The
	// offset in bytes is SrcOrigin[2] * SrcSlicePitch + SrcOrigin[1] *
	// SrcRowPitch + SrcOrigin[0].
	SrcOrigin     [3]int64
	SrcRowPitch   int64
	SrcSlicePitch int64

	// The destination offset. For a 2D rectangle DstOrigin[2] should be 0. The
	// offset in bytes is DstOrigin[2] * DstSlicePitch + DstOrigin[1] *
	// DstRowPitch + DstOrigin[0].
	DstOrigin     [3]int64
	DstRowPitch   int64
	DstSlicePitch int64

	// The (width, height, depth) in bytes of the 2D or 3D rectangle being
	// copied. For a 2D rectangle the depth value given by Region[2] should be
	// 1.
	Region [3]int64
}

type MappedBuffer struct {
	pointer unsafe.Pointer
	size    int64
	buffer  *Buffer
}

type MemFlags int

// Bitfield.
const (
	MemReadWrite = MemFlags(clw.MemReadWrite)
	MemWriteOnly = MemFlags(clw.MemWriteOnly)
	MemReadOnly  = MemFlags(clw.MemReadOnly)
)

type MapFlags int

// Bitfield.
const (
	MapRead  = MapFlags(clw.MapRead)
	MapWrite = MapFlags(clw.MapWrite)
)

type BlockingCall clw.Bool

const (
	Blocking    = BlockingCall(clw.True)
	NonBlocking = BlockingCall(clw.False)
)

func (c *Context) CreateDeviceBuffer(size int64, mf MemFlags) (*Buffer, error) {

	memory, err := clw.CreateBuffer(c.id, clw.MemFlags(mf), clw.Size(size), nil)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: size, Flags: mf}, nil
}

func (c *Context) CreateDeviceBufferInitializedBy(mf MemFlags, value interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemCopyHostPointer

	var scratch [scratchSize]byte
	pointer, size := getPointerAndSize(value, unsafe.Pointer(&scratch[0]))

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Flags: mf}, nil
}

func (c *Context) CreateDeviceBufferFromHostMem(mf MemFlags, host interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemUseHostPointer

	value := reflect.ValueOf(host)
	if kind := value.Kind(); kind != reflect.Ptr && kind != reflect.Slice {
		return nil, wrapError(fmt.Errorf("host value not addressable"))
	} else if kind == reflect.Ptr {
		for {
			value = value.Elem()
			if value.Kind() != reflect.Ptr {
				break
			}
		}
	}
	pointer, size := addressablePointerAndSize(value)

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Host: host, Flags: mf}, nil
}

func (c *Context) CreateHostBuffer(size int64, mf MemFlags) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemAllocHostPointer

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), nil)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: size, Flags: mf}, nil
}

func (c *Context) CreateHostBufferInitializedBy(mf MemFlags, value interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemAllocHostPointer | clw.MemCopyHostPointer

	var scratch [scratchSize]byte
	pointer, size := getPointerAndSize(value, unsafe.Pointer(&scratch[0]))

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Flags: mf}, nil
}

func (b *Buffer) CreateSubBuffer(mf MemFlags, origin, size int64) (*Buffer, error) {

	region := clw.BufferRegion{clw.Size(origin), clw.Size(size)}

	memory, err := clw.CreateSubBuffer(b.id, clw.MemFlags(mf), region)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: b.Context, Size: size, Flags: mf, Buffer: b, Origin: origin}, nil
}

func (b *Buffer) Retain() error {
	return clw.RetainMemObject(b.id)
}

func (b *Buffer) Release() error {
	return clw.ReleaseMemObject(b.id)
}

func (cq *CommandQueue) CopyBuffer(src, dst *Buffer, srcOffset, dstOffset, size int64, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandCopyBuffer
		e.CommandQueue = cq
	}

	return clw.EnqueueCopyBuffer(cq.id, src.id, dst.id, clw.Size(srcOffset), clw.Size(dstOffset), clw.Size(size),
		cq.toEvents(waitList), event)
}

func (cq *CommandQueue) CopyBufferRect(src, dst *Buffer, r *Rect, waitList []*Event, e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandCopyBuffer
		e.CommandQueue = cq
	}

	return clw.EnqueueCopyBufferRect(cq.id, src.id, dst.id, r.srcOrigin(), r.dstOrigin(), r.region(), r.srcRowPitch(),
		r.srcSlicePitch(), r.dstRowPitch(), r.dstSlicePitch(), cq.toEvents(waitList), event)
}

func (cq *CommandQueue) MapBuffer(b *Buffer, bc BlockingCall, flags MapFlags, offset, size int64, waitList []*Event,
	e *Event) (*MappedBuffer, error) {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandMapBuffer
		e.CommandQueue = cq
	}

	pointer, err := clw.EnqueueMapBuffer(cq.id, b.id, clw.Bool(bc), clw.MapFlags(flags), clw.Size(offset),
		clw.Size(size), cq.toEvents(waitList), event)
	if err != nil {
		return nil, err
	}

	return &MappedBuffer{pointer, size, b}, nil
}

func (cq *CommandQueue) UnmapBuffer(mb *MappedBuffer, waitList []*Event, e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandUnmapMemoryObject
		e.CommandQueue = cq
	}

	return clw.EnqueueUnmapMemObject(cq.id, mb.buffer.id, mb.pointer, cq.toEvents(waitList), event)
}

func (cq *CommandQueue) ReadBuffer(b *Buffer, bc BlockingCall, offset int64, dst interface{}, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandReadBuffer
		e.CommandQueue = cq
	}

	pointer, size, err := tryPointerAndSize(dst)
	if err != nil {
		return wrapError(err)
	}

	return clw.EnqueueReadBuffer(cq.id, b.id, clw.Bool(bc), clw.Size(offset), clw.Size(size), pointer,
		cq.toEvents(waitList), event)
}

func (cq *CommandQueue) WriteBuffer() {

}

func (cq *CommandQueue) ReadBufferRect() {

}

func (cq *CommandQueue) WriteBufferRect() {

}

func (r *Rect) srcOrigin() [3]clw.Size {
	return [3]clw.Size{clw.Size(r.SrcOrigin[0]), clw.Size(r.SrcOrigin[1]), clw.Size(r.SrcOrigin[2])}
}

func (r *Rect) srcRowPitch() clw.Size {
	return clw.Size(r.SrcRowPitch)
}

func (r *Rect) srcSlicePitch() clw.Size {
	return clw.Size(r.SrcSlicePitch)
}

func (r *Rect) dstOrigin() [3]clw.Size {
	return [3]clw.Size{clw.Size(r.DstOrigin[0]), clw.Size(r.DstOrigin[1]), clw.Size(r.DstOrigin[2])}
}

func (r *Rect) dstRowPitch() clw.Size {
	return clw.Size(r.DstRowPitch)
}

func (r *Rect) dstSlicePitch() clw.Size {
	return clw.Size(r.DstSlicePitch)
}

func (r *Rect) region() [3]clw.Size {
	return [3]clw.Size{clw.Size(r.Region[0]), clw.Size(r.Region[1]), clw.Size(r.Region[2])}
}

func (bm *MappedBuffer) Float32Slice() []float32 {

	var header reflect.SliceHeader
	header.Data = uintptr(bm.pointer)
	size := int(bm.size / int64(float32Size))
	header.Len = size
	header.Cap = size

	return *(*[]float32)(unsafe.Pointer(&header))
}
<commit_msg>Think I have a method for dealing with garbage collection and async mem copy operations.<commit_after>package cl11

import (
	"fmt"
	"reflect"
	"unsafe"

	clw "github.com/rdwilliamson/clw11"
)

// A buffer object stores a one-dimensional collection of elements. Elements of
// a buffer object can be a scalar data type (such as an int, float), vector
// data type, or a user-defined structure.
type Buffer struct {
	id clw.Mem

	// The context the buffer was created on.
	Context *Context

	// The size of the buffer in bytes.
	Size int64

	// Usage information for the buffer from the device's point of view.
	Flags MemFlags

	// The host backed memory for the buffer, if applicable.
	Host interface{}

	// The parent buffer, if applicable.
	Buffer *Buffer

	// The offset in bytes within the parent buffer.
	Origin int64
}

// A source and destination rectangular region.
type Rect struct {

	// The source offset in bytes. For a 2D rectangle SrcOrigin[2] should be 0.
	// The offset in bytes is SrcOrigin[2] * SrcSlicePitch + SrcOrigin[1] *
	// SrcRowPitch + SrcOrigin[0].
	SrcOrigin     [3]int64
	SrcRowPitch   int64
	SrcSlicePitch int64

	// The destination offset in bytes. For a 2D rectangle DstOrigin[2] should
	// be 0. The offset in bytes is DstOrigin[2] * DstSlicePitch + DstOrigin[1]
	// * DstRowPitch + DstOrigin[0].
	DstOrigin     [3]int64
	DstRowPitch   int64
	DstSlicePitch int64

	// The (width, height, depth) in bytes of the 2D or 3D rectangle being
	// copied. For a 2D rectangle the depth value given by Region[2] should be
	// 1.
	Region [3]int64
}

type MappedBuffer struct {
	pointer unsafe.Pointer
	size    int64
	buffer  *Buffer
}

type MemFlags int

// Bitfield.
const (
	MemReadWrite = MemFlags(clw.MemReadWrite)
	MemWriteOnly = MemFlags(clw.MemWriteOnly)
	MemReadOnly  = MemFlags(clw.MemReadOnly)
)

type MapFlags int

// Bitfield.
const (
	MapRead  = MapFlags(clw.MapRead)
	MapWrite = MapFlags(clw.MapWrite)
)

type BlockingCall clw.Bool

const (
	Blocking    = BlockingCall(clw.True)
	NonBlocking = BlockingCall(clw.False)
)

func (c *Context) CreateDeviceBuffer(size int64, mf MemFlags) (*Buffer, error) {

	memory, err := clw.CreateBuffer(c.id, clw.MemFlags(mf), clw.Size(size), nil)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: size, Flags: mf}, nil
}

func (c *Context) CreateDeviceBufferInitializedBy(mf MemFlags, value interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemCopyHostPointer

	var scratch [scratchSize]byte
	pointer, size := getPointerAndSize(value, unsafe.Pointer(&scratch[0]))

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Flags: mf}, nil
}

func (c *Context) CreateDeviceBufferFromHostMem(mf MemFlags, host interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemUseHostPointer

	value := reflect.ValueOf(host)
	if kind := value.Kind(); kind != reflect.Ptr && kind != reflect.Slice {
		return nil, wrapError(fmt.Errorf("host value not addressable"))
	} else if kind == reflect.Ptr {
		for {
			value = value.Elem()
			if value.Kind() != reflect.Ptr {
				break
			}
		}
	}
	pointer, size := addressablePointerAndSize(value)

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Host: host, Flags: mf}, nil
}

func (c *Context) CreateHostBuffer(size int64, mf MemFlags) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemAllocHostPointer

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), nil)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: size, Flags: mf}, nil
}

func (c *Context) CreateHostBufferInitializedBy(mf MemFlags, value interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemAllocHostPointer | clw.MemCopyHostPointer

	var scratch [scratchSize]byte
	pointer, size := getPointerAndSize(value, unsafe.Pointer(&scratch[0]))

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Flags: mf}, nil
}

func (b *Buffer) CreateSubBuffer(mf MemFlags, origin, size int64) (*Buffer, error) {

	region := clw.BufferRegion{clw.Size(origin), clw.Size(size)}

	memory, err := clw.CreateSubBuffer(b.id, clw.MemFlags(mf), region)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: b.Context, Size: size, Flags: mf, Buffer: b, Origin: origin}, nil
}

func (b *Buffer) Retain() error {
	return clw.RetainMemObject(b.id)
}

func (b *Buffer) Release() error {
	return clw.ReleaseMemObject(b.id)
}

func (cq *CommandQueue) CopyBuffer(src, dst *Buffer, srcOffset, dstOffset, size int64, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandCopyBuffer
		e.CommandQueue = cq
	}

	return clw.EnqueueCopyBuffer(cq.id, src.id, dst.id, clw.Size(srcOffset), clw.Size(dstOffset), clw.Size(size),
		cq.toEvents(waitList), event)
}

func (cq *CommandQueue) CopyBufferRect(src, dst *Buffer, r *Rect, waitList []*Event, e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandCopyBuffer
		e.CommandQueue = cq
	}

	return clw.EnqueueCopyBufferRect(cq.id, src.id, dst.id, r.srcOrigin(), r.dstOrigin(), r.region(), r.srcRowPitch(),
		r.srcSlicePitch(), r.dstRowPitch(), r.dstSlicePitch(), cq.toEvents(waitList), event)
}

func (cq *CommandQueue) MapBuffer(b *Buffer, bc BlockingCall, flags MapFlags, offset, size int64, waitList []*Event,
	e *Event) (*MappedBuffer, error) {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandMapBuffer
		e.CommandQueue = cq
	}

	pointer, err := clw.EnqueueMapBuffer(cq.id, b.id, clw.Bool(bc), clw.MapFlags(flags), clw.Size(offset),
		clw.Size(size), cq.toEvents(waitList), event)
	if err != nil {
		return nil, err
	}

	return &MappedBuffer{pointer, size, b}, nil
}

func (cq *CommandQueue) UnmapBuffer(mb *MappedBuffer, waitList []*Event, e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandUnmapMemoryObject
		e.CommandQueue = cq
	}

	return clw.EnqueueUnmapMemObject(cq.id, mb.buffer.id, mb.pointer, cq.toEvents(waitList), event)
}

// TODO see write notes on garbage collection.
func (cq *CommandQueue) ReadBuffer(b *Buffer, bc BlockingCall, offset int64, dst interface{}, waitList []*Event,
	e *Event) error {

	if e == nil && bc == NonBlocking {
		e = &Event{}
	}

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandReadBuffer
		e.CommandQueue = cq
	}

	pointer, size, err := tryPointerAndSize(dst)
	if err != nil {
		return wrapError(err)
	}

	err = clw.EnqueueReadBuffer(cq.id, b.id, clw.Bool(bc), clw.Size(offset), clw.Size(size), pointer,
		cq.toEvents(waitList), event)
	if err != nil {
		return err
	}

	if bc == NonBlocking {
		err = e.SetCallback(noOpEventCallback, dst)
		if err != nil {
			return err
		}
	}

	return nil
}

// It is the user's responsibility to ensure the source is valid at the time the
// write is actually performed. If the source is addressable a reference will be
// held to prevent it being garbage collected (but not overwritten), if it isn't
// addressable a copy will be created that is elegiable for garbage collection
// once the write has completed (via an event callback).
func (cq *CommandQueue) WriteBuffer(b *Buffer, bc BlockingCall, offset int64, src interface{}, waitList []*Event,
	e *Event) error {

	// Ensure we always have an event if not blocking, need this to set a
	// callback with the source as user data to prevent garbage collection. This
	// way the source data is guaranteed to be referenced somewhere and will not
	// be garbage collected. Once the event has completed the callback is
	// triggered, doing nothing, but removing the reference to the source data
	// allowing it to be garbage collected.
	if e == nil && bc == NonBlocking {
		e = &Event{}
	}

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandWriteBuffer
		e.CommandQueue = cq
	}

	pointer, size, err := tryPointerAndSize(src)

	// The source value is not addressable so create a local copy of it.
	if err != nil {
		var scratch [scratchSize]byte
		pointer, size = getPointerAndSize(src, unsafe.Pointer(&scratch[0]))
		src = &scratch
	}

	err = clw.EnqueueWriteBuffer(cq.id, b.id, clw.Bool(bc), clw.Size(offset), clw.Size(size), pointer,
		cq.toEvents(waitList), event)
	if err != nil {
		return err
	}

	// Set a no-op callback, just need hold a reference to the source data.
	if bc == NonBlocking {
		err = e.SetCallback(noOpEventCallback, src)
		if err != nil {
			return err
		}
	}

	return nil
}

func (cq *CommandQueue) ReadBufferRect() {

	// var event *clw.Event
	// if e != nil {
	// 	event = &e.id
	// 	e.Context = cq.Context
	// 	e.CommandType = CommandReadBuffer
	// 	e.CommandQueue = cq
	// }

}

func (cq *CommandQueue) WriteBufferRect() {

	// var event *clw.Event
	// if e != nil {
	// 	event = &e.id
	// 	e.Context = cq.Context
	// 	e.CommandType = CommandWriteBuffer
	// 	e.CommandQueue = cq
	// }

}

func (r *Rect) srcOrigin() [3]clw.Size {
	return [3]clw.Size{clw.Size(r.SrcOrigin[0]), clw.Size(r.SrcOrigin[1]), clw.Size(r.SrcOrigin[2])}
}

func (r *Rect) srcRowPitch() clw.Size {
	return clw.Size(r.SrcRowPitch)
}

func (r *Rect) srcSlicePitch() clw.Size {
	return clw.Size(r.SrcSlicePitch)
}

func (r *Rect) dstOrigin() [3]clw.Size {
	return [3]clw.Size{clw.Size(r.DstOrigin[0]), clw.Size(r.DstOrigin[1]), clw.Size(r.DstOrigin[2])}
}

func (r *Rect) dstRowPitch() clw.Size {
	return clw.Size(r.DstRowPitch)
}

func (r *Rect) dstSlicePitch() clw.Size {
	return clw.Size(r.DstSlicePitch)
}

func (r *Rect) region() [3]clw.Size {
	return [3]clw.Size{clw.Size(r.Region[0]), clw.Size(r.Region[1]), clw.Size(r.Region[2])}
}

func (bm *MappedBuffer) Float32Slice() []float32 {

	var header reflect.SliceHeader
	header.Data = uintptr(bm.pointer)
	size := int(bm.size / int64(float32Size))
	header.Len = size
	header.Cap = size

	return *(*[]float32)(unsafe.Pointer(&header))
}
<|endoftext|>


<commit_before>package dicom

import (
	"fmt"
	"io"
	"bytes"
	"encoding/binary"
)

type Encoder struct {
	err     error
	buf     *bytes.Buffer
	bo  binary.ByteOrder
}

func NewEncoder(bo binary.ByteOrder) *Encoder {
	return &Encoder{
		err: nil,
		buf : &bytes.Buffer{},
		bo: bo}
}

func (e *Encoder) SetError(err error) {
	if e.err == nil {
		e.err = err
	}
}

func (e *Encoder) Finish() ([]byte, error) {
	return e.buf.Bytes(), e.err
}

func (e *Encoder) EncodeByte(v byte) {
	binary.Write(e.buf, e.bo, &v)
}

func (e *Encoder) EncodeUint16(v uint16) {
	binary.Write(e.buf, e.bo, &v)
}

func (e *Encoder) EncodeUint32(v uint32) {
	binary.Write(e.buf, e.bo, &v)
}

func (e *Encoder) EncodeString(v string) {
	e.buf.Write([]byte(v))
}

func (e *Encoder) EncodeZeros(len int) {
	// TODO(saito) reuse the buffer!
	zeros := make([]byte, len)
	e.buf.Write(zeros)
}

func (e *Encoder) EncodeBytes(v []byte) {
	e.buf.Write(v)
}

type Decoder struct {
	in        io.Reader
	err       error

	bo       binary.ByteOrder
	implicit bool

	// Cumulative # bytes read.
	pos    int64
	// Max bytes to read. PushLimit() will add a new limit, and PopLimit()
	// will restore the old limit. The newest limit is at the end.
	//
	// INVARIANT: limits[] store values in decreasing order.
	limits []int64
}

func NewDecoder(
	in io.Reader,
	limit int64,
	bo binary.ByteOrder,
	implicit bool) *Decoder {
	return &Decoder{
		in: in,
		err: nil,
		bo: bo,
		implicit: implicit,
		pos: 0,
		limits: []int64{limit},
	}
}

func (d *Decoder) SetError(err error) {
 	if d.err == nil {
 		d.err = err
 	}
}

func (d *Decoder) PushLimit(limit int64) {
	d.limits = append(d.limits, d.pos + limit)
}

func (d *Decoder) PopLimit() {
	d.limits = d.limits[:len(d.limits)-1]
}

func (d *Decoder) Pos() int64 { return d.pos }

func (d *Decoder) Error() error { return d.err }

func (d *Decoder) Finish() error {
	if d.err != nil {
		return d.err
	}
	if d.Len() != 0 {
		return fmt.Errorf("Decoder found junk (%d bytes remaining)", d.Len())
	}
	return nil
}

// io.Reader implementation
func (d *Decoder) Read(p []byte) (int, error) {
	desired := d.Len()
	if desired == 0 {
		if len(p)==0 {return 0, nil}
		return 0, io.EOF
	}
	if desired < int64(len(p)) {
		p = p[:desired]
		desired = int64(len(p))
	}
	n, err := d.in.Read(p)
	if err == nil {
		d.pos += int64(n)
	}
	return n, err
}

func (d *Decoder) Len() int64 {
	return d.limits[len(d.limits)-1] - d.pos
}

func (d *Decoder) DecodeByte() (v byte) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
		return 0
	}
	return v
}

func (d *Decoder) DecodeUInt32() (v uint32) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeInt32() (v int32) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeUInt16() (v uint16) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeInt16() (v int16) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeFloat32() (v float32) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeFloat64() (v float64) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeString(length int) string {
	return string(d.DecodeBytes(length))
}

func (d *Decoder) DecodeBytes(length int) []byte {
	v := make([]byte, length)
	remaining := v
	for len(remaining) > 0 {
		n, err := d.Read(v)
		if err != nil {
			d.err = err
			break
		}
		remaining = remaining[n:]
	}
	//doassert(d.err==nil)
	if len(remaining) > 0 {
		d.err = fmt.Errorf("DecodeBytes: requested %d, remaining %d",
			length, len(remaining))
		panic(d.err)  // TODO(saito) remove
	}
	return v
}

func (d *Decoder) Skip(bytes int) {
	junk := make([]byte, bytes)
	n, err := d.Read(junk)
	if err != nil {
		d.err = err
		return
	}
	if n != bytes {
		d.err = fmt.Errorf("Failed to skip %d bytes (read %d bytes instead)", bytes, n)
		return
	}
}
<commit_msg>Updated the doc.<commit_after>package dicom

import (
	"fmt"
	"io"
	"bytes"
	"encoding/binary"
)

type Encoder struct {
	err     error
	buf     *bytes.Buffer
	bo  binary.ByteOrder
}

func NewEncoder(bo binary.ByteOrder) *Encoder {
	return &Encoder{
		err: nil,
		buf : &bytes.Buffer{},
		bo: bo}
}

func (e *Encoder) SetError(err error) {
	if e.err == nil {
		e.err = err
	}
}

func (e *Encoder) Finish() ([]byte, error) {
	return e.buf.Bytes(), e.err
}

func (e *Encoder) EncodeByte(v byte) {
	binary.Write(e.buf, e.bo, &v)
}

func (e *Encoder) EncodeUint16(v uint16) {
	binary.Write(e.buf, e.bo, &v)
}

func (e *Encoder) EncodeUint32(v uint32) {
	binary.Write(e.buf, e.bo, &v)
}

func (e *Encoder) EncodeString(v string) {
	e.buf.Write([]byte(v))
}

// Encode an array of zero bytes.
func (e *Encoder) EncodeZeros(len int) {
	// TODO(saito) reuse the buffer!
	zeros := make([]byte, len)
	e.buf.Write(zeros)
}

// Copy the given data to the output.
func (e *Encoder) EncodeBytes(v []byte) {
	e.buf.Write(v)
}

type Decoder struct {
	in        io.Reader
	err       error

	bo       binary.ByteOrder
	implicit bool

	// Cumulative # bytes read.
	pos    int64
	// Max bytes to read. PushLimit() will add a new limit, and PopLimit()
	// will restore the old limit. The newest limit is at the end.
	//
	// INVARIANT: limits[] store values in decreasing order.
	limits []int64
}

// limit is the maximum number of read from "in". Don't pass just an arbitrary
// large number as the limit. The underlying code assumes that "limit"
// accurately bounds the end of the data.
func NewDecoder(
	in io.Reader,
	limit int64,
	bo binary.ByteOrder,
	implicit bool) *Decoder {
	return &Decoder{
		in: in,
		err: nil,
		bo: bo,
		implicit: implicit,
		pos: 0,
		limits: []int64{limit},
	}
}

func (d *Decoder) SetError(err error) {
 	if d.err == nil {
 		d.err = err
 	}
}

// Temporarily override the end of the buffer.
//
// REQUIRES: limit must be smaller than the current limit
func (d *Decoder) PushLimit(limit int64) {
	d.limits = append(d.limits, d.pos + limit)
}

// Restore the old limit overridden by PushLimit.
func (d *Decoder) PopLimit() {
	d.limits = d.limits[:len(d.limits)-1]
}

// Pos() returns the cumulative number of bytes read so far.
func (d *Decoder) Pos() int64 { return d.pos }

func (d *Decoder) Error() error { return d.err }

// Finish() must be called after using the decoder. It returns any error
// encountered during decoding.
func (d *Decoder) Finish() error {
	if d.err != nil {
		return d.err
	}
	if d.Len() != 0 {
		return fmt.Errorf("Decoder found junk (%d bytes remaining)", d.Len())
	}
	return nil
}

// io.Reader implementation
func (d *Decoder) Read(p []byte) (int, error) {
	desired := d.Len()
	if desired == 0 {
		if len(p)==0 {return 0, nil}
		return 0, io.EOF
	}
	if desired < int64(len(p)) {
		p = p[:desired]
		desired = int64(len(p))
	}
	n, err := d.in.Read(p)
	if err == nil {
		d.pos += int64(n)
	}
	return n, err
}

// Len() returns the number of bytes yet unread.
func (d *Decoder) Len() int64 {
	return d.limits[len(d.limits)-1] - d.pos
}

// DecodeByte() reads a single byte from the buffer. On EOF, it returns a junk
// value, and sets an error to be returned by Error() or Finish().
func (d *Decoder) DecodeByte() (v byte) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
		return 0
	}
	return v
}

func (d *Decoder) DecodeUInt32() (v uint32) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeInt32() (v int32) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeUInt16() (v uint16) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeInt16() (v int16) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeFloat32() (v float32) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeFloat64() (v float64) {
	err := binary.Read(d, d.bo, &v)
	if err != nil {
		d.err = err
	}
	return v
}

func (d *Decoder) DecodeString(length int) string {
	return string(d.DecodeBytes(length))
}

func (d *Decoder) DecodeBytes(length int) []byte {
	v := make([]byte, length)
	remaining := v
	for len(remaining) > 0 {
		n, err := d.Read(v)
		if err != nil {
			d.err = err
			break
		}
		remaining = remaining[n:]
	}
	//doassert(d.err==nil)
	if len(remaining) > 0 {
		d.err = fmt.Errorf("DecodeBytes: requested %d, remaining %d",
			length, len(remaining))
		panic(d.err)  // TODO(saito) remove
	}
	return v
}

func (d *Decoder) Skip(bytes int) {
	junk := make([]byte, bytes)
	n, err := d.Read(junk)
	if err != nil {
		d.err = err
		return
	}
	if n != bytes {
		d.err = fmt.Errorf("Failed to skip %d bytes (read %d bytes instead)", bytes, n)
		return
	}
}
<|endoftext|>


<commit_before>package memio

import "io"

// Buffer grants a byte slice very straightforward IO methods.
type Buffer []byte

// Read satisfies the io.Reader interface
func (s *Buffer) Read(p []byte) (int, error) {
	if len(p) == 0 {
		return 0, nil
	}
	if len(*s) == 0 {
		return 0, io.EOF
	}
	n := copy(p, *s)
	*s = (*s)[n:]
	return n, nil
}

// WriteTo satisfies the io.WriterTo interface
func (s *Buffer) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write([]byte(*s))
	*s = (*s)[n:]
	return int64(n), err
}

// Write satisfies the io.Writer interface
func (s *Buffer) Write(p []byte) (int, error) {
	*s = append(*s, p...)
	return len(p), nil
}

// ReadByte satisfies the io.ByteReader interface
func (s *Buffer) ReadByte() (byte, error) {
	if len(*s) == 0 {
		return 0, io.EOF
	}
	b := (*s)[0]
	*s = (*s)[1:]
	return b, nil
}

// WriteByte satisfies the io.ByteWriter interface
func (s *Buffer) WriteByte(b byte) error {
	*s = append(*s, b)
	return nil
}

// Close satisfies the io.Closer interface
func (s *Buffer) Close() error {
	*s = nil
	return nil
}
<commit_msg>added WriteString and Peek methods to Buffer<commit_after>package memio

import "io"

// Buffer grants a byte slice very straightforward IO methods.
type Buffer []byte

// Read satisfies the io.Reader interface
func (s *Buffer) Read(p []byte) (int, error) {
	if len(p) == 0 {
		return 0, nil
	}
	if len(*s) == 0 {
		return 0, io.EOF
	}
	n := copy(p, *s)
	*s = (*s)[n:]
	return n, nil
}

// WriteTo satisfies the io.WriterTo interface
func (s *Buffer) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write([]byte(*s))
	*s = (*s)[n:]
	return int64(n), err
}

// Write satisfies the io.Writer interface
func (s *Buffer) Write(p []byte) (int, error) {
	*s = append(*s, p...)
	return len(p), nil
}

// WriteString writes a string to the buffer without casting to a byte slice
func (s *Buffer) WriteString(str string) (int, error) {
	*s = append(*s, str...)
	return len(str), nil
}

// ReadByte satisfies the io.ByteReader interface
func (s *Buffer) ReadByte() (byte, error) {
	if len(*s) == 0 {
		return 0, io.EOF
	}
	b := (*s)[0]
	*s = (*s)[1:]
	return b, nil
}

// WriteByte satisfies the io.ByteWriter interface
func (s *Buffer) WriteByte(b byte) error {
	*s = append(*s, b)
	return nil
}

// Peek reads the next n bytes without advancing the position
func (s *Buffer) Peek(n int) ([]byte, error) {
	if *s == nil {
		return nil, ErrClosed
	} else if n > len(*s) {
		return *s, io.EOF
	}
	return (*s)[:n], nil
}

// Close satisfies the io.Closer interface
func (s *Buffer) Close() error {
	*s = nil
	return nil
}
<|endoftext|>


<commit_before>/* Copyright (c) 2013 Matthias S. Benkmann
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this file (originally named buffer.go) and associated documentation files
 * (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is furnished
 * to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

// Alternative to the Go standard lib's bytes package that avoids the GC problems.
package bytes

/*
#include <stdlib.h>
*/
import "C"
import "unsafe"
import "runtime"
import "errors"

const mAX_GROW_SIZE = 1024*1024
const maxInt = int(^uint(0) >> 1)

// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("bytes.Buffer: too large")

type Buffer struct {
  ptr unsafe.Pointer
  sz int
  capa int
}

// Returns nil if no space is currently allocated for the Buffer,
// otherwise a raw memory pointer to the buffer space.
func (b *Buffer) Pointer() unsafe.Pointer {
  return b.ptr
}

// Returns the total buffer space (used + unused).
func (b *Buffer) Capacity() int { return b.capa }

// grow grows the buffer to guarantee space for n more bytes.
// It returns the index where bytes should be written.
// If the buffer can't grow it will panic with ErrTooLarge.
func (b *Buffer) Grow(n int) int {
  if n > 0 {
    rest := b.capa - b.sz
    if rest < n {

      rest = n - rest  // minimum number of bytes we need to grow by

      // if we're allocating memory for the first time, set a finalizer
      if b.capa == 0 { runtime.SetFinalizer(b, (*Buffer).Reset) }
      growth := b.capa
      if growth > mAX_GROW_SIZE { growth = mAX_GROW_SIZE }
      if rest > growth { growth = rest }
      b.capa += growth
      p := C.realloc(b.ptr, C.size_t(b.capa))
      if p == nil { panic(ErrTooLarge) }
      b.ptr = p
    }
  } else if n < 0 { panic(ErrTooLarge) } // not really too large, but this case is just a precaution.

  return b.sz
}

// WriteByte appends the byte c to the buffer.
// The returned error is always nil, but is included to match bufio.Writer's
// WriteByte. If the buffer becomes too large, WriteByte will panic with ErrTooLarge.
func (b *Buffer) WriteByte(c byte) error {
  b.Grow(1)
  ((*[maxInt]byte)(b.ptr))[b.sz] = c
  b.sz++
  return nil
}

// Appends the bytes from p to the buffer. Always returns len(p), nil.
// If out of memory occurs trying to grow the buffer, the function will
// panic with ErrTooLarge.
func (b *Buffer) Write(p []byte) (n int, err error) {
  if len(p) == 0 { return 0, nil }
  b.Grow(len(p))
  b.sz += copy(((*[maxInt]byte)(b.ptr))[b.sz:b.capa], p)
  return len(p),nil
}

// Appends the bytes from s to the buffer. Always returns len(s), nil.
// If out of memory occurs trying to grow the buffer, the function will
// panic with ErrTooLarge.
func (b *Buffer) WriteString(s string) (n int, err error) {
  if len(s) == 0 { return 0, nil }
  b.Grow(len(s))
  b.sz += copy(((*[maxInt]byte)(b.ptr))[b.sz:b.capa], s)
  return len(s),nil
}

// Returns a copy of the Buffer contents as string.
func (b *Buffer) String() string {
  return C.GoStringN((*C.char)(b.ptr), C.int(b.sz))
}

// Split slices the buffer into all substrings separated by sep and returns
// a slice of the substrings between those separators. The buffer itself is
// unchanged and the strings are copies (of course).
// If sep is empty, this function panics.
func (b *Buffer) Split(sep string) []string {
  if sep == "" { panic("UTF-8 splitting like strings.Split() has is not implemented") }
  if b.sz == 0 { return []string{} }

  result := make([]string,0,2)
  buf := ((*[maxInt]byte)(b.ptr))[0:b.sz]
  last_idx := 0
  idx := 0
  ch := sep[0]
  last_possible_idx := len(buf)-len(sep)
  for idx <= last_possible_idx {
    if buf[idx] == ch {
      for k := 1; k < len(sep); k++ {
        if buf[idx+k] != sep[k] { goto notfound }
      }
      result = append(result, string(buf[last_idx:idx]))
      last_idx = idx + len(sep)
      idx = last_idx - 1
    }
  notfound:
    idx++
  }
  result = append(result, string(buf[last_idx:]))
  return result
}

// Frees the memory held by the Buffer. The Buffer remains valid and is ready
// to take new data.
func (b *Buffer) Reset() {
  C.free(b.ptr)
  b.ptr = nil
  b.sz = 0
  b.capa = 0
  runtime.SetFinalizer(b, nil)
}
<commit_msg>Rewrite of LDIFToHash to reduce GC leaks. Big improvement to issue #101 but not completely fixed.<commit_after>/* Copyright (c) 2013 Matthias S. Benkmann
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this file (originally named buffer.go) and associated documentation files
 * (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is furnished
 * to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

// Alternative to the Go standard lib's bytes package that avoids the GC problems.
package bytes

/*
#include <stdlib.h>
*/
import "C"
import "unsafe"
import "runtime"
import "errors"

const mAX_GROW_SIZE = 1024*1024
const maxInt = int(^uint(0) >> 1)

// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("bytes.Buffer: too large")

type Buffer struct {
  ptr unsafe.Pointer
  sz int
  capa int
}

// Returns nil if no space is currently allocated for the Buffer,
// otherwise a raw memory pointer to the buffer space.
func (b *Buffer) Pointer() unsafe.Pointer {
  return b.ptr
}

// Returns the total buffer space (used + unused).
func (b *Buffer) Capacity() int { return b.capa }

// grow grows the buffer to guarantee space for n more bytes.
// It returns the index where bytes should be written.
// If the buffer can't grow it will panic with ErrTooLarge.
func (b *Buffer) Grow(n int) int {
  if n > 0 {
    rest := b.capa - b.sz
    if rest < n {

      rest = n - rest  // minimum number of bytes we need to grow by

      // if we're allocating memory for the first time, set a finalizer
      if b.capa == 0 { runtime.SetFinalizer(b, (*Buffer).Reset) }
      growth := b.capa
      if growth > mAX_GROW_SIZE { growth = mAX_GROW_SIZE }
      if rest > growth { growth = rest }
      b.capa += growth
      p := C.realloc(b.ptr, C.size_t(b.capa))
      if p == nil { panic(ErrTooLarge) }
      b.ptr = p
    }
  } else if n < 0 { panic(ErrTooLarge) } // not really too large, but this case is just a precaution.

  return b.sz
}

// WriteByte appends the byte c to the buffer.
// The returned error is always nil, but is included to match bufio.Writer's
// WriteByte. If the buffer becomes too large, WriteByte will panic with ErrTooLarge.
func (b *Buffer) WriteByte(c byte) error {
  b.Grow(1)
  ((*[maxInt]byte)(b.ptr))[b.sz] = c
  b.sz++
  return nil
}

// Appends the bytes from p to the buffer. Always returns len(p), nil.
// If out of memory occurs trying to grow the buffer, the function will
// panic with ErrTooLarge.
func (b *Buffer) Write(p []byte) (n int, err error) {
  if len(p) == 0 { return 0, nil }
  b.Grow(len(p))
  b.sz += copy(((*[maxInt]byte)(b.ptr))[b.sz:b.capa], p)
  return len(p),nil
}

// Appends the bytes from s to the buffer. Always returns len(s), nil.
// If out of memory occurs trying to grow the buffer, the function will
// panic with ErrTooLarge.
func (b *Buffer) WriteString(s string) (n int, err error) {
  if len(s) == 0 { return 0, nil }
  b.Grow(len(s))
  b.sz += copy(((*[maxInt]byte)(b.ptr))[b.sz:b.capa], s)
  return len(s),nil
}

// Returns a copy of the Buffer contents as string.
func (b *Buffer) String() string {
  return C.GoStringN((*C.char)(b.ptr), C.int(b.sz))
}

// Returns a byte slice that directly accesses the buffer's data.
// WARNING! Appending anything to the buffer invalidates any
// slices obtained via this function. They may end up pointing at
// invalid memory locations.
//
// NOTE: The return value is always a valid slice, even if the
// buffer is empty. The function never returns nil.
func (b *Buffer) Bytes() []byte {
  if b.ptr == nil { return []byte{} }
  return ((*[maxInt]byte)(b.ptr))[0:b.sz]
}

// Split slices the buffer into all substrings separated by sep and returns
// a slice of the substrings between those separators. The buffer itself is
// unchanged and the strings are copies (of course).
// If sep is empty, this function panics.
func (b *Buffer) Split(sep string) []string {
  if sep == "" { panic("UTF-8 splitting like strings.Split() has is not implemented") }
  if b.sz == 0 { return []string{} }

  result := make([]string,0,2)
  buf := ((*[maxInt]byte)(b.ptr))[0:b.sz]
  last_idx := 0
  idx := 0
  ch := sep[0]
  last_possible_idx := len(buf)-len(sep)
  for idx <= last_possible_idx {
    if buf[idx] == ch {
      for k := 1; k < len(sep); k++ {
        if buf[idx+k] != sep[k] { goto notfound }
      }
      result = append(result, string(buf[last_idx:idx]))
      last_idx = idx + len(sep)
      idx = last_idx - 1
    }
  notfound:
    idx++
  }
  result = append(result, string(buf[last_idx:]))
  return result
}

// Frees the memory held by the Buffer. The Buffer remains valid and is ready
// to take new data.
func (b *Buffer) Reset() {
  C.free(b.ptr)
  b.ptr = nil
  b.sz = 0
  b.capa = 0
  runtime.SetFinalizer(b, nil)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package search

import (
	"fmt"
	"reflect"

	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/size"
)

var reflectStaticSizeDocumentMatch int
var reflectStaticSizeSearchContext int
var reflectStaticSizeLocation int

func init() {
	var dm DocumentMatch
	reflectStaticSizeDocumentMatch = int(reflect.TypeOf(dm).Size())
	var sc SearchContext
	reflectStaticSizeSearchContext = int(reflect.TypeOf(sc).Size())
	var l Location
	reflectStaticSizeLocation = int(reflect.TypeOf(l).Size())
}

type ArrayPositions []uint64

func (ap ArrayPositions) Equals(other ArrayPositions) bool {
	if len(ap) != len(other) {
		return false
	}
	for i := range ap {
		if ap[i] != other[i] {
			return false
		}
	}
	return true
}

type Location struct {
	// Pos is the position of the term within the field, starting at 1
	Pos uint64 `json:"pos"`

	// Start and End are the byte offsets of the term in the field
	Start uint64 `json:"start"`
	End   uint64 `json:"end"`

	// ArrayPositions contains the positions of the term within any elements.
	ArrayPositions ArrayPositions `json:"array_positions"`
}

func (l *Location) Size() int {
	return reflectStaticSizeLocation + size.SizeOfPtr +
		len(l.ArrayPositions)*size.SizeOfUint64
}

type Locations []*Location

type TermLocationMap map[string]Locations

func (t TermLocationMap) AddLocation(term string, location *Location) {
	t[term] = append(t[term], location)
}

type FieldTermLocationMap map[string]TermLocationMap

type FieldTermLocation struct {
	Field    string
	Term     string
	Location Location
}

type FieldFragmentMap map[string][]string

type DocumentMatch struct {
	Index           string                `json:"index,omitempty"`
	ID              string                `json:"id"`
	IndexInternalID index.IndexInternalID `json:"-"`
	Score           float64               `json:"score"`
	Expl            *Explanation          `json:"explanation,omitempty"`
	Locations       FieldTermLocationMap  `json:"locations,omitempty"`
	Fragments       FieldFragmentMap      `json:"fragments,omitempty"`
	Sort            []string              `json:"sort,omitempty"`

	// Fields contains the values for document fields listed in
	// SearchRequest.Fields. Text fields are returned as strings, numeric
	// fields as float64s and date fields as time.RFC3339 formatted strings.
	Fields map[string]interface{} `json:"fields,omitempty"`

	// if we load the document for this hit, remember it so we dont load again
	Document *document.Document `json:"-"`

	// used to maintain natural index order
	HitNumber uint64 `json:"-"`

	// used to temporarily hold field term location information during
	// search processing in an efficient, recycle-friendly manner, to
	// be later incorporated into the Locations map when search
	// results are completed
	FieldTermLocations []FieldTermLocation `json:"-"`
}

func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) {
	if dm.Fields == nil {
		dm.Fields = make(map[string]interface{})
	}
	existingVal, ok := dm.Fields[name]
	if !ok {
		dm.Fields[name] = value
		return
	}

	valSlice, ok := existingVal.([]interface{})
	if ok {
		// already a slice, append to it
		valSlice = append(valSlice, value)
	} else {
		// create a slice
		valSlice = []interface{}{existingVal, value}
	}
	dm.Fields[name] = valSlice
}

// Reset allows an already allocated DocumentMatch to be reused
func (dm *DocumentMatch) Reset() *DocumentMatch {
	// remember the []byte used for the IndexInternalID
	indexInternalID := dm.IndexInternalID
	// remember the []interface{} used for sort
	sort := dm.Sort
	// remember the FieldTermLocations backing array
	ftls := dm.FieldTermLocations
	for i := range ftls { // recycle the ArrayPositions of each location
		ftls[i].Location.ArrayPositions = ftls[i].Location.ArrayPositions[:0]
	}
	// idiom to copy over from empty DocumentMatch (0 allocations)
	*dm = DocumentMatch{}
	// reuse the []byte already allocated (and reset len to 0)
	dm.IndexInternalID = indexInternalID[:0]
	// reuse the []interface{} already allocated (and reset len to 0)
	dm.Sort = sort[:0]
	// reuse the FieldTermLocations already allocated (and reset len to 0)
	dm.FieldTermLocations = ftls[:0]
	return dm
}

func (dm *DocumentMatch) Size() int {
	sizeInBytes := reflectStaticSizeDocumentMatch + size.SizeOfPtr +
		len(dm.Index) +
		len(dm.ID) +
		len(dm.IndexInternalID)

	if dm.Expl != nil {
		sizeInBytes += dm.Expl.Size()
	}

	for k, v := range dm.Locations {
		sizeInBytes += size.SizeOfString + len(k)
		for k1, v1 := range v {
			sizeInBytes += size.SizeOfString + len(k1) +
				size.SizeOfSlice
			for _, entry := range v1 {
				sizeInBytes += entry.Size()
			}
		}
	}

	for k, v := range dm.Fragments {
		sizeInBytes += size.SizeOfString + len(k) +
			size.SizeOfSlice

		for _, entry := range v {
			sizeInBytes += size.SizeOfString + len(entry)
		}
	}

	for _, entry := range dm.Sort {
		sizeInBytes += size.SizeOfString + len(entry)
	}

	for k, _ := range dm.Fields {
		sizeInBytes += size.SizeOfString + len(k) +
			size.SizeOfPtr
	}

	if dm.Document != nil {
		sizeInBytes += dm.Document.Size()
	}

	return sizeInBytes
}

// Complete performs final preparation & transformation of the
// DocumentMatch at the end of search processing, also allowing the
// caller to provide an optional preallocated locations slice
func (dm *DocumentMatch) Complete(prealloc []Location) []Location {
	// transform the FieldTermLocations slice into the Locations map
	nlocs := len(dm.FieldTermLocations)
	if nlocs > 0 {
		if cap(prealloc) < nlocs {
			prealloc = make([]Location, nlocs)
		}
		prealloc = prealloc[:nlocs]

		var lastField string
		var tlm TermLocationMap

		for i, ftl := range dm.FieldTermLocations {
			if lastField != ftl.Field {
				lastField = ftl.Field

				if dm.Locations == nil {
					dm.Locations = make(FieldTermLocationMap)
				}

				tlm = dm.Locations[ftl.Field]
				if tlm == nil {
					tlm = make(TermLocationMap)
					dm.Locations[ftl.Field] = tlm
				}
			}

			loc := &prealloc[i]
			*loc = ftl.Location

			if len(loc.ArrayPositions) > 0 { // copy
				loc.ArrayPositions = append(ArrayPositions(nil), loc.ArrayPositions...)
			}

			tlm[ftl.Term] = append(tlm[ftl.Term], loc)

			dm.FieldTermLocations[i] = FieldTermLocation{ // recycle
				Location: Location{
					ArrayPositions: ftl.Location.ArrayPositions[:0],
				},
			}
		}
	}

	dm.FieldTermLocations = dm.FieldTermLocations[:0] // recycle

	return prealloc
}

func (dm *DocumentMatch) String() string {
	return fmt.Sprintf("[%s-%f]", string(dm.IndexInternalID), dm.Score)
}

type DocumentMatchCollection []*DocumentMatch

func (c DocumentMatchCollection) Len() int           { return len(c) }
func (c DocumentMatchCollection) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }
func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score }

type Searcher interface {
	Next(ctx *SearchContext) (*DocumentMatch, error)
	Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error)
	Close() error
	Weight() float64
	SetQueryNorm(float64)
	Count() uint64
	Min() int
	Size() int

	DocumentMatchPoolSize() int
}

type SearcherOptions struct {
	Explain            bool
	IncludeTermVectors bool
}

// SearchContext represents the context around a single search
type SearchContext struct {
	DocumentMatchPool *DocumentMatchPool
}

func (sc *SearchContext) Size() int {
	sizeInBytes := reflectStaticSizeSearchContext + size.SizeOfPtr +
		reflectStaticSizeDocumentMatchPool + size.SizeOfPtr

	if sc.DocumentMatchPool != nil {
		for _, entry := range sc.DocumentMatchPool.avail {
			if entry != nil {
				sizeInBytes += entry.Size()
			}
		}
	}

	return sizeInBytes
}
<commit_msg>cleaning up DocumentMatch<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package search

import (
	"fmt"
	"reflect"

	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/size"
)

var reflectStaticSizeDocumentMatch int
var reflectStaticSizeSearchContext int
var reflectStaticSizeLocation int

func init() {
	var dm DocumentMatch
	reflectStaticSizeDocumentMatch = int(reflect.TypeOf(dm).Size())
	var sc SearchContext
	reflectStaticSizeSearchContext = int(reflect.TypeOf(sc).Size())
	var l Location
	reflectStaticSizeLocation = int(reflect.TypeOf(l).Size())
}

type ArrayPositions []uint64

func (ap ArrayPositions) Equals(other ArrayPositions) bool {
	if len(ap) != len(other) {
		return false
	}
	for i := range ap {
		if ap[i] != other[i] {
			return false
		}
	}
	return true
}

type Location struct {
	// Pos is the position of the term within the field, starting at 1
	Pos uint64 `json:"pos"`

	// Start and End are the byte offsets of the term in the field
	Start uint64 `json:"start"`
	End   uint64 `json:"end"`

	// ArrayPositions contains the positions of the term within any elements.
	ArrayPositions ArrayPositions `json:"array_positions"`
}

func (l *Location) Size() int {
	return reflectStaticSizeLocation + size.SizeOfPtr +
		len(l.ArrayPositions)*size.SizeOfUint64
}

type Locations []*Location

type TermLocationMap map[string]Locations

func (t TermLocationMap) AddLocation(term string, location *Location) {
	t[term] = append(t[term], location)
}

type FieldTermLocationMap map[string]TermLocationMap

type FieldTermLocation struct {
	Field    string
	Term     string
	Location Location
}

type FieldFragmentMap map[string][]string

type DocumentMatch struct {
	Index           string                `json:"index,omitempty"`
	ID              string                `json:"id"`
	IndexInternalID index.IndexInternalID `json:"-"`
	Score           float64               `json:"score"`
	Expl            *Explanation          `json:"explanation,omitempty"`
	Locations       FieldTermLocationMap  `json:"locations,omitempty"`
	Fragments       FieldFragmentMap      `json:"fragments,omitempty"`
	Sort            []string              `json:"sort,omitempty"`

	// Fields contains the values for document fields listed in
	// SearchRequest.Fields. Text fields are returned as strings, numeric
	// fields as float64s and date fields as time.RFC3339 formatted strings.
	Fields map[string]interface{} `json:"fields,omitempty"`

	// used to maintain natural index order
	HitNumber uint64 `json:"-"`

	// used to temporarily hold field term location information during
	// search processing in an efficient, recycle-friendly manner, to
	// be later incorporated into the Locations map when search
	// results are completed
	FieldTermLocations []FieldTermLocation `json:"-"`
}

func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) {
	if dm.Fields == nil {
		dm.Fields = make(map[string]interface{})
	}
	existingVal, ok := dm.Fields[name]
	if !ok {
		dm.Fields[name] = value
		return
	}

	valSlice, ok := existingVal.([]interface{})
	if ok {
		// already a slice, append to it
		valSlice = append(valSlice, value)
	} else {
		// create a slice
		valSlice = []interface{}{existingVal, value}
	}
	dm.Fields[name] = valSlice
}

// Reset allows an already allocated DocumentMatch to be reused
func (dm *DocumentMatch) Reset() *DocumentMatch {
	// remember the []byte used for the IndexInternalID
	indexInternalID := dm.IndexInternalID
	// remember the []interface{} used for sort
	sort := dm.Sort
	// remember the FieldTermLocations backing array
	ftls := dm.FieldTermLocations
	for i := range ftls { // recycle the ArrayPositions of each location
		ftls[i].Location.ArrayPositions = ftls[i].Location.ArrayPositions[:0]
	}
	// idiom to copy over from empty DocumentMatch (0 allocations)
	*dm = DocumentMatch{}
	// reuse the []byte already allocated (and reset len to 0)
	dm.IndexInternalID = indexInternalID[:0]
	// reuse the []interface{} already allocated (and reset len to 0)
	dm.Sort = sort[:0]
	// reuse the FieldTermLocations already allocated (and reset len to 0)
	dm.FieldTermLocations = ftls[:0]
	return dm
}

func (dm *DocumentMatch) Size() int {
	sizeInBytes := reflectStaticSizeDocumentMatch + size.SizeOfPtr +
		len(dm.Index) +
		len(dm.ID) +
		len(dm.IndexInternalID)

	if dm.Expl != nil {
		sizeInBytes += dm.Expl.Size()
	}

	for k, v := range dm.Locations {
		sizeInBytes += size.SizeOfString + len(k)
		for k1, v1 := range v {
			sizeInBytes += size.SizeOfString + len(k1) +
				size.SizeOfSlice
			for _, entry := range v1 {
				sizeInBytes += entry.Size()
			}
		}
	}

	for k, v := range dm.Fragments {
		sizeInBytes += size.SizeOfString + len(k) +
			size.SizeOfSlice

		for _, entry := range v {
			sizeInBytes += size.SizeOfString + len(entry)
		}
	}

	for _, entry := range dm.Sort {
		sizeInBytes += size.SizeOfString + len(entry)
	}

	for k, _ := range dm.Fields {
		sizeInBytes += size.SizeOfString + len(k) +
			size.SizeOfPtr
	}

	return sizeInBytes
}

// Complete performs final preparation & transformation of the
// DocumentMatch at the end of search processing, also allowing the
// caller to provide an optional preallocated locations slice
func (dm *DocumentMatch) Complete(prealloc []Location) []Location {
	// transform the FieldTermLocations slice into the Locations map
	nlocs := len(dm.FieldTermLocations)
	if nlocs > 0 {
		if cap(prealloc) < nlocs {
			prealloc = make([]Location, nlocs)
		}
		prealloc = prealloc[:nlocs]

		var lastField string
		var tlm TermLocationMap

		for i, ftl := range dm.FieldTermLocations {
			if lastField != ftl.Field {
				lastField = ftl.Field

				if dm.Locations == nil {
					dm.Locations = make(FieldTermLocationMap)
				}

				tlm = dm.Locations[ftl.Field]
				if tlm == nil {
					tlm = make(TermLocationMap)
					dm.Locations[ftl.Field] = tlm
				}
			}

			loc := &prealloc[i]
			*loc = ftl.Location

			if len(loc.ArrayPositions) > 0 { // copy
				loc.ArrayPositions = append(ArrayPositions(nil), loc.ArrayPositions...)
			}

			tlm[ftl.Term] = append(tlm[ftl.Term], loc)

			dm.FieldTermLocations[i] = FieldTermLocation{ // recycle
				Location: Location{
					ArrayPositions: ftl.Location.ArrayPositions[:0],
				},
			}
		}
	}

	dm.FieldTermLocations = dm.FieldTermLocations[:0] // recycle

	return prealloc
}

func (dm *DocumentMatch) String() string {
	return fmt.Sprintf("[%s-%f]", string(dm.IndexInternalID), dm.Score)
}

type DocumentMatchCollection []*DocumentMatch

func (c DocumentMatchCollection) Len() int           { return len(c) }
func (c DocumentMatchCollection) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }
func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score }

type Searcher interface {
	Next(ctx *SearchContext) (*DocumentMatch, error)
	Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error)
	Close() error
	Weight() float64
	SetQueryNorm(float64)
	Count() uint64
	Min() int
	Size() int

	DocumentMatchPoolSize() int
}

type SearcherOptions struct {
	Explain            bool
	IncludeTermVectors bool
}

// SearchContext represents the context around a single search
type SearchContext struct {
	DocumentMatchPool *DocumentMatchPool
}

func (sc *SearchContext) Size() int {
	sizeInBytes := reflectStaticSizeSearchContext + size.SizeOfPtr +
		reflectStaticSizeDocumentMatchPool + size.SizeOfPtr

	if sc.DocumentMatchPool != nil {
		for _, entry := range sc.DocumentMatchPool.avail {
			if entry != nil {
				sizeInBytes += entry.Size()
			}
		}
	}

	return sizeInBytes
}
<|endoftext|>


<commit_before>package vantage

import (
	"fmt"
	"io"
	"log"
	//"sort"
	"time"
)

const PAGE_COUNT = 513
const PAGE_SIZE = 267
const RECORDS_PER_PAGE = 5
const DATA_RECORD_LENGTH = 52

// Rev B
type ArchiveRecord struct {
	ArchivePage       int
	ArchivePageRecord int
	ArchiveTime       time.Time
	OutsideTemp       float32
	HighOutsideTemp   float32
	LowOutsideTemp    float32
	Rainfall          int
	HighRainRate      int
	Barometer         float32
	SolarRad          int
	WindSamples       int
	InsideTemp        float32
	InsideHumidity    int
	OutsideHumidity   int
	WindAvg           int
	WindMax           int
	WindMaxDir        int
	WindDir           int
	UVIndexAvg        float32
	ET                float32
	HighSolarRad      int
	UVIndexMax        int
	ForecastRule      int
	LeafTemp          []int //2
	LeafWetness       []int //2
	SoilTemp          []int //4
	RecordType        int
	ExtraHumidities   []int //2
	ExtraTemps        []int //3
	SoilMoistures     []int //4
}

type sortedArchive []*ArchiveRecord

func (sa sortedArchive) Len() int           { return len(sa) }
func (sa sortedArchive) Swap(i, j int)      { sa[i], sa[j] = sa[j], sa[i] }
func (sa sortedArchive) Less(i, j int) bool { return sa[i].ArchiveTime.Before(sa[j].ArchiveTime) }

func (vc *Conn) GetArchiveRecords() ([]*ArchiveRecord, error) {
	ars := make(sortedArchive, 0, PAGE_COUNT*RECORDS_PER_PAGE)
	archiveChan := make(chan *ArchiveRecord, 10)
	errChan := make(chan error, 1)

	err := vc.GetArchiveStream(archiveChan, errChan)
	if err != nil {
		return nil, err
	}
	for {
		select {
		case ar := <-archiveChan:
			if ar == nil {
				// Channel closed
				//sort.Sort(ars)
				return ars, nil
			}
			ars = append(ars, ar)
		case err = <-errChan:
			return ars, err
		}
	}
}

func (vc *Conn) GetArchiveStream(archiveChan chan *ArchiveRecord, errChan chan error) error {
	err := vc.sendAckCommand("DMP\n")
	if err != nil {
		return fmt.Errorf("DMP command failed: %v", err)
	}
	go vc.dmpArchive(archiveChan, errChan)
	return nil
}

func (vc *Conn) dmpArchive(archiveChan chan *ArchiveRecord, errChan chan error) {
	_, err := vc.conn.Write([]byte{ACK})
	if err != nil {
		errChan <- fmt.Errorf("Error first DMP ACK: %v\n", err)
		return
	}
	/*
		j := 0
		for {
			vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
			n, err := vc.conn.Read(pkt[j:])
			if err != nil {
				errChan <- fmt.Errorf("Err: %v\nDMP data: %v\n", err, pkt[:j+n])
				return
			}
			j += n
		}
	*/
	pkt := make([]byte, PAGE_SIZE)
	for i := 0; i < PAGE_COUNT; i++ {
		vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
		c, err := io.ReadFull(vc.buf, pkt)
		if err != nil {
			if c > 0 {
				log.Printf("Got bytes: %v", pkt[:c])
			}
			errChan <- fmt.Errorf("Error during DMP read: %v\n", err)
			return
		}
		log.Printf("%v Got pkt: %v\n", i, pkt[0:10])
		ars, err := parseArchive(pkt)
		if err != nil {
			//TODO
		}
		for _, ar := range ars {
			archiveChan <- ar
		}
		var toSend byte = ACK
		if i > 5 {
			toSend = ESC
		}
		_, err = vc.conn.Write([]byte{toSend})
		if err != nil {
			errChan <- fmt.Errorf("Error during DMP ACK: %v\n", err)
			return
		}
		if toSend == ESC {
			break
		}
	}
	close(archiveChan)
}

func parseArchive(pkt []byte) ([]*ArchiveRecord, error) {
	ret := make([]*ArchiveRecord, 0, 5)
	for i := 0; i < 5; i++ {
		dr := pkt[i*DATA_RECORD_LENGTH+1 : (i+1)*DATA_RECORD_LENGTH]
		tm := parseArchiveTime(toInt(dr[0], dr[1]), toInt(dr[2], dr[3]))
		if tm == (time.Time{}) {
			continue
		}
		// TODO CRC
		ar := &ArchiveRecord{
			ArchivePage:       int(pkt[0]),
			ArchivePageRecord: i,
			ArchiveTime:       tm,
			OutsideTemp:       float32(toInt(dr[4], dr[5])) / 10,
			HighOutsideTemp:   float32(toInt(dr[6], dr[7])) / 10,
			LowOutsideTemp:    float32(toInt(dr[8], dr[9])) / 10,
			Rainfall:          toInt(dr[10], dr[11]),
			HighRainRate:      toInt(dr[12], dr[13]),
			Barometer:         float32(toInt(dr[14], dr[15])) / 1000,
			SolarRad:          toInt(dr[16], dr[17]),
			WindSamples:       toInt(dr[18], dr[19]),
			InsideTemp:        float32(toInt(dr[20], dr[21])) / 10,
			InsideHumidity:    int(dr[22]),
			OutsideHumidity:   int(dr[23]),
			WindAvg:           int(dr[24]),
			WindMax:           int(dr[25]),
			WindMaxDir:        archiveDirectionLookup[int(26)],
			WindDir:           archiveDirectionLookup[int(27)],
			UVIndexAvg:        float32(int(dr[28])) / 10,
			ET:                float32(int(dr[29])) / 1000,
			HighSolarRad:      toInt(dr[30], dr[31]),
			UVIndexMax:        int(dr[32]),
			ForecastRule:      int(dr[33]),
			LeafTemp:          nil,
			LeafWetness:       nil,
			SoilTemp:          nil,
			RecordType:        int(dr[42]),
			ExtraHumidities:   nil,
			ExtraTemps:        nil,
			SoilMoistures:     nil,
		}
		ret = append(ret, ar)
	}
	return ret, nil
}

var archiveDirectionLookup map[int]int = map[int]int{
	0:   0,   // N
	1:   22,  // NNE
	2:   45,  // NE
	3:   67,  // ENE
	4:   90,  // E
	5:   112, // ESE
	6:   135, // SE
	7:   157, // SSE
	8:   180, // S
	9:   202, // SSW
	10:  225, // SW
	11:  247, // WSW
	12:  270, // W
	13:  292, // WNW
	14:  315, // NW
	15:  337, // NNW
	255: 0,
}

func parseArchiveTime(dt, tm int) time.Time {
	if dt == 0 {
		return time.Time{}
	}
	day := dt & 0x1f                     // lower 5 bits
	month := time.Month((dt >> 5) & 0xF) // 4 bits
	year := (dt >> 9) + 2000             // 7 bits
	hour := tm / 100
	min := tm - hour

	return time.Date(year, month, day, hour, min, 0, 0, time.Local)
}
<commit_msg>more parsing debug<commit_after>package vantage

import (
	"fmt"
	"io"
	"log"
	//"sort"
	"time"
)

const PAGE_COUNT = 513
const PAGE_SIZE = 267
const RECORDS_PER_PAGE = 5
const DATA_RECORD_LENGTH = 52

// Rev B
type ArchiveRecord struct {
	ArchivePage       int
	ArchivePageRecord int
	ArchiveTime       time.Time
	OutsideTemp       float32
	HighOutsideTemp   float32
	LowOutsideTemp    float32
	Rainfall          int
	HighRainRate      int
	Barometer         float32
	SolarRad          int
	WindSamples       int
	InsideTemp        float32
	InsideHumidity    int
	OutsideHumidity   int
	WindAvg           int
	WindMax           int
	WindMaxDir        int
	WindDir           int
	UVIndexAvg        float32
	ET                float32
	HighSolarRad      int
	UVIndexMax        int
	ForecastRule      int
	LeafTemp          []int //2
	LeafWetness       []int //2
	SoilTemp          []int //4
	RecordType        int
	ExtraHumidities   []int //2
	ExtraTemps        []int //3
	SoilMoistures     []int //4
}

type sortedArchive []*ArchiveRecord

func (sa sortedArchive) Len() int           { return len(sa) }
func (sa sortedArchive) Swap(i, j int)      { sa[i], sa[j] = sa[j], sa[i] }
func (sa sortedArchive) Less(i, j int) bool { return sa[i].ArchiveTime.Before(sa[j].ArchiveTime) }

func (vc *Conn) GetArchiveRecords() ([]*ArchiveRecord, error) {
	ars := make(sortedArchive, 0, PAGE_COUNT*RECORDS_PER_PAGE)
	archiveChan := make(chan *ArchiveRecord, 10)
	errChan := make(chan error, 1)

	err := vc.GetArchiveStream(archiveChan, errChan)
	if err != nil {
		return nil, err
	}
	for {
		select {
		case ar := <-archiveChan:
			if ar == nil {
				// Channel closed
				//sort.Sort(ars)
				return ars, nil
			}
			ars = append(ars, ar)
		case err = <-errChan:
			return ars, err
		}
	}
}

func (vc *Conn) GetArchiveStream(archiveChan chan *ArchiveRecord, errChan chan error) error {
	err := vc.sendAckCommand("DMP\n")
	if err != nil {
		return fmt.Errorf("DMP command failed: %v", err)
	}
	go vc.dmpArchive(archiveChan, errChan)
	return nil
}

func (vc *Conn) dmpArchive(archiveChan chan *ArchiveRecord, errChan chan error) {
	_, err := vc.conn.Write([]byte{ACK})
	if err != nil {
		errChan <- fmt.Errorf("Error first DMP ACK: %v\n", err)
		return
	}
	/*
		j := 0
		for {
			vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
			n, err := vc.conn.Read(pkt[j:])
			if err != nil {
				errChan <- fmt.Errorf("Err: %v\nDMP data: %v\n", err, pkt[:j+n])
				return
			}
			j += n
		}
	*/
	pkt := make([]byte, PAGE_SIZE)
	for i := 0; i < PAGE_COUNT; i++ {
		vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
		c, err := io.ReadFull(vc.buf, pkt)
		if err != nil {
			if c > 0 {
				log.Printf("Got bytes: %v", pkt[:c])
			}
			errChan <- fmt.Errorf("Error during DMP read: %v\n", err)
			return
		}
		log.Printf("%v of %v Got pkt: %v..%v\n", i, c, pkt[0:10], pkt[len(pkt)-10:])
		ars, err := parseArchive(pkt)
		if err != nil {
			//TODO
		}
		for _, ar := range ars {
			archiveChan <- ar
		}
		var toSend byte = ACK
		if i > 5 {
			toSend = ESC
		}
		_, err = vc.conn.Write([]byte{toSend})
		if err != nil {
			errChan <- fmt.Errorf("Error during DMP ACK: %v\n", err)
			return
		}
		if toSend == ESC {
			break
		}
	}
	close(archiveChan)
}

func parseArchive(pkt []byte) ([]*ArchiveRecord, error) {
	ret := make([]*ArchiveRecord, 0, 5)
	for i := 0; i < 5; i++ {
		dr := pkt[i*DATA_RECORD_LENGTH+1 : (i+1)*DATA_RECORD_LENGTH]
		tm := parseArchiveTime(toInt(dr[0], dr[1]), toInt(dr[2], dr[3]))
		if tm == (time.Time{}) {
			continue
		}
		// TODO CRC
		ar := &ArchiveRecord{
			ArchivePage:       int(pkt[0]),
			ArchivePageRecord: i,
			ArchiveTime:       tm,
			OutsideTemp:       float32(toInt(dr[4], dr[5])) / 10,
			HighOutsideTemp:   float32(toInt(dr[6], dr[7])) / 10,
			LowOutsideTemp:    float32(toInt(dr[8], dr[9])) / 10,
			Rainfall:          toInt(dr[10], dr[11]),
			HighRainRate:      toInt(dr[12], dr[13]),
			Barometer:         float32(toInt(dr[14], dr[15])) / 1000,
			SolarRad:          toInt(dr[16], dr[17]),
			WindSamples:       toInt(dr[18], dr[19]),
			InsideTemp:        float32(toInt(dr[20], dr[21])) / 10,
			InsideHumidity:    int(dr[22]),
			OutsideHumidity:   int(dr[23]),
			WindAvg:           int(dr[24]),
			WindMax:           int(dr[25]),
			WindMaxDir:        archiveDirectionLookup[int(26)],
			WindDir:           archiveDirectionLookup[int(27)],
			UVIndexAvg:        float32(int(dr[28])) / 10,
			ET:                float32(int(dr[29])) / 1000,
			HighSolarRad:      toInt(dr[30], dr[31]),
			UVIndexMax:        int(dr[32]),
			ForecastRule:      int(dr[33]),
			LeafTemp:          nil,
			LeafWetness:       nil,
			SoilTemp:          nil,
			RecordType:        int(dr[42]),
			ExtraHumidities:   nil,
			ExtraTemps:        nil,
			SoilMoistures:     nil,
		}
		ret = append(ret, ar)
	}
	return ret, nil
}

var archiveDirectionLookup map[int]int = map[int]int{
	0:   0,   // N
	1:   22,  // NNE
	2:   45,  // NE
	3:   67,  // ENE
	4:   90,  // E
	5:   112, // ESE
	6:   135, // SE
	7:   157, // SSE
	8:   180, // S
	9:   202, // SSW
	10:  225, // SW
	11:  247, // WSW
	12:  270, // W
	13:  292, // WNW
	14:  315, // NW
	15:  337, // NNW
	255: 0,
}

func parseArchiveTime(dt, tm int) time.Time {
	if dt == 0 {
		return time.Time{}
	}
	day := dt & 0x1f                     // lower 5 bits
	month := time.Month((dt >> 5) & 0xF) // 4 bits
	year := (dt >> 9) + 2000             // 7 bits
	hour := tm / 100
	min := tm - hour

	return time.Date(year, month, day, hour, min, 0, 0, time.Local)
}
<|endoftext|>


<commit_before><commit_msg>fix issue in case of missing workerLifeCycleFunc<commit_after><|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nat

/*
client/server for testing CLOSE_WAIT timeout condition in iptables NAT.

client              server
  |                   |
  |<--tcp handshake-->|
  |<-------fin--------| half-close from server
  |                   | client is in CLOSE_WAIT
*/

import (
	"errors"
	"io"
	"log"
	"net"
	"time"

	"k8s.io/kubernetes/test/images/net/common"
)

// leakedConnection is a
var leakedConnection *net.TCPConn

// Server JSON options.
type CloseWaitServerOptions struct {
	// Address to bind for the test
	LocalAddr string
	// Timeout to wait after sending the FIN.
	PostFinTimeoutSeconds int
}

type closeWaitServer struct {
	options *CloseWaitServerOptions
}

// NewCloseWaitServer returns a new Runner.
func NewCloseWaitServer() common.Runner {
	return &closeWaitServer{}
}

// NewOptions allocates new options structure.
func (server *closeWaitServer) NewOptions() interface{} {
	return &CloseWaitServerOptions{}
}

// Run the server-side of the test.
func (server *closeWaitServer) Run(logger *log.Logger, rawOptions interface{}) error {
	if options, ok := rawOptions.(*CloseWaitServerOptions); ok {
		server.options = options
	} else {
		return errors.New("invalid type")
	}

	logger.Printf("Run %v", server.options)

	addr, err := net.ResolveTCPAddr("tcp", server.options.LocalAddr)
	if err != nil {
		return err
	}

	listener, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return err
	}
	defer listener.Close()

	logger.Printf("Server listening on %v", addr)

	conn, err := listener.AcceptTCP()
	if err != nil {
		return err
	}
	defer conn.Close()

	logger.Printf("Client connected")

	// Send client half-close FIN so client is now in CLOSE_WAIT. We keep
	// the client -> server pipe open to verify whether or not the NAT
	// dropped our connection.
	if err := conn.CloseWrite(); err != nil {
		return err
	}

	logger.Printf("Server sent FIN, waiting %v seconds",
		server.options.PostFinTimeoutSeconds)

	<-time.After(time.Duration(server.options.PostFinTimeoutSeconds) * time.Second)

	logger.Printf("Done")

	return nil
}

// Client JSON options
type CloseWaitClientOptions struct {
	// RemoteAddr of the server to connect to.
	RemoteAddr string
	// TimeoutSeconds on I/O with the server.
	TimeoutSeconds int
	// Half-close timeout (to give the test time to check the status of the
	// conntrack table entry.
	PostFinTimeoutSeconds int
	// Leak connection (assign to global variable so connection persists
	// as long as the process remains.
	LeakConnection bool
}

type closeWaitClient struct {
	options *CloseWaitClientOptions
}

// NewCloseWaitClient creates a new runner
func NewCloseWaitClient() common.Runner {
	return &closeWaitClient{}
}

// NewOptions allocates new options structure.
func (client *closeWaitClient) NewOptions() interface{} {
	return &CloseWaitClientOptions{}
}

// Run the client.m
func (client *closeWaitClient) Run(logger *log.Logger, rawOptions interface{}) error {
	if options, ok := rawOptions.(*CloseWaitClientOptions); ok {
		client.options = options
	} else {
		return errors.New("invalid type")
	}

	logger.Printf("Run %v", client.options)

	addr, err := net.ResolveTCPAddr("tcp", client.options.RemoteAddr)
	if err != nil {
		return err
	}

	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return err
	}
	defer conn.Close()

	logger.Printf("Connected to server")

	if client.options.TimeoutSeconds > 0 {
		delay := time.Duration(client.options.TimeoutSeconds) * time.Second
		conn.SetReadDeadline(time.Now().Add(delay))
	}

	buf := make([]byte, 1, 1)
	size, err := conn.Read(buf)

	if err != nil && err != io.EOF {
		return err
	}

	if size != 0 {
		return errors.New("Got data but expected EOF")
	}

	logger.Printf("Server has half-closed the connection, waiting %v seconds",
		client.options.PostFinTimeoutSeconds)

	if client.options.LeakConnection {
		logger.Printf("Leaking client connection (assigning to global variable)")
		leakedConnection = conn
	}

	<-time.After(
		time.Duration(client.options.PostFinTimeoutSeconds) * time.Second)

	logger.Printf("Done")

	return nil
}
<commit_msg>Should not defer conn.Close when leaking the connection<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nat

/*
client/server for testing CLOSE_WAIT timeout condition in iptables NAT.

client              server
  |                   |
  |<--tcp handshake-->|
  |<-------fin--------| half-close from server
  |                   | client is in CLOSE_WAIT
*/

import (
	"errors"
	"io"
	"log"
	"net"
	"time"

	"k8s.io/kubernetes/test/images/net/common"
)

// leakedConnection is a global variable that should leak the active
// connection assigned here.
var leakedConnection *net.TCPConn

// Server JSON options.
type CloseWaitServerOptions struct {
	// Address to bind for the test
	LocalAddr string
	// Timeout to wait after sending the FIN.
	PostFinTimeoutSeconds int
}

type closeWaitServer struct {
	options *CloseWaitServerOptions
}

// NewCloseWaitServer returns a new Runner.
func NewCloseWaitServer() common.Runner {
	return &closeWaitServer{}
}

// NewOptions allocates new options structure.
func (server *closeWaitServer) NewOptions() interface{} {
	return &CloseWaitServerOptions{}
}

// Run the server-side of the test.
func (server *closeWaitServer) Run(logger *log.Logger, rawOptions interface{}) error {
	if options, ok := rawOptions.(*CloseWaitServerOptions); ok {
		server.options = options
	} else {
		return errors.New("invalid type")
	}

	logger.Printf("Run %v", server.options)

	addr, err := net.ResolveTCPAddr("tcp", server.options.LocalAddr)
	if err != nil {
		return err
	}

	listener, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return err
	}
	defer listener.Close()

	logger.Printf("Server listening on %v", addr)

	conn, err := listener.AcceptTCP()
	if err != nil {
		return err
	}
	defer conn.Close()

	logger.Printf("Client connected")

	// Send client half-close FIN so client is now in CLOSE_WAIT. We keep
	// the client -> server pipe open to verify whether or not the NAT
	// dropped our connection.
	if err := conn.CloseWrite(); err != nil {
		return err
	}

	logger.Printf("Server sent FIN, waiting %v seconds",
		server.options.PostFinTimeoutSeconds)

	<-time.After(time.Duration(server.options.PostFinTimeoutSeconds) * time.Second)

	logger.Printf("Done")

	return nil
}

// Client JSON options
type CloseWaitClientOptions struct {
	// RemoteAddr of the server to connect to.
	RemoteAddr string
	// TimeoutSeconds on I/O with the server.
	TimeoutSeconds int
	// Half-close timeout (to give the test time to check the status of the
	// conntrack table entry.
	PostFinTimeoutSeconds int
	// Leak connection (assign to global variable so connection persists
	// as long as the process remains.
	LeakConnection bool
}

type closeWaitClient struct {
	options *CloseWaitClientOptions
}

// NewCloseWaitClient creates a new runner
func NewCloseWaitClient() common.Runner {
	return &closeWaitClient{}
}

// NewOptions allocates new options structure.
func (client *closeWaitClient) NewOptions() interface{} {
	return &CloseWaitClientOptions{}
}

// Run the client.m
func (client *closeWaitClient) Run(logger *log.Logger, rawOptions interface{}) error {
	if options, ok := rawOptions.(*CloseWaitClientOptions); ok {
		client.options = options
	} else {
		return errors.New("invalid type")
	}

	logger.Printf("Run %v", client.options)

	addr, err := net.ResolveTCPAddr("tcp", client.options.RemoteAddr)
	if err != nil {
		return err
	}

	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return err
	}
	if !client.options.LeakConnection {
		defer conn.Close()
	}

	logger.Printf("Connected to server")

	if client.options.TimeoutSeconds > 0 {
		delay := time.Duration(client.options.TimeoutSeconds) * time.Second
		conn.SetReadDeadline(time.Now().Add(delay))
	}

	buf := make([]byte, 1, 1)
	size, err := conn.Read(buf)

	if err != nil && err != io.EOF {
		return err
	}

	if size != 0 {
		return errors.New("Got data but expected EOF")
	}

	logger.Printf("Server has half-closed the connection, waiting %v seconds",
		client.options.PostFinTimeoutSeconds)

	if client.options.LeakConnection {
		logger.Printf("Leaking client connection (assigning to global variable)")
		leakedConnection = conn
	}

	<-time.After(
		time.Duration(client.options.PostFinTimeoutSeconds) * time.Second)

	logger.Printf("Done")

	return nil
}
<|endoftext|>


<commit_before><commit_msg>Remove unnecessary error checking<commit_after><|endoftext|>