<commit_before>package vm

import (
	"fmt"
	"github.com/goby-lang/goby/compiler/bytecode"
	"github.com/goby-lang/goby/vm/classes"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

// Version stores current Goby version
const Version = "0.1.8.1"

// These are the enums for marking parser's mode, which decides whether it should pop unused values.
const (
	NormalMode int = iota
	REPLMode
	TestMode
)

type isIndexTable struct {
	Data map[string]int
}

func newISIndexTable() *isIndexTable {
	return &isIndexTable{Data: make(map[string]int)}
}

type isTable map[string][]*instructionSet

type filename = string

var standardLibraries = map[string]func(*VM){
	"net/http":           initHTTPClass,
	"net/simple_server":  initSimpleServerClass,
	"uri":                initURIClass,
	"db":                 initDBClass,
	"plugin":             initPluginClass,
	"json":               initJSONClass,
	"concurrent/array":   initConcurrentArrayClass,
	"concurrent/hash":    initConcurrentHashClass,
	"concurrent/rw_lock": initConcurrentRWLockClass,
	"spec":               initSpecClass,
}

// VM represents a stack based virtual machine.
type VM struct {
	mainObj     *RObject
	mainThread  *thread
	objectClass *RClass
	// a map holds different types of instruction set tables
	isTables map[setType]isTable
	// method instruction set table
	methodISIndexTables map[filename]*isIndexTable
	// class instruction set table
	classISIndexTables map[filename]*isIndexTable
	// block instruction set table
	blockTables map[filename]map[string]*instructionSet
	// fileDir indicates executed file's directory
	fileDir string
	// args are command line arguments
	args []string
	// projectRoot is goby root's absolute path, which is $GOROOT/src/github.com/goby-lang/goby
	projectRoot string

	channelObjectMap *objectMap

	mode int

	libFiles []string
}

// New initializes a vm to initialize state and returns it.
func New(fileDir string, args []string) (vm *VM, e error) {
	vm = &VM{args: args}
	vm.mainThread = vm.newThread()

	vm.methodISIndexTables = map[filename]*isIndexTable{
		fileDir: newISIndexTable(),
	}
	vm.classISIndexTables = map[filename]*isIndexTable{
		fileDir: newISIndexTable(),
	}
	vm.blockTables = make(map[filename]map[string]*instructionSet)
	vm.isTables = map[setType]isTable{
		bytecode.MethodDef: make(isTable),
		bytecode.ClassDef:  make(isTable),
	}
	vm.fileDir = fileDir

	gobyRoot := os.Getenv("GOBY_ROOT")

	if len(gobyRoot) == 0 {
		vm.projectRoot = fmt.Sprintf("/usr/local/Cellar/goby/%s", Version)

		_, err := os.Stat(vm.projectRoot)

		if err != nil {
			path, _ := filepath.Abs("$GOPATH/src/github.com/goby-lang/goby")
			_, err = os.Stat(path)

			if err != nil {
				e = fmt.Errorf("You haven't set $GOBY_ROOT properly")
				return nil, e
			}

			vm.projectRoot = path
		}
	} else {
		vm.projectRoot = gobyRoot
	}

	vm.initConstants()
	vm.mainObj = vm.initMainObj()
	vm.channelObjectMap = &objectMap{store: &sync.Map{}}

	for _, fn := range vm.libFiles {
		err := vm.mainThread.execGobyLib(fn)
		if err != nil {
			fmt.Printf("An error occurs when loading lib file %s:\n", string(fn))
			fmt.Println(err.Error())
			break
		}
	}

	return
}

func (vm *VM) newThread() *thread {
	s := &stack{RWMutex: new(sync.RWMutex)}
	cfs := &callFrameStack{callFrames: []callFrame{}}
	t := &thread{stack: s, callFrameStack: cfs, sp: 0, cfp: 0}
	s.thread = t
	cfs.thread = t
	t.vm = vm
	return t
}

// ExecInstructions accepts a sequence of bytecodes and use vm to evaluate them.
func (vm *VM) ExecInstructions(sets []*bytecode.InstructionSet, fn string) {
	translator := newInstructionTranslator(fn)
	translator.vm = vm
	translator.transferInstructionSets(sets)

	// Keep instruction set table updated after parsed new files.
	// TODO: Find more efficient way to do this.
	for setType, table := range translator.setTable {
		for name, is := range table {
			vm.isTables[setType][name] = is
		}
	}

	vm.blockTables[translator.filename] = translator.blockTable
	vm.SetClassISIndexTable(translator.filename)
	vm.SetMethodISIndexTable(translator.filename)

	cf := newNormalCallFrame(translator.program, translator.filename, 1)
	cf.self = vm.mainObj
	vm.mainThread.callFrameStack.push(cf)

	defer func() {
		err, ok := recover().(*Error)

		if ok && vm.mode == NormalMode {
			fmt.Println(err.Message())
		}
	}()

	vm.mainThread.startFromTopFrame()
}

// SetClassISIndexTable adds new instruction set's index table to vm.classISIndexTables
func (vm *VM) SetClassISIndexTable(fn filename) {
	vm.classISIndexTables[fn] = newISIndexTable()
}

// SetMethodISIndexTable adds new instruction set's index table to vm.methodISIndexTables
func (vm *VM) SetMethodISIndexTable(fn filename) {
	vm.methodISIndexTables[fn] = newISIndexTable()
}

// main object singleton methods -----------------------------------------------------
func builtinMainObjSingletonMethods() []*BuiltinMethodObject {
	return []*BuiltinMethodObject{
		{
			Name: "to_s",
			Fn: func(receiver Object, sourceLine int) builtinMethodBody {
				return func(thread *thread, objects []Object, frame *normalCallFrame) Object {
					return thread.vm.initStringObject("main")
				}
			},
		},
	}
}

func (vm *VM) initMainObj() *RObject {
	obj := vm.objectClass.initializeInstance()
	singletonClass := vm.initializeClass(fmt.Sprintf("#<Class:%s>", obj.toString()), false)
	singletonClass.Methods.set("include", vm.topLevelClass(classes.ClassClass).lookupMethod("include"))
	singletonClass.setBuiltinMethods(builtinMainObjSingletonMethods(), false)
	obj.singletonClass = singletonClass

	return obj
}

func (vm *VM) initConstants() {
	// Init Class and Object
	cClass := initClassClass()
	vm.objectClass = initObjectClass(cClass)
	vm.topLevelClass(classes.ObjectClass).setClassConstant(cClass)

	// Init builtin classes
	builtinClasses := []*RClass{
		vm.initIntegerClass(),
		vm.initFloatClass(),
		vm.initStringClass(),
		vm.initBoolClass(),
		vm.initNullClass(),
		vm.initArrayClass(),
		vm.initHashClass(),
		vm.initRangeClass(),
		vm.initMethodClass(),
		vm.initBlockClass(),
		vm.initChannelClass(),
		vm.initGoClass(),
		vm.initFileClass(),
		vm.initRegexpClass(),
		vm.initMatchDataClass(),
		vm.initGoMapClass(),
		vm.initDecimalClass(),
	}

	// Init error classes
	vm.initErrorClasses()

	for _, c := range builtinClasses {
		vm.objectClass.setClassConstant(c)
	}

	// Init ARGV
	args := []Object{}

	for _, arg := range vm.args {
		args = append(args, vm.initStringObject(arg))
	}

	vm.objectClass.constants["ARGV"] = &Pointer{Target: vm.initArrayObject(args)}

	// Init ENV
	envs := map[string]Object{}

	for _, e := range os.Environ() {
		pair := strings.Split(e, "=")
		envs[pair[0]] = vm.initStringObject(pair[1])
	}

	vm.objectClass.constants["ENV"] = &Pointer{Target: vm.initHashObject(envs)}
	vm.objectClass.constants["STDOUT"] = &Pointer{Target: vm.initFileObject(os.Stdout)}
	vm.objectClass.constants["STDERR"] = &Pointer{Target: vm.initFileObject(os.Stderr)}
	vm.objectClass.constants["STDIN"] = &Pointer{Target: vm.initFileObject(os.Stdin)}
}

func (vm *VM) topLevelClass(cn string) *RClass {
	objClass := vm.objectClass

	if cn == classes.ObjectClass {
		return objClass
	}

	return objClass.constants[cn].Target.(*RClass)
}

func (vm *VM) currentFilePath() string {
	frame := vm.mainThread.callFrameStack.top()
	return frame.FileName()
}

// loadConstant makes sure we don't create a class twice.
func (vm *VM) loadConstant(name string, isModule bool) *RClass {
	var c *RClass
	var ptr *Pointer

	ptr = vm.objectClass.constants[name]

	if ptr == nil {
		c = vm.initializeClass(name, isModule)
		vm.objectClass.setClassConstant(c)
	} else {
		c = ptr.Target.(*RClass)
	}

	return c
}

func (vm *VM) lookupConstant(cf callFrame, constName string) (constant *Pointer) {
	var namespace *RClass
	var hasNamespace bool

	top := vm.mainThread.stack.top()

	if top == nil {
		hasNamespace = false
	} else {
		namespace, hasNamespace = top.Target.(*RClass)
	}

	if hasNamespace {
		constant = namespace.lookupConstantUnderAllScope(constName)

		if constant != nil {
			return
		}
	}

	constant = cf.lookupConstantUnderAllScope(constName)

	if constant == nil {
		constant = vm.objectClass.constants[constName]
	}

	if constName == classes.ObjectClass {
		constant = &Pointer{Target: vm.objectClass}
	}

	return
}
<commit_msg>Bump version to 0.1.9<commit_after>package vm

import (
	"fmt"
	"github.com/goby-lang/goby/compiler/bytecode"
	"github.com/goby-lang/goby/vm/classes"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

// Version stores current Goby version
const Version = "0.1.9"

// These are the enums for marking parser's mode, which decides whether it should pop unused values.
const (
	NormalMode int = iota
	REPLMode
	TestMode
)

type isIndexTable struct {
	Data map[string]int
}

func newISIndexTable() *isIndexTable {
	return &isIndexTable{Data: make(map[string]int)}
}

type isTable map[string][]*instructionSet

type filename = string

var standardLibraries = map[string]func(*VM){
	"net/http":           initHTTPClass,
	"net/simple_server":  initSimpleServerClass,
	"uri":                initURIClass,
	"db":                 initDBClass,
	"plugin":             initPluginClass,
	"json":               initJSONClass,
	"concurrent/array":   initConcurrentArrayClass,
	"concurrent/hash":    initConcurrentHashClass,
	"concurrent/rw_lock": initConcurrentRWLockClass,
	"spec":               initSpecClass,
}

// VM represents a stack based virtual machine.
type VM struct {
	mainObj     *RObject
	mainThread  *thread
	objectClass *RClass
	// a map holds different types of instruction set tables
	isTables map[setType]isTable
	// method instruction set table
	methodISIndexTables map[filename]*isIndexTable
	// class instruction set table
	classISIndexTables map[filename]*isIndexTable
	// block instruction set table
	blockTables map[filename]map[string]*instructionSet
	// fileDir indicates executed file's directory
	fileDir string
	// args are command line arguments
	args []string
	// projectRoot is goby root's absolute path, which is $GOROOT/src/github.com/goby-lang/goby
	projectRoot string

	channelObjectMap *objectMap

	mode int

	libFiles []string
}

// New initializes a vm to initialize state and returns it.
func New(fileDir string, args []string) (vm *VM, e error) {
	vm = &VM{args: args}
	vm.mainThread = vm.newThread()

	vm.methodISIndexTables = map[filename]*isIndexTable{
		fileDir: newISIndexTable(),
	}
	vm.classISIndexTables = map[filename]*isIndexTable{
		fileDir: newISIndexTable(),
	}
	vm.blockTables = make(map[filename]map[string]*instructionSet)
	vm.isTables = map[setType]isTable{
		bytecode.MethodDef: make(isTable),
		bytecode.ClassDef:  make(isTable),
	}
	vm.fileDir = fileDir

	gobyRoot := os.Getenv("GOBY_ROOT")

	if len(gobyRoot) == 0 {
		vm.projectRoot = fmt.Sprintf("/usr/local/Cellar/goby/%s", Version)

		_, err := os.Stat(vm.projectRoot)

		if err != nil {
			path, _ := filepath.Abs("$GOPATH/src/github.com/goby-lang/goby")
			_, err = os.Stat(path)

			if err != nil {
				e = fmt.Errorf("You haven't set $GOBY_ROOT properly")
				return nil, e
			}

			vm.projectRoot = path
		}
	} else {
		vm.projectRoot = gobyRoot
	}

	vm.initConstants()
	vm.mainObj = vm.initMainObj()
	vm.channelObjectMap = &objectMap{store: &sync.Map{}}

	for _, fn := range vm.libFiles {
		err := vm.mainThread.execGobyLib(fn)
		if err != nil {
			fmt.Printf("An error occurs when loading lib file %s:\n", string(fn))
			fmt.Println(err.Error())
			break
		}
	}

	return
}

func (vm *VM) newThread() *thread {
	s := &stack{RWMutex: new(sync.RWMutex)}
	cfs := &callFrameStack{callFrames: []callFrame{}}
	t := &thread{stack: s, callFrameStack: cfs, sp: 0, cfp: 0}
	s.thread = t
	cfs.thread = t
	t.vm = vm
	return t
}

// ExecInstructions accepts a sequence of bytecodes and use vm to evaluate them.
func (vm *VM) ExecInstructions(sets []*bytecode.InstructionSet, fn string) {
	translator := newInstructionTranslator(fn)
	translator.vm = vm
	translator.transferInstructionSets(sets)

	// Keep instruction set table updated after parsed new files.
	// TODO: Find more efficient way to do this.
	for setType, table := range translator.setTable {
		for name, is := range table {
			vm.isTables[setType][name] = is
		}
	}

	vm.blockTables[translator.filename] = translator.blockTable
	vm.SetClassISIndexTable(translator.filename)
	vm.SetMethodISIndexTable(translator.filename)

	cf := newNormalCallFrame(translator.program, translator.filename, 1)
	cf.self = vm.mainObj
	vm.mainThread.callFrameStack.push(cf)

	defer func() {
		err, ok := recover().(*Error)

		if ok && vm.mode == NormalMode {
			fmt.Println(err.Message())
		}
	}()

	vm.mainThread.startFromTopFrame()
}

// SetClassISIndexTable adds new instruction set's index table to vm.classISIndexTables
func (vm *VM) SetClassISIndexTable(fn filename) {
	vm.classISIndexTables[fn] = newISIndexTable()
}

// SetMethodISIndexTable adds new instruction set's index table to vm.methodISIndexTables
func (vm *VM) SetMethodISIndexTable(fn filename) {
	vm.methodISIndexTables[fn] = newISIndexTable()
}

// main object singleton methods -----------------------------------------------------
func builtinMainObjSingletonMethods() []*BuiltinMethodObject {
	return []*BuiltinMethodObject{
		{
			Name: "to_s",
			Fn: func(receiver Object, sourceLine int) builtinMethodBody {
				return func(thread *thread, objects []Object, frame *normalCallFrame) Object {
					return thread.vm.initStringObject("main")
				}
			},
		},
	}
}

func (vm *VM) initMainObj() *RObject {
	obj := vm.objectClass.initializeInstance()
	singletonClass := vm.initializeClass(fmt.Sprintf("#<Class:%s>", obj.toString()), false)
	singletonClass.Methods.set("include", vm.topLevelClass(classes.ClassClass).lookupMethod("include"))
	singletonClass.setBuiltinMethods(builtinMainObjSingletonMethods(), false)
	obj.singletonClass = singletonClass

	return obj
}

func (vm *VM) initConstants() {
	// Init Class and Object
	cClass := initClassClass()
	vm.objectClass = initObjectClass(cClass)
	vm.topLevelClass(classes.ObjectClass).setClassConstant(cClass)

	// Init builtin classes
	builtinClasses := []*RClass{
		vm.initIntegerClass(),
		vm.initFloatClass(),
		vm.initStringClass(),
		vm.initBoolClass(),
		vm.initNullClass(),
		vm.initArrayClass(),
		vm.initHashClass(),
		vm.initRangeClass(),
		vm.initMethodClass(),
		vm.initBlockClass(),
		vm.initChannelClass(),
		vm.initGoClass(),
		vm.initFileClass(),
		vm.initRegexpClass(),
		vm.initMatchDataClass(),
		vm.initGoMapClass(),
		vm.initDecimalClass(),
	}

	// Init error classes
	vm.initErrorClasses()

	for _, c := range builtinClasses {
		vm.objectClass.setClassConstant(c)
	}

	// Init ARGV
	args := []Object{}

	for _, arg := range vm.args {
		args = append(args, vm.initStringObject(arg))
	}

	vm.objectClass.constants["ARGV"] = &Pointer{Target: vm.initArrayObject(args)}

	// Init ENV
	envs := map[string]Object{}

	for _, e := range os.Environ() {
		pair := strings.Split(e, "=")
		envs[pair[0]] = vm.initStringObject(pair[1])
	}

	vm.objectClass.constants["ENV"] = &Pointer{Target: vm.initHashObject(envs)}
	vm.objectClass.constants["STDOUT"] = &Pointer{Target: vm.initFileObject(os.Stdout)}
	vm.objectClass.constants["STDERR"] = &Pointer{Target: vm.initFileObject(os.Stderr)}
	vm.objectClass.constants["STDIN"] = &Pointer{Target: vm.initFileObject(os.Stdin)}
}

func (vm *VM) topLevelClass(cn string) *RClass {
	objClass := vm.objectClass

	if cn == classes.ObjectClass {
		return objClass
	}

	return objClass.constants[cn].Target.(*RClass)
}

func (vm *VM) currentFilePath() string {
	frame := vm.mainThread.callFrameStack.top()
	return frame.FileName()
}

// loadConstant makes sure we don't create a class twice.
func (vm *VM) loadConstant(name string, isModule bool) *RClass {
	var c *RClass
	var ptr *Pointer

	ptr = vm.objectClass.constants[name]

	if ptr == nil {
		c = vm.initializeClass(name, isModule)
		vm.objectClass.setClassConstant(c)
	} else {
		c = ptr.Target.(*RClass)
	}

	return c
}

func (vm *VM) lookupConstant(cf callFrame, constName string) (constant *Pointer) {
	var namespace *RClass
	var hasNamespace bool

	top := vm.mainThread.stack.top()

	if top == nil {
		hasNamespace = false
	} else {
		namespace, hasNamespace = top.Target.(*RClass)
	}

	if hasNamespace {
		constant = namespace.lookupConstantUnderAllScope(constName)

		if constant != nil {
			return
		}
	}

	constant = cf.lookupConstantUnderAllScope(constName)

	if constant == nil {
		constant = vm.objectClass.constants[constName]
	}

	if constName == classes.ObjectClass {
		constant = &Pointer{Target: vm.objectClass}
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/golang/glog"
	"github.com/zefer/gompd/mpd"
)

type watchConn struct {
	watcher *mpd.Watcher
}

func newWatchConn(addr string) (*watchConn, error) {
	w, err := mpd.NewWatcher("tcp", addr, "")
	if err != nil {
		return nil, err
	}
	conn := &watchConn{w}
	go conn.errorLoop()
	go conn.eventLoop()

	return conn, nil
}

func (w *watchConn) Close() {
	w.watcher.Close()
}

func (w *watchConn) eventLoop() {
	for subsystem := range w.watcher.Event {
		glog.Info("Changed subsystem:", subsystem)
		broadcastStatus()
	}
}

func (w *watchConn) errorLoop() {
	for err := range w.watcher.Error {
		glog.Errorf("Watcher: %v", err)
	}
}
<commit_msg>Retry watcher connection on start-up<commit_after>package main

import (
	"time"

	"github.com/golang/glog"
	"github.com/zefer/gompd/mpd"
)

const retryDur time.Duration = time.Second * 3

type watchConn struct {
	watcher *mpd.Watcher
	addr    string
}

func newWatchConn(addr string) (*watchConn, error) {
	conn := &watchConn{addr: addr}
	conn.retryConnect()
	go conn.errorLoop()
	go conn.eventLoop()
	return conn, nil
}

func (w *watchConn) connect() error {
	watcher, err := mpd.NewWatcher("tcp", w.addr, "")
	if err != nil {
		return err
	}
	w.watcher = watcher
	return nil
}

func (w *watchConn) retryConnect() {
	for {
		err := w.connect()
		if err == nil {
			glog.Infof("Watcher: connected to %s", w.addr)
			return
		}
		glog.Errorf("Watcher: connect failed. Waiting then retrying. %v", err)
		time.Sleep(retryDur)
	}
}

func (w *watchConn) Close() {
	w.watcher.Close()
}

func (w *watchConn) eventLoop() {
	for subsystem := range w.watcher.Event {
		glog.Info("Changed subsystem:", subsystem)
		broadcastStatus()
	}
}

func (w *watchConn) errorLoop() {
	for err := range w.watcher.Error {
		glog.Errorf("Watcher: %v", err)
	}
}
<|endoftext|>


<commit_before>// Copyright Peter Herth 2016

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

func readWord(reader *bufio.Reader) (word string, eof bool) {
	var b bytes.Buffer
	eof = skipWS(reader)
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == 32 {
			break
		}
		if rune == 13 {
			break
		}
		if rune == 10 {
			break
		}

		b.WriteRune(rune)
	}
	return b.String(), eof
}

func readLong(reader *bufio.Reader) (word string, eof bool) {
	var b bytes.Buffer
	eof = skipWS(reader)
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == '~' {
			break
		}
		b.WriteRune(rune)
	}
	return b.String(), eof
}

func readEOL(reader *bufio.Reader) (word string, eof bool) {
	var b bytes.Buffer
	eof = false
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == '\r' {
			continue
		}
		if rune == '\n' {
			break
		}
		b.WriteRune(rune)
	}
	return b.String(), eof
}

func skipTo(reader *bufio.Reader, stopRune rune) (eof bool) {
	eof = false
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == stopRune {
			reader.UnreadRune()
			break
		}
	}
	return eof
}

func skipWS(reader *bufio.Reader) (eof bool) {
	eof = false
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == 32 {
			continue
		}
		if rune == 13 {
			continue
		}
		if rune == 10 {
			continue
		}
		if rune == 9 {
			continue
		}
		reader.UnreadRune()
		break
	}
	return eof
}

type Mob struct {
	ID          string
	Name        string
	ShortDesc   string
	LongDesc    string
	Description string
	Race        string
}

type Item struct {
	ID        string
	Name      string
	ShortDesc string
	LongDesc  string
}

type Exit struct {
	Description string
	Keyword     string
	State       string
	Key         string
	To          string
}

type ExtraDescription struct {
	Keyword     string
	Description string
}

type Room struct {
	ID          string
	Name        string
	Description string
	Exit        [6]Exit
	Extra       []ExtraDescription
}

type Area struct {
	Name     string
	FileName string
	Nr       int
	Flags    string
	Mobs     [](*Mob)
	Items    [](*Item)
	Rooms    [](*Room)
}

var areas [](*Area)

func (mob *Mob) read(r *bufio.Reader) {
	mob.Name, _ = readLong(r)
	mob.ShortDesc, _ = readLong(r)
	mob.LongDesc, _ = readLong(r)
	mob.Description, _ = readLong(r)
	mob.Race, _ = readLong(r)
	skipTo(r, '#')
	return
}

func (area *Area) readMobs(r *bufio.Reader) {
	for {
		vnum, eof := readWord(r)
		if vnum == "#0" {
			return
		}

		if eof {
			fmt.Println("Eof while reading mobiles")
			return
		}
		if len(vnum) < 1 {
			fmt.Println("empty vnum")
			return
		}

		//fmt.Printf("vnum:%v<<<\n", vnum)
		mob := Mob{ID: vnum[1:]}
		mob.read(r)
		//fmt.Println("read mob:", mob.Vnum, mob.Name)
		area.Mobs = append(area.Mobs, &mob)
		//fmt.Println("mobs:", area.Mobs)
	}
}

func (item *Item) read(r *bufio.Reader) {
	item.Name, _ = readLong(r)
	item.ShortDesc, _ = readLong(r)
	item.LongDesc, _ = readLong(r)
	skipTo(r, '#')
	return
}

func (area *Area) readItems(r *bufio.Reader) {
	for {
		vnum, eof := readWord(r)
		if vnum == "#0" {
			return
		}
		if eof {
			fmt.Println("Eof while reading items in area", area.FileName)
			//panic("items eof")
			return
		}
		if len(vnum) < 1 {
			fmt.Println("empty vnum")
			return
		}

		//fmt.Printf("vnum:%v<<<\n", vnum)
		item := Item{ID: vnum[1:]}
		item.read(r)
		//fmt.Println("read item:", item.Vnum, item.Name)
		area.Items = append(area.Items, &item)
		//fmt.Println("items:", area.Items)
	}
}

func (room *Room) read(r *bufio.Reader) {
	room.Name, _ = readLong(r)
	room.Description, _ = readLong(r)
	readWord(r)
	readWord(r)
	readWord(r)
	for {
		entry, _ := readWord(r)
		//fmt.Println("entry", entry)
		e := entry[0]
		if e == 'D' { // exit
			dir := entry[1] - '0'
			//fmt.Println("dir=", dir)
			desc, _ := readLong(r)
			//fmt.Println("desc=", desc)
			keyword, _ := readLong(r)
			//fmt.Println("keyword=", keyword)
			state, _ := readWord(r)
			//fmt.Println("state=", state)
			key, _ := readWord(r)
			//fmt.Println("key=", key)
			toRoom, _ := readWord(r)
			//fmt.Println("toRooom=", toRoom)
			ex := Exit{Description: desc, Keyword: keyword, State: state, Key: key, To: toRoom}
			//fmt.Println("exx=", ex)

			//fmt.Println()

			room.Exit[dir] = ex
			//fmt.Println("exits", room.Exit)
			continue
		}
		if entry == "S" {
			skipTo(r, '#')
			return
		}
		if entry == "E" {
			k, _ := readLong(r)
			d, _ := readLong(r)
			room.Extra = append(room.Extra, ExtraDescription{Keyword: k, Description: d})
			continue
		}
		if entry == "H" {
			readWord(r)
			continue
		}
		if entry == "M" {
			readWord(r)
			continue
		}
		break
		// case "H":
		// 	break
		// case "M":
		// 	break
		// case "K":
		// 	break
		// case "amap":
		// 	break
		// case "WLH":
		// 	break
		// case "X":
		// 	break
		// case "watches":
		// 	break
		// case "clan":
		// 	break
		// case "O":
		// 	break
		// case "A":
		// 	break
		// case "damage":
		// 	break
		// case "encounters":
		// 	break
		// case "encountr":
		// 	break
		// case "populate":
		// 	break
		// case "religion":
		// 	break

	}
	skipTo(r, '#')
	return
}

func (area *Area) readRooms(r *bufio.Reader) {
	for {
		vnum, eof := readWord(r)
		if vnum == "#0" {
			return
		}

		if eof {
			fmt.Println("Eof while reading rooms")
			return
		}
		if len(vnum) < 1 {
			fmt.Println("empty vnum")
			return
		}

		// //fmt.Printf("vnum:%v<<<\n", vnum)
		room := Room{ID: vnum[1:]}
		room.read(r)
		//fmt.Println("read room:", room.Vnum, room.Name, room)
		area.Rooms = append(area.Rooms, &room)
	}
}

func (area *Area) readResets(r *bufio.Reader) {
	for {
		code, eof := readWord(r)
		if code == "S" {
			return
		}
		if eof {
			fmt.Println("Eof while reading resets")
			return
		}
		_, _ = readEOL(r)

	}
}

func (area *Area) load() {
	file, err := os.Open("./area/" + area.FileName)
	if err != nil {
		return
	}
	defer file.Close()

	//	fmt.Println("reading", area.FileName)

	r := bufio.NewReader(file)
	for {
		word, eof := readWord(r)
		//fmt.Println("word", word)
		switch word {
		case "#AREA":
			_, _ = readLong(r)
			area.Name, _ = readLong(r)
			_, _ = readLong(r)
			continue
		case "#MOBILES":
			area.readMobs(r)
		case "#OBJECTS":
			area.readItems(r)
		case "#ROOMS":
			area.readRooms(r)
		case "#RESETS":
			area.readResets(r)
		default:
			//fmt.Println("skipping word", word)
		}

		if eof {
			return
		}
	}

}

func loadAreas() {
	file, err := os.Open("./area/area.list")
	n := 0
	if err != nil {
		return
	}
	var c chan (int)
	parallel := true
	if parallel {
		c = make(chan int)
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		l := scanner.Text()
		//fmt.Println("area:", l)
		if l[0] != '#' {
			words := bufio.NewScanner(strings.NewReader(l))
			words.Split(bufio.ScanWords)
			words.Scan()
			nr, _ := strconv.Atoi(words.Text())
			words.Scan()
			flags := words.Text()
			words.Scan()
			fileName := words.Text()
			//parts := strings.Split(l, " ")
			//fmt.Println("parts:", len(parts), parts)
			area := Area{Nr: nr, FileName: fileName, Flags: flags}

			areas = append(areas, &area)
			//			area.load()
			//fmt.Println("loaded", area)
			if parallel {
				go func() {
					area.load()
					c <- 1 // mark area loaded
				}()
			} else {
				area.load()
			}
			n++
		}
	}
	if parallel {
		for i := 0; i < n; i++ {
			<-c // wait till all areas sent load signal
		}
	}
	fmt.Println(n, "areas loaded", len(areas))
	//fmt.Println(areas)
}

func handler(c net.Conn) {
	//buffer := make([]byte, 1024)
	scanner := bufio.NewScanner(c)
	log.Printf("New connection from %v\n", c.RemoteAddr())
	for scanner.Scan() {
		line := scanner.Text()
		log.Println("read:", line)
	}

	// for {
	// 	n, err := c.Read(buffer)

	// 	if err == io.EOF {
	// 		log.Printf("socket closed %v\n", c.RemoteAddr())
	// 		c.Close()
	// 		return
	// 	}
	// 	if err != nil {
	// 		log.Println("error reading socket", err)
	// 		c.Close()
	// 		return
	// 	}
	// 	if n == 0 {
	// 		log.Println("zero read")
	// 		c.Close()
	// 		return
	// 	}
	// 	log.Println("read", string(buffer[:n]))
	// }
	log.Printf("socket closed %v\n", c.RemoteAddr())
}

func serve() {
	listener, err := net.Listen("tcp", ":8000")
	if err != nil {
		log.Fatal(err)
	}
	defer listener.Close()
	for {
		// Wait for a connection.
		conn, err := listener.Accept()
		if err != nil {
			log.Fatal(err)
		}
		go handler(conn)
	}
}

func main() {
	t1 := time.Now()
	//	runtime.ThreadCreateProfile()
	loadAreas()

	//time.Sleep(1 * time.Second)
	// for _, area := range areas {
	// 	//fmt.Println(area)
	// 	fmt.Println(area.Name, len(area.Mobs), "mobs", len(area.Items), "items", len(area.Rooms), "rooms")
	// }
	fmt.Fprintf(os.Stderr, "loading:  %v\n", time.Since(t1))
	fmt.Println(runtime.NumCPU(), "cpus")

	serve()
}
<commit_msg>cleanup<commit_after>// Copyright Peter Herth 2016

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

func readWord(reader *bufio.Reader) (word string, eof bool) {
	var b bytes.Buffer
	eof = skipWS(reader)
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == 32 {
			break
		}
		if rune == 13 {
			break
		}
		if rune == 10 {
			break
		}

		b.WriteRune(rune)
	}
	return b.String(), eof
}

func readLong(reader *bufio.Reader) (word string, eof bool) {
	var b bytes.Buffer
	eof = skipWS(reader)
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == '~' {
			break
		}
		b.WriteRune(rune)
	}
	return b.String(), eof
}

func readEOL(reader *bufio.Reader) (word string, eof bool) {
	var b bytes.Buffer
	eof = false
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == '\r' {
			continue
		}
		if rune == '\n' {
			break
		}
		b.WriteRune(rune)
	}
	return b.String(), eof
}

func skipTo(reader *bufio.Reader, stopRune rune) (eof bool) {
	eof = false
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == stopRune {
			reader.UnreadRune()
			break
		}
	}
	return eof
}

func skipWS(reader *bufio.Reader) (eof bool) {
	eof = false
	for {
		rune, size, _ := reader.ReadRune()
		if size == 0 {
			eof = true
			break
		}
		if rune == 32 {
			continue
		}
		if rune == 13 {
			continue
		}
		if rune == 10 {
			continue
		}
		if rune == 9 {
			continue
		}
		reader.UnreadRune()
		break
	}
	return eof
}

type Mob struct {
	ID          string
	Name        string
	ShortDesc   string
	LongDesc    string
	Description string
	Race        string
}

type Item struct {
	ID        string
	Name      string
	ShortDesc string
	LongDesc  string
}

type Exit struct {
	Description string
	Keyword     string
	State       string
	Key         string
	To          string
}

type ExtraDescription struct {
	Keyword     string
	Description string
}

type Room struct {
	ID          string
	Name        string
	Description string
	Exit        [6]Exit
	Extra       []ExtraDescription
}

type Area struct {
	Name     string
	FileName string
	Nr       int
	Flags    string
	Mobs     [](*Mob)
	Items    [](*Item)
	Rooms    [](*Room)
}

var areas [](*Area)

func (mob *Mob) read(r *bufio.Reader) {
	mob.Name, _ = readLong(r)
	mob.ShortDesc, _ = readLong(r)
	mob.LongDesc, _ = readLong(r)
	mob.Description, _ = readLong(r)
	mob.Race, _ = readLong(r)
	skipTo(r, '#')
	return
}

func (area *Area) readMobs(r *bufio.Reader) {
	for {
		vnum, eof := readWord(r)
		if vnum == "#0" {
			return
		}

		if eof {
			fmt.Println("Eof while reading mobiles")
			return
		}
		if len(vnum) < 1 {
			fmt.Println("empty vnum")
			return
		}

		//fmt.Printf("vnum:%v<<<\n", vnum)
		mob := Mob{ID: vnum[1:]}
		mob.read(r)
		//fmt.Println("read mob:", mob.Vnum, mob.Name)
		area.Mobs = append(area.Mobs, &mob)
		//fmt.Println("mobs:", area.Mobs)
	}
}

func (item *Item) read(r *bufio.Reader) {
	item.Name, _ = readLong(r)
	item.ShortDesc, _ = readLong(r)
	item.LongDesc, _ = readLong(r)
	skipTo(r, '#')
	return
}

func (area *Area) readItems(r *bufio.Reader) {
	for {
		vnum, eof := readWord(r)
		if vnum == "#0" {
			return
		}
		if eof {
			fmt.Println("Eof while reading items in area", area.FileName)
			//panic("items eof")
			return
		}
		if len(vnum) < 1 {
			fmt.Println("empty vnum")
			return
		}

		//fmt.Printf("vnum:%v<<<\n", vnum)
		item := Item{ID: vnum[1:]}
		item.read(r)
		//fmt.Println("read item:", item.Vnum, item.Name)
		area.Items = append(area.Items, &item)
		//fmt.Println("items:", area.Items)
	}
}

func (room *Room) read(r *bufio.Reader) {
	room.Name, _ = readLong(r)
	room.Description, _ = readLong(r)
	readWord(r)
	readWord(r)
	readWord(r)
	for {
		entry, _ := readWord(r)
		//fmt.Println("entry", entry)
		e := entry[0]
		if e == 'D' { // exit
			dir := entry[1] - '0'
			//fmt.Println("dir=", dir)
			desc, _ := readLong(r)
			//fmt.Println("desc=", desc)
			keyword, _ := readLong(r)
			//fmt.Println("keyword=", keyword)
			state, _ := readWord(r)
			//fmt.Println("state=", state)
			key, _ := readWord(r)
			//fmt.Println("key=", key)
			toRoom, _ := readWord(r)
			//fmt.Println("toRooom=", toRoom)
			ex := Exit{Description: desc, Keyword: keyword, State: state, Key: key, To: toRoom}
			//fmt.Println("exx=", ex)

			//fmt.Println()

			room.Exit[dir] = ex
			//fmt.Println("exits", room.Exit)
			continue
		}
		if entry == "S" {
			skipTo(r, '#')
			return
		}
		if entry == "E" {
			k, _ := readLong(r)
			d, _ := readLong(r)
			room.Extra = append(room.Extra, ExtraDescription{Keyword: k, Description: d})
			continue
		}
		if entry == "H" {
			readWord(r)
			continue
		}
		if entry == "M" {
			readWord(r)
			continue
		}
		break
		// case "H":
		// 	break
		// case "M":
		// 	break
		// case "K":
		// 	break
		// case "amap":
		// 	break
		// case "WLH":
		// 	break
		// case "X":
		// 	break
		// case "watches":
		// 	break
		// case "clan":
		// 	break
		// case "O":
		// 	break
		// case "A":
		// 	break
		// case "damage":
		// 	break
		// case "encounters":
		// 	break
		// case "encountr":
		// 	break
		// case "populate":
		// 	break
		// case "religion":
		// 	break

	}
	skipTo(r, '#')
	return
}

func (area *Area) readRooms(r *bufio.Reader) {
	for {
		vnum, eof := readWord(r)
		if vnum == "#0" {
			return
		}

		if eof {
			fmt.Println("Eof while reading rooms")
			return
		}
		if len(vnum) < 1 {
			fmt.Println("empty vnum")
			return
		}

		// //fmt.Printf("vnum:%v<<<\n", vnum)
		room := Room{ID: vnum[1:]}
		room.read(r)
		//fmt.Println("read room:", room.Vnum, room.Name, room)
		area.Rooms = append(area.Rooms, &room)
	}
}

func (area *Area) readResets(r *bufio.Reader) {
	for {
		code, eof := readWord(r)
		if code == "S" {
			return
		}
		if eof {
			fmt.Println("Eof while reading resets")
			return
		}
		_, _ = readEOL(r)

	}
}

func (area *Area) load() {
	file, err := os.Open("./area/" + area.FileName)
	if err != nil {
		return
	}
	defer file.Close()

	//	fmt.Println("reading", area.FileName)

	r := bufio.NewReader(file)
	for {
		word, eof := readWord(r)
		//fmt.Println("word", word)
		switch word {
		case "#AREA":
			_, _ = readLong(r)
			area.Name, _ = readLong(r)
			_, _ = readLong(r)
			continue
		case "#MOBILES":
			area.readMobs(r)
		case "#OBJECTS":
			area.readItems(r)
		case "#ROOMS":
			area.readRooms(r)
		case "#RESETS":
			area.readResets(r)
		default:
			//fmt.Println("skipping word", word)
		}

		if eof {
			return
		}
	}

}

func loadAreas() {
	file, err := os.Open("./area/area.list")
	n := 0
	if err != nil {
		return
	}
	var c chan (int)
	parallel := true
	if parallel {
		c = make(chan int)
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		l := scanner.Text()
		//fmt.Println("area:", l)
		if l[0] != '#' {
			words := bufio.NewScanner(strings.NewReader(l))
			words.Split(bufio.ScanWords)
			words.Scan()
			nr, _ := strconv.Atoi(words.Text())
			words.Scan()
			flags := words.Text()
			words.Scan()
			fileName := words.Text()
			//parts := strings.Split(l, " ")
			//fmt.Println("parts:", len(parts), parts)
			area := Area{Nr: nr, FileName: fileName, Flags: flags}

			areas = append(areas, &area)
			//			area.load()
			//fmt.Println("loaded", area)
			if parallel {
				go func() {
					area.load()
					c <- 1 // mark area loaded
				}()
			} else {
				area.load()
			}
			n++
		}
	}
	if parallel {
		for i := 0; i < n; i++ {
			<-c // wait till all areas sent load signal
		}
	}
	fmt.Println(n, "areas loaded", len(areas))
	//fmt.Println(areas)
}

func handler(c net.Conn) {
	//buffer := make([]byte, 1024)
	scanner := bufio.NewScanner(c)
	log.Printf("New connection from %v\n", c.RemoteAddr())
	for scanner.Scan() {
		line := scanner.Text()
		log.Println("read:", line)
	}

	// for {
	// 	n, err := c.Read(buffer)

	// 	if err == io.EOF {
	// 		log.Printf("socket closed %v\n", c.RemoteAddr())
	// 		c.Close()
	// 		return
	// 	}
	// 	if err != nil {
	// 		log.Println("error reading socket", err)
	// 		c.Close()
	// 		return
	// 	}
	// 	if n == 0 {
	// 		log.Println("zero read")
	// 		c.Close()
	// 		return
	// 	}
	// 	log.Println("read", string(buffer[:n]))
	// }
	log.Printf("socket closed %v\n", c.RemoteAddr())
}

func serve() {
	listener, err := net.Listen("tcp", ":8000")
	if err != nil {
		log.Fatal(err)
	}
	defer listener.Close()
	for {
		// Wait for a connection.
		conn, err := listener.Accept()
		if err != nil {
			log.Fatal(err)
		}
		go handler(conn)
	}
}

func main() {
	t1 := time.Now()
	loadAreas()
	fmt.Fprintf(os.Stderr, "loading:  %v\n", time.Since(t1))
	fmt.Println(runtime.NumCPU(), "cpus")

	serve()
}
<|endoftext|>


<commit_before>// This file is part of fs1up.
// Copyright (C) 2014 Andreas Klauer <Andreas.Klauer@metamorpher.de>
// License: GPL-2

// Package nbd uses the Linux NBD layer to emulate a block device in user space
package nbd

import (
	"encoding/binary"
	//"fmt"
	"os"
	"runtime"
	"syscall"
)

const (
	// Defined in <linux/fs.h>:
	BLKROSET = 4701
	// Defined in <linux/nbd.h>:
	NBD_SET_SOCK        = 43776
	NBD_SET_BLKSIZE     = 43777
	NBD_SET_SIZE        = 43778
	NBD_DO_IT           = 43779
	NBD_CLEAR_SOCK      = 43780
	NBD_CLEAR_QUE       = 43781
	NBD_PRINT_DEBUG     = 43782
	NBD_SET_SIZE_BLOCKS = 43783
	NBD_DISCONNECT      = 43784
	NBD_SET_TIMEOUT     = 43785
	NBD_SET_FLAGS       = 43786
	// enum
	NBD_CMD_READ  = 0
	NBD_CMD_WRITE = 1
	NBD_CMD_DISC  = 2
	NBD_CMD_FLUSH = 3
	NBD_CMD_TRIM  = 4
	// values for flags field
	NBD_FLAG_HAS_FLAGS  = (1 << 0) // nbd-server supports flags
	NBD_FLAG_READ_ONLY  = (1 << 1) // device is read-only
	NBD_FLAG_SEND_FLUSH = (1 << 2) // can flush writeback cache
	// there is a gap here to match userspace
	NBD_FLAG_SEND_TRIM = (1 << 5) // send trim/discard
	// These are sent over the network in the request/reply magic fields
	NBD_REQUEST_MAGIC = 0x25609513
	NBD_REPLY_MAGIC   = 0x67446698
	// Do *not* use magics: 0x12560953 0x96744668.
)

// Device interface is a subset of os.File.
type Device interface {
	ReadAt(b []byte, off int64) (n int, err error)
	WriteAt(b []byte, off int64) (n int, err error)
}

type request struct {
	magic  uint32
	typus  uint32
	handle uint64
	from   uint64
	len    uint32
}

func handle(fd int, d Device) {
	buf := make([]byte, 2<<19)
	var x request

	for {
		n, err := syscall.Read(fd, buf[0:28])
		if err != nil {
			n += 1
		}
		//fmt.Println("actually read", n, err)
		x.magic = binary.BigEndian.Uint32(buf)
		x.typus = binary.BigEndian.Uint32(buf[4:8])
		x.handle = binary.BigEndian.Uint64(buf[8:16])
		x.from = binary.BigEndian.Uint64(buf[16:24])
		x.len = binary.BigEndian.Uint32(buf[24:28])

		//fmt.Println("read", x)

		switch x.magic {
		case NBD_REPLY_MAGIC:
			fallthrough
		case NBD_REQUEST_MAGIC:
			switch x.typus {
			case NBD_CMD_READ:
				n, _ = d.ReadAt(buf[16:16+x.len], int64(x.from))
				//fmt.Println("got", n, "bytes to send back")
				binary.BigEndian.PutUint32(buf[0:4], NBD_REPLY_MAGIC)
				binary.BigEndian.PutUint32(buf[4:8], 0)
				n, err = syscall.Write(fd, buf[0:16+x.len])
				//fmt.Println("actually wrote", n-16, err)
			case NBD_CMD_WRITE:
				//fmt.Println("write", x)
			case NBD_CMD_DISC:
				panic("Disconnect")
			case NBD_CMD_FLUSH:
				//fmt.Println("flush", x)
			case NBD_CMD_TRIM:
				//fmt.Println("trim", x)
			default:
				panic("unknown command")
			}
		default:
			panic("Invalid packet")
		}

		// syscall.Write(fd, buf[0:n])
		// fmt.Println("wrote", buf[0:n])
	}
}

func Client(d Device, offset int64, size int64) {
	nbd, _ := os.Open("/dev/nbd0") // TODO: find a free one
	fd, _ := syscall.Socketpair(syscall.SOCK_STREAM, syscall.AF_UNIX, 0)
	go handle(fd[1], d)
	runtime.LockOSThread()
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_SET_SOCK, uintptr(fd[0]))
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_SET_BLKSIZE, 4096)
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_SET_SIZE_BLOCKS, uintptr(size/4096))
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_SET_FLAGS, 1)
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), BLKROSET, 0)  // || 1
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_DO_IT, 0) // doesn't return
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_DISCONNECT, 0)
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_CLEAR_SOCK, 0)
	runtime.UnlockOSThread()
}
<commit_msg>nbd write<commit_after>// This file is part of fs1up.
// Copyright (C) 2014 Andreas Klauer <Andreas.Klauer@metamorpher.de>
// License: GPL-2

// Package nbd uses the Linux NBD layer to emulate a block device in user space
package nbd

import (
	"encoding/binary"
	//"fmt"
	"os"
	"runtime"
	"syscall"
)

const (
	// Defined in <linux/fs.h>:
	BLKROSET = 4701
	// Defined in <linux/nbd.h>:
	NBD_SET_SOCK        = 43776
	NBD_SET_BLKSIZE     = 43777
	NBD_SET_SIZE        = 43778
	NBD_DO_IT           = 43779
	NBD_CLEAR_SOCK      = 43780
	NBD_CLEAR_QUE       = 43781
	NBD_PRINT_DEBUG     = 43782
	NBD_SET_SIZE_BLOCKS = 43783
	NBD_DISCONNECT      = 43784
	NBD_SET_TIMEOUT     = 43785
	NBD_SET_FLAGS       = 43786
	// enum
	NBD_CMD_READ  = 0
	NBD_CMD_WRITE = 1
	NBD_CMD_DISC  = 2
	NBD_CMD_FLUSH = 3
	NBD_CMD_TRIM  = 4
	// values for flags field
	NBD_FLAG_HAS_FLAGS  = (1 << 0) // nbd-server supports flags
	NBD_FLAG_READ_ONLY  = (1 << 1) // device is read-only
	NBD_FLAG_SEND_FLUSH = (1 << 2) // can flush writeback cache
	// there is a gap here to match userspace
	NBD_FLAG_SEND_TRIM = (1 << 5) // send trim/discard
	// These are sent over the network in the request/reply magic fields
	NBD_REQUEST_MAGIC = 0x25609513
	NBD_REPLY_MAGIC   = 0x67446698
	// Do *not* use magics: 0x12560953 0x96744668.
)

// Device interface is a subset of os.File.
type Device interface {
	ReadAt(b []byte, off int64) (n int, err error)
	WriteAt(b []byte, off int64) (n int, err error)
}

type request struct {
	magic  uint32
	typus  uint32
	handle uint64
	from   uint64
	len    uint32
}

type reply struct {
	magic  uint32
	error  uint32
	handle uint64
}

func handle(fd int, d Device) {
	buf := make([]byte, 2<<19)
	var x request

	for {
		syscall.Read(fd, buf[0:28])

		x.magic = binary.BigEndian.Uint32(buf)
		x.typus = binary.BigEndian.Uint32(buf[4:8])
		x.handle = binary.BigEndian.Uint64(buf[8:16])
		x.from = binary.BigEndian.Uint64(buf[16:24])
		x.len = binary.BigEndian.Uint32(buf[24:28])

		switch x.magic {
		case NBD_REPLY_MAGIC:
			fallthrough
		case NBD_REQUEST_MAGIC:
			switch x.typus {
			case NBD_CMD_READ:
				d.ReadAt(buf[16:16+x.len], int64(x.from))
				binary.BigEndian.PutUint32(buf[0:4], NBD_REPLY_MAGIC)
				binary.BigEndian.PutUint32(buf[4:8], 0)
				syscall.Write(fd, buf[0:16+x.len])
			case NBD_CMD_WRITE:
				n, _ := syscall.Read(fd, buf[28:28+x.len])
				for uint32(n) < x.len {
					m, _ := syscall.Read(fd, buf[28+n:28+x.len])
					n += m
				}
				d.WriteAt(buf[28:28+x.len], int64(x.from))
				binary.BigEndian.PutUint32(buf[0:4], NBD_REPLY_MAGIC)
				binary.BigEndian.PutUint32(buf[4:8], 0)
				syscall.Write(fd, buf[0:16])
			case NBD_CMD_DISC:
				panic("Disconnect")
			case NBD_CMD_FLUSH:
			case NBD_CMD_TRIM:
			default:
				panic("unknown command")
			}
		default:
			panic("Invalid packet")
		}

		// syscall.Write(fd, buf[0:n])
		// fmt.Println("wrote", buf[0:n])
	}
}

func Client(d Device, offset int64, size int64) {
	nbd, _ := os.Open("/dev/nbd0") // TODO: find a free one
	fd, _ := syscall.Socketpair(syscall.SOCK_STREAM, syscall.AF_UNIX, 0)
	go handle(fd[1], d)
	runtime.LockOSThread()
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_SET_SOCK, uintptr(fd[0]))
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_SET_BLKSIZE, 4096)
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_SET_SIZE_BLOCKS, uintptr(size/4096))
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_SET_FLAGS, 1)
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), BLKROSET, 0)  // || 1
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_DO_IT, 0) // doesn't return
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_DISCONNECT, 0)
	syscall.Syscall(syscall.SYS_IOCTL, nbd.Fd(), NBD_CLEAR_SOCK, 0)
	runtime.UnlockOSThread()
}
<|endoftext|>


<commit_before>package ncp

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// NCp values:
// client http.Client with cookie
type NCp struct {
	client http.Client
}

// Topic from forum
type Topic struct {
	Href    string
	Name    string
	Year    string
	Quality string
	Body    []byte
}

// Film all values
// ID            id
// Name          Название
// EngName       Английское название
// Href          Ссылка
// Year          Год
// Genre         Жанр
// Country       Производство
// Director      Режиссер
// Producer      Продюсер
// Actors        Актеры
// Description   Описание
// Age           Возраст
// ReleaseDate   Дата мировой премьеры
// RussianDate   Дата премьеры в России
// Duration      Продолжительность
// Quality       Качество видео
// Translation   Перевод
// SubtitlesType Вид субтитров
// Subtitles     Субтитры
// Video         Видео
// Resolution    Разрешение видео
// Audio         Аудио
// Kinopoisk     Рейтинг кинопоиска
// Imdb          Рейтинг IMDb
// NNM           Рейтинг nnm-club
// Sound         Звук
// Size          Размер
// DateCreate    Дата создания раздачи
// Torrent       Ссылка на torrent
// Poster        Ссылка на постер
// Seeders       Количество раздающих
// Leechers      Количество скачивающих
// UpdatedAt     Дата обновления записи БД
// CreatedAt     Дата создания записи БД
type Film struct {
	ID            int64     `gorm:"column:id"             db:"id"             sql:"AUTO_INCREMENT"`
	Name          string    `gorm:"column:name"           db:"name"           sql:"type:text"`
	EngName       string    `gorm:"column:eng_name"       db:"eng_name"       sql:"type:text"`
	Href          string    `gorm:"column:href"           db:"href"           sql:"type:text"`
	Year          int64     `gorm:"column:year"           db:"year"`
	Genre         string    `gorm:"column:genre"          db:"genre"          sql:"type:text"`
	Country       string    `gorm:"column:country"        db:"country"        sql:"type:text"`
	Director      string    `gorm:"column:director"       db:"director"       sql:"type:text"`
	Producer      string    `gorm:"column:producer"       db:"producer"       sql:"type:text"`
	Actors        string    `gorm:"column:actors"         db:"actors"         sql:"type:text"`
	Description   string    `gorm:"column:description"    db:"description"    sql:"type:text"`
	Age           string    `gorm:"column:age"            db:"age"            sql:"type:text"`
	ReleaseDate   string    `gorm:"column:release_date"   db:"release_date"   sql:"type:text"`
	RussianDate   string    `gorm:"column:russian_date"   db:"russian_date"   sql:"type:text"`
	Duration      int64     `gorm:"column:duration"       db:"duration"`
	Quality       string    `gorm:"column:quality"        db:"quality"        sql:"type:text"`
	Translation   string    `gorm:"column:translation"    db:"translation"    sql:"type:text"`
	SubtitlesType string    `gorm:"column:subtitles_type" db:"subtitles_type" sql:"type:text"`
	Subtitles     string    `gorm:"column:subtitles"      db:"subtitles"      sql:"type:text"`
	Video         string    `gorm:"column:video"          db:"video"          sql:"type:text"`
	Resolution    string    `gorm:"column:resolution"     db:"resolution"     sql:"type:text"`
	Audio         string    `gorm:"column:audio"          db:"audio"          sql:"type:text"`
	Kinopoisk     float64   `gorm:"column:kinopoisk"      db:"kinopoisk"`
	IMDb          float64   `gorm:"column:imdb"           db:"imdb"`
	NNM           float64   `gorm:"column:nnm"            db:"nnm"`
	Sound         string    `gorm:"column:sound"          db:"sound"          sql:"type:text"`
	Size          int64     `gorm:"column:size"           db:"size"`
	DateCreate    string    `gorm:"column:date_create"    db:"date_create"    sql:"type:text"`
	Torrent       string    `gorm:"column:torrent"        db:"torrent"        sql:"type:text"`
	Poster        string    `gorm:"column:poster"         db:"poster"         sql:"type:text"`
	Seeders       int64     `gorm:"column:seeders"        db:"seeders"`
	Leechers      int64     `gorm:"column:leechers"       db:"leechers"`
	UpdatedAt     time.Time `gorm:"column:updated_at"     db:"updated_at"`
	CreatedAt     time.Time `gorm:"column:created_at"     db:"created_at"`
}

// Init nnmc with login password
func Init(login string, password string) (*NCp, error) {
	var client http.Client
	cookieJar, _ := cookiejar.New(nil)
	client.Jar = cookieJar
	urlPost := "http://nnm-club.me/forum/login.php"
	form := url.Values{}
	form.Set("username", login)
	form.Add("password", password)
	form.Add("redirect", "")
	form.Add("login", "âõîä")
	req, _ := http.NewRequest("POST", urlPost, bytes.NewBufferString(form.Encode()))
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Content-Length", strconv.Itoa(len(form.Encode())))
	_, err := client.Do(req)
	return &NCp{client: client}, err
}

// getHTML get body from url
func getHTML(url string, n *NCp) ([]byte, error) {
	resp, err := n.client.Get(url)
	if err != nil {
		log.Println("client Get error:", err)
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("ioutil.ReadAll error:", err)
	}
	buffer := bytes.NewBufferString("")
	for _, char := range body {
		var ch = Utf(char)
		fmt.Fprintf(buffer, "%c", ch)
	}
	doc := buffer.Bytes()
	doc = replaceAll(doc, " ", " ")
	doc = replaceAll(doc, "&", "&")
	doc = replaceAll(doc, "
", "")
	return doc, nil
}

// ParseForumTree get topics from forumTree
func (n *NCp) ParseForumTree(url string) ([]Topic, error) {
	var (
		topics []Topic
		reTree = regexp.MustCompile(`<a href="(viewtopic.php\?t=\d+)"class="topictitle">(.+?)\s\((\d{4})\)\s(.+?)</a>`)
	)
	body, err := getHTML(url, n)
	if err != nil {
		return topics, err
	}
	if reTree.Match(body) == false {
		return topics, fmt.Errorf("No topic in body")
	}
	findResult := reTree.FindAllSubmatch(body, -1)
	for _, v := range findResult {
		var t Topic
		t.Href = "http://nnm-club.me/forum/" + string(v[1])
		t.Name = string(v[2])
		t.Year = string(v[3])
		t.Quality = string(v[4])
		topics = append(topics, t)
	}
	return topics, nil
}

// ParseTopic get film from topic
func (n *NCp) ParseTopic(topic Topic) (Film, error) {
	var (
		film Film
	)
	name := strings.Split(topic.Name, " / ")
	switch len(name) {
	case 1:
		film.Name = strings.Trim(name[0], " ")
	case 2:
		film.Name = strings.Trim(name[0], " ")
		film.EngName = strings.Trim(name[1], " ")
	case 3:
		film.Name = strings.Trim(name[0], " ")
		film.EngName = strings.Trim(name[1], " ")
	}
	film.Href = topic.Href
	if year64, err := strconv.ParseInt(topic.Year, 10, 64); err == nil {
		film.Year = year64
	}
	body, err := getHTML(film.Href, n)
	if err != nil {
		return film, err
	}
	topic.Body = body
	film.Country = topic.getCountry()
	film.Genre = topic.getGenre()
	film.Director = topic.getDirector()
	film.Producer = topic.getProducer()
	film.Actors = topic.getActors()
	film.Description = topic.getDescription()
	film.Age = topic.getAge()
	film.ReleaseDate = topic.getReleaseDate()
	film.RussianDate = topic.getRussianDate()
	film.Duration = topic.getDuration()
	film.Quality = topic.getQuality()
	film.Translation = topic.getTranslation()
	film.SubtitlesType = topic.getSubtitlesType()
	film.Subtitles = topic.getSubtitles()
	film.Video = topic.getVideo()
	film.Resolution = getResolution(film.Video)
	film.Audio = topic.getAudio()
	film.Torrent = topic.getTorrent()
	film.DateCreate = topic.getDate()
	film.Size = topic.getSize()
	film.NNM = topic.getRating()
	film.Poster = topic.getPoster()
	film.Seeders = topic.getSeeds()
	film.Leechers = topic.getLeechs()
	return film, nil
}

func replaceAll(body []byte, from string, to string) []byte {
	var reStr = regexp.MustCompile(from)
	result := reStr.ReplaceAll(body, []byte(to))
	return result
}

func replaceDate(s string) string {
	s = strings.Replace(s, " Янв ", ".01.", -1)
	s = strings.Replace(s, " Фев ", ".02.", -1)
	s = strings.Replace(s, " Мар ", ".03.", -1)
	s = strings.Replace(s, " Апр ", ".04.", -1)
	s = strings.Replace(s, " Май ", ".05.", -1)
	s = strings.Replace(s, " Июн ", ".06.", -1)
	s = strings.Replace(s, " Июл ", ".07.", -1)
	s = strings.Replace(s, " Авг ", ".08.", -1)
	s = strings.Replace(s, " Сен ", ".09.", -1)
	s = strings.Replace(s, " Окт ", ".10.", -1)
	s = strings.Replace(s, " Ноя ", ".11.", -1)
	s = strings.Replace(s, " Дек ", ".12.", -1)
	return s
}

func caseInsensitiveContains(s, substr string) bool {
	s, substr = strings.ToUpper(s), strings.ToUpper(substr)
	return strings.Contains(s, substr)
}

func cleanStr(str string) string {
	var reSpan = regexp.MustCompile("")
	str = reSpan.ReplaceAllString(str, "")
	str = strings.Trim(str, " ")
	return str
}
<commit_msg>more clean body<commit_after>package ncp

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// NCp values:
// client http.Client with cookie
type NCp struct {
	client http.Client
}

// Topic from forum
type Topic struct {
	Href    string
	Name    string
	Year    string
	Quality string
	Body    []byte
}

// Film all values
// ID            id
// Name          Название
// EngName       Английское название
// Href          Ссылка
// Year          Год
// Genre         Жанр
// Country       Производство
// Director      Режиссер
// Producer      Продюсер
// Actors        Актеры
// Description   Описание
// Age           Возраст
// ReleaseDate   Дата мировой премьеры
// RussianDate   Дата премьеры в России
// Duration      Продолжительность
// Quality       Качество видео
// Translation   Перевод
// SubtitlesType Вид субтитров
// Subtitles     Субтитры
// Video         Видео
// Resolution    Разрешение видео
// Audio         Аудио
// Kinopoisk     Рейтинг кинопоиска
// Imdb          Рейтинг IMDb
// NNM           Рейтинг nnm-club
// Sound         Звук
// Size          Размер
// DateCreate    Дата создания раздачи
// Torrent       Ссылка на torrent
// Poster        Ссылка на постер
// Seeders       Количество раздающих
// Leechers      Количество скачивающих
// UpdatedAt     Дата обновления записи БД
// CreatedAt     Дата создания записи БД
type Film struct {
	ID            int64     `gorm:"column:id"             db:"id"             sql:"AUTO_INCREMENT"`
	Name          string    `gorm:"column:name"           db:"name"           sql:"type:text"`
	EngName       string    `gorm:"column:eng_name"       db:"eng_name"       sql:"type:text"`
	Href          string    `gorm:"column:href"           db:"href"           sql:"type:text"`
	Year          int64     `gorm:"column:year"           db:"year"`
	Genre         string    `gorm:"column:genre"          db:"genre"          sql:"type:text"`
	Country       string    `gorm:"column:country"        db:"country"        sql:"type:text"`
	Director      string    `gorm:"column:director"       db:"director"       sql:"type:text"`
	Producer      string    `gorm:"column:producer"       db:"producer"       sql:"type:text"`
	Actors        string    `gorm:"column:actors"         db:"actors"         sql:"type:text"`
	Description   string    `gorm:"column:description"    db:"description"    sql:"type:text"`
	Age           string    `gorm:"column:age"            db:"age"            sql:"type:text"`
	ReleaseDate   string    `gorm:"column:release_date"   db:"release_date"   sql:"type:text"`
	RussianDate   string    `gorm:"column:russian_date"   db:"russian_date"   sql:"type:text"`
	Duration      int64     `gorm:"column:duration"       db:"duration"`
	Quality       string    `gorm:"column:quality"        db:"quality"        sql:"type:text"`
	Translation   string    `gorm:"column:translation"    db:"translation"    sql:"type:text"`
	SubtitlesType string    `gorm:"column:subtitles_type" db:"subtitles_type" sql:"type:text"`
	Subtitles     string    `gorm:"column:subtitles"      db:"subtitles"      sql:"type:text"`
	Video         string    `gorm:"column:video"          db:"video"          sql:"type:text"`
	Resolution    string    `gorm:"column:resolution"     db:"resolution"     sql:"type:text"`
	Audio         string    `gorm:"column:audio"          db:"audio"          sql:"type:text"`
	Kinopoisk     float64   `gorm:"column:kinopoisk"      db:"kinopoisk"`
	IMDb          float64   `gorm:"column:imdb"           db:"imdb"`
	NNM           float64   `gorm:"column:nnm"            db:"nnm"`
	Sound         string    `gorm:"column:sound"          db:"sound"          sql:"type:text"`
	Size          int64     `gorm:"column:size"           db:"size"`
	DateCreate    string    `gorm:"column:date_create"    db:"date_create"    sql:"type:text"`
	Torrent       string    `gorm:"column:torrent"        db:"torrent"        sql:"type:text"`
	Poster        string    `gorm:"column:poster"         db:"poster"         sql:"type:text"`
	Seeders       int64     `gorm:"column:seeders"        db:"seeders"`
	Leechers      int64     `gorm:"column:leechers"       db:"leechers"`
	UpdatedAt     time.Time `gorm:"column:updated_at"     db:"updated_at"`
	CreatedAt     time.Time `gorm:"column:created_at"     db:"created_at"`
}

// Init nnmc with login password
func Init(login string, password string) (*NCp, error) {
	var client http.Client
	cookieJar, _ := cookiejar.New(nil)
	client.Jar = cookieJar
	urlPost := "http://nnm-club.me/forum/login.php"
	form := url.Values{}
	form.Set("username", login)
	form.Add("password", password)
	form.Add("redirect", "")
	form.Add("login", "âõîä")
	req, _ := http.NewRequest("POST", urlPost, bytes.NewBufferString(form.Encode()))
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Content-Length", strconv.Itoa(len(form.Encode())))
	_, err := client.Do(req)
	return &NCp{client: client}, err
}

// getHTML get body from url
func getHTML(url string, n *NCp) ([]byte, error) {
	resp, err := n.client.Get(url)
	if err != nil {
		log.Println("client Get error:", err)
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("ioutil.ReadAll error:", err)
	}
	buffer := bytes.NewBufferString("")
	for _, char := range body {
		var ch = Utf(char)
		fmt.Fprintf(buffer, "%c", ch)
	}
	doc := buffer.Bytes()
	doc = replaceAll(doc, " ", " ")
	doc = replaceAll(doc, "&", "&")
	doc = replaceAll(doc, "
", " ")
	doc = replaceAll(body, ":", ":")
	doc = replaceAll(doc, "  ", " ")
	doc = removeTag(doc, `<span style="text-decoration:.+?">(.+?)</span>`)
	return doc, nil
}

// ParseForumTree get topics from forumTree
func (n *NCp) ParseForumTree(url string) ([]Topic, error) {
	var (
		topics []Topic
		reTree = regexp.MustCompile(`<a href="(viewtopic.php\?t=\d+)"class="topictitle">(.+?)\s\((\d{4})\)\s(.+?)</a>`)
	)
	body, err := getHTML(url, n)
	if err != nil {
		return topics, err
	}
	if reTree.Match(body) == false {
		return topics, fmt.Errorf("No topic in body")
	}
	findResult := reTree.FindAllSubmatch(body, -1)
	for _, v := range findResult {
		var t Topic
		t.Href = "http://nnm-club.me/forum/" + string(v[1])
		t.Name = string(v[2])
		t.Year = string(v[3])
		t.Quality = string(v[4])
		topics = append(topics, t)
	}
	return topics, nil
}

// ParseTopic get film from topic
func (n *NCp) ParseTopic(topic Topic) (Film, error) {
	var (
		film Film
	)
	name := strings.Split(topic.Name, " / ")
	switch len(name) {
	case 1:
		film.Name = strings.Trim(name[0], " ")
	case 2:
		film.Name = strings.Trim(name[0], " ")
		film.EngName = strings.Trim(name[1], " ")
	case 3:
		film.Name = strings.Trim(name[0], " ")
		film.EngName = strings.Trim(name[1], " ")
	}
	film.Href = topic.Href
	if year64, err := strconv.ParseInt(topic.Year, 10, 64); err == nil {
		film.Year = year64
	}
	body, err := getHTML(film.Href, n)
	if err != nil {
		return film, err
	}
	topic.Body = body
	film.Country = topic.getCountry()
	film.Genre = topic.getGenre()
	film.Director = topic.getDirector()
	film.Producer = topic.getProducer()
	film.Actors = topic.getActors()
	film.Description = topic.getDescription()
	film.Age = topic.getAge()
	film.ReleaseDate = topic.getReleaseDate()
	film.RussianDate = topic.getRussianDate()
	film.Duration = topic.getDuration()
	film.Quality = topic.getQuality()
	film.Translation = topic.getTranslation()
	film.SubtitlesType = topic.getSubtitlesType()
	film.Subtitles = topic.getSubtitles()
	film.Video = topic.getVideo()
	film.Resolution = getResolution(film.Video)
	film.Audio = topic.getAudio()
	film.Torrent = topic.getTorrent()
	film.DateCreate = topic.getDate()
	film.Size = topic.getSize()
	film.NNM = topic.getRating()
	film.Poster = topic.getPoster()
	film.Seeders = topic.getSeeds()
	film.Leechers = topic.getLeechs()
	return film, nil
}

func replaceAll(body []byte, from string, to string) []byte {
	var reStr = regexp.MustCompile(from)
	if reStr.Match(body) == false {
		return body
	}
	return reStr.ReplaceAll(body, []byte(to))
}

func replaceDate(s string) string {
	s = strings.Replace(s, " Янв ", ".01.", -1)
	s = strings.Replace(s, " Фев ", ".02.", -1)
	s = strings.Replace(s, " Мар ", ".03.", -1)
	s = strings.Replace(s, " Апр ", ".04.", -1)
	s = strings.Replace(s, " Май ", ".05.", -1)
	s = strings.Replace(s, " Июн ", ".06.", -1)
	s = strings.Replace(s, " Июл ", ".07.", -1)
	s = strings.Replace(s, " Авг ", ".08.", -1)
	s = strings.Replace(s, " Сен ", ".09.", -1)
	s = strings.Replace(s, " Окт ", ".10.", -1)
	s = strings.Replace(s, " Ноя ", ".11.", -1)
	s = strings.Replace(s, " Дек ", ".12.", -1)
	return s
}

func caseInsensitiveContains(s, substr string) bool {
	s, substr = strings.ToUpper(s), strings.ToUpper(substr)
	return strings.Contains(s, substr)
}

func cleanStr(str string) string {
	var reSpan = regexp.MustCompile("")
	str = reSpan.ReplaceAllString(str, "")
	str = strings.Trim(str, " ")
	return str
}

func removeTag(body []byte, tag string) []byte {
	var reTag = regexp.MustCompile(tag)
	if reTag.Match(body) == false {
		return body
	}
	tags := reTag.FindAllSubmatch(body, -1)
	for _, item := range tags {
		body = bytes.Replace(body, item[0], item[1], 1)
	}
	return body
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tag

import (
	"bytes"
	"errors"
	"io"
)

const (
	idType      int = 1
	commentType int = 3
)

// ReadOGGTags reads OGG metadata from the io.ReadSeeker, returning the resulting
// metadata in a Metadata implementation, or non-nil error if there was a problem.
// See http://www.xiph.org/vorbis/doc/Vorbis_I_spec.html
// and http://www.xiph.org/ogg/doc/framing.html for details.
func ReadOGGTags(r io.ReadSeeker) (Metadata, error) {
	oggs, err := readString(r, 4)
	if err != nil {
		return nil, err
	}
	if oggs != "OggS" {
		return nil, errors.New("expected 'OggS'")
	}

	// Skip 22 bytes of Page header to read page_segments length byte at position 26
	// See http://www.xiph.org/ogg/doc/framing.html
	_, err = r.Seek(22, io.SeekCurrent)
	if err != nil {
		return nil, err
	}

	nS, err := readInt(r, 1)
	if err != nil {
		return nil, err
	}

	// Seek and discard the segments
	_, err = r.Seek(int64(nS), io.SeekCurrent)
	if err != nil {
		return nil, err
	}

	// First packet type is identification, type 1
	t, err := readInt(r, 1)
	if err != nil {
		return nil, err
	}
	if t != idType {
		return nil, errors.New("expected 'vorbis' identification type 1")
	}

	// Seek and discard 29 bytes from common and identification header
	// See http://www.xiph.org/vorbis/doc/Vorbis_I_spec.html#x1-610004.2
	_, err = r.Seek(29, io.SeekCurrent)
	if err != nil {
		return nil, err
	}

	// Read comment header packet. May include setup header packet, if it is on the
	// same page. First audio packet is guaranteed to be on the separate page.
	// See https://www.xiph.org/vorbis/doc/Vorbis_I_spec.html#x1-132000A.2
	ch, err := readPackets(r)
	if err != nil {
		return nil, err
	}
	chr := bytes.NewReader(ch)

	// First packet type is comment, type 3
	t, err = readInt(chr, 1)
	if err != nil {
		return nil, err
	}
	if t != commentType {
		return nil, errors.New("expected 'vorbis' comment type 3")
	}

	// Seek and discard 6 bytes from common header
	_, err = chr.Seek(6, io.SeekCurrent)
	if err != nil {
		return nil, err
	}

	m := &metadataOGG{
		newMetadataVorbis(),
	}

	err = m.readVorbisComment(chr)
	return m, err
}

// readPackets reads vorbis header packets from contiguous ogg pages in ReadSeeker.
// The pages are considered contiguous, if the first lacing value in second
// page's segment table continues rather than begins a packet. This is indicated
// by setting header_type_flag 0x1 (continued packet).
// See https://www.xiph.org/ogg/doc/framing.html on packets spanning pages.
func readPackets(r io.ReadSeeker) ([]byte, error) {
	buf := &bytes.Buffer{}

	firstPage := true
	for {
		// Read capture pattern
		oggs, err := readString(r, 4)
		if err != nil {
			return nil, err
		}
		if oggs != "OggS" {
			return nil, errors.New("expected 'OggS'")
		}

		// Read page header
		head, err := readBytes(r, 22)
		if err != nil {
			return nil, err
		}
		headerTypeFlag := head[1]

		continuation := headerTypeFlag&0x1 > 0
		if !(firstPage || continuation) {
			// Rewind to the beginning of the page
			_, err = r.Seek(-26, io.SeekCurrent)
			if err != nil {
				return nil, err
			}
			break
		}
		firstPage = false

		// Read the number of segments
		nS, err := readUint(r, 1)
		if err != nil {
			return nil, err
		}

		// Read segment table
		segments, err := readBytes(r, nS)
		if err != nil {
			return nil, err
		}

		// Calculate remaining page size
		pageSize := 0
		for i := uint(0); i < nS; i++ {
			pageSize += int(segments[i])
		}

		_, err = io.CopyN(buf, r, int64(pageSize))
		if err != nil {
			return nil, err
		}
	}

	return buf.Bytes(), nil
}

type metadataOGG struct {
	*metadataVorbis
}

func (m *metadataOGG) FileType() FileType {
	return OGG
}
<commit_msg>Add Ogg Opus metadata support (#69)<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tag

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
)

var (
	vorbisCommentPrefix = []byte("\x03vorbis")
	opusTagsPrefix      = []byte("OpusTags")
)

var oggCRC32Poly04c11db7 = oggCRCTable(0x04c11db7)

type crc32Table [256]uint32

func oggCRCTable(poly uint32) *crc32Table {
	var t crc32Table

	for i := 0; i < 256; i++ {
		crc := uint32(i) << 24
		for j := 0; j < 8; j++ {
			if crc&0x80000000 != 0 {
				crc = (crc << 1) ^ poly
			} else {
				crc <<= 1
			}
		}
		t[i] = crc
	}

	return &t
}

func oggCRCUpdate(crc uint32, tab *crc32Table, p []byte) uint32 {
	for _, v := range p {
		crc = (crc << 8) ^ tab[byte(crc>>24)^v]
	}
	return crc
}

type oggPageHeader struct {
	Magic           [4]byte // "OggS"
	Version         uint8
	Flags           uint8
	GranulePosition uint64
	SerialNumber    uint32
	SequenceNumber  uint32
	CRC             uint32
	Segments        uint8
}

type oggDemuxer struct {
	packetBufs map[uint32]*bytes.Buffer
}

// Read ogg packets, can return empty slice of packets and nil err
// if more data is needed
func (o *oggDemuxer) Read(r io.Reader) ([][]byte, error) {
	headerBuf := &bytes.Buffer{}
	var oh oggPageHeader
	if err := binary.Read(io.TeeReader(r, headerBuf), binary.LittleEndian, &oh); err != nil {
		return nil, err
	}

	if bytes.Compare(oh.Magic[:], []byte("OggS")) != 0 {
		// TODO: seek for syncword?
		return nil, errors.New("expected 'OggS'")
	}

	segmentTable := make([]byte, oh.Segments)
	if _, err := io.ReadFull(r, segmentTable); err != nil {
		return nil, err
	}
	var segmentsSize int64
	for _, s := range segmentTable {
		segmentsSize += int64(s)
	}
	segmentsData := make([]byte, segmentsSize)
	if _, err := io.ReadFull(r, segmentsData); err != nil {
		return nil, err
	}

	headerBytes := headerBuf.Bytes()
	// reset CRC to zero in header before checksum
	headerBytes[22] = 0
	headerBytes[23] = 0
	headerBytes[24] = 0
	headerBytes[25] = 0
	crc := oggCRCUpdate(0, oggCRC32Poly04c11db7, headerBytes)
	crc = oggCRCUpdate(crc, oggCRC32Poly04c11db7, segmentTable)
	crc = oggCRCUpdate(crc, oggCRC32Poly04c11db7, segmentsData)
	if crc != oh.CRC {
		return nil, fmt.Errorf("expected crc %x != %x", oh.CRC, crc)
	}

	if o.packetBufs == nil {
		o.packetBufs = map[uint32]*bytes.Buffer{}
	}

	var packetBuf *bytes.Buffer
	continued := oh.Flags&0x1 != 0
	if continued {
		if b, ok := o.packetBufs[oh.SerialNumber]; ok {
			packetBuf = b
		} else {
			return nil, fmt.Errorf("could not find continued packet %d", oh.SerialNumber)
		}
	} else {
		packetBuf = &bytes.Buffer{}
	}

	var packets [][]byte
	var p int
	for _, s := range segmentTable {
		packetBuf.Write(segmentsData[p : p+int(s)])
		if s < 255 {
			packets = append(packets, packetBuf.Bytes())
			packetBuf = &bytes.Buffer{}
		}
		p += int(s)
	}

	o.packetBufs[oh.SerialNumber] = packetBuf

	return packets, nil
}

// ReadOGGTags reads OGG metadata from the io.ReadSeeker, returning the resulting
// metadata in a Metadata implementation, or non-nil error if there was a problem.
// See http://www.xiph.org/vorbis/doc/Vorbis_I_spec.html
// and http://www.xiph.org/ogg/doc/framing.html for details.
// For Opus see https://tools.ietf.org/html/rfc7845
func ReadOGGTags(r io.Reader) (Metadata, error) {
	od := &oggDemuxer{}
	for {
		bs, err := od.Read(r)
		if err != nil {
			return nil, err
		}

		for _, b := range bs {
			switch {
			case bytes.HasPrefix(b, vorbisCommentPrefix):
				m := &metadataOGG{
					newMetadataVorbis(),
				}
				err = m.readVorbisComment(bytes.NewReader(b[len(vorbisCommentPrefix):]))
				return m, err
			case bytes.HasPrefix(b, opusTagsPrefix):
				m := &metadataOGG{
					newMetadataVorbis(),
				}
				err = m.readVorbisComment(bytes.NewReader(b[len(opusTagsPrefix):]))
				return m, err
			}
		}
	}
}

type metadataOGG struct {
	*metadataVorbis
}

func (m *metadataOGG) FileType() FileType {
	return OGG
}
<|endoftext|>


<commit_before>package php

import (
	"fmt"
	"strings"
	"unicode"

	"github.com/stephens2424/php/token"
)

const shortPHPBegin = "<?"
const longPHPBegin = "<?php"
const phpEnd = "?>"

const eof = -1

// lexHTML consumes and emits an html t until it
// finds a php begin
func lexHTML(l *lexer) stateFn {
	for {
		if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
			if l.pos > l.start {
				l.emit(token.HTML)
			}
			return lexPHPBegin
		}
		if l.next() == eof {
			break
		}
	}
	if l.pos > l.start {
		l.emit(token.HTML)
	}
	l.emit(token.EOF)
	return nil
}

func lexPHPBegin(l *lexer) stateFn {
	if strings.HasPrefix(l.input[l.pos:], longPHPBegin) {
		l.pos += len(longPHPBegin)
	}
	if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
		l.pos += len(shortPHPBegin)
	}
	l.emit(token.PHPBegin)
	return lexPHP
}

func lexPHP(l *lexer) stateFn {
	l.skipSpace()

	if r := l.peek(); unicode.IsDigit(r) {
		return lexNumberLiteral
	} else if r == '.' {
		l.next() // must advance because we only peeked before
		secondR := l.peek()
		l.backup()
		if unicode.IsDigit(secondR) {
			return lexNumberLiteral
		}
	}

	if strings.HasPrefix(l.input[l.pos:], "<<<") {
		return lexDoc
	}

	if strings.HasPrefix(l.input[l.pos:], "?>") {
		return lexPHPEnd
	}

	if strings.HasPrefix(l.input[l.pos:], "#") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "//") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "/*") {
		return lexBlockComment
	}

	if l.peek() == eof {
		l.emit(token.EOF)
		return nil
	}

	if l.peek() == '`' {
		return lexShellCommand
	}

	if l.peek() == '\'' {
		return lexSingleQuotedStringLiteral
	}

	if l.peek() == '"' {
		return lexDoubleQuotedStringLiteral
	}

	for _, tokenString := range token.TokenList {
		t := token.TokenMap[tokenString]
		potentialToken := l.input[l.pos:]
		if len(potentialToken) > len(tokenString) {
			potentialToken = potentialToken[:len(tokenString)]
		}
		if strings.HasPrefix(strings.ToLower(potentialToken), tokenString) {
			prev := l.previous()
			if isKeyword(t, tokenString) && prev == '$' {
				break
			}
			l.pos += len(tokenString)
			if isKeyword(t, tokenString) && l.accept(alphabet+underscore+digits) {
				l.backup() // to account for the character consumed by accept
				l.pos -= len(tokenString)
				break
			}
			l.emit(t)
			return lexPHP
		}
	}

	l.acceptRun(alphabet + underscore + digits + "\\")
	l.emit(token.Identifier)
	return lexPHP
}

func lexNumberLiteral(l *lexer) stateFn {
	if l.accept("0") {
		// binary?
		if l.accept("b") {
			l.acceptRun("01")
			l.emit(token.NumberLiteral)
			return lexPHP
		}
		// hexadecimal?
		if l.accept("xX") {
			l.acceptRun(digits + "abcdefABCDEF")
			l.emit(token.NumberLiteral)
			return lexPHP
		}
	}
	// is decimal?
	l.acceptRun(digits)
	if l.accept(".") {
		l.acceptRun(digits)
	}

	if l.accept("E") {
		l.acceptRun(digits)
	}

	l.emit(token.NumberLiteral)
	return lexPHP
}

func lexShellCommand(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '`':
			l.emit(token.ShellCommand)
			return lexPHP
		case eof:
			l.emit(token.ShellCommand)
			return nil
		}
	}
}

func lexSingleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '\'':
			l.emit(token.StringLiteral)
			return lexPHP
		}
	}
}

func lexDoubleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '"':
			l.emit(token.StringLiteral)
			return lexPHP
		}
	}
}

const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const digits = "0123456789"
const underscore = "_"

func lexIdentifier(l *lexer) stateFn {
	l.accept("$")
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	l.emit(token.VariableOperator)
	return lexPHP
}

// lexPHPEnd lexes the end of a PHP section returning the context to HTML
func lexPHPEnd(l *lexer) stateFn {
	l.pos += len(phpEnd)
	l.emit(token.PHPEnd)
	return lexHTML
}

func lexLineComment(l *lexer) stateFn {
	lineLength := strings.Index(l.input[l.pos:], "\n") + 1
	if lineLength == 0 {
		// this is the last line, so lex until the end
		lineLength = len(l.input[l.pos:])
	}
	// don't lex php end
	if phpEndLength := strings.Index(l.input[l.pos:l.pos+lineLength], phpEnd); phpEndLength >= 0 && phpEndLength < lineLength {
		lineLength = phpEndLength
	}
	l.pos += lineLength
	l.ignore()
	return lexPHP
}

func lexBlockComment(l *lexer) stateFn {
	commentLength := strings.Index(l.input[l.pos:], "*/") + 2
	if commentLength == 1 {
		// the file ends before we find */
		commentLength = len(l.input[l.pos:])
	}
	l.pos += commentLength
	l.ignore()
	return lexPHP
}

func lexDoc(l *lexer) stateFn {
	var nowDoc bool
	l.pos += len("<<<")
	l.skipSpace()
	if strings.HasPrefix(l.input[l.pos:], "'") {
		nowDoc = true
		l.pos += len("'")
	}
	labelPos := l.pos
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	endMarker := fmt.Sprintf("\n%s", l.input[labelPos:l.pos])
	if nowDoc {
		l.accept("'")
	}
	l.accept("\n")
	for !strings.HasPrefix(l.input[l.pos:], endMarker) {
		l.next()
	}
	l.pos += len(endMarker)
	l.emit(token.StringLiteral)
	return lexPHP
}
<commit_msg>Adding lowercase e to scientific notation lexing<commit_after>package php

import (
	"fmt"
	"strings"
	"unicode"

	"github.com/stephens2424/php/token"
)

const shortPHPBegin = "<?"
const longPHPBegin = "<?php"
const phpEnd = "?>"

const eof = -1

// lexHTML consumes and emits an html t until it
// finds a php begin
func lexHTML(l *lexer) stateFn {
	for {
		if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
			if l.pos > l.start {
				l.emit(token.HTML)
			}
			return lexPHPBegin
		}
		if l.next() == eof {
			break
		}
	}
	if l.pos > l.start {
		l.emit(token.HTML)
	}
	l.emit(token.EOF)
	return nil
}

func lexPHPBegin(l *lexer) stateFn {
	if strings.HasPrefix(l.input[l.pos:], longPHPBegin) {
		l.pos += len(longPHPBegin)
	}
	if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
		l.pos += len(shortPHPBegin)
	}
	l.emit(token.PHPBegin)
	return lexPHP
}

func lexPHP(l *lexer) stateFn {
	l.skipSpace()

	if r := l.peek(); unicode.IsDigit(r) {
		return lexNumberLiteral
	} else if r == '.' {
		l.next() // must advance because we only peeked before
		secondR := l.peek()
		l.backup()
		if unicode.IsDigit(secondR) {
			return lexNumberLiteral
		}
	}

	if strings.HasPrefix(l.input[l.pos:], "<<<") {
		return lexDoc
	}

	if strings.HasPrefix(l.input[l.pos:], "?>") {
		return lexPHPEnd
	}

	if strings.HasPrefix(l.input[l.pos:], "#") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "//") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "/*") {
		return lexBlockComment
	}

	if l.peek() == eof {
		l.emit(token.EOF)
		return nil
	}

	if l.peek() == '`' {
		return lexShellCommand
	}

	if l.peek() == '\'' {
		return lexSingleQuotedStringLiteral
	}

	if l.peek() == '"' {
		return lexDoubleQuotedStringLiteral
	}

	for _, tokenString := range token.TokenList {
		t := token.TokenMap[tokenString]
		potentialToken := l.input[l.pos:]
		if len(potentialToken) > len(tokenString) {
			potentialToken = potentialToken[:len(tokenString)]
		}
		if strings.HasPrefix(strings.ToLower(potentialToken), tokenString) {
			prev := l.previous()
			if isKeyword(t, tokenString) && prev == '$' {
				break
			}
			l.pos += len(tokenString)
			if isKeyword(t, tokenString) && l.accept(alphabet+underscore+digits) {
				l.backup() // to account for the character consumed by accept
				l.pos -= len(tokenString)
				break
			}
			l.emit(t)
			return lexPHP
		}
	}

	l.acceptRun(alphabet + underscore + digits + "\\")
	l.emit(token.Identifier)
	return lexPHP
}

func lexNumberLiteral(l *lexer) stateFn {
	if l.accept("0") {
		// binary?
		if l.accept("b") {
			l.acceptRun("01")
			l.emit(token.NumberLiteral)
			return lexPHP
		}
		// hexadecimal?
		if l.accept("xX") {
			l.acceptRun(digits + "abcdefABCDEF")
			l.emit(token.NumberLiteral)
			return lexPHP
		}
	}
	// is decimal?
	l.acceptRun(digits)
	if l.accept(".") {
		l.acceptRun(digits)
	}

	if l.accept("eE") {
		l.acceptRun(digits)
	}

	l.emit(token.NumberLiteral)
	return lexPHP
}

func lexShellCommand(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '`':
			l.emit(token.ShellCommand)
			return lexPHP
		case eof:
			l.emit(token.ShellCommand)
			return nil
		}
	}
}

func lexSingleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '\'':
			l.emit(token.StringLiteral)
			return lexPHP
		}
	}
}

func lexDoubleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '"':
			l.emit(token.StringLiteral)
			return lexPHP
		}
	}
}

const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const digits = "0123456789"
const underscore = "_"

func lexIdentifier(l *lexer) stateFn {
	l.accept("$")
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	l.emit(token.VariableOperator)
	return lexPHP
}

// lexPHPEnd lexes the end of a PHP section returning the context to HTML
func lexPHPEnd(l *lexer) stateFn {
	l.pos += len(phpEnd)
	l.emit(token.PHPEnd)
	return lexHTML
}

func lexLineComment(l *lexer) stateFn {
	lineLength := strings.Index(l.input[l.pos:], "\n") + 1
	if lineLength == 0 {
		// this is the last line, so lex until the end
		lineLength = len(l.input[l.pos:])
	}
	// don't lex php end
	if phpEndLength := strings.Index(l.input[l.pos:l.pos+lineLength], phpEnd); phpEndLength >= 0 && phpEndLength < lineLength {
		lineLength = phpEndLength
	}
	l.pos += lineLength
	l.ignore()
	return lexPHP
}

func lexBlockComment(l *lexer) stateFn {
	commentLength := strings.Index(l.input[l.pos:], "*/") + 2
	if commentLength == 1 {
		// the file ends before we find */
		commentLength = len(l.input[l.pos:])
	}
	l.pos += commentLength
	l.ignore()
	return lexPHP
}

func lexDoc(l *lexer) stateFn {
	var nowDoc bool
	l.pos += len("<<<")
	l.skipSpace()
	if strings.HasPrefix(l.input[l.pos:], "'") {
		nowDoc = true
		l.pos += len("'")
	}
	labelPos := l.pos
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	endMarker := fmt.Sprintf("\n%s", l.input[labelPos:l.pos])
	if nowDoc {
		l.accept("'")
	}
	l.accept("\n")
	for !strings.HasPrefix(l.input[l.pos:], endMarker) {
		l.next()
	}
	l.pos += len(endMarker)
	l.emit(token.StringLiteral)
	return lexPHP
}
<|endoftext|>


<commit_before>package CloudForest

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"strconv"
	"strings"
)

func ParseLibSVM(input io.Reader) *FeatureMatrix {
	reader := bufio.NewReader(input)

	data := make([]Feature, 0, 100)
	lookup := make(map[string]int, 0)
	labels := make([]string, 0, 0)

	i := 0
	ncases := 8100000
	allthedata := make([]uint8, 784*ncases, 784*ncases)
	for {

		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			log.Print("Error:", err)
			return nil
		}

		vals := strings.Fields(line)

		if i == 0 {
			if strings.Contains(vals[0], ".") {
				//looks like a float...add dense float64 feature regression
				data = append(data, &DenseNumFeature{
					make([]float64, 0, 0),
					make([]bool, 0, 0),
					"0"})

			} else {
				//doesn't look like a float...add dense catagorical
				data = append(data, &DenseCatFeature{
					&CatMap{make(map[string]int, 0),
						make([]string, 0, 0)},
					make([]int, 0, 0),
					make([]bool, 0, 0),
					"0",
					false})
			}
		}
		data[0].Append(vals[0])

		for _, v := range vals[1:] {
			parts := strings.Split(v, ":")
			xi, err := strconv.Atoi(parts[0])
			if err != nil {
				log.Print("Atoi error: ", err, " Line ", i, " Parsing: ", v)
			}
			//pad out the data to include this feature
			for xi >= len(data) {
				// data = append(data, &SparseNumFeature{
				// 	make(map[int]float64),
				// 	make(map[int]bool),
				// 	fmt.Sprintf("%v", len(data))})

				data = append(data, &DenseUint8Feature{
					allthedata[(len(data)-1)*ncases : len(data)*ncases],
					make(map[int]bool),
					fmt.Sprintf("%v", len(data))})
				/*
					data = append(data, &SparseUint8Feature{
						make(map[int]uint8),
						make(map[int]bool),
						fmt.Sprintf("%v", len(data))})
				*/

			}
			data[xi].PutStr(i, parts[1])

		}
		label := fmt.Sprintf("%v", i)
		lookup[label] = i
		labels = append(labels, label)

		i++

	}

	fm := &FeatureMatrix{data, lookup, labels}

	return fm

}
<commit_msg>fix to libsvm parseing<commit_after>package CloudForest

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"strconv"
	"strings"
)

func ParseLibSVM(input io.Reader) *FeatureMatrix {
	reader := bufio.NewReader(input)

	data := make([]Feature, 0, 100)
	lookup := make(map[string]int, 0)
	labels := make([]string, 0, 0)

	i := 0
	//ncases := 8100000
	//allthedata := make([]uint8, 784*ncases, 784*ncases)
	for {

		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			log.Print("Error:", err)
			return nil
		}

		vals := strings.Fields(line)

		if i == 0 {
			if strings.Contains(vals[0], ".") {
				//looks like a float...add dense float64 feature regression
				data = append(data, &DenseNumFeature{
					make([]float64, 0, 0),
					make([]bool, 0, 0),
					"0"})

			} else {
				//doesn't look like a float...add dense catagorical
				data = append(data, &DenseCatFeature{
					&CatMap{make(map[string]int, 0),
						make([]string, 0, 0)},
					make([]int, 0, 0),
					make([]bool, 0, 0),
					"0",
					false})
			}
		}
		data[0].Append(vals[0])

		for _, v := range vals[1:] {
			parts := strings.Split(v, ":")
			xi, err := strconv.Atoi(parts[0])
			if err != nil {
				log.Print("Atoi error: ", err, " Line ", i, " Parsing: ", v)
			}
			//pad out the data to include this feature
			for xi >= len(data) {
				data = append(data, &SparseNumFeature{
					make(map[int]float64),
					make(map[int]bool),
					fmt.Sprintf("%v", len(data))})

				/*
					data = append(data, &DenseUint8Feature{
						allthedata[(len(data)-1)*ncases : len(data)*ncases],
						make(map[int]bool),
						fmt.Sprintf("%v", len(data))})
					/*
						data = append(data, &SparseUint8Feature{
							make(map[int]uint8),
							make(map[int]bool),
							fmt.Sprintf("%v", len(data))})
				*/

			}
			data[xi].PutStr(i, parts[1])

		}
		label := fmt.Sprintf("%v", i)
		lookup[label] = i
		labels = append(labels, label)

		i++

	}

	fm := &FeatureMatrix{data, lookup, labels}

	return fm

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"compress/zlib"
	"encoding/binary"
	"fmt"
	"io"
	"os"
)

type PrtGenerator struct {
	enclosedsChan chan *EnclosedChunkJob
	completeChan  chan bool

	outFile *os.File
	w       *bufio.Writer
	zw      io.WriteCloser

	particleCount int64
	total         int
	boundary      *BoundaryLocator
}

func (o *PrtGenerator) Start(outFilename string, total int, maxProcs int, boundary *BoundaryLocator) {
	o.enclosedsChan = make(chan *EnclosedChunkJob, maxProcs*2)
	o.completeChan = make(chan bool)
	o.total = total
	o.boundary = boundary

	maxProcs = 1
	for i := 0; i < maxProcs; i++ {
		go o.chunkProcessor()
	}

	var openErr os.Error

	o.outFile, openErr = os.Create(outFilename)
	if openErr != nil {
		fmt.Fprintln(os.Stderr, openErr) // TODO: return openErr
		return
	}

	o.w = bufio.NewWriter(o.outFile)
	WriteHeader(o.w, -1, []ChannelDefinition{{"Position", 4, 3, 0}, {"BlockID", 1, 1, 12}})

	var zErr os.Error
	o.zw, zErr = zlib.NewWriterLevel(o.w, zlib.NoCompression)
	if zErr != nil {
		fmt.Fprintln(os.Stderr, zErr) // TODO: return zErr
		return
	}
}

func (o *PrtGenerator) chunkProcessor() {
	var chunkCount = 0
	for {
		var job = <-o.enclosedsChan

		var e = job.enclosed

		for i := 0; i < len(e.blocks); i += 128 {
			var x, z = (i / 128) / 16, (i / 128) % 16

			var column = BlockColumn(e.blocks[i : i+128])
			for y, blockId := range column {
				if y < yMin {
					continue
				}

				switch {
				case o.boundary.IsBoundary(blockId, e.Get(x, y-1, z)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x, y+1, z)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x-1, y, z)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x+1, y, z)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x, y, z-1)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x, y, z+1)):
					o.particleCount++
					var (
						xa = x + e.xPos*16
						ya = y - 64
						za = -(z + e.zPos*16)
					)
					binary.Write(o.zw, binary.LittleEndian, float32(xa*2))
					binary.Write(o.zw, binary.LittleEndian, float32(za*2))
					binary.Write(o.zw, binary.LittleEndian, float32(ya*2))
					binary.Write(o.zw, binary.LittleEndian, int32(blockId))
				}
			}
		}

		chunkCount++
		fmt.Printf("%4v/%-4v (%3v,%3v) Particles: %d\n", chunkCount, o.total, job.enclosed.xPos, job.enclosed.zPos, o.particleCount)

		if job.last {
			o.completeChan <- true
			break
		}
	}
}

func (o *PrtGenerator) Close() {
	o.zw.Close()
	o.w.Flush()
	UpdateParticleCount(o.outFile, o.particleCount)
	o.outFile.Close()
}

func (o *PrtGenerator) GetEnclosedJobsChan() chan *EnclosedChunkJob {
	return o.enclosedsChan
}

func (o *PrtGenerator) GetCompleteChan() chan bool {
	return o.completeChan
}

type ChannelDefinition struct {
	Name                    string // max of 31 characters and must meet the regex [a-zA-Z_][0-9a-zA-Z_]*
	DataType, Arity, Offset int32
}

// http://software.primefocusworld.com/software/support/krakatoa/prt_file_format.php
func WriteHeader(w io.Writer, particleCount int64, channels []ChannelDefinition) {
	// Header (56 bytes)
	var magic = []byte{192, 'P', 'R', 'T', '\r', '\n', 26, '\n'}
	w.Write(magic)

	var headerLength = uint32(56)
	binary.Write(w, binary.LittleEndian, headerLength)

	var signature = make([]byte, 32)
	copy(signature, []byte("Extensible Particle Format"))
	w.Write(signature)

	var version = uint32(1)
	binary.Write(w, binary.LittleEndian, version)

	binary.Write(w, binary.LittleEndian, particleCount)

	// Reserved bytes (4 bytes)
	var reserved = int32(4)
	binary.Write(w, binary.LittleEndian, reserved)

	// Channel definition header (8 bytes)
	binary.Write(w, binary.LittleEndian, int32(len(channels)))

	var channelDefinitionSize = int32(44)
	binary.Write(w, binary.LittleEndian, channelDefinitionSize)

	for _, channel := range channels {
		var nameBytes = make([]byte, 32)
		copy(nameBytes, []byte(channel.Name))
		w.Write(nameBytes)

		binary.Write(w, binary.LittleEndian, channel.DataType)
		binary.Write(w, binary.LittleEndian, channel.Arity)
		binary.Write(w, binary.LittleEndian, channel.Offset)
	}
}

func UpdateParticleCount(file *os.File, particleCount int64) os.Error {
	var storedOffset, err = file.Seek(0, 1)
	if err != nil {
		return err
	}
	_, err = file.Seek(0x30, 0)
	if err != nil {
		return err
	}
	err = binary.Write(file, binary.LittleEndian, particleCount)
	if err != nil {
		return err
	}
	_, err = file.Seek(storedOffset, 0)
	return err
}
<commit_msg>Remove the 2x scaling for prt<commit_after>package main

import (
	"bufio"
	"compress/zlib"
	"encoding/binary"
	"fmt"
	"io"
	"os"
)

type PrtGenerator struct {
	enclosedsChan chan *EnclosedChunkJob
	completeChan  chan bool

	outFile *os.File
	w       *bufio.Writer
	zw      io.WriteCloser

	particleCount int64
	total         int
	boundary      *BoundaryLocator
}

func (o *PrtGenerator) Start(outFilename string, total int, maxProcs int, boundary *BoundaryLocator) {
	o.enclosedsChan = make(chan *EnclosedChunkJob, maxProcs*2)
	o.completeChan = make(chan bool)
	o.total = total
	o.boundary = boundary

	maxProcs = 1
	for i := 0; i < maxProcs; i++ {
		go o.chunkProcessor()
	}

	var openErr os.Error

	o.outFile, openErr = os.Create(outFilename)
	if openErr != nil {
		fmt.Fprintln(os.Stderr, openErr) // TODO: return openErr
		return
	}

	o.w = bufio.NewWriter(o.outFile)
	WriteHeader(o.w, -1, []ChannelDefinition{{"Position", 4, 3, 0}, {"BlockID", 1, 1, 12}})

	var zErr os.Error
	o.zw, zErr = zlib.NewWriterLevel(o.w, zlib.NoCompression)
	if zErr != nil {
		fmt.Fprintln(os.Stderr, zErr) // TODO: return zErr
		return
	}
}

func (o *PrtGenerator) chunkProcessor() {
	var chunkCount = 0
	for {
		var job = <-o.enclosedsChan

		var e = job.enclosed

		for i := 0; i < len(e.blocks); i += 128 {
			var x, z = (i / 128) / 16, (i / 128) % 16

			var column = BlockColumn(e.blocks[i : i+128])
			for y, blockId := range column {
				if y < yMin {
					continue
				}

				switch {
				case o.boundary.IsBoundary(blockId, e.Get(x, y-1, z)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x, y+1, z)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x-1, y, z)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x+1, y, z)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x, y, z-1)):
					fallthrough
				case o.boundary.IsBoundary(blockId, e.Get(x, y, z+1)):
					o.particleCount++
					var (
						xa = x + e.xPos*16
						ya = y - 64
						za = -(z + e.zPos*16)
					)
					binary.Write(o.zw, binary.LittleEndian, float32(xa))
					binary.Write(o.zw, binary.LittleEndian, float32(za))
					binary.Write(o.zw, binary.LittleEndian, float32(ya))
					binary.Write(o.zw, binary.LittleEndian, int32(blockId))
				}
			}
		}

		chunkCount++
		fmt.Printf("%4v/%-4v (%3v,%3v) Particles: %d\n", chunkCount, o.total, job.enclosed.xPos, job.enclosed.zPos, o.particleCount)

		if job.last {
			o.completeChan <- true
			break
		}
	}
}

func (o *PrtGenerator) Close() {
	o.zw.Close()
	o.w.Flush()
	UpdateParticleCount(o.outFile, o.particleCount)
	o.outFile.Close()
}

func (o *PrtGenerator) GetEnclosedJobsChan() chan *EnclosedChunkJob {
	return o.enclosedsChan
}

func (o *PrtGenerator) GetCompleteChan() chan bool {
	return o.completeChan
}

type ChannelDefinition struct {
	Name                    string // max of 31 characters and must meet the regex [a-zA-Z_][0-9a-zA-Z_]*
	DataType, Arity, Offset int32
}

// http://software.primefocusworld.com/software/support/krakatoa/prt_file_format.php
func WriteHeader(w io.Writer, particleCount int64, channels []ChannelDefinition) {
	// Header (56 bytes)
	var magic = []byte{192, 'P', 'R', 'T', '\r', '\n', 26, '\n'}
	w.Write(magic)

	var headerLength = uint32(56)
	binary.Write(w, binary.LittleEndian, headerLength)

	var signature = make([]byte, 32)
	copy(signature, []byte("Extensible Particle Format"))
	w.Write(signature)

	var version = uint32(1)
	binary.Write(w, binary.LittleEndian, version)

	binary.Write(w, binary.LittleEndian, particleCount)

	// Reserved bytes (4 bytes)
	var reserved = int32(4)
	binary.Write(w, binary.LittleEndian, reserved)

	// Channel definition header (8 bytes)
	binary.Write(w, binary.LittleEndian, int32(len(channels)))

	var channelDefinitionSize = int32(44)
	binary.Write(w, binary.LittleEndian, channelDefinitionSize)

	for _, channel := range channels {
		var nameBytes = make([]byte, 32)
		copy(nameBytes, []byte(channel.Name))
		w.Write(nameBytes)

		binary.Write(w, binary.LittleEndian, channel.DataType)
		binary.Write(w, binary.LittleEndian, channel.Arity)
		binary.Write(w, binary.LittleEndian, channel.Offset)
	}
}

func UpdateParticleCount(file *os.File, particleCount int64) os.Error {
	var storedOffset, err = file.Seek(0, 1)
	if err != nil {
		return err
	}
	_, err = file.Seek(0x30, 0)
	if err != nil {
		return err
	}
	err = binary.Write(file, binary.LittleEndian, particleCount)
	if err != nil {
		return err
	}
	_, err = file.Seek(storedOffset, 0)
	return err
}
<|endoftext|>