<commit_before>package main

import (
	"bytes"
	"github.com/Sirupsen/logrus"
	"io"
	"net"
	"strconv"
	"sync"
)

const (
	NETBUF_SIZE = 2048 // bytes
)

var (
	poolNetBuffers sync.Pool
)

// var-constants
var (
	HEAD_CONNECTION            = []byte("CONNECTION")
	HEAD_CONNECTION_KEEP_ALIVE = []byte("KEEP-ALIVE")
	HEAD_CONTENT_LENGTH        = []byte("CONTENT-LENGTH")
)

// Get or create network buffer for proxy
func netbufGet() (buf []byte) {
	bufInterface := poolNetBuffers.Get()
	if bufInterface == nil {
		buf = make([]byte, NETBUF_SIZE)
	} else {
		buf = bufInterface.([]byte)
		// prevent data leak
		for i := range buf {
			buf[i] = 0
		}

	}
	return buf
}

func netbufPut(buf []byte) {
	poolNetBuffers.Put(buf)
}

func proxyHTTPHeaders(targetConn net.Conn, sourceConn net.Conn) (keepalive bool, contentLength int64) {
	buf := netbufGet()
	defer netbufPut(buf)
	var totalReadBytes int

	// Read lines
readHeaderLines:
	for {
		var i int
		var headerStart []byte
		for i = 0; i < len(buf); i++ {
			readBytes, err := sourceConn.Read(buf[i : i+1])
			totalReadBytes += readBytes
			if err != nil {
				if err == io.EOF && totalReadBytes == 0 {
					logrus.Debugf("Normal close connection from '%v' to '%v', becouse eof and zero headers readed", sourceConn.RemoteAddr(), targetConn.RemoteAddr())
				} else {
					logrus.Infof("Error while read header from '%v': %v", sourceConn.RemoteAddr(), err)
				}
				targetConn.Close()
				sourceConn.Close()
				return
			}
			if readBytes != 1 {
				logrus.Infof("Can't read a byte from header from '%v'", sourceConn.RemoteAddr())
				targetConn.Close()
				sourceConn.Close()
				return
			}
			if buf[i] == ':' || buf[i] == '\n' {
				headerStart = buf[:i+1]
				logrus.Debugf("Header Name '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), buf[:i])
				break
			}
		}
		if len(headerStart) == 0 {
			logrus.Infof("Header line longer then buffer (%v bytes). Force close connection. '%v' -> '%v'.", len(buf), sourceConn.RemoteAddr(), targetConn.RemoteAddr())
			targetConn.Close()
			sourceConn.Close()
			return
		}

		// Empty line - end http headers
		if bytes.Equal(headerStart, []byte("\n")) || bytes.Equal(headerStart, []byte("\r\n")) {
			break readHeaderLines
		}

		headerName := bytes.ToUpper(headerStart[:len(headerStart)-1]) // Cut trailing colon from start

		skipHeader := false
		for _, ownHeader := range cutHeaders {
			if bytes.Equal(ownHeader, headerName) {
				skipHeader = true
				break
			}
			ownHeaderS := string(ownHeader)
			headerNameS := string(headerName)
			skipHeader = ownHeaderS == headerNameS
		}

		if skipHeader {
			logrus.Debugf("Skip header: '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerName)
			buf[0] = headerStart[len(headerStart)-1]

			for buf[0] != '\n' {
				_, err := sourceConn.Read(buf[:1])
				if err != nil {
					logrus.Infof("Error read header. Close connections. '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
					sourceConn.Close()
					targetConn.Close()
					return
				}
			}
			continue readHeaderLines
		}

		logrus.Debugf("Copy header: '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerName)

		// copy header without changes
		_, err := targetConn.Write(headerStart)
		if err != nil {
			logrus.Infof("Write header start, from '%v' to '%v', headerStart='%s': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerStart, err)
			sourceConn.Close()
			targetConn.Close()
			return
		}

		needHeaderContent := bytes.Equal(headerName, HEAD_CONTENT_LENGTH) || bytes.Equal(headerName, HEAD_CONNECTION)
		headerContent := bytes.NewBuffer(buf[1:])
		headerContent.Reset()

		for buf[0] != '\n' {
			readBytes, err := sourceConn.Read(buf[:1])
			if err != nil {
				logrus.Infof("Error read header to copy. Close connections. '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			if readBytes != 1 {
				logrus.Infof("Header copy read bytes != 1. Error. Close connections. '%v' -> '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr())
				sourceConn.Close()
				targetConn.Close()
				return
			}
			_, err = targetConn.Write(buf[:1])
			if err != nil {
				logrus.Infof("Error write header. Close connections. '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			if needHeaderContent {
				headerContent.WriteByte(buf[0])
			}
		}
		if needHeaderContent {
			switch {
			case bytes.Equal(headerName, HEAD_CONNECTION):
				keepalive = bytes.EqualFold(HEAD_CONNECTION_KEEP_ALIVE, bytes.TrimSpace(headerContent.Bytes()))

			case bytes.Equal(headerName, HEAD_CONTENT_LENGTH):
				contentLength, err = strconv.ParseInt(string(bytes.TrimSpace(headerContent.Bytes())), 10, 64)
				if err == nil {
					logrus.Debugf("Header content-length parsed from '%v' to '%v': %v", sourceConn.RemoteAddr(),
						targetConn.RemoteAddr(), contentLength)
				} else {
					logrus.Infof("Can't header content-length parsed from '%v' to '%v' content '%s': %v", sourceConn.RemoteAddr(),
						targetConn.RemoteAddr(), headerContent.Bytes(), err)
					contentLength = 0
				}

			default:
				logrus.Debug("ERROR. Unknow why i need header content. Code error.")
			}
		}
	}

	remoteTcpAddr := sourceConn.RemoteAddr().(*net.TCPAddr)
	remoteAddrString := remoteTcpAddr.IP.String()

	headerBuf := bytes.NewBuffer(buf)
	headerBuf.Reset()

	// Write real IP
	for _, header := range realIPHeaderNames {
		headerBuf.Write(header)
		headerBuf.WriteByte(':')
		headerBuf.WriteString(remoteAddrString)
		headerBuf.Write([]byte("\r\n"))
	}
	headerBuf.Write(additionalHeaders)
	headerBuf.Write([]byte("\r\n")) // end http headers
	logrus.Debugf("Add headers. '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerBuf.Bytes())

	_, err := targetConn.Write(headerBuf.Bytes())
	if err != nil {
		logrus.Infof("Error while write real ip headers to target '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
	}

	return
}

func startProxy(targetAddr net.TCPAddr, in net.Conn) {
	targetConnCommon, err := net.DialTimeout("tcp", targetAddr.String(), *targetConnTimeout)
	if err != nil {
		logrus.Warnf("Can't connect to target '%v': %v", targetAddr.String(), err)
		return
	}

	targetConn, ok := targetConnCommon.(*net.TCPConn)
	if !ok {
		logrus.Errorf("Can't cast connection to tcp connection, target '%v'", targetAddr.String())
		return
	}

	switch *proxyMode {
	case "http":
		startProxyHTTP(targetConn, in)
	case "tcp":
		startProxyTCP(targetConn, in)
	default:
		logrus.Panicf("Unknow proxy mode: %v", *proxyMode)
	}
}

func startProxyHTTP(targetConn net.Conn, sourceConn net.Conn) {
	logrus.Infof("Start http-proxy connection from '%v' to'%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr())

	// answer from server proxy without changes
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(sourceConn, targetConn, buf)
		logrus.Debugf("Connection closed with error1 '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
		sourceConn.Close()
		targetConn.Close()
	}()

	// proxy incoming traffic, parse every headers
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)
		var summBytesCopied int64
		for {
			keepalive, contentLength := proxyHTTPHeaders(targetConn, sourceConn)
			if keepalive {
				logrus.Debugf("Start keep-alieved proxy. '%v' -> '%v', content-length '%v'", sourceConn.RemoteAddr(),
					targetConn.RemoteAddr(), contentLength)

				bytesCopied, err := io.CopyBuffer(targetConn, io.LimitReader(sourceConn, contentLength), buf)
				summBytesCopied += bytesCopied
				logrus.Debugf("Connection chunk copied '%v' -> '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), bytesCopied, summBytesCopied, err)
				if err != nil {
					logrus.Debugf("Connection closed '%v' -> '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), bytesCopied, summBytesCopied, err)
				}
			} else {
				logrus.Debugf("Start proxy without support keepalive middle headers '%v' -> '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr())
				bytesCopied, err := io.CopyBuffer(targetConn, sourceConn, buf)
				summBytesCopied += bytesCopied
				logrus.Debugf("Connection closed '%v' -> '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), bytesCopied, summBytesCopied, err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
		}
	}()
}

func startProxyTCP(targetConn net.Conn, sourceConn net.Conn) {
	logrus.Infof("Start tcp-proxy connection from '%v' to'%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr())

	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(targetConn, sourceConn, buf)
		logrus.Debugf("Connection closed with error2 '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
		sourceConn.Close()
		targetConn.Close()
	}()
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(sourceConn, targetConn, buf)
		logrus.Debugf("Connection closed with error3 '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
		sourceConn.Close()
		targetConn.Close()
	}()
}
<commit_msg>lower log level for read error in header<commit_after>package main

import (
	"bytes"
	"github.com/Sirupsen/logrus"
	"io"
	"net"
	"strconv"
	"sync"
)

const (
	NETBUF_SIZE = 2048 // bytes
)

var (
	poolNetBuffers sync.Pool
)

// var-constants
var (
	HEAD_CONNECTION            = []byte("CONNECTION")
	HEAD_CONNECTION_KEEP_ALIVE = []byte("KEEP-ALIVE")
	HEAD_CONTENT_LENGTH        = []byte("CONTENT-LENGTH")
)

// Get or create network buffer for proxy
func netbufGet() (buf []byte) {
	bufInterface := poolNetBuffers.Get()
	if bufInterface == nil {
		buf = make([]byte, NETBUF_SIZE)
	} else {
		buf = bufInterface.([]byte)
		// prevent data leak
		for i := range buf {
			buf[i] = 0
		}

	}
	return buf
}

func netbufPut(buf []byte) {
	poolNetBuffers.Put(buf)
}

func proxyHTTPHeaders(targetConn net.Conn, sourceConn net.Conn) (keepalive bool, contentLength int64) {
	buf := netbufGet()
	defer netbufPut(buf)
	var totalReadBytes int

	// Read lines
readHeaderLines:
	for {
		var i int
		var headerStart []byte
		for i = 0; i < len(buf); i++ {
			readBytes, err := sourceConn.Read(buf[i : i+1])
			totalReadBytes += readBytes
			if err != nil {
				logrus.Debugf("Error while read header from '%v': %v", sourceConn.RemoteAddr(), err)
				targetConn.Close()
				sourceConn.Close()
				return
			}
			if readBytes != 1 {
				logrus.Infof("Can't read a byte from header from '%v'", sourceConn.RemoteAddr())
				targetConn.Close()
				sourceConn.Close()
				return
			}
			if buf[i] == ':' || buf[i] == '\n' {
				headerStart = buf[:i+1]
				logrus.Debugf("Header Name '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), buf[:i])
				break
			}
		}
		if len(headerStart) == 0 {
			logrus.Infof("Header line longer then buffer (%v bytes). Force close connection. '%v' -> '%v'.", len(buf), sourceConn.RemoteAddr(), targetConn.RemoteAddr())
			targetConn.Close()
			sourceConn.Close()
			return
		}

		// Empty line - end http headers
		if bytes.Equal(headerStart, []byte("\n")) || bytes.Equal(headerStart, []byte("\r\n")) {
			break readHeaderLines
		}

		headerName := bytes.ToUpper(headerStart[:len(headerStart)-1]) // Cut trailing colon from start

		skipHeader := false
		for _, ownHeader := range cutHeaders {
			if bytes.Equal(ownHeader, headerName) {
				skipHeader = true
				break
			}
			ownHeaderS := string(ownHeader)
			headerNameS := string(headerName)
			skipHeader = ownHeaderS == headerNameS
		}

		if skipHeader {
			logrus.Debugf("Skip header: '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerName)
			buf[0] = headerStart[len(headerStart)-1]

			for buf[0] != '\n' {
				_, err := sourceConn.Read(buf[:1])
				if err != nil {
					logrus.Infof("Error read header. Close connections. '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
					sourceConn.Close()
					targetConn.Close()
					return
				}
			}
			continue readHeaderLines
		}

		logrus.Debugf("Copy header: '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerName)

		// copy header without changes
		_, err := targetConn.Write(headerStart)
		if err != nil {
			logrus.Infof("Write header start, from '%v' to '%v', headerStart='%s': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerStart, err)
			sourceConn.Close()
			targetConn.Close()
			return
		}

		needHeaderContent := bytes.Equal(headerName, HEAD_CONTENT_LENGTH) || bytes.Equal(headerName, HEAD_CONNECTION)
		headerContent := bytes.NewBuffer(buf[1:])
		headerContent.Reset()

		for buf[0] != '\n' {
			readBytes, err := sourceConn.Read(buf[:1])
			if err != nil {
				logrus.Infof("Error read header to copy. Close connections. '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			if readBytes != 1 {
				logrus.Infof("Header copy read bytes != 1. Error. Close connections. '%v' -> '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr())
				sourceConn.Close()
				targetConn.Close()
				return
			}
			_, err = targetConn.Write(buf[:1])
			if err != nil {
				logrus.Infof("Error write header. Close connections. '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			if needHeaderContent {
				headerContent.WriteByte(buf[0])
			}
		}
		if needHeaderContent {
			switch {
			case bytes.Equal(headerName, HEAD_CONNECTION):
				keepalive = bytes.EqualFold(HEAD_CONNECTION_KEEP_ALIVE, bytes.TrimSpace(headerContent.Bytes()))

			case bytes.Equal(headerName, HEAD_CONTENT_LENGTH):
				contentLength, err = strconv.ParseInt(string(bytes.TrimSpace(headerContent.Bytes())), 10, 64)
				if err == nil {
					logrus.Debugf("Header content-length parsed from '%v' to '%v': %v", sourceConn.RemoteAddr(),
						targetConn.RemoteAddr(), contentLength)
				} else {
					logrus.Infof("Can't header content-length parsed from '%v' to '%v' content '%s': %v", sourceConn.RemoteAddr(),
						targetConn.RemoteAddr(), headerContent.Bytes(), err)
					contentLength = 0
				}

			default:
				logrus.Debug("ERROR. Unknow why i need header content. Code error.")
			}
		}
	}

	remoteTcpAddr := sourceConn.RemoteAddr().(*net.TCPAddr)
	remoteAddrString := remoteTcpAddr.IP.String()

	headerBuf := bytes.NewBuffer(buf)
	headerBuf.Reset()

	// Write real IP
	for _, header := range realIPHeaderNames {
		headerBuf.Write(header)
		headerBuf.WriteByte(':')
		headerBuf.WriteString(remoteAddrString)
		headerBuf.Write([]byte("\r\n"))
	}
	headerBuf.Write(additionalHeaders)
	headerBuf.Write([]byte("\r\n")) // end http headers
	logrus.Debugf("Add headers. '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerBuf.Bytes())

	_, err := targetConn.Write(headerBuf.Bytes())
	if err != nil {
		logrus.Infof("Error while write real ip headers to target '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
	}

	return
}

func startProxy(targetAddr net.TCPAddr, in net.Conn) {
	targetConnCommon, err := net.DialTimeout("tcp", targetAddr.String(), *targetConnTimeout)
	if err != nil {
		logrus.Warnf("Can't connect to target '%v': %v", targetAddr.String(), err)
		return
	}

	targetConn, ok := targetConnCommon.(*net.TCPConn)
	if !ok {
		logrus.Errorf("Can't cast connection to tcp connection, target '%v'", targetAddr.String())
		return
	}

	switch *proxyMode {
	case "http":
		startProxyHTTP(targetConn, in)
	case "tcp":
		startProxyTCP(targetConn, in)
	default:
		logrus.Panicf("Unknow proxy mode: %v", *proxyMode)
	}
}

func startProxyHTTP(targetConn net.Conn, sourceConn net.Conn) {
	logrus.Infof("Start http-proxy connection from '%v' to'%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr())

	// answer from server proxy without changes
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(sourceConn, targetConn, buf)
		logrus.Debugf("Connection closed with error1 '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
		sourceConn.Close()
		targetConn.Close()
	}()

	// proxy incoming traffic, parse every headers
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)
		var summBytesCopied int64
		for {
			keepalive, contentLength := proxyHTTPHeaders(targetConn, sourceConn)
			if keepalive {
				logrus.Debugf("Start keep-alieved proxy. '%v' -> '%v', content-length '%v'", sourceConn.RemoteAddr(),
					targetConn.RemoteAddr(), contentLength)

				bytesCopied, err := io.CopyBuffer(targetConn, io.LimitReader(sourceConn, contentLength), buf)
				summBytesCopied += bytesCopied
				logrus.Debugf("Connection chunk copied '%v' -> '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), bytesCopied, summBytesCopied, err)
				if err != nil {
					logrus.Debugf("Connection closed '%v' -> '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), bytesCopied, summBytesCopied, err)
				}
			} else {
				logrus.Debugf("Start proxy without support keepalive middle headers '%v' -> '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr())
				bytesCopied, err := io.CopyBuffer(targetConn, sourceConn, buf)
				summBytesCopied += bytesCopied
				logrus.Debugf("Connection closed '%v' -> '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), bytesCopied, summBytesCopied, err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
		}
	}()
}

func startProxyTCP(targetConn net.Conn, sourceConn net.Conn) {
	logrus.Infof("Start tcp-proxy connection from '%v' to'%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr())

	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(targetConn, sourceConn, buf)
		logrus.Debugf("Connection closed with error2 '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
		sourceConn.Close()
		targetConn.Close()
	}()
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(sourceConn, targetConn, buf)
		logrus.Debugf("Connection closed with error3 '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
		sourceConn.Close()
		targetConn.Close()
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"onestepback.org/assert"
)
import "strconv"

type StringIo struct {
	data string
}

func (self *StringIo) WriteString(s string) (int, error) {
	self.data += s
	return 0, nil
}

func TestTag(t *testing.T) {
	tag := NewTag("file.go")
	tag.Add("fun", "def fun", Location { 10, 123 })
	tag.Add("g", "def g", Location { 23, 150 })

	s := StringIo { "" }
	tag.WriteOn(&s)

	defstring :=
		"def fun\x7ffun\x0110,123\n" +
		"def g\x7fg\x0123,150\n"
	expected := "\x0c\n" +
		"file.go," + strconv.Itoa(len(defstring)) + "\n" +
		defstring

	assert.StringEqual(t, expected, s.data)
}
<commit_msg>Test for empty definitions in tag<commit_after>package main

import (
	"testing"
	"onestepback.org/assert"
)
import "strconv"

type StringIo struct {
	data string
}

func (self *StringIo) WriteString(s string) (int, error) {
	self.data += s
	return 0, nil
}

func TestTag(t *testing.T) {
	tag := NewTag("file.go")
	tag.Add("fun", "def fun", Location { 10, 123 })
	tag.Add("g", "def g", Location { 23, 150 })

	s := StringIo { "" }
	tag.WriteOn(&s)

	defstring :=
		"def fun\x7ffun\x0110,123\n" +
		"def g\x7fg\x0123,150\n"
	expected := "\x0c\n" +
		"file.go," + strconv.Itoa(len(defstring)) + "\n" +
		defstring

	assert.StringEqual(t, expected, s.data)
}

func TestTagWithNoDefs(t *testing.T) {
	tag := NewTag("empty.go")

	s := StringIo { "" }
	tag.WriteOn(&s)

	expected := ""

	assert.StringEqual(t, expected, s.data)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"neon/build"
	_ "neon/builtin"
	_ "neon/task"
	"neon/util"
	"os"
	"path/filepath"
)

const (
	DEFAULT_BUILD_FILE = "build.yml"
)

func ParseCommandLine() (string, bool, bool, bool, string, bool, bool, string, []string) {
	file := flag.String("file", DEFAULT_BUILD_FILE, "Build file to run")
	help := flag.Bool("build", false, "Print build help")
	debug := flag.Bool("debug", false, "Output debugging information")
	tasks := flag.Bool("tasks", false, "Print tasks list")
	task := flag.String("task", "", "Print help on given task")
	targs := flag.Bool("targets", false, "Print targets list")
	builtins := flag.Bool("builtins", false, "Print builtins list")
	builtin := flag.String("builtin", "", "Print help on given builtin")
	flag.Parse()
	targets := flag.Args()
	return *file, *help, *debug, *tasks, *task, *targs, *builtins, *builtin, targets
}

func FindBuildFile(name string) (string, error) {
	absolute, err := filepath.Abs(name)
	if err != nil {
		return "", fmt.Errorf("getting build file path: %v", err)
	}
	file := filepath.Base(absolute)
	dir := filepath.Dir(absolute)
	for {
		path := filepath.Join(dir, file)
		if util.FileExists(path) {
			return path, nil
		} else {
			parent := filepath.Dir(dir)
			if parent == dir {
				return "", fmt.Errorf("build file not found")
			}
			dir = parent
		}
	}
}

func main() {
	file, help, debug, tasks, task, targs, builtins, builtin, targets := ParseCommandLine()
	path, err := FindBuildFile(file)
	if err != nil {
		util.PrintError(err.Error())
		os.Exit(1)
	}
	build, err := build.NewBuild(path, debug)
	if err != nil {
		util.PrintError(err.Error())
		os.Exit(2)
	}
	if help {
		err = build.Help()
		if err != nil {
			util.PrintError(err.Error())
			os.Exit(3)
		}
	} else if tasks {
		build.PrintTasks()
	} else if task != "" {
		build.PrintHelpTask(task)
	} else if builtins {
		build.PrintBuiltins()
	} else if builtin != "" {
		build.PrintHelpBuiltin(builtin)
	} else if targs {
		build.PrintTargets()
	} else {
		err = build.Run(targets)
		if err == nil {
			util.PrintOK()
		} else {
			util.PrintError(err.Error())
			os.Exit(4)
		}
	}
}
<commit_msg>Debug option renamed verbose<commit_after>package main

import (
	"flag"
	"fmt"
	"neon/build"
	_ "neon/builtin"
	_ "neon/task"
	"neon/util"
	"os"
	"path/filepath"
)

const (
	DEFAULT_BUILD_FILE = "build.yml"
)

func ParseCommandLine() (string, bool, bool, bool, string, bool, bool, string, []string) {
	file := flag.String("file", DEFAULT_BUILD_FILE, "Build file to run")
	help := flag.Bool("build", false, "Print build help")
	verbose := flag.Bool("verbose", false, "Verbose build output")
	tasks := flag.Bool("tasks", false, "Print tasks list")
	task := flag.String("task", "", "Print help on given task")
	targs := flag.Bool("targets", false, "Print targets list")
	builtins := flag.Bool("builtins", false, "Print builtins list")
	builtin := flag.String("builtin", "", "Print help on given builtin")
	flag.Parse()
	targets := flag.Args()
	return *file, *help, *verbose, *tasks, *task, *targs, *builtins, *builtin, targets
}

func FindBuildFile(name string) (string, error) {
	absolute, err := filepath.Abs(name)
	if err != nil {
		return "", fmt.Errorf("getting build file path: %v", err)
	}
	file := filepath.Base(absolute)
	dir := filepath.Dir(absolute)
	for {
		path := filepath.Join(dir, file)
		if util.FileExists(path) {
			return path, nil
		} else {
			parent := filepath.Dir(dir)
			if parent == dir {
				return "", fmt.Errorf("build file not found")
			}
			dir = parent
		}
	}
}

func main() {
	file, help, verbose, tasks, task, targs, builtins, builtin, targets := ParseCommandLine()
	path, err := FindBuildFile(file)
	if err != nil {
		util.PrintError(err.Error())
		os.Exit(1)
	}
	build, err := build.NewBuild(path, verbose)
	if err != nil {
		util.PrintError(err.Error())
		os.Exit(2)
	}
	if help {
		err = build.Help()
		if err != nil {
			util.PrintError(err.Error())
			os.Exit(3)
		}
	} else if tasks {
		build.PrintTasks()
	} else if task != "" {
		build.PrintHelpTask(task)
	} else if builtins {
		build.PrintBuiltins()
	} else if builtin != "" {
		build.PrintHelpBuiltin(builtin)
	} else if targs {
		build.PrintTargets()
	} else {
		err = build.Run(targets)
		if err == nil {
			util.PrintOK()
		} else {
			util.PrintError(err.Error())
			os.Exit(4)
		}
	}
}
<|endoftext|>


<commit_before>package profile

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/tormoder/fit/internal/base"
)

func TransformTypes(ptypes []*PType) (map[string]*Type, error) {
	types := make(map[string]*Type)
	for _, pt := range ptypes {
		t := Type{data: pt}
		skip, err := t.transform()
		if err != nil {
			return nil, err
		}
		if skip {
			continue
		}
		types[t.CCName] = &t
	}

	return types, nil
}

func (t *Type) transform() (skip bool, err error) {
	_, found := timestampTypes[t.data.Header[tNAME]]
	if found {
		return true, nil
	}

	t.Name = t.data.Header[tNAME]
	if len(t.Name) == 0 {
		return false, fmt.Errorf(
			"found empty type name in header %q",
			t.data.Header)
	}
	t.OrigName = t.Name
	t.CCName = toCamelCase(t.Name)

	t.PkgName = strings.ToLower(t.CCName)

	t.BaseType, err = base.TypeFromString(t.data.Header[tBTYPE])
	if err != nil {
		return false, err
	}

	for _, f := range t.data.Fields {
		vt := ValueTriple{
			Name:    toCamelCase(f[tVALNAME]),
			Value:   f[tVAL],
			Comment: f[tCOMMENT],
		}
		t.Values = append(t.Values, vt)
	}

	if renamed, found := typeQuirks[t.Name]; found {
		t.Name = renamed
		t.CCName = toCamelCase(t.Name)
		t.PkgName = strings.ToLower(t.CCName)
	}

	return false, nil
}

func TransformMsgs(pmsgs []*PMsg, types map[string]*Type) ([]*Msg, error) {
	var msgs []*Msg
	for _, pmsg := range pmsgs {
		msg := Msg{
			Name:        pmsg.Header[mMSGNAME],
			FieldByName: make(map[string]*Field),
		}

		if len(msg.Name) == 0 {
			return nil, fmt.Errorf(
				"found empty message name in header %q",
				pmsg.Header)
		}
		msg.CCName = toCamelCase(msg.Name)
		debugln("transforming message", msg.CCName)

		for _, pfield := range pmsg.Fields {
			f := &Field{data: pfield.Field}
			skip, err := f.transform(false, types)
			if err != nil {
				return nil, err
			}
			if skip {
				continue
			}
			msg.Fields = append(msg.Fields, f)
			msg.FieldByName[f.CCName] = f
			if len(pfield.Subfields) == 0 {
				continue
			}

			for _, sfield := range pfield.Subfields {
				sf := &Field{data: sfield}
				skip, err := sf.transform(true, types)
				if err != nil {
					return nil, fmt.Errorf("error parsing subfield: %v", err)
				}
				if skip {
					continue
				}
				f.Subfields = append(f.Subfields, sf)
			}
		}
		msgs = append(msgs, &msg)
	}

	return msgs, nil
}

func (f *Field) transform(subfield bool, types map[string]*Type) (skip bool, err error) {
	if f.data[mEXAMPLE] == "" {
		return true, nil
	}

	f.DefNum = f.data[mFDEFN]
	f.Name = f.data[mFNAME]
	f.CCName = toCamelCase(f.Name)

	err = f.parseBaseType(types)
	if err != nil {
		return false, err
	}

	err = f.parseType(types)
	if err != nil {
		return false, err
	}

	f.parseArray()

	f.Units = f.data[mUNITS]
	f.Comment = f.data[mCOMMENT]
	f.Example = f.data[mEXAMPLE]

	if subfield {
		f.parseRefFields()
	}

	if f.data[mCOMPS] == "" {
		f.parseScaleOffset()
		return false, nil
	}

	return false, f.parseComponents(types)
}

func (f *Field) parseType(types map[string]*Type) error {
	switch {
	case strings.HasSuffix(f.Name, "_lat"):
		f.Type = "Latitude"
		f.GoType = "lat"
		f.GoInvalid = "NewLatitudeInvalid()"
	case strings.HasSuffix(f.Name, "_long"):
		f.Type = "Longitude"
		f.GoType = "lng"
		f.GoInvalid = "NewLongitudeInvalid()"
	default:
		ft := f.data[mFTYPE]
		_, found := timestampTypes[ft]
		if found {
			f.Type = "time.Time"
			f.GoInvalid = "timeBase"
			if strings.HasPrefix(ft, "local") {
				f.GoType = "timelocal"
			} else {
				f.GoType = "timeutc"
			}
			return nil
		}

		f.GoType = "fit"

		if renamed, shouldRename := typeQuirks[ft]; shouldRename {
			f.Type = toCamelCase(renamed)
		} else {
			if btype, isBaseType := baseTypeToGoType[ft]; isBaseType {
				f.Type = btype
			} else {
				f.Type = toCamelCase(ft)
			}
		}

		tdef, tfound := types[f.Type]
		if tfound {
			f.GoInvalid = tdef.BaseType.GoInvalidValue()
			return nil
		}

		if f.Type == "Bool" { // Special case for now.
			f.GoInvalid = base.Enum.GoInvalidValue()
			return nil
		}

		// Assume base type.
		val, vfound := goBaseTypeToInvalidValue[f.Type]
		if !vfound {
			return fmt.Errorf(
				"base type for type %q not found",
				f.Type,
			)

		}
		f.GoInvalid = val
		f.BTInvalid = val // GoInvalid may be overwritten in parseArray.
	}

	return nil
}

func (f *Field) parseBaseType(types map[string]*Type) error {
	typeName := f.data[mFTYPE]
	if rewrite, tfound := typeQuirks[typeName]; tfound {
		typeName = rewrite
	}
	typeDef, found := types[toCamelCase(typeName)]
	if found {
		f.BaseType = typeDef.BaseType
	} else {
		_, found = timestampTypes[typeName]
		if found {
			f.BaseType = base.Uint32
		} else if typeName == "bool" {
			f.BaseType = base.Enum
		} else {
			var err error
			f.BaseType, err = base.TypeFromString(typeName)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (f *Field) parseArray() {
	arrayStr := strings.TrimFunc(
		f.data[mARRAY], func(r rune) bool {
			if r == '[' || r == ']' {
				return true
			}
			return false
		})
	switch arrayStr {
	case "":
		f.Array = "0"
	case "N":
		f.Array = "255"
		f.GoInvalid = "nil"
	default:
		f.Array = arrayStr
		f.GoInvalid = "nil"
	}
}

func (f *Field) parseRefFields() {
	f.RefFieldName = strings.Split(f.data[mRFNAME], ",")
	if len(f.RefFieldName) > 0 {
		for i, rfn := range f.RefFieldName {
			tmp := strings.TrimSpace(rfn)
			f.RefFieldName[i] = toCamelCase(tmp)
		}
	}

	f.RefFieldValue = strings.Split(f.data[mRFVAL], ",")
	if len(f.RefFieldValue) > 0 {
		for i, rfv := range f.RefFieldValue {
			tmp := strings.TrimSpace(rfv)
			f.RefFieldValue[i] = toCamelCase(tmp)
		}
	}
}

func (f *Field) parseScaleOffset() {
	if f.data[mSCALE] == "" {
		return
	}
	f.Scale = f.data[mSCALE]
	if f.data[mOFFSET] != "" {
		f.Offset = f.data[mOFFSET]
	}
}

func (f *Field) parseComponents(types map[string]*Type) error {
	if f.data[mCOMPS] == "" {
		return nil
	}

	debugln("parsing components for field", f.CCName)

	switch f.BaseType {
	case base.Uint8, base.Uint16, base.Uint32:
	case base.Byte:
		if !f.IsArray() {
			return fmt.Errorf("parseComponents: base type was byte but not an array")
		}
	default:
		return fmt.Errorf(
			"parseComponents: unhandled base type (%s) for field %s",
			f.BaseType, f.CCName)
	}

	components := strings.Split(f.data[mCOMPS], ",")
	if len(components) == 0 {
		return fmt.Errorf("parseComponents: zero components after string split")
	}

	bits := strings.Split(f.data[mBITS], ",")
	if len(components) != len(bits) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and bits (%d) differ",
			len(components), len(bits))
	}

	accumulate := strings.Split(f.data[mACCUMU], ",")
	if len(accumulate) == 1 && accumulate[0] == "" {
		accumulate = nil
	}

	if len(accumulate) > 0 && (len(accumulate) != len(components)) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and accumulate flags (%d) differ",
			len(components), len(accumulate))
	}

	f.Components = make([]Component, len(components))

	var (
		err       error
		bitsTotal int
	)

	for i, comp := range components {
		f.Components[i].Name = strings.TrimSpace(comp)
		f.Components[i].Name = toCamelCase(f.Components[i].Name)
		f.Components[i].Bits = strings.TrimSpace(bits[i])
		f.Components[i].BitsInt, err = strconv.Atoi(f.Components[i].Bits)
		if err != nil {
			return fmt.Errorf("parseComponents: error converting bit to integer: %v", err)
		}
		bitsTotal += f.Components[i].BitsInt
		if len(accumulate) == len(components) {
			tmp := strings.TrimSpace(accumulate[i])
			f.Components[i].Accumulate, err = strconv.ParseBool(tmp)
			if err != nil {
				return fmt.Errorf("parseComponents: %v", err)
			}
		}
	}

	if bitsTotal > 32 || bitsTotal < 0 {
		return fmt.Errorf("parseComponents: illegal size for total number of bits: %d", bitsTotal)
	}

	if len(components) == 1 {
		// Set any scale on the "main" field.
		// TODO(tormoder): Verify that this is correct.
		f.parseScaleOffset()
		return nil
	}

	cscale := strings.Split(f.data[mSCALE], ",")
	coffset := strings.Split(f.data[mOFFSET], ",")

	if len(coffset) == 1 && coffset[0] == "" {
		coffset = nil
	}
	if len(cscale) != len(components) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and scales (%d) differ",
			len(components), len(cscale))
	}
	if len(coffset) != 0 && len(coffset) != len(components) {
		return fmt.Errorf(
			"parseComponents: #offset != 0 and number of components (%d) and offsets (%d) differ",
			len(components), len(coffset))
	}

	for i := range f.Components {
		f.Components[i].Scale = strings.TrimSpace(cscale[i])
		if len(coffset) == 0 {
			continue
		}
		f.Components[i].Offset = strings.TrimSpace(coffset[i])
	}

	return nil
}
<commit_msg>cmd/fitgen/internal/profile: extract two strings to a const<commit_after>package profile

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/tormoder/fit/internal/base"
)

func TransformTypes(ptypes []*PType) (map[string]*Type, error) {
	types := make(map[string]*Type)
	for _, pt := range ptypes {
		t := Type{data: pt}
		skip, err := t.transform()
		if err != nil {
			return nil, err
		}
		if skip {
			continue
		}
		types[t.CCName] = &t
	}

	return types, nil
}

func (t *Type) transform() (skip bool, err error) {
	_, found := timestampTypes[t.data.Header[tNAME]]
	if found {
		return true, nil
	}

	t.Name = t.data.Header[tNAME]
	if len(t.Name) == 0 {
		return false, fmt.Errorf(
			"found empty type name in header %q",
			t.data.Header)
	}
	t.OrigName = t.Name
	t.CCName = toCamelCase(t.Name)

	t.PkgName = strings.ToLower(t.CCName)

	t.BaseType, err = base.TypeFromString(t.data.Header[tBTYPE])
	if err != nil {
		return false, err
	}

	for _, f := range t.data.Fields {
		vt := ValueTriple{
			Name:    toCamelCase(f[tVALNAME]),
			Value:   f[tVAL],
			Comment: f[tCOMMENT],
		}
		t.Values = append(t.Values, vt)
	}

	if renamed, found := typeQuirks[t.Name]; found {
		t.Name = renamed
		t.CCName = toCamelCase(t.Name)
		t.PkgName = strings.ToLower(t.CCName)
	}

	return false, nil
}

func TransformMsgs(pmsgs []*PMsg, types map[string]*Type) ([]*Msg, error) {
	var msgs []*Msg
	for _, pmsg := range pmsgs {
		msg := Msg{
			Name:        pmsg.Header[mMSGNAME],
			FieldByName: make(map[string]*Field),
		}

		if len(msg.Name) == 0 {
			return nil, fmt.Errorf(
				"found empty message name in header %q",
				pmsg.Header)
		}
		msg.CCName = toCamelCase(msg.Name)
		debugln("transforming message", msg.CCName)

		for _, pfield := range pmsg.Fields {
			f := &Field{data: pfield.Field}
			skip, err := f.transform(false, types)
			if err != nil {
				return nil, err
			}
			if skip {
				continue
			}
			msg.Fields = append(msg.Fields, f)
			msg.FieldByName[f.CCName] = f
			if len(pfield.Subfields) == 0 {
				continue
			}

			for _, sfield := range pfield.Subfields {
				sf := &Field{data: sfield}
				skip, err := sf.transform(true, types)
				if err != nil {
					return nil, fmt.Errorf("error parsing subfield: %v", err)
				}
				if skip {
					continue
				}
				f.Subfields = append(f.Subfields, sf)
			}
		}
		msgs = append(msgs, &msg)
	}

	return msgs, nil
}

func (f *Field) transform(subfield bool, types map[string]*Type) (skip bool, err error) {
	if f.data[mEXAMPLE] == "" {
		return true, nil
	}

	f.DefNum = f.data[mFDEFN]
	f.Name = f.data[mFNAME]
	f.CCName = toCamelCase(f.Name)

	err = f.parseBaseType(types)
	if err != nil {
		return false, err
	}

	err = f.parseType(types)
	if err != nil {
		return false, err
	}

	f.parseArray()

	f.Units = f.data[mUNITS]
	f.Comment = f.data[mCOMMENT]
	f.Example = f.data[mEXAMPLE]

	if subfield {
		f.parseRefFields()
	}

	if f.data[mCOMPS] == "" {
		f.parseScaleOffset()
		return false, nil
	}

	return false, f.parseComponents(types)
}

func (f *Field) parseType(types map[string]*Type) error {
	switch {
	case strings.HasSuffix(f.Name, "_lat"):
		f.Type = "Latitude"
		f.GoType = "lat"
		f.GoInvalid = "NewLatitudeInvalid()"
	case strings.HasSuffix(f.Name, "_long"):
		f.Type = "Longitude"
		f.GoType = "lng"
		f.GoInvalid = "NewLongitudeInvalid()"
	default:
		ft := f.data[mFTYPE]
		_, found := timestampTypes[ft]
		if found {
			f.Type = "time.Time"
			f.GoInvalid = "timeBase"
			if strings.HasPrefix(ft, "local") {
				f.GoType = "timelocal"
			} else {
				f.GoType = "timeutc"
			}
			return nil
		}

		f.GoType = "fit"

		if renamed, shouldRename := typeQuirks[ft]; shouldRename {
			f.Type = toCamelCase(renamed)
		} else {
			if btype, isBaseType := baseTypeToGoType[ft]; isBaseType {
				f.Type = btype
			} else {
				f.Type = toCamelCase(ft)
			}
		}

		tdef, tfound := types[f.Type]
		if tfound {
			f.GoInvalid = tdef.BaseType.GoInvalidValue()
			return nil
		}

		if f.Type == "Bool" { // Special case for now.
			f.GoInvalid = base.Enum.GoInvalidValue()
			return nil
		}

		// Assume base type.
		val, vfound := goBaseTypeToInvalidValue[f.Type]
		if !vfound {
			return fmt.Errorf(
				"base type for type %q not found",
				f.Type,
			)

		}
		f.GoInvalid = val
		f.BTInvalid = val // GoInvalid may be overwritten in parseArray.
	}

	return nil
}

func (f *Field) parseBaseType(types map[string]*Type) error {
	typeName := f.data[mFTYPE]
	if rewrite, tfound := typeQuirks[typeName]; tfound {
		typeName = rewrite
	}
	typeDef, found := types[toCamelCase(typeName)]
	if found {
		f.BaseType = typeDef.BaseType
	} else {
		_, found = timestampTypes[typeName]
		if found {
			f.BaseType = base.Uint32
		} else if typeName == "bool" {
			f.BaseType = base.Enum
		} else {
			var err error
			f.BaseType, err = base.TypeFromString(typeName)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (f *Field) parseArray() {
	const nilString = "nil"
	arrayStr := strings.TrimFunc(
		f.data[mARRAY], func(r rune) bool {
			if r == '[' || r == ']' {
				return true
			}
			return false
		})
	switch arrayStr {
	case "":
		f.Array = "0"
	case "N":
		f.Array = "255"
		f.GoInvalid = nilString
	default:
		f.Array = arrayStr
		f.GoInvalid = nilString
	}
}

func (f *Field) parseRefFields() {
	f.RefFieldName = strings.Split(f.data[mRFNAME], ",")
	if len(f.RefFieldName) > 0 {
		for i, rfn := range f.RefFieldName {
			tmp := strings.TrimSpace(rfn)
			f.RefFieldName[i] = toCamelCase(tmp)
		}
	}

	f.RefFieldValue = strings.Split(f.data[mRFVAL], ",")
	if len(f.RefFieldValue) > 0 {
		for i, rfv := range f.RefFieldValue {
			tmp := strings.TrimSpace(rfv)
			f.RefFieldValue[i] = toCamelCase(tmp)
		}
	}
}

func (f *Field) parseScaleOffset() {
	if f.data[mSCALE] == "" {
		return
	}
	f.Scale = f.data[mSCALE]
	if f.data[mOFFSET] != "" {
		f.Offset = f.data[mOFFSET]
	}
}

func (f *Field) parseComponents(types map[string]*Type) error {
	if f.data[mCOMPS] == "" {
		return nil
	}

	debugln("parsing components for field", f.CCName)

	switch f.BaseType {
	case base.Uint8, base.Uint16, base.Uint32:
	case base.Byte:
		if !f.IsArray() {
			return fmt.Errorf("parseComponents: base type was byte but not an array")
		}
	default:
		return fmt.Errorf(
			"parseComponents: unhandled base type (%s) for field %s",
			f.BaseType, f.CCName)
	}

	components := strings.Split(f.data[mCOMPS], ",")
	if len(components) == 0 {
		return fmt.Errorf("parseComponents: zero components after string split")
	}

	bits := strings.Split(f.data[mBITS], ",")
	if len(components) != len(bits) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and bits (%d) differ",
			len(components), len(bits))
	}

	accumulate := strings.Split(f.data[mACCUMU], ",")
	if len(accumulate) == 1 && accumulate[0] == "" {
		accumulate = nil
	}

	if len(accumulate) > 0 && (len(accumulate) != len(components)) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and accumulate flags (%d) differ",
			len(components), len(accumulate))
	}

	f.Components = make([]Component, len(components))

	var (
		err       error
		bitsTotal int
	)

	for i, comp := range components {
		f.Components[i].Name = strings.TrimSpace(comp)
		f.Components[i].Name = toCamelCase(f.Components[i].Name)
		f.Components[i].Bits = strings.TrimSpace(bits[i])
		f.Components[i].BitsInt, err = strconv.Atoi(f.Components[i].Bits)
		if err != nil {
			return fmt.Errorf("parseComponents: error converting bit to integer: %v", err)
		}
		bitsTotal += f.Components[i].BitsInt
		if len(accumulate) == len(components) {
			tmp := strings.TrimSpace(accumulate[i])
			f.Components[i].Accumulate, err = strconv.ParseBool(tmp)
			if err != nil {
				return fmt.Errorf("parseComponents: %v", err)
			}
		}
	}

	if bitsTotal > 32 || bitsTotal < 0 {
		return fmt.Errorf("parseComponents: illegal size for total number of bits: %d", bitsTotal)
	}

	if len(components) == 1 {
		// Set any scale on the "main" field.
		// TODO(tormoder): Verify that this is correct.
		f.parseScaleOffset()
		return nil
	}

	cscale := strings.Split(f.data[mSCALE], ",")
	coffset := strings.Split(f.data[mOFFSET], ",")

	if len(coffset) == 1 && coffset[0] == "" {
		coffset = nil
	}
	if len(cscale) != len(components) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and scales (%d) differ",
			len(components), len(cscale))
	}
	if len(coffset) != 0 && len(coffset) != len(components) {
		return fmt.Errorf(
			"parseComponents: #offset != 0 and number of components (%d) and offsets (%d) differ",
			len(components), len(coffset))
	}

	for i := range f.Components {
		f.Components[i].Scale = strings.TrimSpace(cscale[i])
		if len(coffset) == 0 {
			continue
		}
		f.Components[i].Offset = strings.TrimSpace(coffset[i])
	}

	return nil
}
<|endoftext|>


<commit_before>package metadata

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/jacksontj/dataman/src/datamantype"
)

func NewCollection(name string) *Collection {
	return &Collection{
		Name:    name,
		Indexes: make(map[string]*CollectionIndex),
	}
}

type Collection struct {
	ID   int64  `json:"_id,omitempty"`
	Name string `json:"name"`

	// NOTE: we reserve the "_" namespace for fields for our own data (created, etc.)
	// All the columns in this table
	Fields map[string]*CollectionField `json:"fields"`

	// map of name -> index
	Indexes map[string]*CollectionIndex `json:"indexes,omitempty"`
	// Link directly to primary index (for convenience)
	PrimaryIndex *CollectionIndex `json:"-"`

	ProvisionState ProvisionState `json:"provision_state"`
}

func (c *Collection) UnmarshalJSON(data []byte) error {
	type Alias Collection
	aux := &struct {
		*Alias
	}{
		Alias: (*Alias)(c),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	for _, index := range c.Indexes {
		if index.Primary {
			if c.PrimaryIndex == nil {
				c.PrimaryIndex = index
			} else {
				return fmt.Errorf("Collections can only have one primary index")
			}
		}
	}
	if c.PrimaryIndex == nil {
		return fmt.Errorf("Collection %s missing primary index", c.Name)
	}

	return nil
}

func (c *Collection) GetFieldByName(name string) *CollectionField {
	return c.GetField(strings.Split(name, "."))
}

func (c *Collection) GetField(nameParts []string) *CollectionField {
	field := c.Fields[nameParts[0]]

	for _, part := range nameParts[1:] {
		field = field.SubFields[part]
	}

	return field
}

func (c *Collection) Equal(o *Collection) bool {
	if c.Name != o.Name {
		return false
	}

	return true
}

func (c *Collection) ListIndexes() []string {
	indexes := make([]string, 0, len(c.Indexes))
	for name, _ := range c.Indexes {
		indexes = append(indexes, name)
	}
	return indexes
}

func (c *Collection) ValidateRecordInsert(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	// TODO: We need to check that we where given no more than the Fields we know about
	for fieldName, field := range c.Fields {
		switch field.FieldType.DatamanType {
		case datamantype.Serial:
			// TODO: check the serial type
		default:
			if v, ok := record[fieldName]; ok {
				normalizedValue, normalizationResult := field.Normalize(v)
				result.Fields[fieldName] = normalizationResult
				if normalizationResult.IsValid() {
					record[fieldName] = normalizedValue
				}
			} else {
				if field.NotNull && field.Default == nil {
					result.Fields[fieldName] = &ValidationResult{
						Error: fmt.Sprintf("Missing required field %s", fieldName),
					}
				}
				// TODO: include an empty result? Not sure if an empty one is any good (also-- check for subfields?)
			}
		}
	}
	return result
}

// TODO: underlying datasources should know how to do this-- us doing it shouldn't
// be necessary
// For updates we want to ensure that all fields we where given are (1) valid and (2) are ones we know about
func (c *Collection) ValidateRecordUpdate(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	for fieldName, field := range c.Fields {
		if v, ok := record[fieldName]; ok {
			normalizedValue, normalizationResult := field.Normalize(v)
			result.Fields[fieldName] = normalizationResult
			if normalizationResult.IsValid() {
				record[fieldName] = normalizedValue
			}
		}
	}
	return result
}
<commit_msg>Return if non-existant instead of panic-ing<commit_after>package metadata

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/jacksontj/dataman/src/datamantype"
)

func NewCollection(name string) *Collection {
	return &Collection{
		Name:    name,
		Indexes: make(map[string]*CollectionIndex),
	}
}

type Collection struct {
	ID   int64  `json:"_id,omitempty"`
	Name string `json:"name"`

	// NOTE: we reserve the "_" namespace for fields for our own data (created, etc.)
	// All the columns in this table
	Fields map[string]*CollectionField `json:"fields"`

	// map of name -> index
	Indexes map[string]*CollectionIndex `json:"indexes,omitempty"`
	// Link directly to primary index (for convenience)
	PrimaryIndex *CollectionIndex `json:"-"`

	ProvisionState ProvisionState `json:"provision_state"`
}

func (c *Collection) UnmarshalJSON(data []byte) error {
	type Alias Collection
	aux := &struct {
		*Alias
	}{
		Alias: (*Alias)(c),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	for _, index := range c.Indexes {
		if index.Primary {
			if c.PrimaryIndex == nil {
				c.PrimaryIndex = index
			} else {
				return fmt.Errorf("Collections can only have one primary index")
			}
		}
	}
	if c.PrimaryIndex == nil {
		return fmt.Errorf("Collection %s missing primary index", c.Name)
	}

	return nil
}

func (c *Collection) GetFieldByName(name string) *CollectionField {
	return c.GetField(strings.Split(name, "."))
}

func (c *Collection) GetField(nameParts []string) *CollectionField {
	field := c.Fields[nameParts[0]]

	for _, part := range nameParts[1:] {
	    if field == nil {
	        return field
	    }
		field = field.SubFields[part]
	}

	return field
}

func (c *Collection) Equal(o *Collection) bool {
	if c.Name != o.Name {
		return false
	}

	return true
}

func (c *Collection) ListIndexes() []string {
	indexes := make([]string, 0, len(c.Indexes))
	for name, _ := range c.Indexes {
		indexes = append(indexes, name)
	}
	return indexes
}

func (c *Collection) ValidateRecordInsert(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	// TODO: We need to check that we where given no more than the Fields we know about
	for fieldName, field := range c.Fields {
		switch field.FieldType.DatamanType {
		case datamantype.Serial:
			// TODO: check the serial type
		default:
			if v, ok := record[fieldName]; ok {
				normalizedValue, normalizationResult := field.Normalize(v)
				result.Fields[fieldName] = normalizationResult
				if normalizationResult.IsValid() {
					record[fieldName] = normalizedValue
				}
			} else {
				if field.NotNull && field.Default == nil {
					result.Fields[fieldName] = &ValidationResult{
						Error: fmt.Sprintf("Missing required field %s", fieldName),
					}
				}
				// TODO: include an empty result? Not sure if an empty one is any good (also-- check for subfields?)
			}
		}
	}
	return result
}

// TODO: underlying datasources should know how to do this-- us doing it shouldn't
// be necessary
// For updates we want to ensure that all fields we where given are (1) valid and (2) are ones we know about
func (c *Collection) ValidateRecordUpdate(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	for fieldName, field := range c.Fields {
		if v, ok := record[fieldName]; ok {
			normalizedValue, normalizationResult := field.Normalize(v)
			result.Fields[fieldName] = normalizationResult
			if normalizationResult.IsValid() {
				record[fieldName] = normalizedValue
			}
		}
	}
	return result
}
<|endoftext|>


<commit_before>package collectors

import (
	"fmt"
	"strings"
	"time"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
)

const (
	ifAlias              = ".1.3.6.1.2.1.31.1.1.1.18"
	ifDescr              = ".1.3.6.1.2.1.2.2.1.2"
	ifHCInBroadcastPkts  = ".1.3.6.1.2.1.31.1.1.1.9"
	ifHCInMulticastPkts  = ".1.3.6.1.2.1.31.1.1.1.8"
	ifHCInUcastPkts      = ".1.3.6.1.2.1.31.1.1.1.7"
	ifHCOutBroadcastPkts = ".1.3.6.1.2.1.31.1.1.1.13"
	ifHCOutMulticastPkts = ".1.3.6.1.2.1.31.1.1.1.12"
	ifHCOutOctets        = ".1.3.6.1.2.1.31.1.1.1.10"
	ifHCOutUcastPkts     = ".1.3.6.1.2.1.31.1.1.1.11"
	ifHCinOctets         = ".1.3.6.1.2.1.31.1.1.1.6"
	ifInDiscards         = ".1.3.6.1.2.1.2.2.1.13"
	ifInErrors           = ".1.3.6.1.2.1.2.2.1.14"
	ifOutDiscards        = ".1.3.6.1.2.1.2.2.1.19"
	ifOutErrors          = ".1.3.6.1.2.1.2.2.1.20"
)

// SNMPIfaces registers a SNMP Interfaces collector for the given community and host.
func SNMPIfaces(community, host string) {
	collectors = append(collectors, &IntervalCollector{
		F: func() (opentsdb.MultiDataPoint, error) {
			return c_snmp_ifaces(community, host)
		},
		Interval: time.Second * 30,
		name:     fmt.Sprintf("snmp-ifaces-%s", host),
	})
}

func switch_bond(metric, iname string) string {
	if strings.Contains(iname, "port-channel") {
		return "os.net.bond" + strings.TrimPrefix(metric, "os.net")
	}
	return metric
}

func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) {
	n, err := snmp_subtree(host, community, ifDescr)
	if err != nil {
		return nil, err
	}
	a, err := snmp_subtree(host, community, ifAlias)
	if err != nil {
		return nil, err
	}
	names := make(map[interface{}]string, len(n))
	aliases := make(map[interface{}]string, len(a))
	for k, v := range n {
		names[k] = fmt.Sprintf("%s", v)
	}
	for k, v := range a {
		// In case clean would come up empty, prevent the point from being removed
		// by setting our own empty case.
		aliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v))
		if aliases[k] == "" {
			aliases[k] = "NA"
		}
	}
	var md opentsdb.MultiDataPoint
	add := func(oid, metric, dir string) error {
		m, err := snmp_subtree(host, community, oid)
		if err != nil {
			return err
		}
		for k, v := range m {
			tags := opentsdb.TagSet{
				"host":      host,
				"direction": dir,
				"iface":     fmt.Sprintf("%d", k),
				"iname":     names[k],
			}
			Add(&md, switch_bond(metric, names[k]), v, tags, metadata.Unknown, metadata.None, "")
			metadata.AddMeta("", tags, "alias", aliases[k], false)
		}
		return nil
	}
	oids := []snmpAdd{
		{ifHCInBroadcastPkts, osNetBroadcast, "in"},
		{ifHCInMulticastPkts, osNetMulticast, "in"},
		{ifHCInUcastPkts, osNetUnicast, "in"},
		{ifHCOutBroadcastPkts, osNetBroadcast, "out"},
		{ifHCOutMulticastPkts, osNetMulticast, "out"},
		{ifHCOutOctets, osNetBytes, "out"},
		{ifHCOutUcastPkts, osNetUnicast, "out"},
		{ifHCinOctets, osNetBytes, "in"},
		{ifInDiscards, osNetDropped, "in"},
		{ifInErrors, osNetErrors, "in"},
		{ifOutDiscards, osNetDropped, "out"},
		{ifOutErrors, osNetErrors, "out"},
	}
	for _, o := range oids {
		if err := add(o.oid, o.metric, o.dir); err != nil {
			return nil, err
		}
	}
	return md, nil
}

type snmpAdd struct {
	oid    string
	metric string
	dir    string
}
<commit_msg>cmd/scollector: Use ifName first for snmp<commit_after>package collectors

import (
	"fmt"
	"strings"
	"time"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
)

const (
	ifAlias              = ".1.3.6.1.2.1.31.1.1.1.18"
	ifDescr              = ".1.3.6.1.2.1.2.2.1.2"
	ifHCInBroadcastPkts  = ".1.3.6.1.2.1.31.1.1.1.9"
	ifHCInMulticastPkts  = ".1.3.6.1.2.1.31.1.1.1.8"
	ifHCInUcastPkts      = ".1.3.6.1.2.1.31.1.1.1.7"
	ifHCOutBroadcastPkts = ".1.3.6.1.2.1.31.1.1.1.13"
	ifHCOutMulticastPkts = ".1.3.6.1.2.1.31.1.1.1.12"
	ifHCOutOctets        = ".1.3.6.1.2.1.31.1.1.1.10"
	ifHCOutUcastPkts     = ".1.3.6.1.2.1.31.1.1.1.11"
	ifHCinOctets         = ".1.3.6.1.2.1.31.1.1.1.6"
	ifInDiscards         = ".1.3.6.1.2.1.2.2.1.13"
	ifInErrors           = ".1.3.6.1.2.1.2.2.1.14"
	ifName               = ".1.3.6.1.2.1.31.1.1.1.1"
	ifOutDiscards        = ".1.3.6.1.2.1.2.2.1.19"
	ifOutErrors          = ".1.3.6.1.2.1.2.2.1.20"
)

// SNMPIfaces registers a SNMP Interfaces collector for the given community and host.
func SNMPIfaces(community, host string) {
	collectors = append(collectors, &IntervalCollector{
		F: func() (opentsdb.MultiDataPoint, error) {
			return c_snmp_ifaces(community, host)
		},
		Interval: time.Second * 30,
		name:     fmt.Sprintf("snmp-ifaces-%s", host),
	})
}

func switch_bond(metric, iname string) string {
	if strings.Contains(iname, "port-channel") {
		return "os.net.bond" + strings.TrimPrefix(metric, "os.net")
	}
	return metric
}

func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) {
	n, err := snmp_subtree(host, community, ifName)
	if err != nil || len(n) == 0 {
		n, err = snmp_subtree(host, community, ifDescr)
		if err != nil {
			return nil, err
		}
	}
	a, err := snmp_subtree(host, community, ifAlias)
	if err != nil {
		return nil, err
	}
	names := make(map[interface{}]string, len(n))
	aliases := make(map[interface{}]string, len(a))
	for k, v := range n {
		names[k] = fmt.Sprintf("%s", v)
	}
	for k, v := range a {
		// In case clean would come up empty, prevent the point from being removed
		// by setting our own empty case.
		aliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v))
		if aliases[k] == "" {
			aliases[k] = "NA"
		}
	}
	var md opentsdb.MultiDataPoint
	add := func(oid, metric, dir string) error {
		m, err := snmp_subtree(host, community, oid)
		if err != nil {
			return err
		}
		for k, v := range m {
			tags := opentsdb.TagSet{
				"host":      host,
				"direction": dir,
				"iface":     fmt.Sprintf("%d", k),
				"iname":     names[k],
			}
			Add(&md, switch_bond(metric, names[k]), v, tags, metadata.Unknown, metadata.None, "")
			metadata.AddMeta("", tags, "alias", aliases[k], false)
		}
		return nil
	}
	oids := []snmpAdd{
		{ifHCInBroadcastPkts, osNetBroadcast, "in"},
		{ifHCInMulticastPkts, osNetMulticast, "in"},
		{ifHCInUcastPkts, osNetUnicast, "in"},
		{ifHCOutBroadcastPkts, osNetBroadcast, "out"},
		{ifHCOutMulticastPkts, osNetMulticast, "out"},
		{ifHCOutOctets, osNetBytes, "out"},
		{ifHCOutUcastPkts, osNetUnicast, "out"},
		{ifHCinOctets, osNetBytes, "in"},
		{ifInDiscards, osNetDropped, "in"},
		{ifInErrors, osNetErrors, "in"},
		{ifOutDiscards, osNetDropped, "out"},
		{ifOutErrors, osNetErrors, "out"},
	}
	for _, o := range oids {
		if err := add(o.oid, o.metric, o.dir); err != nil {
			return nil, err
		}
	}
	return md, nil
}

type snmpAdd struct {
	oid    string
	metric string
	dir    string
}
<|endoftext|>


<commit_before>package main

import (
	"ndb/common"
	"ndb/operate"
	"fmt"
)

func execute(node *common.Node, query string) []*common.Node{
	return nil
}

func read(filename string) (*common.Node, error) {
	return common.Read(filename)
}

func main() {
	node, err := common.Read("example.ndb")
	if err == nil {
		children := operate.Select(node, "root->parent->child->name:lily")
		if len(children) == 1 {
			person := children[0]
			fmt.Println(person.GetValues())
		}
		children = operate.Select(node, "root->parent->child->name:m$ && sex:^m")
		fmt.Println(len(children))

		node = operate.Update(node, "root->parent->child->name:lily", "age=33, phone=13343351822")
		children = operate.Select(node, "root->parent->child->name:lily")
		if len(children) == 1 {
			person := children[0]
			fmt.Println(person.GetValues())
		}

		node = operate.Delete(node, "root->parent->child->name:lily", "[age, phone]")
		children = operate.Select(node, "root->parent->child->name:lily")
		if len(children) == 1 {
			person := children[0]
			fmt.Println(person.GetValues())
		}

		node = operate.Delete(node, "root->parent->child->name:lily", "block")
		children = operate.Select(node, "root->parent->child->name:lily")
		if len(children) == 0 {
			fmt.Println("Not found : Lily")
		}
	}

}<commit_msg>Update statement<commit_after>package ndb

import (
	"strings"
	"ndb/common"
	"ndb/operate"
)

func Execute(node *common.Node, query string) interface{} {
	command := query

	if strings.Contains(query, ":") {
		command = strings.TrimSpace(query[0 : strings.Index(query, ":")])
		query = strings.TrimSpace(query[strings.Index(query, ":") + 1 : len(query)])
	}

	queryItems := strings.Split(query, "!!")

	if queryItems != nil && len(queryItems) > 0 {
		path := strings.TrimSpace(queryItems[0])

		value := ""
		if len(queryItems) > 1 {
			value = strings.TrimSpace(queryItems[1])
		}

		if command != "" {
			command = strings.ToLower(command)
			if command == "select" || command == "one" || command == "exist" {
				result := operate.Select(node, path)

				if command == "one" {
					if result != nil && len(result) > 0 {
						return result[0]
					} else {
						return nil
					}
				} else if command == "exist"{
					if  result != nil && len(result) > 0 {
						return true
					} else {
						return false
					}
				}

				return result
			} else if command == "update" {
				return operate.Update(node, path, value)
			} else if command == "delete" {
				return operate.Delete(node, path, value)
			} else if command == "insert" {
				return operate.Insert(node, path, value)
			}
		}
	}

    return nil
}

func Read(filename string) (*common.Node, error) {
	return common.Read(filename)
}<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build dragonfly || freebsd || netbsd

package os

import (
	"runtime"
	"syscall"
)

const _P_PID = 0

// blockUntilWaitable attempts to block until a call to p.Wait will
// succeed immediately, and reports whether it has done so.
// It does not actually call p.Wait.
func (p *Process) blockUntilWaitable() (bool, error) {
	var errno syscall.Errno
	for {
		// The arguments on 32-bit FreeBSD look like the following:
		// - freebsd32_wait6_args{ idtype, id1, id2, status, options, wrusage, info } or
		// - freebsd32_wait6_args{ idtype, pad, id1, id2, status, options, wrusage, info } when PAD64_REQUIRED=1 on ARM, MIPS or PowerPC
		if runtime.GOOS == "freebsd" && runtime.GOARCH == "386" {
			_, _, errno = syscall.Syscall9(syscall.SYS_WAIT6, _P_PID, uintptr(p.Pid), 0, 0, syscall.WEXITED|syscall.WNOWAIT, 0, 0, 0, 0)
		} else if runtime.GOOS == "freebsd" && runtime.GOARCH == "arm" {
			_, _, errno = syscall.Syscall9(syscall.SYS_WAIT6, _P_PID, 0, uintptr(p.Pid), 0, 0, syscall.WEXITED|syscall.WNOWAIT, 0, 0, 0)
		} else {
			_, _, errno = syscall.Syscall6(syscall.SYS_WAIT6, _P_PID, uintptr(p.Pid), 0, syscall.WEXITED|syscall.WNOWAIT, 0, 0)
		}
		if errno != syscall.EINTR {
			break
		}
	}
	runtime.KeepAlive(p)
	if errno == syscall.ENOSYS {
		return false, nil
	} else if errno != 0 {
		return false, NewSyscallError("wait6", errno)
	}
	return true, nil
}
<commit_msg>os: use the correct constant for P_PID on NetBSD<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build dragonfly || freebsd || netbsd

package os

import (
	"runtime"
	"syscall"
)

const (
	_P_PID        = 0 // everywhere except for NetBSD?
	_P_PID_NETBSD = 1 // on NetBSD, 0 is P_ALL
)

// blockUntilWaitable attempts to block until a call to p.Wait will
// succeed immediately, and reports whether it has done so.
// It does not actually call p.Wait.
func (p *Process) blockUntilWaitable() (bool, error) {
	var errno syscall.Errno
	for {
		// The arguments on 32-bit FreeBSD look like the following:
		// - freebsd32_wait6_args{ idtype, id1, id2, status, options, wrusage, info } or
		// - freebsd32_wait6_args{ idtype, pad, id1, id2, status, options, wrusage, info } when PAD64_REQUIRED=1 on ARM, MIPS or PowerPC
		if runtime.GOOS == "freebsd" && runtime.GOARCH == "386" {
			_, _, errno = syscall.Syscall9(syscall.SYS_WAIT6, _P_PID, uintptr(p.Pid), 0, 0, syscall.WEXITED|syscall.WNOWAIT, 0, 0, 0, 0)
		} else if runtime.GOOS == "freebsd" && runtime.GOARCH == "arm" {
			_, _, errno = syscall.Syscall9(syscall.SYS_WAIT6, _P_PID, 0, uintptr(p.Pid), 0, 0, syscall.WEXITED|syscall.WNOWAIT, 0, 0, 0)
		} else if runtime.GOOS == "netbsd" {
			_, _, errno = syscall.Syscall6(syscall.SYS_WAIT6, _P_PID_NETBSD, uintptr(p.Pid), 0, syscall.WEXITED|syscall.WNOWAIT, 0, 0)
		} else {
			_, _, errno = syscall.Syscall6(syscall.SYS_WAIT6, _P_PID, uintptr(p.Pid), 0, syscall.WEXITED|syscall.WNOWAIT, 0, 0)
		}
		if errno != syscall.EINTR {
			break
		}
	}
	runtime.KeepAlive(p)
	if errno == syscall.ENOSYS {
		return false, nil
	} else if errno != 0 {
		return false, NewSyscallError("wait6", errno)
	}
	return true, nil
}
<|endoftext|>


<commit_before>package jc

import (
	"fmt"
	"reflect"
	"errors"
	"gopkg.in/mgo.v2"
	"github.com/kalcok/jc/tools"
)

type Query struct {
	Database    string
	Collection  string
	filter      interface{}
	limit       int
	skip        int
	singleValue bool
	result      interface{}
	resultType  reflect.Type
}

func NewQuery(result interface{}) (newQuery Query, err error) {
	newQuery.result = result

	internalType := reflect.TypeOf(result).Elem()

	if internalType.Kind() == reflect.Slice {
		internalType = internalType.Elem()
		newQuery.singleValue = false
	} else {
		newQuery.singleValue = true
	}

	prototype := reflect.New(internalType)
	targetInterface := reflect.TypeOf((*document)(nil)).Elem()
	if !prototype.Type().Implements(targetInterface) {
		err = errors.New(
			fmt.Sprintf("Supplied 'result' type does not implement '%s'.", targetInterface.String()))
		return
	}

	newQuery.resultType = internalType
	proto_val := initPrototype(prototype, internalType)

	newQuery.Collection = proto_val.FieldByName("_collectionName").String()
	newQuery.Database = proto_val.FieldByName("_collectionDB").String()

	return
}

func (q *Query) Execute(reuseSocket bool) (err error) {
	var session *mgo.Session
	masterSession, err := tools.GetSession()

	if err != nil {
		return
	}

	if reuseSocket {
		session = masterSession.Clone()
	} else {
		session = masterSession.Copy()
	}
	fmt.Println(q.Collection)
	query := session.DB(q.Database).C(q.Collection).Find(q.filter)

	if q.skip > 0 {
		query = query.Skip(q.skip)
	}

	if q.limit > 0 {
		query = query.Limit(q.limit)
	}

	if q.singleValue {
		err = query.One(q.result)
	} else {
		err = query.All(q.result)
	}
	if err != nil {
		fmt.Println(err)
		return
	}
	q.initResult()
	return
}

func (q *Query) Filter(filter interface{}) *Query {
	q.filter = filter
	return q
}

func (q *Query) Limit(limit int) *Query {
	q.limit = limit
	return q
}

func (q *Query) Skip(skip int) *Query {
	q.skip = skip
	return q
}

func (q *Query) initResult() error {
	var err error
	resultv := reflect.ValueOf(q.result)

	if q.singleValue {
		initPrototype(resultv, q.resultType)
	} else {
		slicev := resultv.Elem()
		for i := 0; i < slicev.Len(); i++ {
			initPrototype(slicev.Index(i).Addr(), q.resultType)
		}
	}

	return err
}
<commit_msg>Remove prints<commit_after>package jc

import (
	"fmt"
	"reflect"
	"errors"
	"gopkg.in/mgo.v2"
	"github.com/kalcok/jc/tools"
)

type Query struct {
	Database    string
	collection  string
	filter      interface{}
	limit       int
	skip        int
	singleValue bool
	result      interface{}
	resultType  reflect.Type
}

func NewQuery(result interface{}) (newQuery Query, err error) {
	newQuery.result = result

	internalType := reflect.TypeOf(result).Elem()

	if internalType.Kind() == reflect.Slice {
		internalType = internalType.Elem()
		newQuery.singleValue = false
	} else {
		newQuery.singleValue = true
	}

	prototype := reflect.New(internalType)
	targetInterface := reflect.TypeOf((*document)(nil)).Elem()
	if !prototype.Type().Implements(targetInterface) {
		err = errors.New(
			fmt.Sprintf("Supplied 'result' type does not implement '%s'.", targetInterface.String()))
		return
	}

	newQuery.resultType = internalType
	proto_val := initPrototype(prototype, internalType)

	newQuery.collection = proto_val.FieldByName("_collectionName").String()
	newQuery.Database = proto_val.FieldByName("_collectionDB").String()

	return
}

func (q *Query) Execute(reuseSocket bool) (err error) {
	var session *mgo.Session
	masterSession, err := tools.GetSession()

	if err != nil {
		return
	}

	if reuseSocket {
		session = masterSession.Clone()
	} else {
		session = masterSession.Copy()
	}
	query := session.DB(q.Database).C(q.collection).Find(q.filter)

	if q.skip > 0 {
		query = query.Skip(q.skip)
	}

	if q.limit > 0 {
		query = query.Limit(q.limit)
	}

	if q.singleValue {
		err = query.One(q.result)
	} else {
		err = query.All(q.result)
	}

	if err != nil {
		return
	}
	q.initResult()
	return
}

func (q *Query) Filter(filter interface{}) *Query {
	q.filter = filter
	return q
}

func (q *Query) Limit(limit int) *Query {
	q.limit = limit
	return q
}

func (q *Query) Skip(skip int) *Query {
	q.skip = skip
	return q
}

func (q *Query) initResult() error {
	var err error
	resultv := reflect.ValueOf(q.result)

	if q.singleValue {
		initPrototype(resultv, q.resultType)
	} else {
		slicev := resultv.Elem()
		for i := 0; i < slicev.Len(); i++ {
			initPrototype(slicev.Index(i).Addr(), q.resultType)
		}
	}

	return err
}
<|endoftext|>


<commit_before>package adapter

import (
	"errors"
	"fmt"
	"reflect"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/gospackler/caddyshack"
	"github.com/gospackler/couchdb"
)

// View Object is placed over here as a query in couch is possible only with a view
// Helps in forming the javascript that can be used for working on stuff.
type ViewObj struct {
	Name     string
	ViewType reflect.Type
}

// FIXME : Assert Kind of the viewObj passed is a pointer.
func NewViewObj(name string, viewObj caddyshack.StoreObject) *ViewObj {
	// Check if View Exists in the DB.
	// Create View Using the tags if thats the case.
	return &ViewObj{
		Name:     name,
		ViewType: reflect.ValueOf(viewObj).Elem().Type(),
	}
}

// Create the View if it does not exist.
func (v *ViewObj) GetCondition() string {
	// Use the tags to form the condition.
	return "Not implemented"
}

// Initial Version
type CouchQuery struct {
	Condition  string // Code for the view RawJson
	ViewName   string
	Store      *CouchStore
	desDoc     *couchdb.DesignDoc
	Params     string
	Skip       int
	Limit      int
	BufferSize int
}

func NewQuery(line string, viewName string, desDoc string, db *CouchStore) (couchQuery *CouchQuery) {
	// Assuming a design doc is already created.
	line = "\"map\" : \"" + line + "\""
	return createCouchQuery(line, viewName, desDoc, db, "")
}

func NewFilterQuery(line string, viewName string, desDoc string, db *CouchStore, params string) (couchQuery *CouchQuery) {
	// Assuming a design doc is already created.
	line = "\"map\" : \"" + line + "\""
	return createCouchQuery(line, viewName, desDoc, db, params)
}

func NewMRQuery(mrCode string, viewName string, desDoc string, db *CouchStore, params string) (couchQuery *CouchQuery) {
	return createCouchQuery(mrCode, viewName, desDoc, db, params)
}

func createCouchQuery(rawCond string, viewName string, desDoc string, db *CouchStore, params string) (couchQuery *CouchQuery) {
	desDocObj := db.GetDesignDoc(desDoc)

	couchQuery = &CouchQuery{
		desDoc:     desDocObj,
		Condition:  rawCond,
		ViewName:   viewName,
		Store:      db,
		Params:     params,
		BufferSize: 50,
	}

	// Correct the code over here.
	newView := &couchdb.View{Name: viewName}
	newView.RawJson = couchQuery.Condition

	index, status := couchQuery.desDoc.CheckExists(viewName)

	log.Debug("Views -->", couchQuery.desDoc.Views)
	log.Debug("LastView -->", couchQuery.desDoc.LastView)
	if status == false {
		couchQuery.desDoc.AddView(newView)

		err := couchQuery.desDoc.SaveDoc()
		if err != nil {
			panic(err)
		}

	} else {
		log.Debug("Index found at ", index)
		if index < 0 {
			couchQuery.desDoc.LastView = newView
		} else {
			couchQuery.desDoc.Views[index] = newView
		}
	}
	return
}

// Use reflection to create the query from the tags.
func NewObjQuery(obj caddyshack.StoreObject, db *CouchStore) (q *CouchQuery) {
	q = new(CouchQuery)
	prefix := "doc"

	viewName := q.GetViewName(obj)
	view := couchdb.NewView(viewName, prefix, q.getCondition(obj, prefix), q.getEmits(obj))
	//Creates the DesignDoc if it does not exist.
	desDoc := db.GetDesignDoc(db.Res.DesDoc)
	index, status := desDoc.CheckExists(viewName)

	log.Debug("Index, Status", index, status)

	if status == false {
		desDoc.AddView(view)
		log.Debug("Added view", desDoc, view)
		// FIXME: Removing object update for now. Saved by the guard.
		err := desDoc.SaveDoc()
		if err != nil {
			panic(err)
		}
	} else {
		if index < 0 {
			desDoc.LastView = view
		} else {
			desDoc.Views[index] = view
		}
	}

	q.desDoc = desDoc
	q.ViewName = viewName
	q.Store = db

	return
}

func (q *CouchQuery) SetCondition(cond string) {
	q.Condition = cond
}

func (q *CouchQuery) GetCondition() string {
	return q.Condition
}

func (q *CouchQuery) GetViewName(obj caddyshack.StoreObject) string {
	structObj := reflect.ValueOf(obj).Elem()
	typeOfObj := structObj.Type()
	return strings.ToLower(typeOfObj.Name())
}

func (q *CouchQuery) getCondition(obj caddyshack.StoreObject, prefix string) (condStr string) {
	structObj := reflect.ValueOf(obj).Elem()
	typeOfObj := structObj.Type()

	firstCond := true

	if structObj.Kind() == reflect.Struct {
		for index := 0; index < typeOfObj.NumField(); index++ {
			structField := typeOfObj.Field(index)
			fieldCond := structField.Tag.Get("condition")
			if fieldCond != "" {
				if firstCond {
					condStr = condStr + prefix + "." + fieldCond
					firstCond = false
				} else {
					condStr = condStr + " && " + prefix + "." + fieldCond
				}
			}
		}
	} else {
		panic(errors.New("Expected a struct pointer as input."))
	}

	return
}

func (q *CouchQuery) getEmits(obj caddyshack.StoreObject) (emits string) {
	structObj := reflect.ValueOf(obj).Elem()
	typeOfObj := structObj.Type()

	firstCond := true

	if structObj.Kind() == reflect.Struct {
		for index := 0; index < typeOfObj.NumField(); index++ {
			structField := typeOfObj.Field(index)
			jsonStr := structField.Tag.Get("json")
			if jsonStr != "" {
				if firstCond {
					emits = emits + "\\\"" + jsonStr + "\\\""
					firstCond = false
				} else {
					emits = emits + ", \\\"" + jsonStr + "\\\""
				}
			}
		}
	} else {
		panic(errors.New("Expected a struct pointer as input."))
	}

	return
}

func (q *CouchQuery) Execute() (error, []caddyshack.StoreObject) {
	// Currently O(n) w.r.t to views
	var extraParams string
	if q.Limit != 0 {
		extraParams = fmt.Sprintf("&limit=%d&skip=%d", q.Limit, q.Skip)
	}

	params := q.Params + extraParams
	data, err := q.desDoc.Db.GetView(q.desDoc.Id, q.ViewName, params)
	if err != nil {
		return errors.New("Error retreiving view : " + err.Error()), nil
	} else {
		// Print for now create store Object later.
		// FIXME Handle unmarshalling over here.
		result, err := q.Store.MarshalStoreObjects(data)
		if err != nil {
			return errors.New("Could not Marshal json" + err.Error()), result
		}
		return nil, result
	}

	// Move this section to New
	// The intutive version would be creating an object and then adding methods to it.

	// If it exists get the view back.
	// Otherwise Get Retrieve the Data and Marshal the store Object from the json..
}
<commit_msg>Add default buffer size to couchQuery<commit_after>package adapter

import (
	"errors"
	"fmt"
	"reflect"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/gospackler/caddyshack"
	"github.com/gospackler/couchdb"
)

const (
	defaultBufferSize = 50
)

// View Object is placed over here as a query in couch is possible only with a view
// Helps in forming the javascript that can be used for working on stuff.
type ViewObj struct {
	Name     string
	ViewType reflect.Type
}

// FIXME : Assert Kind of the viewObj passed is a pointer.
func NewViewObj(name string, viewObj caddyshack.StoreObject) *ViewObj {
	// Check if View Exists in the DB.
	// Create View Using the tags if thats the case.
	return &ViewObj{
		Name:     name,
		ViewType: reflect.ValueOf(viewObj).Elem().Type(),
	}
}

// Create the View if it does not exist.
func (v *ViewObj) GetCondition() string {
	// Use the tags to form the condition.
	return "Not implemented"
}

// Initial Version
type CouchQuery struct {
	Condition  string // Code for the view RawJson
	ViewName   string
	Store      *CouchStore
	desDoc     *couchdb.DesignDoc
	Params     string
	Skip       int
	Limit      int
	BufferSize int
}

func NewQuery(line string, viewName string, desDoc string, db *CouchStore) (couchQuery *CouchQuery) {
	// Assuming a design doc is already created.
	line = "\"map\" : \"" + line + "\""
	return createCouchQuery(line, viewName, desDoc, db, "")
}

func NewFilterQuery(line string, viewName string, desDoc string, db *CouchStore, params string) (couchQuery *CouchQuery) {
	// Assuming a design doc is already created.
	line = "\"map\" : \"" + line + "\""
	return createCouchQuery(line, viewName, desDoc, db, params)
}

func NewMRQuery(mrCode string, viewName string, desDoc string, db *CouchStore, params string) (couchQuery *CouchQuery) {
	return createCouchQuery(mrCode, viewName, desDoc, db, params)
}

func createCouchQuery(rawCond string, viewName string, desDoc string, db *CouchStore, params string) (couchQuery *CouchQuery) {
	desDocObj := db.GetDesignDoc(desDoc)

	couchQuery = &CouchQuery{
		desDoc:     desDocObj,
		Condition:  rawCond,
		ViewName:   viewName,
		Store:      db,
		Params:     params,
		BufferSize: defaultBufferSize,
	}

	// Correct the code over here.
	newView := &couchdb.View{Name: viewName}
	newView.RawJson = couchQuery.Condition

	index, status := couchQuery.desDoc.CheckExists(viewName)

	log.Debug("Views -->", couchQuery.desDoc.Views)
	log.Debug("LastView -->", couchQuery.desDoc.LastView)
	if status == false {
		couchQuery.desDoc.AddView(newView)

		err := couchQuery.desDoc.SaveDoc()
		if err != nil {
			panic(err)
		}

	} else {
		log.Debug("Index found at ", index)
		if index < 0 {
			couchQuery.desDoc.LastView = newView
		} else {
			couchQuery.desDoc.Views[index] = newView
		}
	}
	return
}

// Use reflection to create the query from the tags.
func NewObjQuery(obj caddyshack.StoreObject, db *CouchStore) (q *CouchQuery) {
	q = new(CouchQuery)
	prefix := "doc"

	viewName := q.GetViewName(obj)
	view := couchdb.NewView(viewName, prefix, q.getCondition(obj, prefix), q.getEmits(obj))
	//Creates the DesignDoc if it does not exist.
	desDoc := db.GetDesignDoc(db.Res.DesDoc)
	index, status := desDoc.CheckExists(viewName)

	log.Debug("Index, Status", index, status)

	if status == false {
		desDoc.AddView(view)
		log.Debug("Added view", desDoc, view)
		// FIXME: Removing object update for now. Saved by the guard.
		err := desDoc.SaveDoc()
		if err != nil {
			panic(err)
		}
	} else {
		if index < 0 {
			desDoc.LastView = view
		} else {
			desDoc.Views[index] = view
		}
	}

	q.desDoc = desDoc
	q.ViewName = viewName
	q.Store = db
	q.BufferSize = defaultBufferSize

	return
}

func (q *CouchQuery) SetCondition(cond string) {
	q.Condition = cond
}

func (q *CouchQuery) GetCondition() string {
	return q.Condition
}

func (q *CouchQuery) GetViewName(obj caddyshack.StoreObject) string {
	structObj := reflect.ValueOf(obj).Elem()
	typeOfObj := structObj.Type()
	return strings.ToLower(typeOfObj.Name())
}

func (q *CouchQuery) getCondition(obj caddyshack.StoreObject, prefix string) (condStr string) {
	structObj := reflect.ValueOf(obj).Elem()
	typeOfObj := structObj.Type()

	firstCond := true

	if structObj.Kind() == reflect.Struct {
		for index := 0; index < typeOfObj.NumField(); index++ {
			structField := typeOfObj.Field(index)
			fieldCond := structField.Tag.Get("condition")
			if fieldCond != "" {
				if firstCond {
					condStr = condStr + prefix + "." + fieldCond
					firstCond = false
				} else {
					condStr = condStr + " && " + prefix + "." + fieldCond
				}
			}
		}
	} else {
		panic(errors.New("Expected a struct pointer as input."))
	}

	return
}

func (q *CouchQuery) getEmits(obj caddyshack.StoreObject) (emits string) {
	structObj := reflect.ValueOf(obj).Elem()
	typeOfObj := structObj.Type()

	firstCond := true

	if structObj.Kind() == reflect.Struct {
		for index := 0; index < typeOfObj.NumField(); index++ {
			structField := typeOfObj.Field(index)
			jsonStr := structField.Tag.Get("json")
			if jsonStr != "" {
				if firstCond {
					emits = emits + "\\\"" + jsonStr + "\\\""
					firstCond = false
				} else {
					emits = emits + ", \\\"" + jsonStr + "\\\""
				}
			}
		}
	} else {
		panic(errors.New("Expected a struct pointer as input."))
	}

	return
}

func (q *CouchQuery) Execute() (error, []caddyshack.StoreObject) {
	// Currently O(n) w.r.t to views
	var extraParams string
	if q.Limit != 0 {
		extraParams = fmt.Sprintf("&limit=%d&skip=%d", q.Limit, q.Skip)
	}

	params := q.Params + extraParams
	data, err := q.desDoc.Db.GetView(q.desDoc.Id, q.ViewName, params)
	if err != nil {
		return errors.New("Error retreiving view : " + err.Error()), nil
	} else {
		// Print for now create store Object later.
		// FIXME Handle unmarshalling over here.
		result, err := q.Store.MarshalStoreObjects(data)
		if err != nil {
			return errors.New("Could not Marshal json" + err.Error()), result
		}
		return nil, result
	}

	// Move this section to New
	// The intutive version would be creating an object and then adding methods to it.

	// If it exists get the view back.
	// Otherwise Get Retrieve the Data and Marshal the store Object from the json..
}
<|endoftext|>


<commit_before>/*
  A thread safe queue for Golang.
*/
package goqueue

import (
	"container/list"
	"errors"
	"sync"
	"time"
)

type EmptyQueueError struct{}

type FullQueueError struct{}

func (err *EmptyQueueError) Error() string {
	return "Queue is Empty"
}

func (err *FullQueueError) Error() string {
	return "Queue is Full"
}

type Queue struct {
	mutex          sync.Mutex
	getCond        *sync.Cond
	putCond        *sync.Cond
	taskDoneCond   *sync.Cond
	list           *list.List
	maxSize        int
	unfinishedTask int
}

func NewQueue(maxSize int) *Queue {
	queue := new(Queue)
	queue.mutex = sync.Mutex{}
	queue.getCond = sync.NewCond(&queue.mutex)
	queue.putCond = sync.NewCond(&queue.mutex)
	queue.taskDoneCond = sync.NewCond(&queue.mutex)
	queue.list = list.New()
	queue.maxSize = maxSize
	queue.unfinishedTask = 0
	return queue
}

func (queue *Queue) Size() int {
	queue.mutex.Lock()
	size := queue.qsize()
	queue.mutex.Unlock()
	return size
}

func (queue *Queue) IsEmpty() bool {
	queue.mutex.Lock()
	isEmpty := (queue.qsize() == 0)
	queue.mutex.Unlock()
	return isEmpty
}

func (queue *Queue) IsFull() bool {
	queue.mutex.Lock()
	isFull := (queue.maxSize > 0 && queue.qsize() == queue.maxSize)
	queue.mutex.Unlock()
	return isFull
}

func (queue *Queue) GetNoWait() (*list.Element, error) {
	return queue.Get(false, 0)
}

func (queue *Queue) Get(block bool, timeout float64) (*list.Element, error) {
	queue.getCond.L.Lock()
	defer queue.getCond.L.Unlock()
	emptyQ := false
	if !block {
		if queue.qsize() == 0 {
			emptyQ = true
		}
	} else if timeout == float64(0) {
		for queue.qsize() == 0 {
			queue.getCond.Wait()
		}
	} else if timeout < float64(0) {
		return &list.Element{}, errors.New("'timeout' must be a non-negative number")
	} else {
		timer := time.After(time.Duration(timeout) * time.Second)
		notEmpty := make(chan bool)
		go queue.waitSignal(queue.getCond, notEmpty)
		for queue.qsize() == 0 {
			select {
			case <-timer:
				emptyQ = true
				break
			case <-notEmpty:
				if queue.qsize() == 0 {
					go queue.waitSignal(queue.getCond, notEmpty)
				} else {
					break
				}
			}
		}
	}
	if emptyQ {
		return &list.Element{}, &EmptyQueueError{}
	}
	e := queue.list.Front()
	queue.list.Remove(e)
	queue.putCond.Signal()
	return e, nil
}

func (queue *Queue) PutNoWait(element interface{}) (*list.Element, error) {
	return queue.Put(element, false, 0)
}

func (queue *Queue) Put(element interface{}, block bool, timeout float64) (*list.Element, error) {
	queue.putCond.L.Lock()
	defer queue.putCond.L.Unlock()
	fullQ := false
	if queue.maxSize > 0 {
		if !block {
			if queue.qsize() == queue.maxSize {
				fullQ = true
			}
		} else if timeout == float64(0) {
			for queue.qsize() == queue.maxSize {
				queue.putCond.Wait()
			}
		} else if timeout < float64(0) {
			return &list.Element{}, errors.New("'timeout' must be a non-negative number")
		} else {
			timer := time.After(time.Duration(timeout) * time.Second)
			notFull := make(chan bool)
			go queue.waitSignal(queue.putCond, notFull)
			for queue.qsize() == queue.maxSize {
				select {
				case <-timer:
					fullQ = true
					break
				case <-notFull:
					if queue.qsize() == queue.maxSize {
						go queue.waitSignal(queue.putCond, notFull)
					} else {
						break
					}
				}
			}
		}
	}
	if fullQ {
		return &list.Element{}, &FullQueueError{}
	}
	e := queue.list.PushBack(element)
	queue.unfinishedTask += 1
	queue.getCond.Signal()
	return e, nil
}

func (queue *Queue) TaskDone() {
	queue.taskDoneCond.L.Lock()
	defer queue.taskDoneCond.L.Unlock()
	unfinished := queue.unfinishedTask - 1
	if unfinished <= 0 {
		if unfinished < 0 {
			panic(errors.New("TaskDone() called too many times"))
		}
		queue.taskDoneCond.Broadcast()
	}
	queue.unfinishedTask = unfinished
}

func (queue *Queue) WaitAllComplete() {
	queue.taskDoneCond.L.Lock()
	defer queue.taskDoneCond.L.Unlock()
	for queue.unfinishedTask != 0 {
		queue.taskDoneCond.Wait()
	}
}

func (queue *Queue) waitSignal(cond *sync.Cond, c chan<- bool) {
	cond.Wait()
	c <- true
}

func (queue *Queue) qsize() int {
	return queue.list.Len()
}
<commit_msg>cancel signal wait goroutine<commit_after>/*
  A thread safe queue for Golang.
*/
package goqueue

import (
	"container/list"
	"errors"
	"sync"
	"time"
)

type EmptyQueueError struct{}

type FullQueueError struct{}

func (err *EmptyQueueError) Error() string {
	return "Queue is Empty"
}

func (err *FullQueueError) Error() string {
	return "Queue is Full"
}

type Queue struct {
	mutex          sync.Mutex
	getCond        *sync.Cond
	putCond        *sync.Cond
	taskDoneCond   *sync.Cond
	list           *list.List
	maxSize        int
	unfinishedTask int
}

func NewQueue(maxSize int) *Queue {
	queue := new(Queue)
	queue.mutex = sync.Mutex{}
	queue.getCond = sync.NewCond(&queue.mutex)
	queue.putCond = sync.NewCond(&queue.mutex)
	queue.taskDoneCond = sync.NewCond(&queue.mutex)
	queue.list = list.New()
	queue.maxSize = maxSize
	queue.unfinishedTask = 0
	return queue
}

func (queue *Queue) Size() int {
	queue.mutex.Lock()
	size := queue.qsize()
	queue.mutex.Unlock()
	return size
}

func (queue *Queue) IsEmpty() bool {
	queue.mutex.Lock()
	isEmpty := (queue.qsize() == 0)
	queue.mutex.Unlock()
	return isEmpty
}

func (queue *Queue) IsFull() bool {
	queue.mutex.Lock()
	isFull := (queue.maxSize > 0 && queue.qsize() == queue.maxSize)
	queue.mutex.Unlock()
	return isFull
}

func (queue *Queue) GetNoWait() (*list.Element, error) {
	return queue.Get(false, 0)
}

func (queue *Queue) Get(block bool, timeout float64) (*list.Element, error) {
	queue.getCond.L.Lock()
	defer queue.getCond.L.Unlock()
	emptyQ := false
	if !block {
		if queue.qsize() == 0 {
			emptyQ = true
		}
	} else if timeout == float64(0) {
		for queue.qsize() == 0 {
			queue.getCond.Wait()
		}
	} else if timeout < float64(0) {
		return &list.Element{}, errors.New("'timeout' must be a non-negative number")
	} else {
		timer := time.After(time.Duration(timeout) * time.Second)
		notEmpty := make(chan bool, 1)
		go queue.waitSignal(queue.getCond, notEmpty)
	TIMEOUT:
		for queue.qsize() == 0 {
			select {
			case <-timer:
				emptyQ = true
				queue.cancelWait(queue.getCond, notEmpty)
				break TIMEOUT
			case <-notEmpty:
				if queue.qsize() == 0 {
					go queue.waitSignal(queue.getCond, notEmpty)
				} else {
					break TIMEOUT
				}
			}
		}
		close(notEmpty)
	}
	if emptyQ {
		return &list.Element{}, &EmptyQueueError{}
	}
	e := queue.list.Front()
	queue.list.Remove(e)
	queue.putCond.Signal()
	return e, nil
}

func (queue *Queue) PutNoWait(element interface{}) (*list.Element, error) {
	return queue.Put(element, false, 0)
}

func (queue *Queue) Put(element interface{}, block bool, timeout float64) (*list.Element, error) {
	queue.putCond.L.Lock()
	defer queue.putCond.L.Unlock()
	fullQ := false
	if queue.maxSize > 0 {
		if !block {
			if queue.qsize() == queue.maxSize {
				fullQ = true
			}
		} else if timeout == float64(0) {
			for queue.qsize() == queue.maxSize {
				queue.putCond.Wait()
			}
		} else if timeout < float64(0) {
			return &list.Element{}, errors.New("'timeout' must be a non-negative number")
		} else {
			timer := time.After(time.Duration(timeout) * time.Second)
			notFull := make(chan bool, 1)
			go queue.waitSignal(queue.putCond, notFull)
		TIMEOUT:
			for queue.qsize() == queue.maxSize {
				select {
				case <-timer:
					fullQ = true
					queue.cancelWait(queue.putCond, notFull)
					break TIMEOUT
				case <-notFull:
					if queue.qsize() == queue.maxSize {
						go queue.waitSignal(queue.putCond, notFull)
					} else {
						break TIMEOUT
					}
				}
			}
			close(notFull)
		}
	}
	if fullQ {
		return &list.Element{}, &FullQueueError{}
	}
	e := queue.list.PushBack(element)
	queue.unfinishedTask += 1
	queue.getCond.Signal()
	return e, nil
}

func (queue *Queue) TaskDone() {
	queue.taskDoneCond.L.Lock()
	defer queue.taskDoneCond.L.Unlock()
	unfinished := queue.unfinishedTask - 1
	if unfinished <= 0 {
		if unfinished < 0 {
			panic(errors.New("TaskDone() called too many times"))
		}
		queue.taskDoneCond.Broadcast()
	}
	queue.unfinishedTask = unfinished
}

func (queue *Queue) WaitAllComplete() {
	queue.taskDoneCond.L.Lock()
	defer queue.taskDoneCond.L.Unlock()
	for queue.unfinishedTask != 0 {
		queue.taskDoneCond.Wait()
	}
}

func (queue *Queue) waitSignal(cond *sync.Cond, c chan<- bool) {
	cond.Wait()
	c <- true
}

func (queue *Queue) cancelWait(cond *sync.Cond, c <-chan bool) {
	cond.Signal()
	<-c
}

func (queue *Queue) qsize() int {
	return queue.list.Len()
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Mitchell Cooper
// quiki - a standalone web server for wikifier
package main

import (
	"errors"
	"fmt"
	"github.com/cooper/quiki/config"
	"log"
	"net/http"
	"os"
)

// wikiserver config instance
var conf *config.Config

// wikifier directory path
var wikifierPath string

func main() {

	// find config file
	if len(os.Args) < 2 || os.Args[1] == "" {
		log.Fatal("usage: " + os.Args[0] + " /path/to/wikiserver.conf")
	}

	// parse configuration
	conf = config.New(os.Args[1])
	if err := conf.Parse(); err != nil {
		log.Fatal(err)
	}

	// these are required
	var port string
	if err := conf.RequireMany(map[string]*string{
		"server.http.port":    &port,
		"server.dir.wikifier": &wikifierPath,
	}); err != nil {
		log.Fatal(err)
	}

	// template search directories may have been specified
	templateDirs = conf.Get("server.dir.template")

	// setup the transport
	if err := initTransport(); err != nil {
		log.Fatal(err)
	}

	// set up wikis
	if err := initWikis(); err != nil {
		log.Fatal(err)
	}

	// setup static files from wikifier
	if err := setupStatic(); err != nil {
		log.Fatal(err)
	}

	log.Println("quiki ready")

	// main handler
	http.HandleFunc("/", handleRoot)

	// listen
	log.Fatal(http.ListenAndServe(conf.Get("server.http.bind")+":"+port, nil))
}

func setupStatic() error {
	if wikifierPath == "" {
		return errors.New("can't find wikifier. set @server.dir.wikifier; " +
			"otherwise at least one configured wiki needs @dir.wikifier",
		)
	} else if stat, err := os.Stat(wikifierPath); err != nil || !stat.IsDir() {
		if err == nil {
			err = errors.New("not a directory")
		}
		errStr := fmt.Sprintf(
			"@dir.wikifier (%s) error: %v\n",
			wikifierPath,
			err.Error(),
		)
		return errors.New(errStr)
	}
	fileServer := http.FileServer(http.Dir(wikifierPath + "/static"))
	http.Handle("/static/", http.StripPrefix("/static/", fileServer))
	return nil
}
<commit_msg>wikifierPath is required now, so we don't need this<commit_after>// Copyright (c) 2017, Mitchell Cooper
// quiki - a standalone web server for wikifier
package main

import (
	"errors"
	"fmt"
	"github.com/cooper/quiki/config"
	"log"
	"net/http"
	"os"
)

// wikiserver config instance
var conf *config.Config

// wikifier directory path
var wikifierPath string

func main() {

	// find config file
	if len(os.Args) < 2 || os.Args[1] == "" {
		log.Fatal("usage: " + os.Args[0] + " /path/to/wikiserver.conf")
	}

	// parse configuration
	conf = config.New(os.Args[1])
	if err := conf.Parse(); err != nil {
		log.Fatal(err)
	}

	// these are required
	var port string
	if err := conf.RequireMany(map[string]*string{
		"server.http.port":    &port,
		"server.dir.wikifier": &wikifierPath,
	}); err != nil {
		log.Fatal(err)
	}

	// template search directories may have been specified
	templateDirs = conf.Get("server.dir.template")

	// setup the transport
	if err := initTransport(); err != nil {
		log.Fatal(err)
	}

	// set up wikis
	if err := initWikis(); err != nil {
		log.Fatal(err)
	}

	// setup static files from wikifier
	if err := setupStatic(); err != nil {
		log.Fatal(err)
	}

	log.Println("quiki ready")

	// main handler
	http.HandleFunc("/", handleRoot)

	// listen
	log.Fatal(http.ListenAndServe(conf.Get("server.http.bind")+":"+port, nil))
}

func setupStatic() error {
	if stat, err := os.Stat(wikifierPath); err != nil || !stat.IsDir() {
		if err == nil {
			err = errors.New("not a directory")
		}
		errStr := fmt.Sprintf(
			"@dir.wikifier (%s) error: %v\n",
			wikifierPath,
			err.Error(),
		)
		return errors.New(errStr)
	}
	fileServer := http.FileServer(http.Dir(wikifierPath + "/static"))
	http.Handle("/static/", http.StripPrefix("/static/", fileServer))
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package poll

import "syscall"

// SendFile wraps the TransmitFile call.
func SendFile(fd *FD, src syscall.Handle, n int64) (int64, error) {
	ft, err := syscall.GetFileType(src)
	if err != nil {
		return 0, err
	}
	// TransmitFile does not work with pipes
	if ft == syscall.FILE_TYPE_PIPE {
		return 0, syscall.ESPIPE
	}

	if err := fd.writeLock(); err != nil {
		return 0, err
	}
	defer fd.writeUnlock()

	o := &fd.wop
	o.qty = uint32(n)
	o.handle = src
	done, err := wsrv.ExecIO(o, func(o *operation) error {
		return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND)
	})
	return int64(done), err
}
<commit_msg>internal/poll: specify current file position when calling TransmitFile<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package poll

import "syscall"

// SendFile wraps the TransmitFile call.
func SendFile(fd *FD, src syscall.Handle, n int64) (int64, error) {
	ft, err := syscall.GetFileType(src)
	if err != nil {
		return 0, err
	}
	// TransmitFile does not work with pipes
	if ft == syscall.FILE_TYPE_PIPE {
		return 0, syscall.ESPIPE
	}

	if err := fd.writeLock(); err != nil {
		return 0, err
	}
	defer fd.writeUnlock()

	o := &fd.wop
	o.qty = uint32(n)
	o.handle = src

	// TODO(brainman): skip calling syscall.Seek if OS allows it
	curpos, err := syscall.Seek(o.handle, 0, 1)
	if err != nil {
		return 0, err
	}

	o.o.OffsetHigh = uint32(curpos)
	o.o.Offset = uint32(curpos >> 32)

	done, err := wsrv.ExecIO(o, func(o *operation) error {
		return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND)
	})
	return int64(done), err
}
<|endoftext|>