<commit_before>package webloop

import (
	"errors"

	"github.com/gotk3/gotk3/glib"
	"github.com/sourcegraph/go-webkit2/webkit2"
	"github.com/sqs/gojs"
)

// ErrLoadFailed indicates that the View failed to load the requested resource.
var ErrLoadFailed = errors.New("load failed")

// Context stores common settings for a group of Views.
type Context struct{}

// New creates a new Context.
func New() *Context {
	return &Context{}
}

// NewView creates a new View in the context.
func (c *Context) NewView() *View {
	view := make(chan *View, 1)
	glib.IdleAdd(func() bool {
		webView := webkit2.NewWebView()
		settings := webView.Settings()
		settings.SetEnableWriteConsoleMessagesToStdout(true)
		settings.SetUserAgentWithApplicationDetails("WebLoop", "v1")
		v := &View{WebView: webView}
		loadChangedHandler, _ := webView.Connect("load-changed", func(_ *glib.Object, loadEvent webkit2.LoadEvent) {
			switch loadEvent {
			case webkit2.LoadFinished:
				// If we're here, then the load must not have failed, because
				// otherwise we would've disconnected this handler in the
				// load-failed signal handler.
				v.load <- struct{}{}
			}
		})
		webView.Connect("load-failed", func() {
			v.lastLoadErr = ErrLoadFailed
			webView.HandlerDisconnect(loadChangedHandler)
		})
		view <- v
		return false
	})
	return <-view
}

// View represents a WebKit view that can load resources at a given URL and
// query information about them.
type View struct {
	*webkit2.WebView

	load        chan struct{}
	lastLoadErr error

	destroyed bool
}

// Open starts loading the resource at the specified URL.
func (v *View) Open(url string) {
	v.load = make(chan struct{}, 1)
	v.lastLoadErr = nil
	glib.IdleAdd(func() bool {
		if !v.destroyed {
			v.WebView.LoadURI(url)
		}
		return false
	})
}

// Wait waits for the current page to finish loading.
func (v *View) Wait() error {
	<-v.load
	return v.lastLoadErr
}

// URI returns the URI of the current resource in the view.
func (v *View) URI() string {
	uri := make(chan string, 1)
	glib.IdleAdd(func() bool {
		uri <- v.WebView.URI()
		return false
	})
	return <-uri
}

// Title returns the title of the current resource in the view.
func (v *View) Title() string {
	title := make(chan string, 1)
	glib.IdleAdd(func() bool {
		title <- v.WebView.Title()
		return false
	})
	return <-title
}

// EvaluateJavaScript runs the JavaScript in script in the view's context and
// returns the script's result as a Go value.
func (v *View) EvaluateJavaScript(script string) (result interface{}, err error) {
	resultChan := make(chan interface{}, 1)
	errChan := make(chan error, 1)

	glib.IdleAdd(func() bool {
		v.WebView.RunJavaScript(script, func(result *gojs.Value, err error) {
			glib.IdleAdd(func() bool {
				if err == nil {
					goval, err := result.GoValue()
					if err != nil {
						errChan <- err
						return false
					}
					resultChan <- goval
				} else {
					errChan <- err
				}
				return false
			})
		})
		return false
	})

	select {
	case result = <-resultChan:
		return result, nil
	case err = <-errChan:
		return nil, err
	}
}

// Close closes the view and releases associated resources. Ensure that Close is
// called after all other pending operations on View have returned, or they may
// hang indefinitely.
func (v *View) Close() {
	// TODO(sqs): remove all of the source funcs we added via IdleAdd, etc.,
	// using g_source_remove, to fix "assertion
	// 'WEBKIT_IS_WEB_VIEW(webView) failed" messages.
	v.destroyed = true
	v.Destroy()
}
<commit_msg>add Load function.<commit_after>package webloop

import (
	"errors"

	"github.com/gotk3/gotk3/glib"
	"github.com/sourcegraph/go-webkit2/webkit2"
	"github.com/sqs/gojs"
)

// ErrLoadFailed indicates that the View failed to load the requested resource.
var ErrLoadFailed = errors.New("load failed")

// Context stores common settings for a group of Views.
type Context struct{}

// New creates a new Context.
func New() *Context {
	return &Context{}
}

// NewView creates a new View in the context.
func (c *Context) NewView() *View {
	view := make(chan *View, 1)
	glib.IdleAdd(func() bool {
		webView := webkit2.NewWebView()
		settings := webView.Settings()
		settings.SetEnableWriteConsoleMessagesToStdout(true)
		settings.SetUserAgentWithApplicationDetails("WebLoop", "v1")
		v := &View{WebView: webView}
		loadChangedHandler, _ := webView.Connect("load-changed", func(_ *glib.Object, loadEvent webkit2.LoadEvent) {
			switch loadEvent {
			case webkit2.LoadFinished:
				// If we're here, then the load must not have failed, because
				// otherwise we would've disconnected this handler in the
				// load-failed signal handler.
				v.load <- struct{}{}
			}
		})
		webView.Connect("load-failed", func() {
			v.lastLoadErr = ErrLoadFailed
			webView.HandlerDisconnect(loadChangedHandler)
		})
		view <- v
		return false
	})
	return <-view
}

// View represents a WebKit view that can load resources at a given URL and
// query information about them.
type View struct {
	*webkit2.WebView

	load        chan struct{}
	lastLoadErr error

	destroyed bool
}

// Open starts loading the resource at the specified URL.
func (v *View) Open(url string) {
	v.load = make(chan struct{}, 1)
	v.lastLoadErr = nil
	glib.IdleAdd(func() bool {
		if !v.destroyed {
			v.WebView.LoadURI(url)
		}
		return false
	})
}

func (v *View) Load(content, baseUrl string) {
	v.load = make(chan struct{}, 1)
	v.lastLoadErr = nil
	glib.IdleAdd(func() bool {
		if !v.destroyed {
			v.WebView.LoadHTML(content, baseUrl)
		}
		return false
	})
}

// Wait waits for the current page to finish loading.
func (v *View) Wait() error {
	<-v.load
	return v.lastLoadErr
}

// URI returns the URI of the current resource in the view.
func (v *View) URI() string {
	uri := make(chan string, 1)
	glib.IdleAdd(func() bool {
		uri <- v.WebView.URI()
		return false
	})
	return <-uri
}

// Title returns the title of the current resource in the view.
func (v *View) Title() string {
	title := make(chan string, 1)
	glib.IdleAdd(func() bool {
		title <- v.WebView.Title()
		return false
	})
	return <-title
}

// EvaluateJavaScript runs the JavaScript in script in the view's context and
// returns the script's result as a Go value.
func (v *View) EvaluateJavaScript(script string) (result interface{}, err error) {
	resultChan := make(chan interface{}, 1)
	errChan := make(chan error, 1)

	glib.IdleAdd(func() bool {
		v.WebView.RunJavaScript(script, func(result *gojs.Value, err error) {
			glib.IdleAdd(func() bool {
				if err == nil {
					goval, err := result.GoValue()
					if err != nil {
						errChan <- err
						return false
					}
					resultChan <- goval
				} else {
					errChan <- err
				}
				return false
			})
		})
		return false
	})

	select {
	case result = <-resultChan:
		return result, nil
	case err = <-errChan:
		return nil, err
	}
}

// Close closes the view and releases associated resources. Ensure that Close is
// called after all other pending operations on View have returned, or they may
// hang indefinitely.
func (v *View) Close() {
	// TODO(sqs): remove all of the source funcs we added via IdleAdd, etc.,
	// using g_source_remove, to fix "assertion
	// 'WEBKIT_IS_WEB_VIEW(webView) failed" messages.
	v.destroyed = true
	v.Destroy()
}
<|endoftext|>


<commit_before>// Container provides a Writer which is capable of serializing gogen-avro structs and writing them in the Avro Object Container File (OCF) format
package container

import (
	"bytes"
	"compress/flate"
	"io"

	"github.com/alanctgardner/gogen-avro/container/avro"
)

/*
  A Codec specifies how the blocks within a container file should be compressed.
*/
type Codec string

const (
	// No compression
	Null Codec = "null"
	// Deflate compression
	Deflate Codec = "deflate"
	// Snappy compression
	Snappy Codec = "snappy"
)

type CloseableResettableWriter interface {
	Close() error
	Reset(io.Writer)
}

/*
  Writer wraps an io.Writer and writes the file and block-level framing required for an OCF file
*/
type Writer struct {
	writer           io.Writer
	syncMarker       [16]byte
	codec            Codec
	recordsPerBlock  int64
	blockBuffer      *bytes.Buffer
	compressedWriter io.Writer
	nextBlockRecords int64
}

/*
  Create a new Writer wrapping the provided io.Writer with the given Codec and number of records per block.
  The Writer will lazily write the container file header when WriteRecord is called the first time.
  You must call Flush on the Writer before closing the underlying io.Writer, to ensure the final block is written.
  A schema string must be passed to ensure that a correct header is written even if no records are written. This
  is required to produce valid empty Avro container files.
*/
func NewWriter(writer io.Writer, codec Codec, recordsPerBlock int64, schema string) (*Writer, error) {
	blockBytes := make([]byte, 0)
	blockBuffer := bytes.NewBuffer(blockBytes)

	avroWriter := &Writer{
		writer:          writer,
		syncMarker:      [16]byte{'g', 'o', 'g', 'e', 'n', 'a', 'v', 'r', 'o', 'm', 'a', 'g', 'i', 'c', '1', '0'},
		codec:           codec,
		recordsPerBlock: recordsPerBlock,
		blockBuffer:     blockBuffer,
	}
	var err error
	if codec == Deflate {
		avroWriter.compressedWriter, err = flate.NewWriter(avroWriter.blockBuffer, flate.DefaultCompression)
		if err != nil {
			return nil, err
		}
	} else if codec == Snappy {
		avroWriter.compressedWriter = newSnappyWriter(avroWriter.blockBuffer)
	} else if codec == Null {
		avroWriter.compressedWriter = avroWriter.blockBuffer
	}

	err = avroWriter.writeHeader(schema)
	if err != nil {
		return nil, err
	}

	return avroWriter, nil
}

func (avroWriter *Writer) writeHeader(schema string) error {
	header := &avro.AvroContainerHeader{
		Magic: [4]byte{'O', 'b', 'j', 1},
		Meta: map[string][]byte{
			"avro.schema": []byte(schema),
			"avro.codec":  []byte(avroWriter.codec),
		},
		Sync: avroWriter.syncMarker,
	}
	return header.Serialize(avroWriter.writer)
}

/*
  Write an AvroRecord to the container file. All gogen-avro generated structs
  fulfill the AvroRecord interface. Note that all records in a given container file
  must be of the same Avro type.
*/
func (avroWriter *Writer) WriteRecord(record AvroRecord) error {
	var err error
	// Serialize the new record into the compressed writer
	err = record.Serialize(avroWriter.compressedWriter)
	if err != nil {
		return err
	}
	avroWriter.nextBlockRecords += 1

	// If the block if full, flush and reset the compressed writer,
	// write the header and the block contents
	if avroWriter.nextBlockRecords >= avroWriter.recordsPerBlock {
		return avroWriter.Flush()
	}

	return nil
}

/*
  Write the current block to the file if it has been filled.  It is
  best-practise to always call this before the underlying io.Writer is closed.
*/
func (avroWriter *Writer) Flush() error {
	if avroWriter.nextBlockRecords == 0 {
		return nil
	}

	// Write out all of the buffered records as a new block
	// Must be called before closing to ensure the last block is written
	if fwWriter, ok := avroWriter.compressedWriter.(CloseableResettableWriter); ok {
		fwWriter.Close()
		fwWriter.Reset(avroWriter.blockBuffer)
	}

	if avroWriter.nextBlockRecords > 0 {
		block := &avro.AvroContainerBlock{
			NumRecords:  avroWriter.nextBlockRecords,
			RecordBytes: avroWriter.blockBuffer.Bytes(),
			Sync:        avroWriter.syncMarker,
		}
		err := block.Serialize(avroWriter.writer)
		if err != nil {
			return err
		}
	}

	avroWriter.blockBuffer.Reset()
	avroWriter.nextBlockRecords = 0

	return nil
}
<commit_msg>Avoid nil writer by defaulting to blockBuffer when creating container writer<commit_after>// Container provides a Writer which is capable of serializing gogen-avro structs and writing them in the Avro Object Container File (OCF) format
package container

import (
	"bytes"
	"compress/flate"
	"io"

	"github.com/alanctgardner/gogen-avro/container/avro"
)

/*
  A Codec specifies how the blocks within a container file should be compressed.
*/
type Codec string

const (
	// No compression
	Null Codec = "null"
	// Deflate compression
	Deflate Codec = "deflate"
	// Snappy compression
	Snappy Codec = "snappy"
)

type CloseableResettableWriter interface {
	Close() error
	Reset(io.Writer)
}

/*
  Writer wraps an io.Writer and writes the file and block-level framing required for an OCF file
*/
type Writer struct {
	writer           io.Writer
	syncMarker       [16]byte
	codec            Codec
	recordsPerBlock  int64
	blockBuffer      *bytes.Buffer
	compressedWriter io.Writer
	nextBlockRecords int64
}

/*
  Create a new Writer wrapping the provided io.Writer with the given Codec and number of records per block.
  The Writer will lazily write the container file header when WriteRecord is called the first time.
  You must call Flush on the Writer before closing the underlying io.Writer, to ensure the final block is written.
  A schema string must be passed to ensure that a correct header is written even if no records are written. This
  is required to produce valid empty Avro container files.
*/
func NewWriter(writer io.Writer, codec Codec, recordsPerBlock int64, schema string) (*Writer, error) {
	blockBytes := make([]byte, 0)
	blockBuffer := bytes.NewBuffer(blockBytes)

	avroWriter := &Writer{
		writer:          writer,
		syncMarker:      [16]byte{'g', 'o', 'g', 'e', 'n', 'a', 'v', 'r', 'o', 'm', 'a', 'g', 'i', 'c', '1', '0'},
		codec:           codec,
		recordsPerBlock: recordsPerBlock,
		blockBuffer:     blockBuffer,
	}
	var err error
	if codec == Deflate {
		avroWriter.compressedWriter, err = flate.NewWriter(avroWriter.blockBuffer, flate.DefaultCompression)
		if err != nil {
			return nil, err
		}
	} else if codec == Snappy {
		avroWriter.compressedWriter = newSnappyWriter(avroWriter.blockBuffer)
	} else {
		avroWriter.compressedWriter = avroWriter.blockBuffer
	}

	err = avroWriter.writeHeader(schema)
	if err != nil {
		return nil, err
	}

	return avroWriter, nil
}

func (avroWriter *Writer) writeHeader(schema string) error {
	header := &avro.AvroContainerHeader{
		Magic: [4]byte{'O', 'b', 'j', 1},
		Meta: map[string][]byte{
			"avro.schema": []byte(schema),
			"avro.codec":  []byte(avroWriter.codec),
		},
		Sync: avroWriter.syncMarker,
	}
	return header.Serialize(avroWriter.writer)
}

/*
  Write an AvroRecord to the container file. All gogen-avro generated structs
  fulfill the AvroRecord interface. Note that all records in a given container file
  must be of the same Avro type.
*/
func (avroWriter *Writer) WriteRecord(record AvroRecord) error {
	var err error
	// Serialize the new record into the compressed writer
	err = record.Serialize(avroWriter.compressedWriter)
	if err != nil {
		return err
	}
	avroWriter.nextBlockRecords += 1

	// If the block if full, flush and reset the compressed writer,
	// write the header and the block contents
	if avroWriter.nextBlockRecords >= avroWriter.recordsPerBlock {
		return avroWriter.Flush()
	}

	return nil
}

/*
  Write the current block to the file if it has been filled.  It is
  best-practise to always call this before the underlying io.Writer is closed.
*/
func (avroWriter *Writer) Flush() error {
	if avroWriter.nextBlockRecords == 0 {
		return nil
	}

	// Write out all of the buffered records as a new block
	// Must be called before closing to ensure the last block is written
	if fwWriter, ok := avroWriter.compressedWriter.(CloseableResettableWriter); ok {
		fwWriter.Close()
		fwWriter.Reset(avroWriter.blockBuffer)
	}

	if avroWriter.nextBlockRecords > 0 {
		block := &avro.AvroContainerBlock{
			NumRecords:  avroWriter.nextBlockRecords,
			RecordBytes: avroWriter.blockBuffer.Bytes(),
			Sync:        avroWriter.syncMarker,
		}
		err := block.Serialize(avroWriter.writer)
		if err != nil {
			return err
		}
	}

	avroWriter.blockBuffer.Reset()
	avroWriter.nextBlockRecords = 0

	return nil
}
<|endoftext|>


<commit_before>// Output parsers
package main

import (
	"bufio"
	"fmt"
	"io"
	"regexp"
	"strings"
)

const (
	// gotest regular expressions

	// === RUN TestAdd
	gtStartRE = "^=== RUN:?[[:space:]]+([a-zA-Z_][^[:space:]]*)"

	// --- PASS: TestSub (0.00 seconds)
	// --- FAIL: TestSubFail (0.00 seconds)
	// --- SKIP: TestSubSkip (0.00 seconds)
	gtEndRE = "--- (PASS|FAIL|SKIP):[[:space:]]+([a-zA-Z_][^[:space:]]*) \\((\\d+(.\\d+)?)"

	// FAIL	_/home/miki/Projects/goroot/src/xunit	0.004s
	// ok  	_/home/miki/Projects/goroot/src/anotherTest	0.000s
	gtSuiteRE = "^(ok|FAIL)[ \t]+([^ \t]+)[ \t]+(\\d+.\\d+)"

	// ?       alipay  [no test files]
	gtNoFilesRE = "^\\?.*\\[no test files\\]$"
	// FAIL    node/config [build failed]
	gtBuildFailedRE = `^FAIL.*\[(build|setup) failed\]$`

	// gocheck regular expressions

	// START: mmath_test.go:16: MySuite.TestAdd
	gcStartRE = "START: [^:]+:[^:]+: ([A-Za-z_][[:word:]]*).([A-Za-z_][[:word:]]*)"

	// PASS: mmath_test.go:16: MySuite.TestAdd	0.000s
	// FAIL: mmath_test.go:35: MySuite.TestDiv
	gcEndRE = "(PASS|FAIL|SKIP|PANIC|MISS): [^:]+:[^:]+: ([A-Za-z_][[:word:]]*).([A-Za-z_][[:word:]]*)[[:space:]]?([0-9]+.[0-9]+)?"

	// FAIL	go2xunit/demo-gocheck	0.008s
	// ok  	go2xunit/demo-gocheck	0.008s
	gcSuiteRE = "^(ok|FAIL)[ \t]+([^ \t]+)[ \t]+(\\d+.\\d+)"
)

var (
	matchDatarace = regexp.MustCompile("^WARNING: DATA RACE$").MatchString
)

// LineScanner scans lines and keep track of line numbers
type LineScanner struct {
	*bufio.Scanner
	lnum int
}

// NewLineScanner creates a new line scanner from r
func NewLineScanner(r io.Reader) *LineScanner {
	scan := bufio.NewScanner(r)
	return &LineScanner{scan, 0}
}

// Scan advances to next line
func (ls *LineScanner) Scan() bool {
	val := ls.Scanner.Scan()
	ls.lnum++
	return val
}

// Line returns the current line number
func (ls *LineScanner) Line() int {
	return ls.lnum
}

// hasDatarace checks if there's a data race warning in the line
func hasDatarace(lines []string) bool {
	for _, line := range lines {
		if matchDatarace(line) {
			return true
		}
	}
	return false
}

// gc_Parse parses output of "go test -gocheck.vv", returns a list of tests
// See data/gocheck.out for an example
// TODO: Refactor to shorter ones
func gcParse(rd io.Reader) ([]*Suite, error) {
	findStart := regexp.MustCompile(gcStartRE).FindStringSubmatch
	findEnd := regexp.MustCompile(gcEndRE).FindStringSubmatch
	findSuite := regexp.MustCompile(gcSuiteRE).FindStringSubmatch

	scanner := NewLineScanner(rd)
	var suites = make([]*Suite, 0)
	var suiteName string
	var suite *Suite

	var testName string
	var out []string

	for scanner.Scan() {
		line := scanner.Text()

		tokens := findStart(line)
		if len(tokens) > 0 {
			if tokens[2] == "SetUpTest" || tokens[2] == "TearDownTest" {
				continue
			}
			if testName != "" {
				return nil, fmt.Errorf("%d: start in middle\n", scanner.Line())
			}
			suiteName = tokens[1]
			testName = tokens[2]
			out = []string{}
			continue
		}

		tokens = findEnd(line)
		if len(tokens) > 0 {
			if tokens[3] == "SetUpTest" || tokens[3] == "TearDownTest" {
				continue
			}
			if testName == "" {
				return nil, fmt.Errorf("%d: orphan end", scanner.Line())
			}
			if (tokens[2] != suiteName) || (tokens[3] != testName) {
				return nil, fmt.Errorf("%d: suite/name mismatch", scanner.Line())
			}
			test := &Test{Name: testName}
			test.Message = strings.Join(out, "\n")
			test.Time = tokens[4]
			test.Failed = (tokens[1] == "FAIL") || (tokens[1] == "PANIC")
			test.Passed = (tokens[1] == "PASS")
			test.Skipped = (tokens[1] == "SKIP" || tokens[1] == "MISS")

			if suite == nil || suite.Name != suiteName {
				suite = &Suite{Name: suiteName}
				suites = append(suites, suite)
			}
			suite.Tests = append(suite.Tests, test)

			testName = ""
			suiteName = ""
			out = []string{}

			continue
		}

		// last "suite" is test summary
		tokens = findSuite(line)
		if tokens != nil {
			if suite == nil {
				suite = &Suite{Name: tokens[2], Status: tokens[1], Time: tokens[3]}
				suites = append(suites, suite)
			} else {
				suite.Status = tokens[1]
				suite.Time = tokens[3]
			}

			testName = ""
			suiteName = ""
			out = []string{}

			continue
		}

		if testName != "" {
			out = append(out, line)
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return suites, nil
}

// gt_Parse parser output of gotest
// TODO: Make it shorter
func gtParse(rd io.Reader) ([]*Suite, error) {
	findStart := regexp.MustCompile(gtStartRE).FindStringSubmatch
	findEnd := regexp.MustCompile(gtEndRE).FindStringSubmatch
	findSuite := regexp.MustCompile(gtSuiteRE).FindStringSubmatch
	isNoFiles := regexp.MustCompile(gtNoFilesRE).MatchString
	isBuildFailed := regexp.MustCompile(gtBuildFailedRE).MatchString
	isExit := regexp.MustCompile("^exit status -?\\d+").MatchString

	suites := []*Suite{}
	var curTest *Test
	var curSuite *Suite
	var out []string
	suiteStack := SuiteStack{}
	// Handles a test that ended with a panic.
	handlePanic := func() {
		curTest.Failed = true
		curTest.Skipped = false
		curTest.Time = "N/A"
		curSuite.Tests = append(curSuite.Tests, curTest)
		curTest = nil
	}

	// Appends output to the last test.
	appendError := func() error {
		if len(out) > 0 && curSuite != nil && len(curSuite.Tests) > 0 {
			message := strings.Join(out, "\n")
			if curSuite.Tests[len(curSuite.Tests)-1].Message == "" {
				curSuite.Tests[len(curSuite.Tests)-1].Message = message
			} else {
				curSuite.Tests[len(curSuite.Tests)-1].Message += "\n" + message
			}
		}
		out = []string{}
		return nil
	}

	scanner := NewLineScanner(rd)
	for scanner.Scan() {
		line := scanner.Text()

		// TODO: Only outside a suite/test, report as empty suite?
		if isNoFiles(line) {
			continue
		}

		if isBuildFailed(line) {
			return nil, fmt.Errorf("%d: package build failed: %s", scanner.Line(), line)
		}

		if curSuite == nil {
			curSuite = &Suite{}
		}

		tokens := findStart(line)
		if tokens != nil {
			if curTest != nil {
				// This occurs when the last test ended with a panic.
				if suiteStack.count == 0 {
					suiteStack.Push(curSuite)
					curSuite = &Suite{Name: curTest.Name}
				} else {
					handlePanic()
				}
			}
			if e := appendError(); e != nil {
				return nil, e
			}
			curTest = &Test{
				Name: tokens[1],
			}
			continue
		}

		tokens = findEnd(line)
		if tokens != nil {
			if curTest == nil {
				if suiteStack.count > 0 {
					prevSuite := suiteStack.Pop()
					suites = append(suites, curSuite)
					curSuite = prevSuite
					continue
				} else {
					return nil, fmt.Errorf("%d: orphan end test", scanner.Line())
				}
			}
			if tokens[2] != curTest.Name {
				err := fmt.Errorf("%d: name mismatch (try disabling parallel mode)", scanner.Line())
				return nil, err
			}
			curTest.Failed = (tokens[1] == "FAIL") || (args.failOnRace && hasDatarace(out))
			curTest.Skipped = (tokens[1] == "SKIP")
			curTest.Passed = (tokens[1] == "PASS")
			curTest.Time = tokens[3]
			curTest.Message = strings.Join(out, "\n")
			curSuite.Tests = append(curSuite.Tests, curTest)
			curTest = nil
			out = []string{}
			continue
		}

		tokens = findSuite(line)
		if tokens != nil {
			if curTest != nil {
				// This occurs when the last test ended with a panic.
				handlePanic()
			}
			if e := appendError(); e != nil {
				return nil, e
			}
			curSuite.Name = tokens[2]
			curSuite.Time = tokens[3]
			suites = append(suites, curSuite)
			curSuite = nil
			continue
		}

		if isExit(line) || (line == "FAIL") || (line == "PASS") {
			continue
		}

		out = append(out, line)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return suites, nil
}
<commit_msg>right name in doc<commit_after>// Output parsers
package main

import (
	"bufio"
	"fmt"
	"io"
	"regexp"
	"strings"
)

const (
	// gotest regular expressions

	// === RUN TestAdd
	gtStartRE = "^=== RUN:?[[:space:]]+([a-zA-Z_][^[:space:]]*)"

	// --- PASS: TestSub (0.00 seconds)
	// --- FAIL: TestSubFail (0.00 seconds)
	// --- SKIP: TestSubSkip (0.00 seconds)
	gtEndRE = "--- (PASS|FAIL|SKIP):[[:space:]]+([a-zA-Z_][^[:space:]]*) \\((\\d+(.\\d+)?)"

	// FAIL	_/home/miki/Projects/goroot/src/xunit	0.004s
	// ok  	_/home/miki/Projects/goroot/src/anotherTest	0.000s
	gtSuiteRE = "^(ok|FAIL)[ \t]+([^ \t]+)[ \t]+(\\d+.\\d+)"

	// ?       alipay  [no test files]
	gtNoFilesRE = "^\\?.*\\[no test files\\]$"
	// FAIL    node/config [build failed]
	gtBuildFailedRE = `^FAIL.*\[(build|setup) failed\]$`

	// gocheck regular expressions

	// START: mmath_test.go:16: MySuite.TestAdd
	gcStartRE = "START: [^:]+:[^:]+: ([A-Za-z_][[:word:]]*).([A-Za-z_][[:word:]]*)"

	// PASS: mmath_test.go:16: MySuite.TestAdd	0.000s
	// FAIL: mmath_test.go:35: MySuite.TestDiv
	gcEndRE = "(PASS|FAIL|SKIP|PANIC|MISS): [^:]+:[^:]+: ([A-Za-z_][[:word:]]*).([A-Za-z_][[:word:]]*)[[:space:]]?([0-9]+.[0-9]+)?"

	// FAIL	go2xunit/demo-gocheck	0.008s
	// ok  	go2xunit/demo-gocheck	0.008s
	gcSuiteRE = "^(ok|FAIL)[ \t]+([^ \t]+)[ \t]+(\\d+.\\d+)"
)

var (
	matchDatarace = regexp.MustCompile("^WARNING: DATA RACE$").MatchString
)

// LineScanner scans lines and keep track of line numbers
type LineScanner struct {
	*bufio.Scanner
	lnum int
}

// NewLineScanner creates a new line scanner from r
func NewLineScanner(r io.Reader) *LineScanner {
	scan := bufio.NewScanner(r)
	return &LineScanner{scan, 0}
}

// Scan advances to next line
func (ls *LineScanner) Scan() bool {
	val := ls.Scanner.Scan()
	ls.lnum++
	return val
}

// Line returns the current line number
func (ls *LineScanner) Line() int {
	return ls.lnum
}

// hasDatarace checks if there's a data race warning in the line
func hasDatarace(lines []string) bool {
	for _, line := range lines {
		if matchDatarace(line) {
			return true
		}
	}
	return false
}

// gcParse parses output of "go test -gocheck.vv", returns a list of tests
// See data/gocheck.out for an example
// TODO: Refactor to shorter ones
func gcParse(rd io.Reader) ([]*Suite, error) {
	findStart := regexp.MustCompile(gcStartRE).FindStringSubmatch
	findEnd := regexp.MustCompile(gcEndRE).FindStringSubmatch
	findSuite := regexp.MustCompile(gcSuiteRE).FindStringSubmatch

	scanner := NewLineScanner(rd)
	var suites = make([]*Suite, 0)
	var suiteName string
	var suite *Suite

	var testName string
	var out []string

	for scanner.Scan() {
		line := scanner.Text()

		tokens := findStart(line)
		if len(tokens) > 0 {
			if tokens[2] == "SetUpTest" || tokens[2] == "TearDownTest" {
				continue
			}
			if testName != "" {
				return nil, fmt.Errorf("%d: start in middle\n", scanner.Line())
			}
			suiteName = tokens[1]
			testName = tokens[2]
			out = []string{}
			continue
		}

		tokens = findEnd(line)
		if len(tokens) > 0 {
			if tokens[3] == "SetUpTest" || tokens[3] == "TearDownTest" {
				continue
			}
			if testName == "" {
				return nil, fmt.Errorf("%d: orphan end", scanner.Line())
			}
			if (tokens[2] != suiteName) || (tokens[3] != testName) {
				return nil, fmt.Errorf("%d: suite/name mismatch", scanner.Line())
			}
			test := &Test{Name: testName}
			test.Message = strings.Join(out, "\n")
			test.Time = tokens[4]
			test.Failed = (tokens[1] == "FAIL") || (tokens[1] == "PANIC")
			test.Passed = (tokens[1] == "PASS")
			test.Skipped = (tokens[1] == "SKIP" || tokens[1] == "MISS")

			if suite == nil || suite.Name != suiteName {
				suite = &Suite{Name: suiteName}
				suites = append(suites, suite)
			}
			suite.Tests = append(suite.Tests, test)

			testName = ""
			suiteName = ""
			out = []string{}

			continue
		}

		// last "suite" is test summary
		tokens = findSuite(line)
		if tokens != nil {
			if suite == nil {
				suite = &Suite{Name: tokens[2], Status: tokens[1], Time: tokens[3]}
				suites = append(suites, suite)
			} else {
				suite.Status = tokens[1]
				suite.Time = tokens[3]
			}

			testName = ""
			suiteName = ""
			out = []string{}

			continue
		}

		if testName != "" {
			out = append(out, line)
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return suites, nil
}

// gtParse parser output of gotest
// TODO: Make it shorter
func gtParse(rd io.Reader) ([]*Suite, error) {
	findStart := regexp.MustCompile(gtStartRE).FindStringSubmatch
	findEnd := regexp.MustCompile(gtEndRE).FindStringSubmatch
	findSuite := regexp.MustCompile(gtSuiteRE).FindStringSubmatch
	isNoFiles := regexp.MustCompile(gtNoFilesRE).MatchString
	isBuildFailed := regexp.MustCompile(gtBuildFailedRE).MatchString
	isExit := regexp.MustCompile("^exit status -?\\d+").MatchString

	suites := []*Suite{}
	var curTest *Test
	var curSuite *Suite
	var out []string
	suiteStack := SuiteStack{}
	// Handles a test that ended with a panic.
	handlePanic := func() {
		curTest.Failed = true
		curTest.Skipped = false
		curTest.Time = "N/A"
		curSuite.Tests = append(curSuite.Tests, curTest)
		curTest = nil
	}

	// Appends output to the last test.
	appendError := func() error {
		if len(out) > 0 && curSuite != nil && len(curSuite.Tests) > 0 {
			message := strings.Join(out, "\n")
			if curSuite.Tests[len(curSuite.Tests)-1].Message == "" {
				curSuite.Tests[len(curSuite.Tests)-1].Message = message
			} else {
				curSuite.Tests[len(curSuite.Tests)-1].Message += "\n" + message
			}
		}
		out = []string{}
		return nil
	}

	scanner := NewLineScanner(rd)
	for scanner.Scan() {
		line := scanner.Text()

		// TODO: Only outside a suite/test, report as empty suite?
		if isNoFiles(line) {
			continue
		}

		if isBuildFailed(line) {
			return nil, fmt.Errorf("%d: package build failed: %s", scanner.Line(), line)
		}

		if curSuite == nil {
			curSuite = &Suite{}
		}

		tokens := findStart(line)
		if tokens != nil {
			if curTest != nil {
				// This occurs when the last test ended with a panic.
				if suiteStack.count == 0 {
					suiteStack.Push(curSuite)
					curSuite = &Suite{Name: curTest.Name}
				} else {
					handlePanic()
				}
			}
			if e := appendError(); e != nil {
				return nil, e
			}
			curTest = &Test{
				Name: tokens[1],
			}
			continue
		}

		tokens = findEnd(line)
		if tokens != nil {
			if curTest == nil {
				if suiteStack.count > 0 {
					prevSuite := suiteStack.Pop()
					suites = append(suites, curSuite)
					curSuite = prevSuite
					continue
				} else {
					return nil, fmt.Errorf("%d: orphan end test", scanner.Line())
				}
			}
			if tokens[2] != curTest.Name {
				err := fmt.Errorf("%d: name mismatch (try disabling parallel mode)", scanner.Line())
				return nil, err
			}
			curTest.Failed = (tokens[1] == "FAIL") || (args.failOnRace && hasDatarace(out))
			curTest.Skipped = (tokens[1] == "SKIP")
			curTest.Passed = (tokens[1] == "PASS")
			curTest.Time = tokens[3]
			curTest.Message = strings.Join(out, "\n")
			curSuite.Tests = append(curSuite.Tests, curTest)
			curTest = nil
			out = []string{}
			continue
		}

		tokens = findSuite(line)
		if tokens != nil {
			if curTest != nil {
				// This occurs when the last test ended with a panic.
				handlePanic()
			}
			if e := appendError(); e != nil {
				return nil, e
			}
			curSuite.Name = tokens[2]
			curSuite.Time = tokens[3]
			suites = append(suites, curSuite)
			curSuite = nil
			continue
		}

		if isExit(line) || (line == "FAIL") || (line == "PASS") {
			continue
		}

		out = append(out, line)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return suites, nil
}
<|endoftext|>


<commit_before>package rog

import (
	"fmt"
	"image/color"
)

type Console struct {
	bg, fg   [][]color.Color
	ch       [][]rune
	w, h     int
}

func NewConsole(width, height int) *Console {
	bg := make([][]color.Color, height)
	fg := make([][]color.Color, height)
	ch := make([][]rune, height)

	for y := 0; y < height; y++ {
		bg[y] = make([]color.Color, width)
		fg[y] = make([]color.Color, width)
		ch[y] = make([]rune, width)
	}

	con := &Console{bg, fg, ch, width, height}

	for x := 0; x < con.w; x++ {
		for y := 0; y < con.h; y++ {
			con.ch[y][x] = ' '
			con.fg[y][x] = color.White
			con.bg[y][x] = color.Black
		}
	}

	return con
}

func (con *Console) Fill(x, y, w, h int, ch rune, fg, bg interface{}) {
	for i := x; i < x+w; i++ {
		for j := y; j < y+h; j++ {
			con.Set(i, j, fg, bg, string(ch))
		}
	}
}

func (con *Console) put(x, y int, fg, bg interface{}, ch rune) {
	if ch > 0 {
		con.ch[y][x] = ch
	}

    switch bgcolor := bg.(type) {
    case color.Color:
		con.bg[y][x] = bgcolor
    case Blender:
	    con.bg[y][x] = bgcolor(con.bg[y][x])
    default:
    }

    switch fgcolor := fg.(type) {
    case color.Color:
		con.fg[y][x] = fgcolor
    case Blender:
		con.fg[y][x] = fgcolor(con.bg[y][x])
    default:
    }
}

func (con *Console) Set(x, y int, fg, bg interface{}, data string, rest ...interface{}) {
    runes := []rune(fmt.Sprintf(data, rest...))
    if len(runes) > 0 {
	    for i := 0; i < len(runes); i++ {
		    con.put(i+x, y, fg, bg, runes[i])
	    }
    } else {
		con.put(x, y, fg, bg, -1)
    }
}

func (con *Console) Width() int {
	return con.w
}

func (con *Console) Height() int {
	return con.h
}
<commit_msg>Added Console.SetR(), which forces wrapping inside a rect bounds.<commit_after>package rog

import (
	"fmt"
	"image/color"
)

type Console struct {
	bg, fg   [][]color.Color
	ch       [][]rune
	w, h     int
}

func NewConsole(width, height int) *Console {
	bg := make([][]color.Color, height)
	fg := make([][]color.Color, height)
	ch := make([][]rune, height)

	for y := 0; y < height; y++ {
		bg[y] = make([]color.Color, width)
		fg[y] = make([]color.Color, width)
		ch[y] = make([]rune, width)
	}

	con := &Console{bg, fg, ch, width, height}

	for x := 0; x < con.w; x++ {
		for y := 0; y < con.h; y++ {
			con.ch[y][x] = ' '
			con.fg[y][x] = color.White
			con.bg[y][x] = color.Black
		}
	}

	return con
}

func (con *Console) put(x, y int, fg, bg interface{}, ch rune) {
	if ch > 0 {
		con.ch[y][x] = ch
	}

    switch bgcolor := bg.(type) {
    case color.Color:
		con.bg[y][x] = bgcolor
    case Blender:
	    con.bg[y][x] = bgcolor(con.bg[y][x])
    default:
    }

    switch fgcolor := fg.(type) {
    case color.Color:
		con.fg[y][x] = fgcolor
    case Blender:
		con.fg[y][x] = fgcolor(con.bg[y][x])
    default:
    }
}

func (con *Console) set(i, j, x, y, w, h int, fg, bg interface{}, data string, rest ...interface{}) {
    runes := []rune(fmt.Sprintf(data, rest...))
    if len(runes) > 0 {
        if h == 0 {
            h = con.h - y
        }
	    for k := 0; k < len(runes); k++ {
            if i == x+w {
                j += 1
                i = x
            }
            if j == y + h {
                break
            }
		    con.put(i, j, fg, bg, runes[k])
            i += 1
	    }
    } else {
		con.put(i, j, fg, bg, -1)
    }
}

func (con *Console) Set(x, y int, fg, bg interface{}, data string, rest ...interface{}) {
    con.set(x, y, 0, 0, con.w, con.h, fg, bg, data, rest...)
}

func (con *Console) SetR(x, y, w, h int, fg, bg interface{}, data string, rest ...interface{}) {
    con.set(x, y, x, y, w, h, fg, bg, data, rest...)
}

func (con *Console) Fill(x, y, w, h int, ch rune, fg, bg interface{}) {
	for i := x; i < x+w; i++ {
		for j := y; j < y+h; j++ {
			con.Set(i, j, fg, bg, string(ch))
		}
	}
}

func (con *Console) Width() int {
	return con.w
}

func (con *Console) Height() int {
	return con.h
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"github.com/Mirantis/virtlet/tests/e2e/framework"
	. "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
)

var _ = Describe("Block PVs", func() {
	var (
		vm  *framework.VMInterface
		ssh framework.Executor
	)

	Context("[Local]", func() {
		var (
			virtletNodeName string
			devPath         string
		)

		withLoopbackBlockDevice(&virtletNodeName, &devPath)

		AfterEach(func() {
			if ssh != nil {
				ssh.Close()
			}
			if vm != nil {
				deleteVM(vm)
			}
		})

		It("Should be accessible from within the VM", func() {
			vm = makeVMWithMountAndSymlinkScript(virtletNodeName, []framework.PVCSpec{
				{
					Name:          "block-pv",
					Size:          "10M",
					NodeName:      virtletNodeName,
					Block:         true,
					LocalPath:     devPath,
					ContainerPath: "/dev/testpvc",
				},
			}, nil)
			ssh = waitSSH(vm)
			expectToBeUsableForFilesystem(ssh, "/dev/testpvc")
		})
	})

	Context("[Ceph RBD]", func() {
		var monitorIP string
		withCeph(&monitorIP, nil, "ceph-admin")

		AfterEach(func() {
			if ssh != nil {
				ssh.Close()
			}
			if vm != nil {
				deleteVM(vm)
			}
		})

		It("Should be accessible from within the VM", func() {
			vm = makeVMWithMountAndSymlinkScript("", []framework.PVCSpec{
				{
					Name:           "block-pv",
					Size:           "10M",
					Block:          true,
					CephRBDImage:   "rbd-test-image1",
					CephMonitorIP:  monitorIP,
					CephRBDPool:    "libvirt-pool",
					CephSecretName: "ceph-admin",
					ContainerPath:  "/dev/testpvc",
				},
			}, nil)
			ssh = waitSSH(vm)
			expectToBeUsableForFilesystem(ssh, "/dev/testpvc")
		})
	})
})
<commit_msg>Don't run Ceph RBD test on CircleCI for now<commit_after>/*
Copyright 2018 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"github.com/Mirantis/virtlet/tests/e2e/framework"
	. "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
)

var _ = Describe("Block PVs", func() {
	var (
		vm  *framework.VMInterface
		ssh framework.Executor
	)

	Context("[Local]", func() {
		var (
			virtletNodeName string
			devPath         string
		)

		withLoopbackBlockDevice(&virtletNodeName, &devPath)

		AfterEach(func() {
			if ssh != nil {
				ssh.Close()
			}
			if vm != nil {
				deleteVM(vm)
			}
		})

		It("Should be accessible from within the VM", func() {
			vm = makeVMWithMountAndSymlinkScript(virtletNodeName, []framework.PVCSpec{
				{
					Name:          "block-pv",
					Size:          "10M",
					NodeName:      virtletNodeName,
					Block:         true,
					LocalPath:     devPath,
					ContainerPath: "/dev/testpvc",
				},
			}, nil)
			ssh = waitSSH(vm)
			expectToBeUsableForFilesystem(ssh, "/dev/testpvc")
		})
	})

	Context("[Ceph RBD]", func() {
		var monitorIP string
		withCeph(&monitorIP, nil, "ceph-admin")

		AfterEach(func() {
			if ssh != nil {
				ssh.Close()
			}
			if vm != nil {
				deleteVM(vm)
			}
		})

		// FIXME: the test is marked Disruptive because rbd
		// hangs on CircleCI for some reason.
		It("[Disruptive] Should be accessible from within the VM", func() {
			vm = makeVMWithMountAndSymlinkScript("", []framework.PVCSpec{
				{
					Name:           "block-pv",
					Size:           "10M",
					Block:          true,
					CephRBDImage:   "rbd-test-image1",
					CephMonitorIP:  monitorIP,
					CephRBDPool:    "libvirt-pool",
					CephSecretName: "ceph-admin",
					ContainerPath:  "/dev/testpvc",
				},
			}, nil)
			ssh = waitSSH(vm)
			expectToBeUsableForFilesystem(ssh, "/dev/testpvc")
		})
	})
})
<|endoftext|>


<commit_before>package godruid

type Filter struct {
	Type      string      `json:"type"`
	Dimension string      `json:"dimension,omitempty"`
	Value     interface{} `json:"value,omitempty"`
	Pattern   string      `json:"pattern,omitempty"`
	Function  string      `json:"function,omitempty"`
	Field     *Filter     `json:"field,omitempty"`
	Fields    []*Filter   `json:"fields,omitempty"`
}

func FilterSelector(dimension string, value interface{}) *Filter {
	return &Filter{
		Type:      "selector",
		Dimension: dimension,
		Value:     value,
	}
}

func FilterRegex(dimension, pattern string) *Filter {
	return &Filter{
		Type:      "regex",
		Dimension: dimension,
		Pattern:   pattern,
	}
}

func FilterJavaScript(dimension, function string) *Filter {
	return &Filter{
		Type:      "javascript",
		Dimension: dimension,
		Function:  function,
	}
}

func FilterAnd(filters ...*Filter) *Filter {
	return joinFilters(filters, "and")
}

func FilterOr(filters ...*Filter) *Filter {
	return joinFilters(filters, "or")
}

func FilterNot(filter *Filter) *Filter {
	return &Filter{
		Type:  "not",
		Field: filter,
	}
}

func joinFilters(filters []*Filter, connector string) *Filter {
	// Remove null filters.
	p := 0
	for _, f := range filters {
		if f != nil {
			filters[p] = f
			p++
		}
	}
	filters = filters[0:p]

	fLen := len(filters)
	if fLen == 0 {
		return nil
	}
	if fLen == 1 {
		return filters[0]
	}

	return &Filter{
		Type:   connector,
		Fields: filters,
	}
}
<commit_msg>add support for in filter<commit_after>package godruid

type Filter struct {
	Type      string      `json:"type"`
	Dimension string      `json:"dimension,omitempty"`
	Value     interface{} `json:"value,omitempty"`
	Values  []interface{} `json:"values,omitempty"`
	Pattern   string      `json:"pattern,omitempty"`
	Function  string      `json:"function,omitempty"`
	Field     *Filter     `json:"field,omitempty"`
	Fields    []*Filter   `json:"fields,omitempty"`
}

func FilterSelector(dimension string, value interface{}) *Filter {
	return &Filter{
		Type:      "selector",
		Dimension: dimension,
		Value:     value,
	}
}

func FilterRegex(dimension, pattern string) *Filter {
	return &Filter{
		Type:      "regex",
		Dimension: dimension,
		Pattern:   pattern,
	}
}

func FilterInClause(dimension, values []interface{}) *Filter {
	return &Filter{
		Type:      "in",
		Dimension: dimension,
		Values:    values,
	}
}

func FilterJavaScript(dimension, function string) *Filter {
	return &Filter{
		Type:      "javascript",
		Dimension: dimension,
		Function:  function,
	}
}

func FilterAnd(filters ...*Filter) *Filter {
	return joinFilters(filters, "and")
}

func FilterOr(filters ...*Filter) *Filter {
	return joinFilters(filters, "or")
}

func FilterNot(filter *Filter) *Filter {
	return &Filter{
		Type:  "not",
		Field: filter,
	}
}

func joinFilters(filters []*Filter, connector string) *Filter {
	// Remove null filters.
	p := 0
	for _, f := range filters {
		if f != nil {
			filters[p] = f
			p++
		}
	}
	filters = filters[0:p]

	fLen := len(filters)
	if fLen == 0 {
		return nil
	}
	if fLen == 1 {
		return filters[0]
	}

	return &Filter{
		Type:   connector,
		Fields: filters,
	}
}
<|endoftext|>


<commit_before>package pdb

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/BurntSushi/bcbgo/matt"
)

// AminoThreeToOne is a map from three letter amino acids to their
// corresponding single letter representation.
var AminoThreeToOne = map[string]byte{
	"ALA": 'A', "ARG": 'R', "ASN": 'N', "ASP": 'D', "CYS": 'C',
	"GLU": 'E', "GLN": 'Q', "GLY": 'G', "HIS": 'H', "ILE": 'I',
	"LEU": 'L', "LYS": 'K', "MET": 'M', "PHE": 'F', "PRO": 'P',
	"SER": 'S', "THR": 'T', "TRP": 'W', "TYR": 'Y', "VAL": 'V',
	"SEC": 'U', "PYL": 'O',
}

// AminoOneToThree is the reverse of AminoThreeToOne. It is created in
// this packages 'init' function.
var AminoOneToThree = map[byte]string{}

func init() {
	// Create a reverse map of AminoThreeToOne.
	for k, v := range AminoThreeToOne {
		AminoOneToThree[v] = k
	}
}

// Entry represents all information known about a particular PDB file (that
// has been implemented in this package).
//
// Currently, a PDB entry is simply a file path and a map of protein chains.
type Entry struct {
	Path   string
	Chains map[byte]*Chain
}

// New creates a new PDB Entry from a file. If the file cannot be read, or there
// is an error parsing the PDB file, an error is returned.
//
// If the file name ends with ".gz", gzip decompression will be used.
func New(fileName string) (*Entry, error) {
	var reader io.Reader
	var err error

	reader, err = os.Open(fileName)
	if err != nil {
		return nil, err
	}

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fileName) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	entry := &Entry{
		Path:   fileName,
		Chains: make(map[byte]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	breader := bufio.NewReaderSize(reader, 1000)
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		// The record name is always in the fix six columns.
		switch strings.TrimSpace(string(line[0:6])) {
		case "SEQRES":
			entry.parseSeqres(line)
		case "ATOM":
			entry.parseAtom(line)
		}
	}

	return entry, nil
}

// PDBArg is a convenience method for creating a PDBArg that can be used in
// the 'matt' package. It sets 'Location' in PDBArg to 'Path' from 'Entry'.
func (e *Entry) PDBArg() matt.PDBArg {
	return matt.PDBArg{Location: e.Path}
}

// PDBArgChain is a convenience method for creating a PDBArg for a specific
// chain that can be used in the 'matt' package.
//
// PDBArgChain panics if 'chainIdent' is not in the chain map for this Entry.
func (e *Entry) PDBArgChain(chainIdent byte) matt.PDBArg {
	if _, ok := e.Chains[chainIdent]; !ok {
		panic(fmt.Sprintf("The chain identifier '%c' was not found in the "+
			"chain map for the '%s' PDB entry.", chainIdent, e.Path))
	}
	return matt.PDBArg{
		Location: e.Path,
		Chain:    chainIdent,
	}
}

// String returns a sort list of all chains, their residue start/stop indics,
// and the amino acid sequence.
func (e *Entry) String() string {
	lines := make([]string, 0)
	for _, chain := range e.Chains {
		lines = append(lines, chain.String())
	}
	sort.Sort(sort.StringSlice(lines))
	return strings.Join(lines, "\n")
}

// getOrMakeChain looks for a chain in the 'Chains' map corresponding to the
// chain indentifier. If one exists, it is returned. If one doesn't exist,
// it is created, memory is allocated and it is returned.
func (e *Entry) getOrMakeChain(ident byte) *Chain {
	if chain, ok := e.Chains[ident]; ok {
		return chain
	}
	e.Chains[ident] = &Chain{
		Ident:            ident,
		Sequence:         make([]byte, 0, 10),
		AtomResidueStart: 0,
		AtomResidueEnd:   0,
	}
	return e.Chains[ident]
}

// parseSeqres loads all pertinent information from SEQRES records in a PDB
// file. In particular, amino acid resides are read and added to the chain's
// "Sequence" field. If a residue isn't a valid amino acid, it is simply
// ignored.
//
// N.B. This assumes that the SEQRES records are in order in the PDB file.
func (e *Entry) parseSeqres(line []byte) {
	chain := e.getOrMakeChain(line[11])

	// Residues are in columns 19-21, 23-25, 27-29, ..., 67-69
	for i := 19; i <= 67; i += 4 {
		end := i + 3

		// If we're passed the end of this line, quit.
		if end >= len(line) {
			break
		}

		// Get the residue. If it's not in our sequence map, skip it.
		residue := strings.TrimSpace(string(line[i:end]))
		if single, ok := AminoThreeToOne[residue]; ok {
			chain.Sequence = append(chain.Sequence, single)
		}
	}
}

// parseAtom loads all pertinent information from ATOM records in a PDB file.
// Currently, this only includes deducing the amino acid residue start and
// stop indices. (Note that the length of the range is not necessarily
// equivalent to the length of the amino acid sequence found in the SEQRES
// records.)
//
// ATOM records without a valid amino acid residue in columns 18-20 are ignored.
func (e *Entry) parseAtom(line []byte) {
	chain := e.getOrMakeChain(line[21])

	// An ATOM record is only processed if it corresponds to an amino acid
	// residue. (Which is in columns 17-19.)
	residue := strings.TrimSpace(string(line[17:20]))
	if _, ok := AminoThreeToOne[residue]; !ok {
		// Sanity check. I'm pretty sure that only amino acids have three
		// letter abbreviations.
		if len(residue) == 3 {
			panic(fmt.Sprintf("The residue '%s' found in PDB file '%s' has "+
				"length 3, but is not in my amino acid map.",
				residue, e.Path))
		}
		return
	}

	// The residue sequence number is in columns 22-25. Grab it, trim it,
	// and look for an integer.
	snum := strings.TrimSpace(string(line[22:26]))
	if num, err := strconv.ParseInt(snum, 10, 32); err == nil {
		inum := int(num)
		switch {
		case chain.AtomResidueStart == 0 || inum < chain.AtomResidueStart:
			chain.AtomResidueStart = inum
		case chain.AtomResidueEnd == 0 || inum > chain.AtomResidueEnd:
			chain.AtomResidueEnd = inum
		}
	}
}

// Chain represents a protein chain or subunit in a PDB file. Each chain has
// its own identifier, amino acid sequence (if its a protein sequence), and
// the start and stop residue indices of the ATOM coordinates.
type Chain struct {
	Ident                            byte
	Sequence                         []byte
	AtomResidueStart, AtomResidueEnd int
}

// String returns a FASTA-like formatted string of this chain and all of its
// related information.
func (c *Chain) String() string {
	return fmt.Sprintf("> Chain %c (%d, %d) :: length %d\n%s",
		c.Ident, c.AtomResidueStart, c.AtomResidueEnd,
		len(c.Sequence), string(c.Sequence))
}
<commit_msg>Typos.<commit_after>package pdb

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/BurntSushi/bcbgo/matt"
)

// AminoThreeToOne is a map from three letter amino acids to their
// corresponding single letter representation.
var AminoThreeToOne = map[string]byte{
	"ALA": 'A', "ARG": 'R', "ASN": 'N', "ASP": 'D', "CYS": 'C',
	"GLU": 'E', "GLN": 'Q', "GLY": 'G', "HIS": 'H', "ILE": 'I',
	"LEU": 'L', "LYS": 'K', "MET": 'M', "PHE": 'F', "PRO": 'P',
	"SER": 'S', "THR": 'T', "TRP": 'W', "TYR": 'Y', "VAL": 'V',
	"SEC": 'U', "PYL": 'O',
}

// AminoOneToThree is the reverse of AminoThreeToOne. It is created in
// this packages 'init' function.
var AminoOneToThree = map[byte]string{}

func init() {
	// Create a reverse map of AminoThreeToOne.
	for k, v := range AminoThreeToOne {
		AminoOneToThree[v] = k
	}
}

// Entry represents all information known about a particular PDB file (that
// has been implemented in this package).
//
// Currently, a PDB entry is simply a file path and a map of protein chains.
type Entry struct {
	Path   string
	Chains map[byte]*Chain
}

// New creates a new PDB Entry from a file. If the file cannot be read, or there
// is an error parsing the PDB file, an error is returned.
//
// If the file name ends with ".gz", gzip decompression will be used.
func New(fileName string) (*Entry, error) {
	var reader io.Reader
	var err error

	reader, err = os.Open(fileName)
	if err != nil {
		return nil, err
	}

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fileName) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	entry := &Entry{
		Path:   fileName,
		Chains: make(map[byte]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	breader := bufio.NewReaderSize(reader, 1000)
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		// The record name is always in the fix six columns.
		switch strings.TrimSpace(string(line[0:6])) {
		case "SEQRES":
			entry.parseSeqres(line)
		case "ATOM":
			entry.parseAtom(line)
		}
	}

	return entry, nil
}

// PDBArg is a convenience method for creating a PDBArg that can be used in
// the 'matt' package. It sets 'Location' in PDBArg to 'Path' from 'Entry'.
func (e *Entry) PDBArg() matt.PDBArg {
	return matt.PDBArg{Location: e.Path}
}

// PDBArgChain is a convenience method for creating a PDBArg for a specific
// chain that can be used in the 'matt' package.
//
// PDBArgChain panics if 'chainIdent' is not in the chain map for this Entry.
func (e *Entry) PDBArgChain(chainIdent byte) matt.PDBArg {
	if _, ok := e.Chains[chainIdent]; !ok {
		panic(fmt.Sprintf("The chain identifier '%c' was not found in the "+
			"chain map for the '%s' PDB entry.", chainIdent, e.Path))
	}
	return matt.PDBArg{
		Location: e.Path,
		Chain:    chainIdent,
	}
}

// String returns a sorted list of all chains, their residue start/stop indices,
// and the amino acid sequence.
func (e *Entry) String() string {
	lines := make([]string, 0)
	for _, chain := range e.Chains {
		lines = append(lines, chain.String())
	}
	sort.Sort(sort.StringSlice(lines))
	return strings.Join(lines, "\n")
}

// getOrMakeChain looks for a chain in the 'Chains' map corresponding to the
// chain indentifier. If one exists, it is returned. If one doesn't exist,
// it is created, memory is allocated and it is returned.
func (e *Entry) getOrMakeChain(ident byte) *Chain {
	if chain, ok := e.Chains[ident]; ok {
		return chain
	}
	e.Chains[ident] = &Chain{
		Ident:            ident,
		Sequence:         make([]byte, 0, 10),
		AtomResidueStart: 0,
		AtomResidueEnd:   0,
	}
	return e.Chains[ident]
}

// parseSeqres loads all pertinent information from SEQRES records in a PDB
// file. In particular, amino acid resides are read and added to the chain's
// "Sequence" field. If a residue isn't a valid amino acid, it is simply
// ignored.
//
// N.B. This assumes that the SEQRES records are in order in the PDB file.
func (e *Entry) parseSeqres(line []byte) {
	chain := e.getOrMakeChain(line[11])

	// Residues are in columns 19-21, 23-25, 27-29, ..., 67-69
	for i := 19; i <= 67; i += 4 {
		end := i + 3

		// If we're passed the end of this line, quit.
		if end >= len(line) {
			break
		}

		// Get the residue. If it's not in our sequence map, skip it.
		residue := strings.TrimSpace(string(line[i:end]))
		if single, ok := AminoThreeToOne[residue]; ok {
			chain.Sequence = append(chain.Sequence, single)
		}
	}
}

// parseAtom loads all pertinent information from ATOM records in a PDB file.
// Currently, this only includes deducing the amino acid residue start and
// stop indices. (Note that the length of the range is not necessarily
// equivalent to the length of the amino acid sequence found in the SEQRES
// records.)
//
// ATOM records without a valid amino acid residue in columns 18-20 are ignored.
func (e *Entry) parseAtom(line []byte) {
	chain := e.getOrMakeChain(line[21])

	// An ATOM record is only processed if it corresponds to an amino acid
	// residue. (Which is in columns 17-19.)
	residue := strings.TrimSpace(string(line[17:20]))
	if _, ok := AminoThreeToOne[residue]; !ok {
		// Sanity check. I'm pretty sure that only amino acids have three
		// letter abbreviations.
		if len(residue) == 3 {
			panic(fmt.Sprintf("The residue '%s' found in PDB file '%s' has "+
				"length 3, but is not in my amino acid map.",
				residue, e.Path))
		}
		return
	}

	// The residue sequence number is in columns 22-25. Grab it, trim it,
	// and look for an integer.
	snum := strings.TrimSpace(string(line[22:26]))
	if num, err := strconv.ParseInt(snum, 10, 32); err == nil {
		inum := int(num)
		switch {
		case chain.AtomResidueStart == 0 || inum < chain.AtomResidueStart:
			chain.AtomResidueStart = inum
		case chain.AtomResidueEnd == 0 || inum > chain.AtomResidueEnd:
			chain.AtomResidueEnd = inum
		}
	}
}

// Chain represents a protein chain or subunit in a PDB file. Each chain has
// its own identifier, amino acid sequence (if its a protein sequence), and
// the start and stop residue indices of the ATOM coordinates.
type Chain struct {
	Ident                            byte
	Sequence                         []byte
	AtomResidueStart, AtomResidueEnd int
}

// String returns a FASTA-like formatted string of this chain and all of its
// related information.
func (c *Chain) String() string {
	return fmt.Sprintf("> Chain %c (%d, %d) :: length %d\n%s",
		c.Ident, c.AtomResidueStart, c.AtomResidueEnd,
		len(c.Sequence), string(c.Sequence))
}
<|endoftext|>


<commit_before>package winston

import (
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"sir"
	"strings"
	"unicode"
)

type indexData map[string][]*document

type index struct {
	data indexData
}

func (i *index) update(w *document) {
	for _, gram := range w.grams {
		if i.data[gram] == nil {
			i.data[gram] = make([]*document, 0)
		}

		i.data[gram] = append(i.data[gram], w)
	}
}

func Query(query string) []string {
	results := make([]string, 0)

	for key, values := range theindex.data {
		if key == query {
			for _, d := range values {
				results = append(results, d.location)
			}
		}
	}

	return results
}

func IndexDataLen() int {
  return len(theindex.data)
}

func Add(website string) {
	var d document
	d.location = website
	d.fetchUrl(website)
	d.CalcGrams()
	documents = append(documents, d)

	theindex.update(&d)
}

type document struct {
	location  string
	text      string
	safeText  string
	sentences []int
	grams     []string
	freq      map[string]int
}

func (d1 *document) CommonFreqKeys(d2 *document) []string {
	common := make([]string, 0)

	for key, _ := range d1.freq {
		if d2.freq[key] != 0 {
			common = append(common, key)
		}
	}

	return common
}

func (w *document) FreqSum() (sum int) {
	for _, count := range w.freq {
		sum += count
	}

	return
}

func (w *document) FreqSquare() (sum float64) {
	for _, count := range w.freq {
		sum += math.Pow(float64(count), 2)
	}

	return
}

func (w1 *document) FreqProduct(w2 *document) (sum int) {
	for _, key := range w1.CommonFreqKeys(w2) {
		sum += w1.freq[key] * w2.freq[key]
	}

	return
}

func (w1 *document) Pearson(w2 *document) float64 {
	sum1 := float64(w1.FreqSum())
	sum2 := float64(w2.FreqSum())
	sumsq1 := w1.FreqSquare()
	sumsq2 := w2.FreqSquare()
	sump := float64(w1.FreqProduct(w2))
	n := float64(len(w1.freq))

	num := sump - ((sum1 * sum2) / n)
	den := math.Sqrt((sumsq1 - (math.Pow(sum1, 2))/n) * (sumsq2 - (math.Pow(sum2, 2))/n))

	if den == 0 {
		return 0
	}

	return num / den
}

func (w *document) CleanText() {
	asciiregexp, err := regexp.Compile("[^A-Za-z ]+")
	sir.CheckError(err)

	tagregexp, err := regexp.Compile("<[^>]+>")
	sir.CheckError(err)

	spaceregexp, err := regexp.Compile("[ ]+")
	sir.CheckError(err)

	w.safeText = tagregexp.ReplaceAllString(w.text, " ")
	w.safeText = asciiregexp.ReplaceAllString(w.safeText, " ")
	w.safeText = spaceregexp.ReplaceAllString(w.safeText, " ")
	w.safeText = strings.Trim(w.safeText, "")
	w.safeText = strings.ToLower(w.safeText)
	w.safeText = strings.TrimSpace(w.safeText)
}

func (w *document) MarkSentenceBoundaries() {
	w.sentences = make([]int, 0)

	for index, r := range w.text {
		if !unicode.IsLetter(r) && r == 46 {
			w.sentences = append(w.sentences, index)
		}
	}
}

func (w *document) FetchSentences() {
	for i := 0; i < (len(w.sentences) - 1); i++ {
		fmt.Println(i, w.text[w.sentences[i]:w.sentences[i+1]])
	}
}

func (d *document) CalcGrams() {
	d.CleanText()

	d.MarkSentenceBoundaries()

	d.grams = strings.Split(d.safeText, ` `)
	d.freq = make(map[string]int)

	for _, gram := range d.grams {
		d.freq[gram] += 1
	}
}

func (w *document) fetchUrl(theurl string) {
	var client *http.Client

	if proxy := os.Getenv("http_proxy"); proxy != `` {
		proxyUrl, err := url.Parse(proxy)
		sir.CheckError(err)

		client = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
	} else {
		client = &http.Client{}
	}

	req, err := http.NewRequest(`GET`, theurl, nil)
	sir.CheckError(err)

	resp, err := client.Do(req)
	sir.CheckError(err)

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	sir.CheckError(err)

	w.text = string(body)
}

var documents []document
var theindex index

func init() {
	documents = make([]document, 0)
	theindex.data = make(map[string][]*document)
}
<commit_msg>timeout<commit_after>package winston

import (
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"sir"
	"strings"
	"time"
	"unicode"
)

type indexData map[string][]*document

type index struct {
	data indexData
}

func (i *index) update(w *document) {
	for _, gram := range w.grams {
		if i.data[gram] == nil {
			i.data[gram] = make([]*document, 0)
		}

		i.data[gram] = append(i.data[gram], w)
	}
}

func Query(query string) []string {
	results := make([]string, 0)

	for key, values := range theindex.data {
		if key == query {
			for _, d := range values {
				results = append(results, d.location)
			}
		}
	}

	return results
}

func IndexDataLen() int {
	return len(theindex.data)
}

func Add(website string) {
	var d document
	d.location = website
	d.fetchUrl(website)
	d.CalcGrams()
	documents = append(documents, d)

	theindex.update(&d)
}

type document struct {
	location  string
	text      string
	safeText  string
	sentences []int
	grams     []string
	freq      map[string]int
}

func (d1 *document) CommonFreqKeys(d2 *document) []string {
	common := make([]string, 0)

	for key, _ := range d1.freq {
		if d2.freq[key] != 0 {
			common = append(common, key)
		}
	}

	return common
}

func (w *document) FreqSum() (sum int) {
	for _, count := range w.freq {
		sum += count
	}

	return
}

func (w *document) FreqSquare() (sum float64) {
	for _, count := range w.freq {
		sum += math.Pow(float64(count), 2)
	}

	return
}

func (w1 *document) FreqProduct(w2 *document) (sum int) {
	for _, key := range w1.CommonFreqKeys(w2) {
		sum += w1.freq[key] * w2.freq[key]
	}

	return
}

func (w1 *document) Pearson(w2 *document) float64 {
	sum1 := float64(w1.FreqSum())
	sum2 := float64(w2.FreqSum())
	sumsq1 := w1.FreqSquare()
	sumsq2 := w2.FreqSquare()
	sump := float64(w1.FreqProduct(w2))
	n := float64(len(w1.freq))

	num := sump - ((sum1 * sum2) / n)
	den := math.Sqrt((sumsq1 - (math.Pow(sum1, 2))/n) * (sumsq2 - (math.Pow(sum2, 2))/n))

	if den == 0 {
		return 0
	}

	return num / den
}

func (w *document) CleanText() {
	asciiregexp, err := regexp.Compile("[^A-Za-z ]+")
	sir.CheckError(err)

	tagregexp, err := regexp.Compile("<[^>]+>")
	sir.CheckError(err)

	spaceregexp, err := regexp.Compile("[ ]+")
	sir.CheckError(err)

	w.safeText = tagregexp.ReplaceAllString(w.text, " ")
	w.safeText = asciiregexp.ReplaceAllString(w.safeText, " ")
	w.safeText = spaceregexp.ReplaceAllString(w.safeText, " ")
	w.safeText = strings.Trim(w.safeText, "")
	w.safeText = strings.ToLower(w.safeText)
	w.safeText = strings.TrimSpace(w.safeText)
}

func (w *document) MarkSentenceBoundaries() {
	w.sentences = make([]int, 0)

	for index, r := range w.text {
		if !unicode.IsLetter(r) && r == 46 {
			w.sentences = append(w.sentences, index)
		}
	}
}

func (w *document) FetchSentences() {
	for i := 0; i < (len(w.sentences) - 1); i++ {
		fmt.Println(i, w.text[w.sentences[i]:w.sentences[i+1]])
	}
}

func (d *document) CalcGrams() {
	d.CleanText()

	d.MarkSentenceBoundaries()

	d.grams = strings.Split(d.safeText, ` `)
	d.freq = make(map[string]int)

	for _, gram := range d.grams {
		d.freq[gram] += 1
	}
}

type Config struct {
	ConnectTimeout   time.Duration
	ReadWriteTimeout time.Duration
}

func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, cTimeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(time.Now().Add(rwTimeout))
		return conn, nil
	}
}

func (w *document) fetchUrl(theurl string) {
	var client *http.Client

	if proxy := os.Getenv("http_proxy"); proxy != `` {
		proxyUrl, err := url.Parse(proxy)
		sir.CheckError(err)

		transport := http.Transport{
			// Dial: TimeoutDialer(connectTimeout, readWriteTimeout),
			Dial:  TimeoutDialer(1*time.Second, 1*time.Second),
			Proxy: http.ProxyURL(proxyUrl),
		}

		client = &http.Client{Transport: &transport}
	} else {
		client = &http.Client{}
	}

	req, err := http.NewRequest(`GET`, theurl, nil)
	sir.CheckError(err)

	resp, err := client.Do(req)
	sir.CheckError(err)

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	sir.CheckError(err)

	w.text = string(body)
}

var documents []document
var theindex index

func init() {
	documents = make([]document, 0)
	theindex.data = make(map[string][]*document)
}
<|endoftext|>


<commit_before>package workman

import "sync"

// Worker Definition
type Worker struct {
	tasks   chan Task
	wg      sync.WaitGroup
	fn      func(Task)
	spawned bool
}

// DefineWorker defines a new worker group
func DefineWorker(fn func(Task)) *Worker {
	return &Worker{
		tasks: make(chan Task),
		fn:    fn,
	}
}

// Spawn some workers
func (w *Worker) Spawn(count int) *Worker {
	w.wg.Add(count)
	for i := 0; i < count; i++ {
		go func(worker int) {
			task, ok := <-w.tasks
			if !ok {
				w.wg.Done()
				return
			}
			w.fn(task)
		}(i)
	}
	w.spawned = true
	return w
}

// AddTask to the worker queue
func (w *Worker) AddTask(task Task) *Worker {
	if !w.spawned {
		panic("workers must be spawned before adding tasks!")
	}
	w.tasks <- task
	return w
}

// CloseTasks tells us that we are donw with our tasks
func (w *Worker) CloseTasks() *Worker {
	close(w.tasks)
	return w
}

// Finish waits for all the workers to finish
func (w *Worker) Finish() {
	w.CloseTasks().Wait()
}

// Wait for the workers to finish
func (w *Worker) Wait() *Worker {
	w.wg.Wait()
	return w
}
<commit_msg>updates<commit_after>package workman

import "sync"

// Worker Definition
type Worker struct {
	tasks   chan Task
	wg      sync.WaitGroup
	fn      func(Task)
	spawned bool
}

// DefineWorker defines a new worker group
func DefineWorker(fn func(Task)) *Worker {
	return &Worker{
		tasks: make(chan Task),
		fn:    fn,
	}
}

// Spawn some workers
func (w *Worker) Spawn(count int) *Worker {
	w.wg.Add(count)
	for i := 0; i < count; i++ {
		go func(worker int) {
			for task := range w.tasks {
				w.fn(task)
			}
			w.wg.Done()
		}(i)
	}
	w.spawned = true
	return w
}

// AddTask to the worker queue
func (w *Worker) AddTask(task Task) *Worker {
	if !w.spawned {
		panic("workers must be spawned before adding tasks!")
	}
	w.tasks <- task
	return w
}

// CloseTasks tells us that we are donw with our tasks
func (w *Worker) CloseTasks() *Worker {
	close(w.tasks)
	return w
}

// Finish waits for all the workers to finish
func (w *Worker) Finish() {
	w.CloseTasks().Wait()
}

// Wait for the workers to finish
func (w *Worker) Wait() *Worker {
	w.wg.Wait()
	return w
}
<|endoftext|>


<commit_before>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

// arpscan implements ARP scanning of all interfaces' local networks using
// gopacket and its subpackages.  This example shows, among other things:
//   * Generating and sending packet data
//   * Reading in packet data and interpreting it
//   * Use of the 'pcap' subpackage for reading/writing
package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"log"
	"net"
	"sync"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
)

func main() {
	// Get a list of all interfaces.
	ifaces, err := net.Interfaces()
	if err != nil {
		panic(err)
	}

	var wg sync.WaitGroup
	for _, iface := range ifaces {
		wg.Add(1)
		// Start up a scan on each interface.
		go func(iface net.Interface) {
			defer wg.Done()
			if err := scan(&iface); err != nil {
				log.Printf("interface %v: %v", iface.Name, err)
			}
		}(iface)
	}
	// Wait for all interfaces' scans to complete.  They'll try to run
	// forever, but will stop on an error, so if we get past this Wait
	// it means all attempts to write have failed.
	wg.Wait()
}

// scan scans an individual interface's local network for machines using ARP requests/replies.
//
// scan loops forever, sending packets out regularly.  It returns an error if
// it's ever unable to write a packet.
func scan(iface *net.Interface) error {
	// We just look for IPv4 addresses, so try to find if the interface has one.
	var addr *net.IPNet
	if addrs, err := iface.Addrs(); err != nil {
		return err
	} else {
		for _, a := range addrs {
			if ipnet, ok := a.(*net.IPNet); ok {
				if ip4 := ipnet.IP.To4(); ip4 != nil {
					addr = &net.IPNet{
						IP:   ip4,
						Mask: ipnet.Mask[len(ipnet.Mask)-4:],
					}
					break
				}
			}
		}
	}
	// Sanity-check that the interface has a good address.
	if addr == nil {
		return errors.New("no good IP network found")
	} else if addr.IP[0] == 127 {
		return errors.New("skipping localhost")
	} else if addr.Mask[0] != 0xff || addr.Mask[1] != 0xff {
		return errors.New("mask means network is too large")
	}
	log.Printf("Using network range %v for interface %v", addr, iface.Name)

	// Open up a pcap handle for packet reads/writes.
	handle, err := pcap.OpenLive(iface.Name, 65536, true, pcap.BlockForever)
	if err != nil {
		return err
	}
	defer handle.Close()

	// Start up a goroutine to read in packet data.
	stop := make(chan struct{})
	go readARP(handle, iface, stop)
	defer close(stop)
	for {
		// Write our scan packets out to the handle.
		if err := writeARP(handle, iface, addr); err != nil {
			log.Printf("error writing packets on %v: %v", iface.Name, err)
			return err
		}
		// We don't know exactly how long it'll take for packets to be
		// sent back to us, but 10 seconds should be more than enough
		// time ;)
		time.Sleep(10 * time.Second)
	}
}

// readARP watches a handle for incoming ARP responses we might care about, and prints them.
//
// readARP loops until 'stop' is closed.
func readARP(handle *pcap.Handle, iface *net.Interface, stop chan struct{}) {
	src := gopacket.NewPacketSource(handle, layers.LayerTypeEthernet)
	in := src.Packets()
	for {
		var packet gopacket.Packet
		select {
		case <-stop:
			return
		case packet = <-in:
			arpLayer := packet.Layer(layers.LayerTypeARP)
			if arpLayer == nil {
				continue
			}
			arp := arpLayer.(*layers.ARP)
			if arp.Operation != layers.ARPReply || bytes.Equal([]byte(iface.HardwareAddr), arp.SourceHwAddress) {
				// This is a packet I sent.
				continue
			}
			// Note:  we might get some packets here that aren't responses to ones we've sent,
			// if for example someone else sends US an ARP request.  Doesn't much matter, though...
			// all information is good information :)
			log.Printf("IP %v is at %v", net.IP(arp.SourceProtAddress), net.HardwareAddr(arp.SourceHwAddress))
		}
	}
}

// writeARP writes an ARP request for each address on our local network to the
// pcap handle.
func writeARP(handle *pcap.Handle, iface *net.Interface, addr *net.IPNet) error {
	// Set up all the layers' fields we can.
	eth := layers.Ethernet{
		SrcMAC:       iface.HardwareAddr,
		DstMAC:       net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
		EthernetType: layers.EthernetTypeARP,
	}
	arp := layers.ARP{
		AddrType:          layers.LinkTypeEthernet,
		Protocol:          layers.EthernetTypeIPv4,
		HwAddressSize:     6,
		ProtAddressSize:   4,
		Operation:         layers.ARPRequest,
		SourceHwAddress:   []byte(iface.HardwareAddr),
		SourceProtAddress: []byte(addr.IP),
		DstHwAddress:      []byte{0, 0, 0, 0, 0, 0},
	}
	// Set up buffer and options for serialization.
	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{
		FixLengths:       true,
		ComputeChecksums: true,
	}
	// Send one packet for every address.
	for _, ip := range ips(addr) {
		arp.DstProtAddress = []byte(ip)
		gopacket.SerializeLayers(buf, opts, &eth, &arp)
		if err := handle.WritePacketData(buf.Bytes()); err != nil {
			return err
		}
	}
	return nil
}

// ips is a simple and not very good method for getting all IPv4 addresses from a
// net.IPNet.  It returns all IPs it can over the channel it sends back, closing
// the channel when done.
func ips(n *net.IPNet) (out []net.IP) {
	num := binary.BigEndian.Uint32([]byte(n.IP))
	mask := binary.BigEndian.Uint32([]byte(n.Mask))
	num &= mask
	for mask < 0xffffffff {
		var buf [4]byte
		binary.BigEndian.PutUint32(buf[:], num)
		out = append(out, net.IP(buf[:]))
		mask += 1
		num += 1
	}
	return
}
<commit_msg>Fix golint warnings<commit_after>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

// arpscan implements ARP scanning of all interfaces' local networks using
// gopacket and its subpackages.  This example shows, among other things:
//   * Generating and sending packet data
//   * Reading in packet data and interpreting it
//   * Use of the 'pcap' subpackage for reading/writing
package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"log"
	"net"
	"sync"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
)

func main() {
	// Get a list of all interfaces.
	ifaces, err := net.Interfaces()
	if err != nil {
		panic(err)
	}

	var wg sync.WaitGroup
	for _, iface := range ifaces {
		wg.Add(1)
		// Start up a scan on each interface.
		go func(iface net.Interface) {
			defer wg.Done()
			if err := scan(&iface); err != nil {
				log.Printf("interface %v: %v", iface.Name, err)
			}
		}(iface)
	}
	// Wait for all interfaces' scans to complete.  They'll try to run
	// forever, but will stop on an error, so if we get past this Wait
	// it means all attempts to write have failed.
	wg.Wait()
}

// scan scans an individual interface's local network for machines using ARP requests/replies.
//
// scan loops forever, sending packets out regularly.  It returns an error if
// it's ever unable to write a packet.
func scan(iface *net.Interface) error {
	// We just look for IPv4 addresses, so try to find if the interface has one.
	var addr *net.IPNet
	if addrs, err := iface.Addrs(); err != nil {
		return err
	} else {
		for _, a := range addrs {
			if ipnet, ok := a.(*net.IPNet); ok {
				if ip4 := ipnet.IP.To4(); ip4 != nil {
					addr = &net.IPNet{
						IP:   ip4,
						Mask: ipnet.Mask[len(ipnet.Mask)-4:],
					}
					break
				}
			}
		}
	}
	// Sanity-check that the interface has a good address.
	if addr == nil {
		return errors.New("no good IP network found")
	} else if addr.IP[0] == 127 {
		return errors.New("skipping localhost")
	} else if addr.Mask[0] != 0xff || addr.Mask[1] != 0xff {
		return errors.New("mask means network is too large")
	}
	log.Printf("Using network range %v for interface %v", addr, iface.Name)

	// Open up a pcap handle for packet reads/writes.
	handle, err := pcap.OpenLive(iface.Name, 65536, true, pcap.BlockForever)
	if err != nil {
		return err
	}
	defer handle.Close()

	// Start up a goroutine to read in packet data.
	stop := make(chan struct{})
	go readARP(handle, iface, stop)
	defer close(stop)
	for {
		// Write our scan packets out to the handle.
		if err := writeARP(handle, iface, addr); err != nil {
			log.Printf("error writing packets on %v: %v", iface.Name, err)
			return err
		}
		// We don't know exactly how long it'll take for packets to be
		// sent back to us, but 10 seconds should be more than enough
		// time ;)
		time.Sleep(10 * time.Second)
	}
}

// readARP watches a handle for incoming ARP responses we might care about, and prints them.
//
// readARP loops until 'stop' is closed.
func readARP(handle *pcap.Handle, iface *net.Interface, stop chan struct{}) {
	src := gopacket.NewPacketSource(handle, layers.LayerTypeEthernet)
	in := src.Packets()
	for {
		var packet gopacket.Packet
		select {
		case <-stop:
			return
		case packet = <-in:
			arpLayer := packet.Layer(layers.LayerTypeARP)
			if arpLayer == nil {
				continue
			}
			arp := arpLayer.(*layers.ARP)
			if arp.Operation != layers.ARPReply || bytes.Equal([]byte(iface.HardwareAddr), arp.SourceHwAddress) {
				// This is a packet I sent.
				continue
			}
			// Note:  we might get some packets here that aren't responses to ones we've sent,
			// if for example someone else sends US an ARP request.  Doesn't much matter, though...
			// all information is good information :)
			log.Printf("IP %v is at %v", net.IP(arp.SourceProtAddress), net.HardwareAddr(arp.SourceHwAddress))
		}
	}
}

// writeARP writes an ARP request for each address on our local network to the
// pcap handle.
func writeARP(handle *pcap.Handle, iface *net.Interface, addr *net.IPNet) error {
	// Set up all the layers' fields we can.
	eth := layers.Ethernet{
		SrcMAC:       iface.HardwareAddr,
		DstMAC:       net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
		EthernetType: layers.EthernetTypeARP,
	}
	arp := layers.ARP{
		AddrType:          layers.LinkTypeEthernet,
		Protocol:          layers.EthernetTypeIPv4,
		HwAddressSize:     6,
		ProtAddressSize:   4,
		Operation:         layers.ARPRequest,
		SourceHwAddress:   []byte(iface.HardwareAddr),
		SourceProtAddress: []byte(addr.IP),
		DstHwAddress:      []byte{0, 0, 0, 0, 0, 0},
	}
	// Set up buffer and options for serialization.
	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{
		FixLengths:       true,
		ComputeChecksums: true,
	}
	// Send one packet for every address.
	for _, ip := range ips(addr) {
		arp.DstProtAddress = []byte(ip)
		gopacket.SerializeLayers(buf, opts, &eth, &arp)
		if err := handle.WritePacketData(buf.Bytes()); err != nil {
			return err
		}
	}
	return nil
}

// ips is a simple and not very good method for getting all IPv4 addresses from a
// net.IPNet.  It returns all IPs it can over the channel it sends back, closing
// the channel when done.
func ips(n *net.IPNet) (out []net.IP) {
	num := binary.BigEndian.Uint32([]byte(n.IP))
	mask := binary.BigEndian.Uint32([]byte(n.Mask))
	num &= mask
	for mask < 0xffffffff {
		var buf [4]byte
		binary.BigEndian.PutUint32(buf[:], num)
		out = append(out, net.IP(buf[:]))
		mask++
		num++
	}
	return
}
<|endoftext|>


<commit_before>package veneur

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	log "github.com/Sirupsen/logrus"
)

// Flush takes the slices of metrics, combines then and marshals them to json
// for posting to Datadog.
func Flush(postMetrics [][]DDMetric) {
	totalCount := 0
	for _, metrics := range postMetrics {
		totalCount += len(metrics)
	}
	finalMetrics := make([]DDMetric, 0, totalCount)
	for _, metrics := range postMetrics {
		finalMetrics = append(finalMetrics, metrics...)
	}
	for i := range finalMetrics {
		finalMetrics[i].Hostname = Config.Hostname
	}
	// Check to see if we have anything to do
	if totalCount == 0 {
		log.Info("Nothing to flush, skipping.")
		return
	}

	postJSON, err := json.Marshal(map[string][]DDMetric{
		"series": finalMetrics,
	})
	if err != nil {
		Stats.Count("flush.error_total", int64(totalCount), []string{"cause:json"}, 1.0)
		log.WithError(err).Error("Error rendering JSON request body")
		return
	}

	resp, err := http.Post(fmt.Sprintf("%s/api/v1/series?api_key=%s", Config.APIHostname, Config.Key), "application/json", bytes.NewReader(postJSON))
	if err != nil {
		Stats.Count("flush.error_total", int64(totalCount), []string{"cause:io"}, 1.0)
		log.WithError(err).Error("Error writing POST request")
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// don't bail out if this errors, we'll just log the body as empty
		log.WithError(err).Error("Error reading response body")
	}
	resultFields := log.Fields{
		"status":   resp.Status,
		"headers":  resp.Header,
		"request":  string(postJSON),
		"response": string(body),
	}

	if resp.StatusCode != http.StatusAccepted {
		Stats.Count("flush.error_total", int64(totalCount), []string{fmt.Sprintf("cause:%d", resp.StatusCode)}, 1.0)
		log.WithFields(resultFields).Error("Error POSTing")
		return
	}

	Stats.Count("flush.error_total", 0, nil, 0.1)
	log.WithField("metrics", len(finalMetrics)).Info("Completed flush to Datadog")
	log.WithFields(resultFields).Debug("POSTing JSON")
}
<commit_msg>Add deflate to request body<commit_after>package veneur

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	log "github.com/Sirupsen/logrus"
)

// Flush takes the slices of metrics, combines then and marshals them to json
// for posting to Datadog.
func Flush(postMetrics [][]DDMetric) {
	totalCount := 0
	for _, metrics := range postMetrics {
		totalCount += len(metrics)
	}
	finalMetrics := make([]DDMetric, 0, totalCount)
	for _, metrics := range postMetrics {
		finalMetrics = append(finalMetrics, metrics...)
	}
	for i := range finalMetrics {
		finalMetrics[i].Hostname = Config.Hostname
	}
	// Check to see if we have anything to do
	if totalCount == 0 {
		log.Info("Nothing to flush, skipping.")
		return
	}

	postJSON, err := json.Marshal(map[string][]DDMetric{
		"series": finalMetrics,
	})
	if err != nil {
		Stats.Count("flush.error_total", int64(totalCount), []string{"cause:json"}, 1.0)
		log.WithError(err).Error("Error rendering JSON request body")
		return
	}

	var reqBody bytes.Buffer
	compressor := zlib.NewWriter(&reqBody)
	// bytes.Buffer never errors
	compressor.Write(postJSON)
	// make sure to flush remaining compressed bytes to the buffer
	compressor.Close()

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/series?api_key=%s", Config.APIHostname, Config.Key), &reqBody)
	if err != nil {
		Stats.Count("flush.error_total", int64(totalCount), []string{"cause:construct"}, 1.0)
		log.WithError(err).Error("Error constructing POST request")
		return
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Content-Encoding", "deflate")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		Stats.Count("flush.error_total", int64(totalCount), []string{"cause:io"}, 1.0)
		log.WithError(err).Error("Error writing POST request")
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// don't bail out if this errors, we'll just log the body as empty
		log.WithError(err).Error("Error reading response body")
	}
	resultFields := log.Fields{
		"status":   resp.Status,
		"headers":  resp.Header,
		"request":  string(postJSON),
		"response": string(body),
	}

	if resp.StatusCode != http.StatusAccepted {
		Stats.Count("flush.error_total", int64(totalCount), []string{fmt.Sprintf("cause:%d", resp.StatusCode)}, 1.0)
		log.WithFields(resultFields).Error("Error POSTing")
		return
	}

	Stats.Count("flush.error_total", 0, nil, 0.1)
	log.WithField("metrics", len(finalMetrics)).Info("Completed flush to Datadog")
	log.WithFields(resultFields).Debug("POSTing JSON")
}
<|endoftext|>


<commit_before>// A URL router implemented by Double-Array Trie.
package doublearray

import (
	"fmt"
	"sort"

	"github.com/naoina/kocha-urlrouter"
)

const (
	// Block size of array of BASE/CHECK of Double-Array.
	blockSize = 256
)

// baseCheck represents a BASE/CHECK node.
type baseCheck struct {
	base  int
	check int
}

// DoubleArray represents a URLRouter by Double-Array.
type DoubleArray struct {
	bc   []baseCheck
	node map[int]*node
}

// NewDoubleArray returns a new DoubleArray with given size.
func New(size int) *DoubleArray {
	return &DoubleArray{
		bc:   newBaseCheckArray(size),
		node: make(map[int]*node),
	}
}

// newBaseCheckArray returns a new slice of baseCheck with given size.
func newBaseCheckArray(size int) []baseCheck {
	bc := make([]baseCheck, size)
	for i := 0; i < len(bc); i++ {
		bc[i].check = -1
	}
	return bc
}

// Lookup returns result data of lookup from Double-Array routing table by given path.
func (da *DoubleArray) Lookup(path string) (data interface{}, params []urlrouter.Param) {
	nodes, idx, values := da.lookup(path, nil, 0)
	if nodes == nil {
		return nil, nil
	}
	nd := nodes[idx]
	if nd == nil {
		return nil, nil
	}
	if len(values) > 0 {
		params = make([]urlrouter.Param, len(values))
		for i, v := range values {
			params[i] = urlrouter.Param{Name: nd.paramNames[i], Value: v}
		}
	}
	return nd.data, params
}

// Build builds Double-Array routing table from records.
func (da *DoubleArray) Build(records []urlrouter.Record) error {
	sortedRecords := makeRecords(records)
	if err := da.build(sortedRecords, 0, 0); err != nil {
		return err
	}
	return nil
}

func (da *DoubleArray) lookup(path string, params []string, idx int) (map[int]*node, int, []string) {
	if path == "" {
		return da.node, idx, params
	}
	c, remaining := path[0], path[1:]
	if next := nextIndex(da.bc[idx].base, c); da.bc[next].check == idx {
		if nodes, idx, params := da.lookup(remaining, params, next); nodes != nil {
			return nodes, idx, params
		}
	}
	if nd := da.node[idx]; nd != nil && nd.paramTree != nil {
		i := urlrouter.NextSeparator(path, 0)
		remaining, params := path[i:], append(params, path[:i])
		if nodes, idx, params := nd.paramTree.lookup(remaining, params, 0); nodes != nil {
			return nodes, idx, params
		}
	}
	if nd := da.node[idx]; nd != nil && nd.wildcardTree != nil {
		return nd.wildcardTree.node, 0, append(params, path)
	}
	return nil, -1, nil
}

func (da *DoubleArray) build(srcs []*Record, idx, depth int) error {
	base, siblings, leaf, err := da.arrange(srcs, idx, depth)
	if err != nil {
		return err
	}
	if leaf != nil {
		nd, err := makeNode(leaf)
		if err != nil {
			return err
		}
		da.node[idx] = nd
	}
	for _, sib := range siblings {
		if !urlrouter.IsMetaChar(sib.c) {
			da.setCheck(nextIndex(base, sib.c), idx)
		}
	}
	for _, sib := range siblings {
		switch records := srcs[sib.start:sib.end]; sib.c {
		case urlrouter.ParamCharacter:
			for _, record := range records {
				next := urlrouter.NextSeparator(record.Key, depth)
				name := record.Key[depth+1 : next]
				record.paramNames = append(record.paramNames, name)
				record.Key = record.Key[next:]
			}
			if da.node[idx] == nil {
				da.node[idx] = &node{}
			}
			da.node[idx].paramTree = New(blockSize)
			if err := da.node[idx].paramTree.build(records, 0, 0); err != nil {
				return err
			}
		case urlrouter.WildcardCharacter:
			if da.node[idx] == nil {
				da.node[idx] = &node{}
			}
			record := records[0]
			name := record.Key[depth+1:]
			record.paramNames = append(record.paramNames, name)
			da.node[idx].wildcardTree = New(0)
			nd, err := makeNode(record)
			if err != nil {
				return err
			}
			da.node[idx].wildcardTree.node[0] = nd
		default:
			if err := da.build(records, nextIndex(base, sib.c), depth+1); err != nil {
				return err
			}
		}
	}
	return nil
}

// setBase sets BASE.
func (da *DoubleArray) setBase(i, base int) {
	da.bc[i].base = base
}

// setCheck sets CHECK.
func (da *DoubleArray) setCheck(i, check int) {
	da.bc[i].check = check
}

// extendBaseCheckArray extends array of BASE/CHECK.
func (da *DoubleArray) extendBaseCheckArray() {
	da.bc = append(da.bc, newBaseCheckArray(blockSize)...)
}

// findEmptyIndex returns an index of unused BASE/CHECK node.
func (da *DoubleArray) findEmptyIndex(start int) int {
	i := start
	for ; i < len(da.bc); i++ {
		if da.bc[i].base == 0 && da.bc[i].check == -1 {
			break
		}
	}
	return i
}

// findBase returns good BASE.
func (da *DoubleArray) findBase(siblings []sibling, start int) (base int) {
	idx := start + 1
	firstChar := siblings[0].c
	for ; idx < len(da.bc); idx = da.findEmptyIndex(idx + 1) {
		base = nextIndex(idx, firstChar)
		i := 0
		for ; i < len(siblings); i++ {
			if urlrouter.IsMetaChar(siblings[i].c) {
				continue
			}
			if next := nextIndex(base, siblings[i].c); da.bc[next].base != 0 || da.bc[next].check != -1 {
				break
			}
		}
		if i == len(siblings) {
			return base
		}
	}
	da.extendBaseCheckArray()
	return nextIndex(idx, firstChar)
}

func (da *DoubleArray) arrange(records []*Record, idx, depth int) (base int, siblings []sibling, leaf *Record, err error) {
	siblings, leaf, err = makeSiblings(records, depth)
	if err != nil {
		return -1, nil, nil, err
	}
	if len(siblings) < 1 {
		return -1, nil, leaf, nil
	}
	base = da.findBase(siblings, idx)
	da.setBase(idx, base)
	return base, siblings, leaf, err
}

// node represents a node of Double-Array.
type node struct {
	data interface{}

	// Tree of path parameter.
	paramTree *DoubleArray

	// Tree of wildcard path parameter.
	wildcardTree *DoubleArray

	// Names of path parameters.
	paramNames []string
}

// makeNode returns a new node from record.
func makeNode(record *Record) (*node, error) {
	dups := make(map[string]bool)
	for _, name := range record.paramNames {
		if dups[name] {
			return nil, fmt.Errorf("path parameter `%v` is duplicated in the key '%v'", name, record.Key)
		}
		dups[name] = true
	}
	return &node{data: record.Value, paramNames: record.paramNames}, nil
}

// sibling represents an intermediate data of build for Double-Array.
type sibling struct {
	// An index of start of duplicated characters.
	start int

	// An index of end of duplicated characters.
	end int

	// A character of sibling.
	c byte
}

// nextIndex returns a next index of array of BASE/CHECK.
func nextIndex(base int, c byte) int {
	return base ^ int(c)
}

// makeSiblings returns slice of sibling.
func makeSiblings(records []*Record, depth int) (sib []sibling, leaf *Record, err error) {
	var (
		pc byte
		n  int
	)
	for i, record := range records {
		if len(record.Key) == depth {
			leaf = record
			continue
		}
		c := record.Key[depth]
		switch {
		case pc < c:
			sib = append(sib, sibling{start: i, c: c})
		case pc == c:
			continue
		default:
			return nil, nil, fmt.Errorf("BUG: routing table hasn't been sorted")
		}
		if n > 0 {
			sib[n-1].end = i
		}
		pc = c
		n++
	}
	if n == 0 {
		return nil, leaf, nil
	}
	sib[n-1].end = len(records)
	return sib, leaf, nil
}

// Record represents a record that use to build the Double-Array.
type Record struct {
	urlrouter.Record
	paramNames []string
}

// RecordSlice represents a slice of Record for sort and implements the sort.Interface.
type RecordSlice []*Record

// makeRecords returns the records that use to build the Double-Array.
func makeRecords(srcs []urlrouter.Record) []*Record {
	records := make([]*Record, len(srcs))
	for i, record := range srcs {
		records[i] = &Record{Record: record}
	}
	sort.Sort(RecordSlice(records))
	return records
}

// Len implements the sort.Interface.Len.
func (rs RecordSlice) Len() int {
	return len(rs)
}

// Less implements the sort.Interface.Less.
func (rs RecordSlice) Less(i, j int) bool {
	return rs[i].Key < rs[j].Key
}

// Swap implements the sort.Interface.Swap.
func (rs RecordSlice) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
}

// DoubleArrayRouter represents the Router of Double-Array.
type DoubleArrayRouter struct{}

// New returns a new URLRouter that implemented by Double-Array.
func (router *DoubleArrayRouter) New() urlrouter.URLRouter {
	return New(blockSize)
}

func init() {
	urlrouter.Register("doublearray", &DoubleArrayRouter{})
}
<commit_msg>doublearray: performance improvement<commit_after>// A URL router implemented by Double-Array Trie.
package doublearray

import (
	"fmt"
	"sort"

	"github.com/naoina/kocha-urlrouter"
)

const (
	// Block size of array of BASE/CHECK of Double-Array.
	blockSize = 256
)

// baseCheck represents a BASE/CHECK node.
type baseCheck struct {
	base        int
	check       int
	hasParams   bool
	hasWildcard bool
}

// DoubleArray represents a URLRouter by Double-Array.
type DoubleArray struct {
	bc   []baseCheck
	node map[int]*node
}

// NewDoubleArray returns a new DoubleArray with given size.
func New(size int) *DoubleArray {
	return &DoubleArray{
		bc:   newBaseCheckArray(size),
		node: make(map[int]*node),
	}
}

// newBaseCheckArray returns a new slice of baseCheck with given size.
func newBaseCheckArray(size int) []baseCheck {
	bc := make([]baseCheck, size)
	for i := 0; i < len(bc); i++ {
		bc[i].check = -1
	}
	return bc
}

// Lookup returns result data of lookup from Double-Array routing table by given path.
func (da *DoubleArray) Lookup(path string) (data interface{}, params []urlrouter.Param) {
	nodes, idx, values := da.lookup(path, nil)
	if nodes == nil {
		return nil, nil
	}
	nd := nodes[idx]
	if nd == nil {
		return nil, nil
	}
	if len(values) > 0 {
		params = make([]urlrouter.Param, len(values))
		for i, v := range values {
			params[i] = urlrouter.Param{Name: nd.paramNames[i], Value: v}
		}
	}
	return nd.data, params
}

// Build builds Double-Array routing table from records.
func (da *DoubleArray) Build(records []urlrouter.Record) error {
	sortedRecords := makeRecords(records)
	if err := da.build(sortedRecords, 0, 0); err != nil {
		return err
	}
	return nil
}

func (da *DoubleArray) lookup(path string, params []string) (map[int]*node, int, []string) {
	idx, indexes, found := da.lookupStatic(path)
	if found {
		return da.node, idx, params
	}
	for i := len(indexes) - 1; i >= 0; i-- {
		curIdx, idx := int((indexes[i]>>32)&0xffffffff), int(indexes[i]&0xffffffff)
		nd := da.node[idx]
		if nd.paramTree != nil {
			i := urlrouter.NextSeparator(path, curIdx)
			remaining, params := path[i:], append(params, path[curIdx:i])
			if nodes, idx, params := nd.paramTree.lookup(remaining, params); nodes != nil {
				return nodes, idx, params
			}
		}
		if nd.wildcardTree != nil {
			return nd.wildcardTree.node, 0, append(params, path[curIdx:])
		}
	}
	return nil, -1, nil
}

func (da *DoubleArray) lookupStatic(path string) (idx int, indexes []int64, found bool) {
	for i := 0; i < len(path); i++ {
		next := nextIndex(da.bc[idx].base, path[i])
		if da.bc[next].check != idx {
			return -1, indexes, false
		}
		idx = next
		if da.bc[idx].hasParams || da.bc[idx].hasWildcard {
			indexes = append(indexes, int64(((i+1)&0xffffffff)<<32)|int64(idx&0xffffffff))
		}
	}
	return idx, nil, true
}

func (da *DoubleArray) build(srcs []*Record, idx, depth int) error {
	base, siblings, leaf, err := da.arrange(srcs, idx, depth)
	if err != nil {
		return err
	}
	if leaf != nil {
		nd, err := makeNode(leaf)
		if err != nil {
			return err
		}
		da.node[idx] = nd
	}
	for _, sib := range siblings {
		if !urlrouter.IsMetaChar(sib.c) {
			da.setCheck(nextIndex(base, sib.c), idx)
		}
	}
	for _, sib := range siblings {
		switch records := srcs[sib.start:sib.end]; sib.c {
		case urlrouter.ParamCharacter:
			for _, record := range records {
				next := urlrouter.NextSeparator(record.Key, depth)
				name := record.Key[depth+1 : next]
				record.paramNames = append(record.paramNames, name)
				record.Key = record.Key[next:]
			}
			if da.node[idx] == nil {
				da.node[idx] = &node{}
			}
			da.node[idx].paramTree = New(blockSize)
			da.bc[idx].hasParams = true
			if err := da.node[idx].paramTree.build(records, 0, 0); err != nil {
				return err
			}
		case urlrouter.WildcardCharacter:
			if da.node[idx] == nil {
				da.node[idx] = &node{}
			}
			record := records[0]
			name := record.Key[depth+1:]
			record.paramNames = append(record.paramNames, name)
			da.node[idx].wildcardTree = New(0)
			nd, err := makeNode(record)
			if err != nil {
				return err
			}
			da.node[idx].wildcardTree.node[0] = nd
			da.bc[idx].hasWildcard = true
		default:
			if err := da.build(records, nextIndex(base, sib.c), depth+1); err != nil {
				return err
			}
		}
	}
	return nil
}

// setBase sets BASE.
func (da *DoubleArray) setBase(i, base int) {
	da.bc[i].base = base
}

// setCheck sets CHECK.
func (da *DoubleArray) setCheck(i, check int) {
	da.bc[i].check = check
}

// extendBaseCheckArray extends array of BASE/CHECK.
func (da *DoubleArray) extendBaseCheckArray() {
	da.bc = append(da.bc, newBaseCheckArray(blockSize)...)
}

// findEmptyIndex returns an index of unused BASE/CHECK node.
func (da *DoubleArray) findEmptyIndex(start int) int {
	i := start
	for ; i < len(da.bc); i++ {
		if da.bc[i].base == 0 && da.bc[i].check == -1 {
			break
		}
	}
	return i
}

// findBase returns good BASE.
func (da *DoubleArray) findBase(siblings []sibling, start int) (base int) {
	idx := start + 1
	firstChar := siblings[0].c
	for ; idx < len(da.bc); idx = da.findEmptyIndex(idx + 1) {
		base = nextIndex(idx, firstChar)
		i := 0
		for ; i < len(siblings); i++ {
			if urlrouter.IsMetaChar(siblings[i].c) {
				continue
			}
			if next := nextIndex(base, siblings[i].c); da.bc[next].base != 0 || da.bc[next].check != -1 {
				break
			}
		}
		if i == len(siblings) {
			return base
		}
	}
	da.extendBaseCheckArray()
	return nextIndex(idx, firstChar)
}

func (da *DoubleArray) arrange(records []*Record, idx, depth int) (base int, siblings []sibling, leaf *Record, err error) {
	siblings, leaf, err = makeSiblings(records, depth)
	if err != nil {
		return -1, nil, nil, err
	}
	if len(siblings) < 1 {
		return -1, nil, leaf, nil
	}
	base = da.findBase(siblings, idx)
	da.setBase(idx, base)
	return base, siblings, leaf, err
}

// node represents a node of Double-Array.
type node struct {
	data interface{}

	// Tree of path parameter.
	paramTree *DoubleArray

	// Tree of wildcard path parameter.
	wildcardTree *DoubleArray

	// Names of path parameters.
	paramNames []string
}

// makeNode returns a new node from record.
func makeNode(record *Record) (*node, error) {
	dups := make(map[string]bool)
	for _, name := range record.paramNames {
		if dups[name] {
			return nil, fmt.Errorf("path parameter `%v` is duplicated in the key '%v'", name, record.Key)
		}
		dups[name] = true
	}
	return &node{data: record.Value, paramNames: record.paramNames}, nil
}

// sibling represents an intermediate data of build for Double-Array.
type sibling struct {
	// An index of start of duplicated characters.
	start int

	// An index of end of duplicated characters.
	end int

	// A character of sibling.
	c byte
}

// nextIndex returns a next index of array of BASE/CHECK.
func nextIndex(base int, c byte) int {
	return base ^ int(c)
}

// makeSiblings returns slice of sibling.
func makeSiblings(records []*Record, depth int) (sib []sibling, leaf *Record, err error) {
	var (
		pc byte
		n  int
	)
	for i, record := range records {
		if len(record.Key) == depth {
			leaf = record
			continue
		}
		c := record.Key[depth]
		switch {
		case pc < c:
			sib = append(sib, sibling{start: i, c: c})
		case pc == c:
			continue
		default:
			return nil, nil, fmt.Errorf("BUG: routing table hasn't been sorted")
		}
		if n > 0 {
			sib[n-1].end = i
		}
		pc = c
		n++
	}
	if n == 0 {
		return nil, leaf, nil
	}
	sib[n-1].end = len(records)
	return sib, leaf, nil
}

// Record represents a record that use to build the Double-Array.
type Record struct {
	urlrouter.Record
	paramNames []string
}

// RecordSlice represents a slice of Record for sort and implements the sort.Interface.
type RecordSlice []*Record

// makeRecords returns the records that use to build the Double-Array.
func makeRecords(srcs []urlrouter.Record) []*Record {
	records := make([]*Record, len(srcs))
	for i, record := range srcs {
		records[i] = &Record{Record: record}
	}
	sort.Sort(RecordSlice(records))
	return records
}

// Len implements the sort.Interface.Len.
func (rs RecordSlice) Len() int {
	return len(rs)
}

// Less implements the sort.Interface.Less.
func (rs RecordSlice) Less(i, j int) bool {
	return rs[i].Key < rs[j].Key
}

// Swap implements the sort.Interface.Swap.
func (rs RecordSlice) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
}

// DoubleArrayRouter represents the Router of Double-Array.
type DoubleArrayRouter struct{}

// New returns a new URLRouter that implemented by Double-Array.
func (router *DoubleArrayRouter) New() urlrouter.URLRouter {
	return New(blockSize)
}

func init() {
	urlrouter.Register("doublearray", &DoubleArrayRouter{})
}
<|endoftext|>