<commit_before>package lumber

import (
	"fmt"
	"os"
	"time"
)

const (
	// mode constants
	BACKUP = -2
	TRUNC  = -1
	APPEND = 0
)

type FileLogger struct {
	out        *os.File
	outLevel   int
	timeFormat string
	prefix     string
	maxLines   int
	lines      int
}

// Creates a new FileLogger with filename f, output level o, mode, and an empty prefix
// Modes are APPEND (append to existing log if it exists), TRUNC (truncate old log file to create
// the new one), BACKUP (moves old log to log.name.1 before creaing new log).
func NewFileLogger(f string, o, mode int) (l *FileLogger, err error) {
	var file *os.File
	switch {
	case mode == TRUNC:
		file, err = os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	case mode == APPEND:
		file, err = os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	case mode > 0:
		file, err = openBackup(f, mode)
	default:
		err = fmt.Errorf("Invalid mode parameter: %d", mode)
		return
	}
	if err != nil {
		err = fmt.Errorf("Error opening file '%s' for logging: %s", f, err)
		return
	}

	l = &FileLogger{file, o, TIMEFORMAT, "", mode, 0}

	return
}

// Attempt to create new log. If the file already exists, backup the old one and create a new file
func openBackup(f string, mode int) (*os.File, error) {
	// First try to open the file with O_EXCL (file must not already exist)
	file, err := os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
	if err == nil {
		return file, nil
	}
	if !os.IsExist(err) {
		return nil, fmt.Errorf("Error opening file for logging: %s", err)
	}

	// The file already exists, we need to back it up
	err = os.Rename(f, fmt.Sprintf("%s.1", f))
	if err != nil {
		backupErr := fmt.Errorf("Error backing up log: %s", err)
		file, err = os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return nil, fmt.Errorf("%s. Error appending to existing log file: %s", backupErr, err)
		}
		return file, backupErr
	}

	// Open new file for log
	file, err = os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return nil, fmt.Errorf("Error opening file for logging: %s", err)
	}

	return file, err
}

// Rename "log.name" to "log.name.1"
func (l *FileLogger) backup() (*os.File, error) {
	err := os.Rename(l.out.Name(), fmt.Sprintf("%s.1", l.out.Name()))
	if err != nil {
		return nil, fmt.Errorf("Error backing up log: %s", err)
	}
	file, err := os.OpenFile(l.out.Name(), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		return nil, fmt.Errorf("Error opening file for logging: %s", err)
	}
	return file, nil
}

// Generic output function. Outputs messages if they are higher level than outLevel for this
// specific logger. If msg does not end with a newline, one will be appended.
func (l *FileLogger) output(msg *Message) {
	if msg.level < l.outLevel {
		return
	}
	if l.lines >= l.maxLines {
		out, err := l.backup()
		if err != nil {
			fmt.Println("Error backing up log:", err)
		} else {
			l.out = out
		}
	}

	buf := []byte{}
	buf = append(buf, msg.time.Format(l.timeFormat)...)
	if l.prefix != "" {
		buf = append(buf, ' ')
		buf = append(buf, l.prefix...)
	}
	buf = append(buf, ' ')
	buf = append(buf, levels[msg.level]...)
	buf = append(buf, ' ')
	buf = append(buf, msg.m...)
	if len(msg.m) > 0 && msg.m[len(msg.m)-1] != '\n' {
		buf = append(buf, '\n')
	}
	l.lines += 1
	l.out.Write(buf)
}

// Sets the output level for this logger
func (l *FileLogger) Level(o int) {
	if o >= TRACE && o <= FATAL {
		l.outLevel = o
	}
}

// Sets the prefix for this logger
func (l *FileLogger) Prefix(p string) {
	l.prefix = p
}

// Sets the time format for this logger
func (l *FileLogger) TimeFormat(f string) {
	l.timeFormat = f
}

// Flush anything that hasn't been written and close the logger
func (l *FileLogger) Close() (err error) {
	err = l.out.Sync()
	if err != nil {
		l.Error("Could not sync log file")
		err = fmt.Errorf("Could not sync log file: %s", err)
	}
	err = l.out.Close()
	if err != nil {
		l.Error("Could not close log file")
		err = fmt.Errorf("Could not close log file: %s", err)
	}
	return
}

// Logging functions
func (l *FileLogger) Fatal(format string, v ...interface{}) {
	l.output(&Message{FATAL, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Error(format string, v ...interface{}) {
	l.output(&Message{ERROR, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Warn(format string, v ...interface{}) {
	l.output(&Message{WARN, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Info(format string, v ...interface{}) {
	l.output(&Message{INFO, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Debug(format string, v ...interface{}) {
	l.output(&Message{DEBUG, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Trace(format string, v ...interface{}) {
	l.output(&Message{TRACE, fmt.Sprintf(format, v...), time.Now()})
}
<commit_msg>Close old file when rotating<commit_after>package lumber

import (
	"fmt"
	"os"
	"time"
)

const (
	// mode constants
	BACKUP = -2
	TRUNC  = -1
	APPEND = 0
)

type FileLogger struct {
	out        *os.File
	outLevel   int
	timeFormat string
	prefix     string
	maxLines   int
	lines      int
}

// Creates a new FileLogger with filename f, output level o, mode, and an empty prefix
// Modes are APPEND (append to existing log if it exists), TRUNC (truncate old log file to create
// the new one), BACKUP (moves old log to log.name.1 before creaing new log).
func NewFileLogger(f string, o, mode int) (l *FileLogger, err error) {
	var file *os.File
	switch {
	case mode == TRUNC:
		file, err = os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	case mode == APPEND:
		file, err = os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	case mode > 0:
		file, err = openBackup(f, mode)
	default:
		err = fmt.Errorf("Invalid mode parameter: %d", mode)
		return
	}
	if err != nil {
		err = fmt.Errorf("Error opening file '%s' for logging: %s", f, err)
		return
	}

	l = &FileLogger{file, o, TIMEFORMAT, "", mode, 0}

	return
}

// Attempt to create new log. If the file already exists, backup the old one and create a new file
func openBackup(f string, mode int) (*os.File, error) {
	// First try to open the file with O_EXCL (file must not already exist)
	file, err := os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
	if err == nil {
		return file, nil
	}
	if !os.IsExist(err) {
		return nil, fmt.Errorf("Error opening file for logging: %s", err)
	}

	// The file already exists, we need to back it up
	err = os.Rename(f, fmt.Sprintf("%s.1", f))
	if err != nil {
		backupErr := fmt.Errorf("Error backing up log: %s", err)
		file, err = os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return nil, fmt.Errorf("%s. Error appending to existing log file: %s", backupErr, err)
		}
		return file, backupErr
	}

	// Open new file for log
	file, err = os.OpenFile(f, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return nil, fmt.Errorf("Error opening file for logging: %s", err)
	}

	return file, err
}

// Rename "log.name" to "log.name.1"
func (l *FileLogger) backup() (*os.File, error) {
	err := os.Rename(l.out.Name(), fmt.Sprintf("%s.1", l.out.Name()))
	if err != nil {
		return nil, fmt.Errorf("Error backing up log: %s", err)
	}
	file, err := os.OpenFile(l.out.Name(), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		return nil, fmt.Errorf("Error opening file for logging: %s", err)
	}
	return file, nil
}

// Generic output function. Outputs messages if they are higher level than outLevel for this
// specific logger. If msg does not end with a newline, one will be appended.
func (l *FileLogger) output(msg *Message) {
	if msg.level < l.outLevel {
		return
	}
	if l.lines >= l.maxLines {
		out, err := l.backup()
		if err != nil {
			fmt.Println("Error backing up log:", err)
		} else {
			tmpOut := l.out
			l.out = out
			if tmpOut != nil {
				tmpOut.Close()
			}
		}
	}

	buf := []byte{}
	buf = append(buf, msg.time.Format(l.timeFormat)...)
	if l.prefix != "" {
		buf = append(buf, ' ')
		buf = append(buf, l.prefix...)
	}
	buf = append(buf, ' ')
	buf = append(buf, levels[msg.level]...)
	buf = append(buf, ' ')
	buf = append(buf, msg.m...)
	if len(msg.m) > 0 && msg.m[len(msg.m)-1] != '\n' {
		buf = append(buf, '\n')
	}
	l.lines += 1
	l.out.Write(buf)
}

// Sets the output level for this logger
func (l *FileLogger) Level(o int) {
	if o >= TRACE && o <= FATAL {
		l.outLevel = o
	}
}

// Sets the prefix for this logger
func (l *FileLogger) Prefix(p string) {
	l.prefix = p
}

// Sets the time format for this logger
func (l *FileLogger) TimeFormat(f string) {
	l.timeFormat = f
}

// Flush anything that hasn't been written and close the logger
func (l *FileLogger) Close() (err error) {
	err = l.out.Sync()
	if err != nil {
		l.Error("Could not sync log file")
		err = fmt.Errorf("Could not sync log file: %s", err)
	}
	err = l.out.Close()
	if err != nil {
		l.Error("Could not close log file")
		err = fmt.Errorf("Could not close log file: %s", err)
	}
	return
}

// Logging functions
func (l *FileLogger) Fatal(format string, v ...interface{}) {
	l.output(&Message{FATAL, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Error(format string, v ...interface{}) {
	l.output(&Message{ERROR, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Warn(format string, v ...interface{}) {
	l.output(&Message{WARN, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Info(format string, v ...interface{}) {
	l.output(&Message{INFO, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Debug(format string, v ...interface{}) {
	l.output(&Message{DEBUG, fmt.Sprintf(format, v...), time.Now()})
}

func (l *FileLogger) Trace(format string, v ...interface{}) {
	l.output(&Message{TRACE, fmt.Sprintf(format, v...), time.Now()})
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package queue

import (
	"bytes"
	"encoding/gob"
	"github.com/globocom/config"
	. "launchpad.net/gocheck"
	"testing"
	"time"
)

func Test(t *testing.T) {
	TestingT(t)
}

type S struct{}

var _ = Suite(&S{})

func (s *S) SetUpSuite(c *C) {
	config.Set("queue-server", "127.0.0.1:11300")

	// Cleaning the queue. All tests must clean its mess, but we can't
	// guarante the state of the queue before running them.
	cleanQ(c)
}

func (s *S) SetUpTest(c *C) {
	conn = nil
}

func (s *S) TestConnection(c *C) {
	cn, err := connection()
	c.Assert(err, IsNil)
	defer cn.Close()
	tubes, err := cn.ListTubes()
	c.Assert(err, IsNil)
	c.Assert(tubes[0], Equals, "default")
}

func (s *S) TestConnectionQueueServerUndefined(c *C) {
	old, _ := config.Get("queue-server")
	config.Unset("queue-server")
	defer config.Set("queue-server", old)
	conn, err := connection()
	c.Assert(conn, IsNil)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, `"queue-server" is not defined in config file.`)
}

func (s *S) TestConnectionResfused(c *C) {
	old, _ := config.Get("queue-server")
	config.Set("queue-server", "127.0.0.1:11301")
	defer config.Set("queue-server", old)
	conn, err := connection()
	c.Assert(conn, IsNil)
	c.Assert(err, NotNil)
}

func (s *S) TestConnectionDoubleCall(c *C) {
	cn1, err := connection()
	c.Assert(err, IsNil)
	defer cn1.Close()
	c.Assert(cn1, Equals, conn)
	cn2, err := connection()
	c.Assert(err, IsNil)
	c.Assert(cn2, Equals, cn1)
}

func (s *S) TestConnectionClosed(c *C) {
	cn1, err := connection()
	c.Assert(err, IsNil)
	cn1.Close()
	cn2, err := connection()
	c.Assert(err, IsNil)
	defer cn2.Close()
	_, err = cn2.ListTubes()
	c.Assert(err, IsNil)
}

func (s *S) TestPut(c *C) {
	msg := Message{
		Action: "regenerate-apprc",
		Args:   []string{"myapp"},
	}
	err := msg.Put("default", 0)
	c.Assert(err, IsNil)
	c.Assert(msg.id, Not(Equals), 0)
	defer conn.Delete(msg.id)
	id, body, err := conn.Reserve(1e6)
	c.Assert(err, IsNil)
	c.Assert(id, Equals, msg.id)
	var got Message
	buf := bytes.NewBuffer(body)
	err = gob.NewDecoder(buf).Decode(&got)
	c.Assert(err, IsNil)
	got.id = msg.id
	c.Assert(got, DeepEquals, msg)
}

func (s *S) TestPutConnectionFailure(c *C) {
	old, _ := config.Get("queue-server")
	defer config.Set("queue-server", old)
	config.Unset("queue-server")
	msg := Message{Action: "regenerate-apprc"}
	err := msg.Put("default", 0)
	c.Assert(err, NotNil)
}

func (s *S) TestPutWithDelay(c *C) {
	msg := Message{
		Action: "do-something",
		Args:   []string{"nothing"},
	}
	err := msg.Put("default", 1e9)
	c.Assert(err, IsNil)
	defer conn.Delete(msg.id)
	_, _, err = conn.Reserve(1e6)
	c.Assert(err, NotNil)
	time.Sleep(1e9)
	id, _, err := conn.Reserve(1e6)
	c.Assert(err, IsNil)
	c.Assert(id, Equals, msg.id)
}

func (s *S) TestPutAndGetFromSpecificQueue(c *C) {
	msg := Message{
		Action: "do-something",
		Args:   []string{"everything"},
	}
	err := msg.Put("here", 0)
	c.Assert(err, IsNil)
	defer msg.Delete()
	_, err = Get("default", 1e6)
	c.Assert(err, NotNil)
	got, err := Get("here", 1e6)
	c.Assert(err, IsNil)
	c.Assert(got.Action, Equals, "do-something")
	c.Assert(got.Args, DeepEquals, []string{"everything"})
}

func (s *S) TestGet(c *C) {
	msg := Message{
		Action: "regenerate-apprc",
		Args:   []string{"myapprc"},
	}
	err := msg.Put("default", 0)
	c.Assert(err, IsNil)
	defer conn.Delete(msg.id)
	got, err := Get("default", 1e6)
	c.Assert(err, IsNil)
	c.Assert(*got, DeepEquals, msg)
}

func (s *S) TestGetConnectionError(c *C) {
	old, _ := config.Get("queue-server")
	defer config.Set("queue-server", old)
	config.Unset("queue-server")
	msg, err := Get("default", 1e6)
	c.Assert(msg, IsNil)
	c.Assert(err, NotNil)
}

func (s *S) TestGetFromEmptyQueue(c *C) {
	msg, err := Get("default", 1e6)
	c.Assert(msg, IsNil)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Timed out waiting for message after 1ms.")
}

func (s *S) TestGetInvalidMessage(c *C) {
	conn, err := connection()
	c.Assert(err, IsNil)
	id, err := conn.Put([]byte("hello world"), 1, 0, 10e9)
	defer conn.Delete(id) // sanity
	msg, err := Get("default", 1e6)
	c.Assert(msg, IsNil)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, `Invalid message: "hello world"`)
	_, _, err = conn.Reserve(1e6)
	c.Assert(err, NotNil)
	c.Assert(err, ErrorMatches, "^.*TIMED_OUT$")
}

func (s *S) TestRelease(c *C) {
	conn, err := connection()
	c.Assert(err, IsNil)
	msg := Message{Action: "do-something"}
	err = msg.Put("default", 0)
	c.Assert(err, IsNil)
	defer msg.Delete()
	copy, err := Get("default", 1e6)
	c.Assert(err, IsNil)
	err = msg.Release(0)
	c.Assert(err, IsNil)
	id, _, err := conn.Reserve(1e6)
	c.Assert(err, IsNil)
	c.Assert(id, Equals, copy.id)
}

func (s *S) TestReleaseWithDelay(c *C) {
	conn, err := connection()
	c.Assert(err, IsNil)
	msg := Message{Action: "do-something"}
	err = msg.Put("default", 0)
	c.Assert(err, IsNil)
	defer msg.Delete()
	copy, err := Get("default", 1e6)
	c.Assert(err, IsNil)
	err = msg.Release(1e9)
	c.Assert(err, IsNil)
	_, _, err = conn.Reserve(1e6)
	c.Assert(err, NotNil)
	time.Sleep(1e9)
	id, _, err := conn.Reserve(1e6)
	c.Assert(err, IsNil)
	c.Assert(id, Equals, copy.id)
}

func (s *S) TestReleaseMessageWithoutId(c *C) {
	msg := Message{Action: "do-something"}
	err := msg.Release(0)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Unknown message.")
}

func (s *S) TestReleaseUnknownMessage(c *C) {
	msg := Message{Action: "do-otherthing", id: 12884}
	err := msg.Release(0)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Message not found.")
}

func (s *S) TestReleaseConnectionError(c *C) {
	old, _ := config.Get("queue-server")
	defer config.Set("queue-server", old)
	config.Unset("queue-server")
	err := (&Message{id: 1}).Release(0)
	c.Assert(err, NotNil)
}

func (s *S) TestDelete(c *C) {
	msg := Message{
		Action: "create-app",
		Args:   []string{"something"},
	}
	err := msg.Put("default", 0)
	c.Assert(err, IsNil)
	defer conn.Delete(msg.id)
	err = msg.Delete()
	c.Assert(err, IsNil)
}

func (s *S) TestDeleteConnectionError(c *C) {
	old, _ := config.Get("queue-server")
	defer config.Set("queue-server", old)
	config.Unset("queue-server")
	err := (&Message{}).Delete()
	c.Assert(err, NotNil)
}

func (s *S) TestDeleteUnknownMessage(c *C) {
	msg := Message{
		Action: "create-app",
		Args:   []string{"something"},
		id:     837826742,
	}
	err := msg.Delete()
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Message not found.")
}

func (s *S) TestDeleteMessageWithoutId(c *C) {
	msg := Message{
		Action: "create-app",
		Args:   []string{"something"},
	}
	err := msg.Delete()
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Unknown message.")
}

func cleanQ(c *C) {
	cn, err := connection()
	c.Assert(err, IsNil)
	var id uint64
	for err == nil {
		if id, _, err = cn.Reserve(1e6); err == nil {
			err = cn.Delete(id)
		}
	}
}
<commit_msg>queue: change TIME_OUT test<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package queue

import (
	"bytes"
	"encoding/gob"
	"github.com/globocom/config"
	. "launchpad.net/gocheck"
	"testing"
	"time"
)

func Test(t *testing.T) {
	TestingT(t)
}

type S struct{}

var _ = Suite(&S{})

func (s *S) SetUpSuite(c *C) {
	config.Set("queue-server", "127.0.0.1:11300")

	// Cleaning the queue. All tests must clean its mess, but we can't
	// guarante the state of the queue before running them.
	cleanQ(c)
}

func (s *S) SetUpTest(c *C) {
	conn = nil
}

func (s *S) TestConnection(c *C) {
	cn, err := connection()
	c.Assert(err, IsNil)
	defer cn.Close()
	tubes, err := cn.ListTubes()
	c.Assert(err, IsNil)
	c.Assert(tubes[0], Equals, "default")
}

func (s *S) TestConnectionQueueServerUndefined(c *C) {
	old, _ := config.Get("queue-server")
	config.Unset("queue-server")
	defer config.Set("queue-server", old)
	conn, err := connection()
	c.Assert(conn, IsNil)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, `"queue-server" is not defined in config file.`)
}

func (s *S) TestConnectionResfused(c *C) {
	old, _ := config.Get("queue-server")
	config.Set("queue-server", "127.0.0.1:11301")
	defer config.Set("queue-server", old)
	conn, err := connection()
	c.Assert(conn, IsNil)
	c.Assert(err, NotNil)
}

func (s *S) TestConnectionDoubleCall(c *C) {
	cn1, err := connection()
	c.Assert(err, IsNil)
	defer cn1.Close()
	c.Assert(cn1, Equals, conn)
	cn2, err := connection()
	c.Assert(err, IsNil)
	c.Assert(cn2, Equals, cn1)
}

func (s *S) TestConnectionClosed(c *C) {
	cn1, err := connection()
	c.Assert(err, IsNil)
	cn1.Close()
	cn2, err := connection()
	c.Assert(err, IsNil)
	defer cn2.Close()
	_, err = cn2.ListTubes()
	c.Assert(err, IsNil)
}

func (s *S) TestPut(c *C) {
	msg := Message{
		Action: "regenerate-apprc",
		Args:   []string{"myapp"},
	}
	err := msg.Put("default", 0)
	c.Assert(err, IsNil)
	c.Assert(msg.id, Not(Equals), 0)
	defer conn.Delete(msg.id)
	id, body, err := conn.Reserve(1e6)
	c.Assert(err, IsNil)
	c.Assert(id, Equals, msg.id)
	var got Message
	buf := bytes.NewBuffer(body)
	err = gob.NewDecoder(buf).Decode(&got)
	c.Assert(err, IsNil)
	got.id = msg.id
	c.Assert(got, DeepEquals, msg)
}

func (s *S) TestPutConnectionFailure(c *C) {
	old, _ := config.Get("queue-server")
	defer config.Set("queue-server", old)
	config.Unset("queue-server")
	msg := Message{Action: "regenerate-apprc"}
	err := msg.Put("default", 0)
	c.Assert(err, NotNil)
}

func (s *S) TestPutWithDelay(c *C) {
	msg := Message{
		Action: "do-something",
		Args:   []string{"nothing"},
	}
	err := msg.Put("default", 1e9)
	c.Assert(err, IsNil)
	defer conn.Delete(msg.id)
	_, _, err = conn.Reserve(1e6)
	c.Assert(err, NotNil)
	time.Sleep(1e9)
	id, _, err := conn.Reserve(1e6)
	c.Assert(err, IsNil)
	c.Assert(id, Equals, msg.id)
}

func (s *S) TestPutAndGetFromSpecificQueue(c *C) {
	msg := Message{
		Action: "do-something",
		Args:   []string{"everything"},
	}
	err := msg.Put("here", 0)
	c.Assert(err, IsNil)
	defer msg.Delete()
	_, err = Get("default", 1e6)
	c.Assert(err, NotNil)
	got, err := Get("here", 1e6)
	c.Assert(err, IsNil)
	c.Assert(got.Action, Equals, "do-something")
	c.Assert(got.Args, DeepEquals, []string{"everything"})
}

func (s *S) TestGet(c *C) {
	msg := Message{
		Action: "regenerate-apprc",
		Args:   []string{"myapprc"},
	}
	err := msg.Put("default", 0)
	c.Assert(err, IsNil)
	defer conn.Delete(msg.id)
	got, err := Get("default", 1e6)
	c.Assert(err, IsNil)
	c.Assert(*got, DeepEquals, msg)
}

func (s *S) TestGetConnectionError(c *C) {
	old, _ := config.Get("queue-server")
	defer config.Set("queue-server", old)
	config.Unset("queue-server")
	msg, err := Get("default", 1e6)
	c.Assert(msg, IsNil)
	c.Assert(err, NotNil)
}

func (s *S) TestGetFromEmptyQueue(c *C) {
	msg, err := Get("default", 1e6)
	c.Assert(msg, IsNil)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Timed out waiting for message after 1ms.")
}

func (s *S) TestGetInvalidMessage(c *C) {
	conn, err := connection()
	c.Assert(err, IsNil)
	id, err := conn.Put([]byte("hello world"), 1, 0, 10e9)
	defer conn.Delete(id) // sanity
	msg, err := Get("default", 1e6)
	c.Assert(msg, IsNil)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, `Invalid message: "hello world"`)
	_, _, err = conn.Reserve(1e6)
	c.Assert(err, NotNil)
	c.Assert(timeoutRegexp.MatchString(err.Error()), Equals, true)
}

func (s *S) TestRelease(c *C) {
	conn, err := connection()
	c.Assert(err, IsNil)
	msg := Message{Action: "do-something"}
	err = msg.Put("default", 0)
	c.Assert(err, IsNil)
	defer msg.Delete()
	copy, err := Get("default", 1e6)
	c.Assert(err, IsNil)
	err = msg.Release(0)
	c.Assert(err, IsNil)
	id, _, err := conn.Reserve(1e6)
	c.Assert(err, IsNil)
	c.Assert(id, Equals, copy.id)
}

func (s *S) TestReleaseWithDelay(c *C) {
	conn, err := connection()
	c.Assert(err, IsNil)
	msg := Message{Action: "do-something"}
	err = msg.Put("default", 0)
	c.Assert(err, IsNil)
	defer msg.Delete()
	copy, err := Get("default", 1e6)
	c.Assert(err, IsNil)
	err = msg.Release(1e9)
	c.Assert(err, IsNil)
	_, _, err = conn.Reserve(1e6)
	c.Assert(err, NotNil)
	time.Sleep(1e9)
	id, _, err := conn.Reserve(1e6)
	c.Assert(err, IsNil)
	c.Assert(id, Equals, copy.id)
}

func (s *S) TestReleaseMessageWithoutId(c *C) {
	msg := Message{Action: "do-something"}
	err := msg.Release(0)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Unknown message.")
}

func (s *S) TestReleaseUnknownMessage(c *C) {
	msg := Message{Action: "do-otherthing", id: 12884}
	err := msg.Release(0)
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Message not found.")
}

func (s *S) TestReleaseConnectionError(c *C) {
	old, _ := config.Get("queue-server")
	defer config.Set("queue-server", old)
	config.Unset("queue-server")
	err := (&Message{id: 1}).Release(0)
	c.Assert(err, NotNil)
}

func (s *S) TestDelete(c *C) {
	msg := Message{
		Action: "create-app",
		Args:   []string{"something"},
	}
	err := msg.Put("default", 0)
	c.Assert(err, IsNil)
	defer conn.Delete(msg.id)
	err = msg.Delete()
	c.Assert(err, IsNil)
}

func (s *S) TestDeleteConnectionError(c *C) {
	old, _ := config.Get("queue-server")
	defer config.Set("queue-server", old)
	config.Unset("queue-server")
	err := (&Message{}).Delete()
	c.Assert(err, NotNil)
}

func (s *S) TestDeleteUnknownMessage(c *C) {
	msg := Message{
		Action: "create-app",
		Args:   []string{"something"},
		id:     837826742,
	}
	err := msg.Delete()
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Message not found.")
}

func (s *S) TestDeleteMessageWithoutId(c *C) {
	msg := Message{
		Action: "create-app",
		Args:   []string{"something"},
	}
	err := msg.Delete()
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "Unknown message.")
}

func cleanQ(c *C) {
	cn, err := connection()
	c.Assert(err, IsNil)
	var id uint64
	for err == nil {
		if id, _, err = cn.Reserve(1e6); err == nil {
			err = cn.Delete(id)
		}
	}
}
<|endoftext|>


<commit_before>package main

import "flag"
import "fmt"
import "sort"
import (
	"bufio"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"
)

func Usage() string {
	return fmt.Sprintf(
		`
    %sUsage of %s:%s

        %sfindref%s %s[options]%s %smatch_regex%s %s[start_dir]%s %s[filename_regex]%s

    %sArguments:%s

        %smatch_regex:  This is an RE2 regular expression that will be matched against lines
                      in each file, with matches being displayed to the user.%s

        %sstart_dir:  This optional argument sets the starting directory to crawl looking
                    for eligible files with lines matching match_regex.  Default value
                    is the current working directory, AKA $PWD or '.'%s

        %sfilename_regex:  This optional argument restricts the set of files checked for
                         matching lines.  Eligible files must match this expression.
                         Default value matches all files%s

    %sOptions:%s
        %s
        -a | --all
							Aggressively search for matches (implies: -i -h)
        -d | --debug
              Enable debug mode
        -f | --filename-only
              Display only filenames with matches, not the matches themselves
        -h | --hidden
              Include hidden files and files in hidden directories
        -i | --ignore-case
              Ignore case in regex (overrides smart-case)
        -m | --match-case
              Match regex case (if unset smart-case is used)
        -n | --no-color
              Disable colorized output
        -s | --stats
              Track basic statistics and print them on exit
        -v | --version
              Print current version and exit
%s
`,
		colors.Red, versionString(false), colors.Restore,
		colors.Brown, colors.Restore,
		colors.Green, colors.Restore,
		colors.Cyan, colors.Restore,
		colors.Blue, colors.Restore,
		colors.Purple, colors.Restore,
		colors.Red, colors.Restore,
		colors.Cyan, colors.Restore,
		colors.Blue, colors.Restore,
		colors.Purple, colors.Restore,
		colors.Red, colors.Restore,
		colors.Green, colors.Restore,
	)
}

const Version = "1.0.1"
const Date = "2020-04-21"

var FILE_PROCESSING_COMPLETE error = nil

var settings *Settings = NewSettings()
var statistics *Statistics = NewStatistics()
var colors *Colors = NewColors()

var filenameOnlyFiles []string = make([]string, 0, 100)
var filesToScan []FileToScan = make([]FileToScan, 0, 100)

func usageAndExit() {
	flag.Usage()
	os.Exit(1)
}

func debug(a ...interface{}) {
	if settings.Debug {
		fmt.Println(a...)
	}
}

func containsNullByte(line []byte) bool {
	for _, el := range line {
		if el == 0 {
			return true
		}
	}
	return false
}

func checkForMatches(path string) []Match {
	debug(colors.Blue+"Checking file for matches:"+colors.Restore, path)
	file, err := os.Open(path)
	if err != nil {
		fmt.Println(colors.Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+colors.Restore, err)
		debug(colors.Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+colors.Restore, err)
		return []Match{Match{path, 0, []byte{}, []int{}}}
	}
	defer func() {
		if path == "src/main/java/com/canopy/service/EFileService.java" {
			fmt.Println("Closing the file: " + path)
		}
		file.Close()
	}()

	retval := make([]Match, 50)

	scanner := bufio.NewScanner(file)
	var lineNumber int = 0
	for scanner.Scan() {
		lineNumber += 1
		line := scanner.Bytes()
		statistics.IncrLineCount()
		if containsNullByte(line) {
			// This is a binary file.  Skip it!
			debug(colors.Blue+"Not processing binary file:"+colors.Restore, path)
			return retval
		}
		if matchIndex := settings.MatchRegex.FindIndex(line); matchIndex != nil {
			// we have a match! loc == nil means no match so just ignore that case
			statistics.IncrMatchCount()
			if settings.FilenameOnly {
				filenameOnlyFiles = append(filenameOnlyFiles, path)
			} else {
				m := Match{path, lineNumber, line, matchIndex}
				m.printMatch()
				retval = append(retval, Match{path, lineNumber, line, matchIndex})
			}
		}
	}

	if err := scanner.Err(); err != nil {
		debug(colors.Red+"Error scanning line from file '"+path+"'. File will be skipped.  Err: "+colors.Restore, err)
	}
	return retval
}

func processFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		debug("filepath.Walk encountered error with path '"+path+"'", err)
		return FILE_PROCESSING_COMPLETE
	}

	if info.IsDir() {
		if settings.IsHidden(path) {
			debug(colors.Blue, "Directory", path, "is hidden and will be pruned", colors.Restore)
			return filepath.SkipDir // skip the whole sub-contents of this hidden directory
		} else {
			return FILE_PROCESSING_COMPLETE
		}
	}

	if settings.PassesFileFilter(path) {
		debug(colors.Blue+"Passes file filter:", path)
		if settings.IsHidden(path) {
			debug(colors.Blue + "Hidden file '" + colors.Restore + path + colors.Blue + "' not processed")
			return FILE_PROCESSING_COMPLETE
		}
		statistics.IncrFilesToScan()
		defer statistics.IncrFileCount()

		filesToScan = append(filesToScan, FileToScan{Path: path, Info: info, Err: err})
	} else {
		debug(colors.Blue + "Ignoring file cause it doesn't match filter: " + colors.Restore + path)
	}
	return FILE_PROCESSING_COMPLETE
}

func getMatchRegex(ignoreCase bool, matchCase bool, usersRegex string) *regexp.Regexp {
	// If ignore case is set, ignore the case of the regex.
	// if match-case is not set, use smart case which means if it's all lower case be case-insensitive,
	// but if there's capitals then be case-sensitive
	if ignoreCase || (!matchCase && !regexp.MustCompile("[A-Z]").MatchString(usersRegex)) {
		debug(colors.Blue, "Match regex will be case-insensitive", colors.Restore)
		return regexp.MustCompile("(?i)" + usersRegex)
	} else {
		debug(colors.Blue, "Match regex will be exactly as user provided", colors.Restore)
		return regexp.MustCompile(usersRegex)
	}
}

func versionString(color bool) string {
	if color {
		return fmt.Sprintf("%s%s%s%s%s%s%s", colors.Cyan, "findref (version ", Version, " released on ", Date, ")", colors.Restore)
	} else {
		return fmt.Sprintf("%s%s%s%s%s", "findref (version ", Version, " released on ", Date, ")")
	}
}

func printVersion() {
	fmt.Println(versionString(true))
}

func uniq(stringSlice []string) []string {
	/* There is no built-in uniq function for slices, so we will use a map */
	stringMap := make(map[string]bool)
	for _, v := range stringSlice {
		stringMap[v] = true
	}
	retval := make([]string, len(stringMap), len(stringMap))
	i := 0
	for key := range stringMap {
		retval[i] = key
		i++
	}
	return retval
}

func finishAndExit() {
	if settings.FilenameOnly {
		filenames := uniq(filenameOnlyFiles)
		sort.Strings(filenames)
		for _, filename := range filenames {
			fmt.Printf("%s%s%s\n", colors.Purple, filename, colors.Restore)
		}
	}

	if settings.TrackStats {
		fmt.Printf("%sElapsed time:%s  %s\n", colors.Cyan, colors.Restore, statistics.ElapsedTime().String())
		fmt.Printf("%sLines scanned:%s %d\n", colors.Cyan, colors.Restore, statistics.LineCount())
		fmt.Printf("%sFiles scanned:%s %d\n", colors.Cyan, colors.Restore, statistics.FileCount())
		fmt.Printf("%sMatches found:%s %d\n", colors.Cyan, colors.Restore, statistics.MatchCount())
	}
}

func worker(id int, jobs <-chan string, results chan<- []Match) {
	for file := range jobs {
		debug(colors.Blue, "Worker number", id, "started file", file, colors.Restore)
		results <- checkForMatches(file)
		debug(colors.Blue, "Worker number", id, "finished file", file, colors.Restore)
	}
}

func main() {
	aPtr := flag.Bool("a", false, "Alias for --all")
	sPtr := flag.Bool("s", false, "Alias for --stats")
	dPtr := flag.Bool("d", false, "Alias for --debug")
	hPtr := flag.Bool("h", false, "Alias for --hidden")
	vPtr := flag.Bool("v", false, "Alias for --version")
	nPtr := flag.Bool("n", false, "Alias for --no-color")
	mPtr := flag.Bool("m", false, "Alias for --match-case")
	iPtr := flag.Bool("i", false, "Alias for --ignore-case")
	fPtr := flag.Bool("f", false, "Alias for --filename-only")
	allPtr := flag.Bool("all", false, "Include hidden files and ignore case (implies: -i -h)")
	helpPtr := flag.Bool("help", false, "Show usage")
	statsPtr := flag.Bool("stats", false, "Track and display statistics")
	debugPtr := flag.Bool("debug", false, "Enable debug mode")
	hiddenPtr := flag.Bool("hidden", false, "Include hidden files and files in hidden directories")
	versionPtr := flag.Bool("version", false, "Print current version and exit")
	nocolorPtr := flag.Bool("no-color", false, "Don't use color in output")
	matchCasePtr := flag.Bool("match-case", false, "Match regex case (if unset smart-case is used)")
	ignoreCasePtr := flag.Bool("ignore-case", false, "Ignore case in regex (overrides smart-case)")
	filenameOnlyPtr := flag.Bool("filename-only", false, "Display only filenames with matches")

	flag.Usage = func() {
		fmt.Print(Usage())
	}
	flag.Parse()

	if *vPtr || *versionPtr {
		printVersion()
		os.Exit(0)
	}

	if *helpPtr {
		usageAndExit()
	}

	if *nPtr || *nocolorPtr {
		debug("Color output is disabled")
		colors.ZeroColors()
	}

	settings.Debug = *debugPtr || *dPtr
	settings.TrackStats = *statsPtr || *sPtr
	settings.FilenameOnly = *filenameOnlyPtr || *fPtr
	settings.IncludeHidden = *hiddenPtr || *hPtr
	settings.IncludeHidden = *allPtr || *aPtr // -a implies -h
	*matchCasePtr = *matchCasePtr || *mPtr
	*ignoreCasePtr = *ignoreCasePtr || *iPtr
	*ignoreCasePtr = *allPtr || *aPtr // -a implies -i

	if settings.TrackStats {
		statistics.startTime = time.Now()
		debug(colors.Blue, "Start time is:", colors.Restore, statistics.startTime.String())
	}

	debug(colors.Blue, "stats enabled: ", colors.Restore, settings.TrackStats)
	debug(colors.Blue, "match-case enabled: ", colors.Restore, *matchCasePtr)
	debug(colors.Blue, "ignore-case enabled: ", colors.Restore, *ignoreCasePtr)
	debug(colors.Blue, "include hidden files: ", colors.Restore, settings.IncludeHidden)
	debug(colors.Blue, "debug mode: ", colors.Restore, settings.Debug)
	debug(colors.Blue, "filename only: ", colors.Restore, settings.FilenameOnly)

	rootDir := "."

	if len(flag.Args()) < 1 {
		fmt.Errorf("%s", "Must specify regex to match against files")
		usageAndExit()
	} else if len(flag.Args()) > 3 {
		fmt.Errorf("%s", "Too many args (expected 1 <= 3)")
		usageAndExit()
	} else {
		settings.MatchRegex = getMatchRegex(*ignoreCasePtr, *matchCasePtr, flag.Args()[0])

		if len(flag.Args()) >= 2 {
			rootDir = flag.Args()[1]
		}
		if len(flag.Args()) == 3 {
			settings.FilenameRegex = regexp.MustCompile(flag.Args()[2])
		}
	}

	debug(colors.Blue, "matchRegex: ", colors.Restore, settings.MatchRegex.String())
	debug(colors.Blue, "rootDir: ", colors.Restore, rootDir)
	debug(colors.Blue, "fileRegex: ", colors.Restore, settings.FilenameRegex.String())

	runtime.GOMAXPROCS(runtime.NumCPU())
	filepath.Walk(rootDir, processFile)

	// TODO: set niceness value to low

	jobs := make(chan string, len(filesToScan))
	results := make(chan []Match, 100)

	// two workers for each core
	numWorkers := runtime.NumCPU() * 1
	for w := 0; w < numWorkers; w++ {
		go worker(w, jobs, results)
	}

	// create a job for each file to scan
	for _, val := range filesToScan {
		jobs <- val.Path
	}
	close(jobs)

	for r := 0; r < len(filesToScan); r++ {
		result := <-results
		for _, res := range result {
			if res.hasMatch() {
				//res.printMatch()
			}
		}
	}

	finishAndExit()
}
<commit_msg>Don't color filename in debug output<commit_after>package main

import "flag"
import "fmt"
import "sort"
import (
	"bufio"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"
)

func Usage() string {
	return fmt.Sprintf(
		`
    %sUsage of %s:%s

        %sfindref%s %s[options]%s %smatch_regex%s %s[start_dir]%s %s[filename_regex]%s

    %sArguments:%s

        %smatch_regex:  This is an RE2 regular expression that will be matched against lines
                      in each file, with matches being displayed to the user.%s

        %sstart_dir:  This optional argument sets the starting directory to crawl looking
                    for eligible files with lines matching match_regex.  Default value
                    is the current working directory, AKA $PWD or '.'%s

        %sfilename_regex:  This optional argument restricts the set of files checked for
                         matching lines.  Eligible files must match this expression.
                         Default value matches all files%s

    %sOptions:%s
        %s
        -a | --all
							Aggressively search for matches (implies: -i -h)
        -d | --debug
              Enable debug mode
        -f | --filename-only
              Display only filenames with matches, not the matches themselves
        -h | --hidden
              Include hidden files and files in hidden directories
        -i | --ignore-case
              Ignore case in regex (overrides smart-case)
        -m | --match-case
              Match regex case (if unset smart-case is used)
        -n | --no-color
              Disable colorized output
        -s | --stats
              Track basic statistics and print them on exit
        -v | --version
              Print current version and exit
%s
`,
		colors.Red, versionString(false), colors.Restore,
		colors.Brown, colors.Restore,
		colors.Green, colors.Restore,
		colors.Cyan, colors.Restore,
		colors.Blue, colors.Restore,
		colors.Purple, colors.Restore,
		colors.Red, colors.Restore,
		colors.Cyan, colors.Restore,
		colors.Blue, colors.Restore,
		colors.Purple, colors.Restore,
		colors.Red, colors.Restore,
		colors.Green, colors.Restore,
	)
}

const Version = "1.0.1"
const Date = "2020-04-21"

var FILE_PROCESSING_COMPLETE error = nil

var settings *Settings = NewSettings()
var statistics *Statistics = NewStatistics()
var colors *Colors = NewColors()

var filenameOnlyFiles []string = make([]string, 0, 100)
var filesToScan []FileToScan = make([]FileToScan, 0, 100)

func usageAndExit() {
	flag.Usage()
	os.Exit(1)
}

func debug(a ...interface{}) {
	if settings.Debug {
		fmt.Println(a...)
	}
}

func containsNullByte(line []byte) bool {
	for _, el := range line {
		if el == 0 {
			return true
		}
	}
	return false
}

func checkForMatches(path string) []Match {
	debug(colors.Blue+"Checking file for matches:"+colors.Restore, path)
	file, err := os.Open(path)
	if err != nil {
		fmt.Println(colors.Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+colors.Restore, err)
		debug(colors.Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+colors.Restore, err)
		return []Match{Match{path, 0, []byte{}, []int{}}}
	}
	defer func() {
		if path == "src/main/java/com/canopy/service/EFileService.java" {
			fmt.Println("Closing the file: " + path)
		}
		file.Close()
	}()

	retval := make([]Match, 50)

	scanner := bufio.NewScanner(file)
	var lineNumber int = 0
	for scanner.Scan() {
		lineNumber += 1
		line := scanner.Bytes()
		statistics.IncrLineCount()
		if containsNullByte(line) {
			// This is a binary file.  Skip it!
			debug(colors.Blue+"Not processing binary file:"+colors.Restore, path)
			return retval
		}
		if matchIndex := settings.MatchRegex.FindIndex(line); matchIndex != nil {
			// we have a match! loc == nil means no match so just ignore that case
			statistics.IncrMatchCount()
			if settings.FilenameOnly {
				filenameOnlyFiles = append(filenameOnlyFiles, path)
			} else {
				m := Match{path, lineNumber, line, matchIndex}
				m.printMatch()
				retval = append(retval, Match{path, lineNumber, line, matchIndex})
			}
		}
	}

	if err := scanner.Err(); err != nil {
		debug(colors.Red+"Error scanning line from file '"+path+"'. File will be skipped.  Err: "+colors.Restore, err)
	}
	return retval
}

func processFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		debug("filepath.Walk encountered error with path '"+path+"'", err)
		return FILE_PROCESSING_COMPLETE
	}

	if info.IsDir() {
		if settings.IsHidden(path) {
			debug(colors.Blue, "Directory", path, "is hidden and will be pruned", colors.Restore)
			return filepath.SkipDir // skip the whole sub-contents of this hidden directory
		} else {
			return FILE_PROCESSING_COMPLETE
		}
	}

	if settings.PassesFileFilter(path) {
		debug(colors.Blue+"Passes file filter:", path)
		if settings.IsHidden(path) {
			debug(colors.Blue + "Hidden file '" + colors.Restore + path + colors.Blue + "' not processed")
			return FILE_PROCESSING_COMPLETE
		}
		statistics.IncrFilesToScan()
		defer statistics.IncrFileCount()

		filesToScan = append(filesToScan, FileToScan{Path: path, Info: info, Err: err})
	} else {
		debug(colors.Blue + "Ignoring file cause it doesn't match filter: " + colors.Restore + path)
	}
	return FILE_PROCESSING_COMPLETE
}

func getMatchRegex(ignoreCase bool, matchCase bool, usersRegex string) *regexp.Regexp {
	// If ignore case is set, ignore the case of the regex.
	// if match-case is not set, use smart case which means if it's all lower case be case-insensitive,
	// but if there's capitals then be case-sensitive
	if ignoreCase || (!matchCase && !regexp.MustCompile("[A-Z]").MatchString(usersRegex)) {
		debug(colors.Blue, "Match regex will be case-insensitive", colors.Restore)
		return regexp.MustCompile("(?i)" + usersRegex)
	} else {
		debug(colors.Blue, "Match regex will be exactly as user provided", colors.Restore)
		return regexp.MustCompile(usersRegex)
	}
}

func versionString(color bool) string {
	if color {
		return fmt.Sprintf("%s%s%s%s%s%s%s", colors.Cyan, "findref (version ", Version, " released on ", Date, ")", colors.Restore)
	} else {
		return fmt.Sprintf("%s%s%s%s%s", "findref (version ", Version, " released on ", Date, ")")
	}
}

func printVersion() {
	fmt.Println(versionString(true))
}

func uniq(stringSlice []string) []string {
	/* There is no built-in uniq function for slices, so we will use a map */
	stringMap := make(map[string]bool)
	for _, v := range stringSlice {
		stringMap[v] = true
	}
	retval := make([]string, len(stringMap), len(stringMap))
	i := 0
	for key := range stringMap {
		retval[i] = key
		i++
	}
	return retval
}

func finishAndExit() {
	if settings.FilenameOnly {
		filenames := uniq(filenameOnlyFiles)
		sort.Strings(filenames)
		for _, filename := range filenames {
			fmt.Printf("%s%s%s\n", colors.Purple, filename, colors.Restore)
		}
	}

	if settings.TrackStats {
		fmt.Printf("%sElapsed time:%s  %s\n", colors.Cyan, colors.Restore, statistics.ElapsedTime().String())
		fmt.Printf("%sLines scanned:%s %d\n", colors.Cyan, colors.Restore, statistics.LineCount())
		fmt.Printf("%sFiles scanned:%s %d\n", colors.Cyan, colors.Restore, statistics.FileCount())
		fmt.Printf("%sMatches found:%s %d\n", colors.Cyan, colors.Restore, statistics.MatchCount())
	}
}

func worker(id int, jobs <-chan string, results chan<- []Match) {
	for file := range jobs {
		debug(colors.Blue, "Worker number", id, "started file", colors.Restore, file)
		results <- checkForMatches(file)
		debug(colors.Blue, "Worker number", id, "finished file", colors.Restore, file)
	}
}

func main() {
	aPtr := flag.Bool("a", false, "Alias for --all")
	sPtr := flag.Bool("s", false, "Alias for --stats")
	dPtr := flag.Bool("d", false, "Alias for --debug")
	hPtr := flag.Bool("h", false, "Alias for --hidden")
	vPtr := flag.Bool("v", false, "Alias for --version")
	nPtr := flag.Bool("n", false, "Alias for --no-color")
	mPtr := flag.Bool("m", false, "Alias for --match-case")
	iPtr := flag.Bool("i", false, "Alias for --ignore-case")
	fPtr := flag.Bool("f", false, "Alias for --filename-only")
	allPtr := flag.Bool("all", false, "Include hidden files and ignore case (implies: -i -h)")
	helpPtr := flag.Bool("help", false, "Show usage")
	statsPtr := flag.Bool("stats", false, "Track and display statistics")
	debugPtr := flag.Bool("debug", false, "Enable debug mode")
	hiddenPtr := flag.Bool("hidden", false, "Include hidden files and files in hidden directories")
	versionPtr := flag.Bool("version", false, "Print current version and exit")
	nocolorPtr := flag.Bool("no-color", false, "Don't use color in output")
	matchCasePtr := flag.Bool("match-case", false, "Match regex case (if unset smart-case is used)")
	ignoreCasePtr := flag.Bool("ignore-case", false, "Ignore case in regex (overrides smart-case)")
	filenameOnlyPtr := flag.Bool("filename-only", false, "Display only filenames with matches")

	flag.Usage = func() {
		fmt.Print(Usage())
	}
	flag.Parse()

	if *vPtr || *versionPtr {
		printVersion()
		os.Exit(0)
	}

	if *helpPtr {
		usageAndExit()
	}

	if *nPtr || *nocolorPtr {
		debug("Color output is disabled")
		colors.ZeroColors()
	}

	settings.Debug = *debugPtr || *dPtr
	settings.TrackStats = *statsPtr || *sPtr
	settings.FilenameOnly = *filenameOnlyPtr || *fPtr
	settings.IncludeHidden = *hiddenPtr || *hPtr
	settings.IncludeHidden = *allPtr || *aPtr // -a implies -h
	*matchCasePtr = *matchCasePtr || *mPtr
	*ignoreCasePtr = *ignoreCasePtr || *iPtr
	*ignoreCasePtr = *allPtr || *aPtr // -a implies -i

	if settings.TrackStats {
		statistics.startTime = time.Now()
		debug(colors.Blue, "Start time is:", colors.Restore, statistics.startTime.String())
	}

	debug(colors.Blue, "stats enabled: ", colors.Restore, settings.TrackStats)
	debug(colors.Blue, "match-case enabled: ", colors.Restore, *matchCasePtr)
	debug(colors.Blue, "ignore-case enabled: ", colors.Restore, *ignoreCasePtr)
	debug(colors.Blue, "include hidden files: ", colors.Restore, settings.IncludeHidden)
	debug(colors.Blue, "debug mode: ", colors.Restore, settings.Debug)
	debug(colors.Blue, "filename only: ", colors.Restore, settings.FilenameOnly)

	rootDir := "."

	if len(flag.Args()) < 1 {
		fmt.Errorf("%s", "Must specify regex to match against files")
		usageAndExit()
	} else if len(flag.Args()) > 3 {
		fmt.Errorf("%s", "Too many args (expected 1 <= 3)")
		usageAndExit()
	} else {
		settings.MatchRegex = getMatchRegex(*ignoreCasePtr, *matchCasePtr, flag.Args()[0])

		if len(flag.Args()) >= 2 {
			rootDir = flag.Args()[1]
		}
		if len(flag.Args()) == 3 {
			settings.FilenameRegex = regexp.MustCompile(flag.Args()[2])
		}
	}

	debug(colors.Blue, "matchRegex: ", colors.Restore, settings.MatchRegex.String())
	debug(colors.Blue, "rootDir: ", colors.Restore, rootDir)
	debug(colors.Blue, "fileRegex: ", colors.Restore, settings.FilenameRegex.String())

	runtime.GOMAXPROCS(runtime.NumCPU())
	filepath.Walk(rootDir, processFile)

	// TODO: set niceness value to low

	jobs := make(chan string, len(filesToScan))
	results := make(chan []Match, 100)

	// two workers for each core
	numWorkers := runtime.NumCPU() * 1
	for w := 0; w < numWorkers; w++ {
		go worker(w, jobs, results)
	}

	// create a job for each file to scan
	for _, val := range filesToScan {
		jobs <- val.Path
	}
	close(jobs)

	for r := 0; r < len(filesToScan); r++ {
		result := <-results
		for _, res := range result {
			if res.hasMatch() {
				//res.printMatch()
			}
		}
	}

	finishAndExit()
}
<|endoftext|>


<commit_before><commit_msg>Set up the basics of the CLI<commit_after>package main

import (
	"github.com/codegangsta/cli"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "flasher"
	app.Version = "0.1.0"
	app.Author = "Alex Jackson"
	app.Email = "alex@alexj.org"

	app.Commands = []cli.Command{
		{
			Name:      "flash",
			ShortName: "f",
			Usage:     "flasher flash [flashcard-file.json]",
			Action:    cliFlash,
		},
	}
	app.Run(os.Args)
}

func cliFlash(c *cli.Context) {
}
<|endoftext|>


<commit_before>package readline

import (
	"context"
	"io"
	"strings"
	"unicode"

	"github.com/atotto/clipboard"
)

func keyFuncEnter(ctx context.Context, this *Buffer) Result { // Ctrl-M
	return ENTER
}

func keyFuncIntr(ctx context.Context, this *Buffer) Result { // Ctrl-C
	this.Buffer = this.Buffer[:0]
	this.Cursor = 0
	this.ViewStart = 0
	this.undoes = nil
	return INTR
}

func keyFuncHead(ctx context.Context, this *Buffer) Result { // Ctrl-A
	this.GotoHead()
	this.Cursor = 0
	this.ViewStart = 0
	this.DrawFromHead()
	return CONTINUE
}

func keyFuncBackward(ctx context.Context, this *Buffer) Result { // Ctrl-B
	if this.Cursor <= 0 {
		return CONTINUE
	}
	this.Cursor--
	if this.Cursor < this.ViewStart {
		this.ViewStart--
		this.DrawFromHead()
	} else {
		this.backspace(GetCharWidth(this.Buffer[this.Cursor]))
	}
	return CONTINUE
}

func keyFuncTail(ctx context.Context, this *Buffer) Result { // Ctrl-E
	allength := this.GetWidthBetween(this.ViewStart, len(this.Buffer))
	if allength < this.ViewWidth() {
		this.puts(this.Buffer[this.Cursor:])
		this.Cursor = len(this.Buffer)
	} else {
		this.backspace(this.GetWidthBetween(this.ViewStart, this.Cursor))
		this.ViewStart = len(this.Buffer) - 1
		w := GetCharWidth(this.Buffer[this.ViewStart])
		for {
			if this.ViewStart <= 0 {
				break
			}
			w_ := w + GetCharWidth(this.Buffer[this.ViewStart-1])
			if w_ >= this.ViewWidth() {
				break
			}
			w = w_
			this.ViewStart--
		}
		this.puts(this.Buffer[this.ViewStart:])
		this.Cursor = len(this.Buffer)
	}
	return CONTINUE
}

func keyFuncForward(ctx context.Context, this *Buffer) Result { // Ctrl-F
	if this.Cursor >= len(this.Buffer) {
		return CONTINUE
	}
	w := this.GetWidthBetween(this.ViewStart, this.Cursor+1)
	if w < this.ViewWidth() {
		// No Scroll
		this.putRune(this.Buffer[this.Cursor])
	} else {
		// Right Scroll
		this.GotoHead()
		if GetCharWidth(this.Buffer[this.Cursor]) > GetCharWidth(this.Buffer[this.ViewStart]) {
			this.ViewStart++
		}
		this.ViewStart++
		this.puts(this.Buffer[this.ViewStart : this.Cursor+1])
		this.Eraseline()
	}
	this.Cursor++
	return CONTINUE
}

func keyFuncBackSpace(ctx context.Context, this *Buffer) Result { // Backspace
	if this.Cursor > 0 {
		this.Cursor--
		delw := this.Delete(this.Cursor, 1)
		if this.Cursor >= this.ViewStart {
			this.backspace(delw)
		} else {
			this.ViewStart = this.Cursor
		}
		this.Repaint(this.Cursor, delw)
	}
	return CONTINUE
}

func keyFuncDelete(ctx context.Context, this *Buffer) Result { // Del
	delw := this.Delete(this.Cursor, 1)
	this.Repaint(this.Cursor, delw)
	return CONTINUE
}

func keyFuncDeleteOrAbort(ctx context.Context, this *Buffer) Result { // Ctrl-D
	if len(this.Buffer) > 0 {
		return keyFuncDelete(ctx, this)
	} else {
		return ABORT
	}
}

func keyFuncInsertSelf(ctx context.Context, this *Buffer, keys string) Result {
	if len(keys) == 2 && keys[0] == '\x1B' { // for AltGr-shift
		keys = keys[1:]
	}
	this.InsertString(this.Cursor, keys)

	w := this.GetWidthBetween(this.ViewStart, this.Cursor)
	w1 := GetStringWidth(keys)
	if w+w1 >= this.ViewWidth() {
		// scroll left
		this.GotoHead()
		this.Cursor += len([]rune(keys))
		this.ResetViewStart()
		this.DrawFromHead()
	} else {
		this.Repaint(this.Cursor, -w1)
		this.Cursor += len([]rune(keys))
	}
	return CONTINUE
}

func keyFuncClearAfter(ctx context.Context, this *Buffer) Result {
	clipboard.WriteAll(this.SubString(this.Cursor, len(this.Buffer)))

	this.Eraseline()
	u := &undo_t{
		pos:  this.Cursor,
		text: string(this.Buffer[this.Cursor:]),
	}
	this.undoes = append(this.undoes, u)
	this.Buffer = this.Buffer[:this.Cursor]
	return CONTINUE
}

func keyFuncClear(ctx context.Context, this *Buffer) Result {
	u := &undo_t{
		pos:  0,
		text: string(this.Buffer),
	}
	this.undoes = append(this.undoes, u)
	this.GotoHead()
	this.Eraseline()
	this.Buffer = this.Buffer[:0]
	this.Cursor = 0
	this.ViewStart = 0
	return CONTINUE
}

func keyFuncWordRubout(ctx context.Context, this *Buffer) Result {
	org_cursor := this.Cursor
	for this.Cursor > 0 && unicode.IsSpace(this.Buffer[this.Cursor-1]) {
		this.Cursor--
	}
	i := this.CurrentWordTop()
	clipboard.WriteAll(this.SubString(i, org_cursor))
	keta := this.Delete(i, org_cursor-i)
	if i >= this.ViewStart {
		this.backspace(keta)
	} else {
		this.backspace(this.GetWidthBetween(this.ViewStart, org_cursor))
	}
	this.Cursor = i
	this.Repaint(i, keta)
	return CONTINUE
}

func keyFuncClearBefore(ctx context.Context, this *Buffer) Result {
	this.GotoHead()
	clipboard.WriteAll(this.SubString(0, this.Cursor))
	this.Delete(0, this.Cursor)
	this.Cursor = 0
	this.ViewStart = 0
	this.DrawFromHead()
	return CONTINUE
}

func keyFuncCLS(ctx context.Context, this *Buffer) Result {
	io.WriteString(this.Out, "\x1B[1;1H\x1B[2J")
	this.RepaintAll()
	return CONTINUE
}

func keyFuncRepaintOnNewline(ctx context.Context, this *Buffer) Result {
	this.Out.WriteByte('\n')
	this.RepaintAll()
	return CONTINUE
}

func keyFuncQuotedInsert(ctx context.Context, this *Buffer) Result {
	io.WriteString(this.Out, ansiCursorOn)
	defer io.WriteString(this.Out, ansiCursorOff)

	this.Out.Flush()
	if key, err := this.GetKey(); err == nil {
		return keyFuncInsertSelf(ctx, this, key)
	} else {
		return CONTINUE
	}
}

func keyFuncPaste(ctx context.Context, this *Buffer) Result {
	text, err := clipboard.ReadAll()
	if err != nil {
		return CONTINUE
	}
	this.InsertAndRepaint(text)
	return CONTINUE
}

func keyFuncPasteQuote(ctx context.Context, this *Buffer) Result {
	text, err := clipboard.ReadAll()
	if err != nil {
		return CONTINUE
	}
	if strings.IndexRune(text, ' ') >= 0 &&
		!strings.HasPrefix(text, `"`) {
		text = `"` + strings.Replace(text, `"`, `""`, -1) + `"`
	}
	this.InsertAndRepaint(text)
	return CONTINUE
}

func maxInt(a, b int) int {
	if a < b {
		return b
	} else {
		return a
	}
}

func keyFuncSwapChar(ctx context.Context, this *Buffer) Result {
	if len(this.Buffer) == this.Cursor {
		if this.Cursor < 2 {
			return CONTINUE
		}
		u := &undo_t{
			pos:  this.Cursor,
			del:  2,
			text: string(this.Buffer[this.Cursor-2 : this.Cursor]),
		}
		this.undoes = append(this.undoes, u)
		this.Buffer[this.Cursor-2], this.Buffer[this.Cursor-1] = this.Buffer[this.Cursor-1], this.Buffer[this.Cursor-2]

		redrawStart := maxInt(this.Cursor-2, this.ViewStart)
		this.backspace(this.GetWidthBetween(redrawStart, this.Cursor))
		this.puts(this.Buffer[redrawStart:this.Cursor])
	} else {
		if this.Cursor < 1 {
			return CONTINUE
		}
		u := &undo_t{
			pos:  this.Cursor - 1,
			del:  2,
			text: string(this.Buffer[this.Cursor-1 : this.Cursor+1]),
		}
		this.undoes = append(this.undoes, u)

		w := this.GetWidthBetween(this.ViewStart, this.Cursor+1)
		this.Buffer[this.Cursor-1], this.Buffer[this.Cursor] = this.Buffer[this.Cursor], this.Buffer[this.Cursor-1]
		if w >= this.ViewWidth() {
			// cursor move right and scroll
			w_1 := w - GetCharWidth(this.Buffer[this.Cursor])
			this.backspace(w_1)
			this.ViewStart++
			this.puts(this.Buffer[this.ViewStart : this.Cursor+1])
		} else {
			// no necessary to scroll
			redrawStart := maxInt(this.Cursor-1, this.ViewStart)
			this.backspace(this.GetWidthBetween(redrawStart, this.Cursor))
			this.puts(this.Buffer[redrawStart : this.Cursor+1])
		}
		this.Cursor++
	}
	return CONTINUE
}

func keyFuncBackwardWord(ctx context.Context, this *Buffer) Result {
	newPos := this.Cursor
	for newPos > 0 && this.Buffer[newPos-1] == ' ' {
		newPos--
	}
	for newPos > 0 && this.Buffer[newPos-1] != ' ' {
		newPos--
	}
	if newPos >= this.ViewStart {
		w := this.GetWidthBetween(newPos, this.Cursor)
		this.backspace(w)
		this.Cursor = newPos
	} else {
		this.GotoHead()
		this.Cursor = newPos
		this.ViewStart = newPos
		this.DrawFromHead()
	}
	return CONTINUE
}

func keyFuncForwardWord(ctx context.Context, this *Buffer) Result {
	newPos := this.Cursor
	for newPos < len(this.Buffer) && this.Buffer[newPos] != ' ' {
		newPos++
	}
	for newPos < len(this.Buffer) && this.Buffer[newPos] == ' ' {
		newPos++
	}
	w := this.GetWidthBetween(this.ViewStart, newPos)
	if w < this.ViewWidth() {
		this.puts(this.Buffer[this.Cursor:newPos])
		this.Cursor = newPos
	} else {
		this.GotoHead()
		this.Cursor = newPos
		this.ResetViewStart()
		this.DrawFromHead()
	}
	return CONTINUE
}

func keyFuncUndo(ctx context.Context, this *Buffer) Result {
	if len(this.undoes) <= 0 {
		io.WriteString(this.Out, "\a")
		return CONTINUE
	}
	u := this.undoes[len(this.undoes)-1]
	this.undoes = this.undoes[:len(this.undoes)-1]

	this.GotoHead()
	if u.del > 0 {
		copy(this.Buffer[u.pos:], this.Buffer[u.pos+u.del:])
		this.Buffer = this.Buffer[:len(this.Buffer)-u.del]
	}
	if u.text != "" {
		t := []rune(u.text)
		// widen buffer
		this.Buffer = append(this.Buffer, t...)
		// make area
		copy(this.Buffer[u.pos+len(t):], this.Buffer[u.pos:])
		copy(this.Buffer[u.pos:], t)
		this.Cursor = u.pos + len(t)
	} else {
		this.Cursor = u.pos
	}
	this.ResetViewStart()
	this.DrawFromHead()
	return CONTINUE
}
<commit_msg>On ALT-Y(paste with quotes), put quotes around CRLF<commit_after>package readline

import (
	"context"
	"io"
	"strings"
	"unicode"

	"github.com/atotto/clipboard"
)

func keyFuncEnter(ctx context.Context, this *Buffer) Result { // Ctrl-M
	return ENTER
}

func keyFuncIntr(ctx context.Context, this *Buffer) Result { // Ctrl-C
	this.Buffer = this.Buffer[:0]
	this.Cursor = 0
	this.ViewStart = 0
	this.undoes = nil
	return INTR
}

func keyFuncHead(ctx context.Context, this *Buffer) Result { // Ctrl-A
	this.GotoHead()
	this.Cursor = 0
	this.ViewStart = 0
	this.DrawFromHead()
	return CONTINUE
}

func keyFuncBackward(ctx context.Context, this *Buffer) Result { // Ctrl-B
	if this.Cursor <= 0 {
		return CONTINUE
	}
	this.Cursor--
	if this.Cursor < this.ViewStart {
		this.ViewStart--
		this.DrawFromHead()
	} else {
		this.backspace(GetCharWidth(this.Buffer[this.Cursor]))
	}
	return CONTINUE
}

func keyFuncTail(ctx context.Context, this *Buffer) Result { // Ctrl-E
	allength := this.GetWidthBetween(this.ViewStart, len(this.Buffer))
	if allength < this.ViewWidth() {
		this.puts(this.Buffer[this.Cursor:])
		this.Cursor = len(this.Buffer)
	} else {
		this.backspace(this.GetWidthBetween(this.ViewStart, this.Cursor))
		this.ViewStart = len(this.Buffer) - 1
		w := GetCharWidth(this.Buffer[this.ViewStart])
		for {
			if this.ViewStart <= 0 {
				break
			}
			w_ := w + GetCharWidth(this.Buffer[this.ViewStart-1])
			if w_ >= this.ViewWidth() {
				break
			}
			w = w_
			this.ViewStart--
		}
		this.puts(this.Buffer[this.ViewStart:])
		this.Cursor = len(this.Buffer)
	}
	return CONTINUE
}

func keyFuncForward(ctx context.Context, this *Buffer) Result { // Ctrl-F
	if this.Cursor >= len(this.Buffer) {
		return CONTINUE
	}
	w := this.GetWidthBetween(this.ViewStart, this.Cursor+1)
	if w < this.ViewWidth() {
		// No Scroll
		this.putRune(this.Buffer[this.Cursor])
	} else {
		// Right Scroll
		this.GotoHead()
		if GetCharWidth(this.Buffer[this.Cursor]) > GetCharWidth(this.Buffer[this.ViewStart]) {
			this.ViewStart++
		}
		this.ViewStart++
		this.puts(this.Buffer[this.ViewStart : this.Cursor+1])
		this.Eraseline()
	}
	this.Cursor++
	return CONTINUE
}

func keyFuncBackSpace(ctx context.Context, this *Buffer) Result { // Backspace
	if this.Cursor > 0 {
		this.Cursor--
		delw := this.Delete(this.Cursor, 1)
		if this.Cursor >= this.ViewStart {
			this.backspace(delw)
		} else {
			this.ViewStart = this.Cursor
		}
		this.Repaint(this.Cursor, delw)
	}
	return CONTINUE
}

func keyFuncDelete(ctx context.Context, this *Buffer) Result { // Del
	delw := this.Delete(this.Cursor, 1)
	this.Repaint(this.Cursor, delw)
	return CONTINUE
}

func keyFuncDeleteOrAbort(ctx context.Context, this *Buffer) Result { // Ctrl-D
	if len(this.Buffer) > 0 {
		return keyFuncDelete(ctx, this)
	} else {
		return ABORT
	}
}

func keyFuncInsertSelf(ctx context.Context, this *Buffer, keys string) Result {
	if len(keys) == 2 && keys[0] == '\x1B' { // for AltGr-shift
		keys = keys[1:]
	}
	this.InsertString(this.Cursor, keys)

	w := this.GetWidthBetween(this.ViewStart, this.Cursor)
	w1 := GetStringWidth(keys)
	if w+w1 >= this.ViewWidth() {
		// scroll left
		this.GotoHead()
		this.Cursor += len([]rune(keys))
		this.ResetViewStart()
		this.DrawFromHead()
	} else {
		this.Repaint(this.Cursor, -w1)
		this.Cursor += len([]rune(keys))
	}
	return CONTINUE
}

func keyFuncClearAfter(ctx context.Context, this *Buffer) Result {
	clipboard.WriteAll(this.SubString(this.Cursor, len(this.Buffer)))

	this.Eraseline()
	u := &undo_t{
		pos:  this.Cursor,
		text: string(this.Buffer[this.Cursor:]),
	}
	this.undoes = append(this.undoes, u)
	this.Buffer = this.Buffer[:this.Cursor]
	return CONTINUE
}

func keyFuncClear(ctx context.Context, this *Buffer) Result {
	u := &undo_t{
		pos:  0,
		text: string(this.Buffer),
	}
	this.undoes = append(this.undoes, u)
	this.GotoHead()
	this.Eraseline()
	this.Buffer = this.Buffer[:0]
	this.Cursor = 0
	this.ViewStart = 0
	return CONTINUE
}

func keyFuncWordRubout(ctx context.Context, this *Buffer) Result {
	org_cursor := this.Cursor
	for this.Cursor > 0 && unicode.IsSpace(this.Buffer[this.Cursor-1]) {
		this.Cursor--
	}
	i := this.CurrentWordTop()
	clipboard.WriteAll(this.SubString(i, org_cursor))
	keta := this.Delete(i, org_cursor-i)
	if i >= this.ViewStart {
		this.backspace(keta)
	} else {
		this.backspace(this.GetWidthBetween(this.ViewStart, org_cursor))
	}
	this.Cursor = i
	this.Repaint(i, keta)
	return CONTINUE
}

func keyFuncClearBefore(ctx context.Context, this *Buffer) Result {
	this.GotoHead()
	clipboard.WriteAll(this.SubString(0, this.Cursor))
	this.Delete(0, this.Cursor)
	this.Cursor = 0
	this.ViewStart = 0
	this.DrawFromHead()
	return CONTINUE
}

func keyFuncCLS(ctx context.Context, this *Buffer) Result {
	io.WriteString(this.Out, "\x1B[1;1H\x1B[2J")
	this.RepaintAll()
	return CONTINUE
}

func keyFuncRepaintOnNewline(ctx context.Context, this *Buffer) Result {
	this.Out.WriteByte('\n')
	this.RepaintAll()
	return CONTINUE
}

func keyFuncQuotedInsert(ctx context.Context, this *Buffer) Result {
	io.WriteString(this.Out, ansiCursorOn)
	defer io.WriteString(this.Out, ansiCursorOff)

	this.Out.Flush()
	if key, err := this.GetKey(); err == nil {
		return keyFuncInsertSelf(ctx, this, key)
	} else {
		return CONTINUE
	}
}

func keyFuncPaste(ctx context.Context, this *Buffer) Result {
	text, err := clipboard.ReadAll()
	if err != nil {
		return CONTINUE
	}
	this.InsertAndRepaint(text)
	return CONTINUE
}

func keyFuncPasteQuote(ctx context.Context, this *Buffer) Result {
	text, err := clipboard.ReadAll()
	if err != nil {
		return CONTINUE
	}
	if strings.IndexRune(text, ' ') >= 0 &&
		!strings.HasPrefix(text, `"`) {

		text = `"` + strings.Replace(text, `"`, `""`, -1) + `"`
		text = strings.Replace(text, "\r\n", "\"\r\n\"", -1)
	}
	this.InsertAndRepaint(text)
	return CONTINUE
}

func maxInt(a, b int) int {
	if a < b {
		return b
	} else {
		return a
	}
}

func keyFuncSwapChar(ctx context.Context, this *Buffer) Result {
	if len(this.Buffer) == this.Cursor {
		if this.Cursor < 2 {
			return CONTINUE
		}
		u := &undo_t{
			pos:  this.Cursor,
			del:  2,
			text: string(this.Buffer[this.Cursor-2 : this.Cursor]),
		}
		this.undoes = append(this.undoes, u)
		this.Buffer[this.Cursor-2], this.Buffer[this.Cursor-1] = this.Buffer[this.Cursor-1], this.Buffer[this.Cursor-2]

		redrawStart := maxInt(this.Cursor-2, this.ViewStart)
		this.backspace(this.GetWidthBetween(redrawStart, this.Cursor))
		this.puts(this.Buffer[redrawStart:this.Cursor])
	} else {
		if this.Cursor < 1 {
			return CONTINUE
		}
		u := &undo_t{
			pos:  this.Cursor - 1,
			del:  2,
			text: string(this.Buffer[this.Cursor-1 : this.Cursor+1]),
		}
		this.undoes = append(this.undoes, u)

		w := this.GetWidthBetween(this.ViewStart, this.Cursor+1)
		this.Buffer[this.Cursor-1], this.Buffer[this.Cursor] = this.Buffer[this.Cursor], this.Buffer[this.Cursor-1]
		if w >= this.ViewWidth() {
			// cursor move right and scroll
			w_1 := w - GetCharWidth(this.Buffer[this.Cursor])
			this.backspace(w_1)
			this.ViewStart++
			this.puts(this.Buffer[this.ViewStart : this.Cursor+1])
		} else {
			// no necessary to scroll
			redrawStart := maxInt(this.Cursor-1, this.ViewStart)
			this.backspace(this.GetWidthBetween(redrawStart, this.Cursor))
			this.puts(this.Buffer[redrawStart : this.Cursor+1])
		}
		this.Cursor++
	}
	return CONTINUE
}

func keyFuncBackwardWord(ctx context.Context, this *Buffer) Result {
	newPos := this.Cursor
	for newPos > 0 && this.Buffer[newPos-1] == ' ' {
		newPos--
	}
	for newPos > 0 && this.Buffer[newPos-1] != ' ' {
		newPos--
	}
	if newPos >= this.ViewStart {
		w := this.GetWidthBetween(newPos, this.Cursor)
		this.backspace(w)
		this.Cursor = newPos
	} else {
		this.GotoHead()
		this.Cursor = newPos
		this.ViewStart = newPos
		this.DrawFromHead()
	}
	return CONTINUE
}

func keyFuncForwardWord(ctx context.Context, this *Buffer) Result {
	newPos := this.Cursor
	for newPos < len(this.Buffer) && this.Buffer[newPos] != ' ' {
		newPos++
	}
	for newPos < len(this.Buffer) && this.Buffer[newPos] == ' ' {
		newPos++
	}
	w := this.GetWidthBetween(this.ViewStart, newPos)
	if w < this.ViewWidth() {
		this.puts(this.Buffer[this.Cursor:newPos])
		this.Cursor = newPos
	} else {
		this.GotoHead()
		this.Cursor = newPos
		this.ResetViewStart()
		this.DrawFromHead()
	}
	return CONTINUE
}

func keyFuncUndo(ctx context.Context, this *Buffer) Result {
	if len(this.undoes) <= 0 {
		io.WriteString(this.Out, "\a")
		return CONTINUE
	}
	u := this.undoes[len(this.undoes)-1]
	this.undoes = this.undoes[:len(this.undoes)-1]

	this.GotoHead()
	if u.del > 0 {
		copy(this.Buffer[u.pos:], this.Buffer[u.pos+u.del:])
		this.Buffer = this.Buffer[:len(this.Buffer)-u.del]
	}
	if u.text != "" {
		t := []rune(u.text)
		// widen buffer
		this.Buffer = append(this.Buffer, t...)
		// make area
		copy(this.Buffer[u.pos+len(t):], this.Buffer[u.pos:])
		copy(this.Buffer[u.pos:], t)
		this.Cursor = u.pos + len(t)
	} else {
		this.Cursor = u.pos
	}
	this.ResetViewStart()
	this.DrawFromHead()
	return CONTINUE
}
<|endoftext|>


<commit_before>package fastx

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"regexp"
	"syscall"

	"github.com/shenwei356/bio/seq"
	"github.com/shenwei356/xopen"
)

// ErrNotFASTXFormat means that the file is not FASTA/Q
var ErrNotFASTXFormat = errors.New("fastx: invalid FASTA/Q format")

// ErrBadFASTQFormat means bad fastq format
var ErrBadFASTQFormat = errors.New("fastx: bad fastq format")

// ErrUnequalSeqAndQual means unequal sequence and quality
var ErrUnequalSeqAndQual = errors.New("fastx: unequal sequence and quality")

var pageSize = syscall.Getpagesize()

// Reader seamlessly parse both FASTA and FASTQ formats
type Reader struct {
	fh *xopen.Reader // file handle, xopen is such a wonderful package

	buf                []byte // for store readed data from fh
	r                  int
	buffer             *bytes.Buffer // buffer of a record
	needMoreCheckOfBuf bool
	lastByte           byte
	checkSeqType       bool
	lastPart           bool
	finished           bool

	firstseq bool // for guess alphabet by the first seq
	delim    byte
	IsFastq  bool // if the file is fastq format

	t        *seq.Alphabet  // alphabet
	IDRegexp *regexp.Regexp // regexp for parsing record id

	head, seq, qual []byte
	seqBuffer       *bytes.Buffer
	qualBuffer      *bytes.Buffer
	record          *Record

	Err error // Current error
}

// regexp for checking idRegexp string.
// The regular expression must contain "(" and ")" to capture matched ID
var reCheckIDregexpStr = regexp.MustCompile(`\(.+\)`)

// DefaultIDRegexp is the default ID parsing regular expression
var DefaultIDRegexp = `^([^\s]+)\s?`
var isUsingDefaultIDRegexp bool

// NewDefaultReader automaticlly recognizes sequence type and parses id with default manner
func NewDefaultReader(file string) (*Reader, error) {
	return NewReader(nil, file, "")
}

// NewReader is constructor of FASTX Reader.
//
// Parameters:
//
//        t            sequence alphabet
//                     if nil is given, it will guess alphabet by the first record
//        file         file name, "-" for stdin
//        idRegexp     id parsing regular expression string, must contains "(" and ")" to capture matched ID
//                     "" for default value: `^([^\s]+)\s?`
//                     if record head does not match the idRegxp, whole name will be the id
//
func NewReader(t *seq.Alphabet, file string, idRegexp string) (*Reader, error) {
	var r *regexp.Regexp
	if idRegexp == "" {
		r = regexp.MustCompile(DefaultIDRegexp)
		isUsingDefaultIDRegexp = true
	} else {
		if !reCheckIDregexpStr.MatchString(idRegexp) {
			return nil, fmt.Errorf(`fastx: regular expression must contain "(" and ")" to capture matched ID. default: %s`, DefaultIDRegexp)
		}
		var err error
		r, err = regexp.Compile(idRegexp)
		if err != nil {
			return nil, fmt.Errorf("fastx: fail to compile regexp: %s", idRegexp)
		}
		if idRegexp == DefaultIDRegexp {
			isUsingDefaultIDRegexp = true
		}
	}

	fh, err := xopen.Ropen(file)
	if err != nil {
		return nil, fmt.Errorf("fastx: %s", err)
	}

	fastxReader := &Reader{
		fh:           fh,
		buf:          make([]byte, pageSize),
		t:            t,
		IDRegexp:     r,
		firstseq:     true,
		checkSeqType: true,
	}
	fastxReader.buffer = bytes.NewBuffer(make([]byte, 0, defaultBytesBufferSize))
	fastxReader.seqBuffer = bytes.NewBuffer(make([]byte, 0, defaultBytesBufferSize))
	fastxReader.qualBuffer = bytes.NewBuffer(make([]byte, 0, defaultBytesBufferSize))

	return fastxReader, nil
}

func (fastxReader *Reader) close() {
	fastxReader.fh.Close()
}

// Read reads and return one FASTA/Q record.
// Note that, similar to bytes.Buffer.Bytes() method,
// the current record will change after your another call of this method.
// So, you could use record.Clone() to make a copy.
func (fastxReader *Reader) Read() (*Record, error) {
	fastxReader.read()
	return fastxReader.record, fastxReader.Err
}

func (fastxReader *Reader) read() {
	if fastxReader.lastPart && fastxReader.finished {
		fastxReader.Err = io.EOF
		return
	}
	if fastxReader.Err != nil {
		return
	}

	var n int
	var err error
	var p []byte

	for {
		if !fastxReader.needMoreCheckOfBuf && !fastxReader.lastPart {
			n, err = fastxReader.fh.Read(fastxReader.buf)
			if err != nil {
				if err == io.EOF {
					fastxReader.lastPart = true
				} else {
					fastxReader.Err = err
					fastxReader.close()
					return
				}
			}

			// last part of file OR just because reader not fulfill the buf,
			// like reading from stdin
			if n < len(fastxReader.buf) {
				// fastxReader.lastPart = true
				fastxReader.buf = fastxReader.buf[0:n] // very important!
			}
			fastxReader.r = 0 /// TO CHECK
		}

		if fastxReader.checkSeqType {
			pn := 0
		FORCHECK:
			for i := range fastxReader.buf {
				switch fastxReader.buf[i] {
				case '>':
					fastxReader.checkSeqType = false
					fastxReader.IsFastq = false
					fastxReader.delim = '>'
					fastxReader.r = i + 1
					break FORCHECK
				case '@':
					fastxReader.IsFastq = true
					fastxReader.delim = '@'
					fastxReader.r = i + 1
					break FORCHECK
				case '\n': // allow some lines
					pn++
					if pn > 100 {
						if i > 10240 { // ErrNotFASTXFormat
							fastxReader.Err = ErrNotFASTXFormat
							fastxReader.close()
							return
						}
					}
					// break FORCHECK
				default: // not typical FASTA/Q
					// if i > 10240 || fastxReader.lastPart { // ErrNotFASTXFormat
					fastxReader.Err = ErrNotFASTXFormat
					fastxReader.close()
					return
					// }
				}
			}
			fastxReader.checkSeqType = false
		}

		var shorterQual bool
	FORSEARCH:
		for {
			if i := bytes.IndexByte(fastxReader.buf[fastxReader.r:], fastxReader.delim); i >= 0 {
				if i > 0 {
					fastxReader.lastByte = fastxReader.buf[fastxReader.r+i-1]
				} else {
					p = fastxReader.buffer.Bytes()
					fastxReader.lastByte = p[len(p)-1]
				}
				if fastxReader.lastByte == '\n' { // yes!
					if i > 0 {
						fastxReader.buffer.Write(dropCR(fastxReader.buf[fastxReader.r : fastxReader.r+i-1]))
					} else {
						fastxReader.buffer.WriteByte('\n')
					}

					// we have to avoid the case of quality line starts with "@"
					shorterQual, err = fastxReader.parseRecord()
					if fastxReader.IsFastq && err != nil && err == ErrUnequalSeqAndQual {
						if shorterQual {
							fastxReader.buffer.WriteByte('\n')
							fastxReader.buffer.WriteByte(fastxReader.delim)
							fastxReader.needMoreCheckOfBuf = true
							fastxReader.r += i + 1
							continue FORSEARCH
						}
						fastxReader.Err = ErrBadFASTQFormat
						return
					}
					fastxReader.buffer.Reset()
					fastxReader.needMoreCheckOfBuf = true
					fastxReader.r += i + 1
					return
				}
				// inline >/@
				fastxReader.buffer.Write(fastxReader.buf[fastxReader.r : fastxReader.r+i+1])
				fastxReader.r += i + 1
				fastxReader.needMoreCheckOfBuf = true
				continue FORSEARCH
			}

			fastxReader.buffer.Write(fastxReader.buf[fastxReader.r:])
			if fastxReader.lastPart {
				_, err = fastxReader.parseRecord()
				if err != nil { // no any chance
					fastxReader.Err = err
					fastxReader.close()
					return
				}
				fastxReader.buffer.Reset()
				fastxReader.close()
				fastxReader.finished = true
				return
			}
			fastxReader.needMoreCheckOfBuf = false
			break FORSEARCH
		}
	}
}

// parseRecord parse a FASTA/Q record from the fastxReader.buffer
func (fastxReader *Reader) parseRecord() (bool, error) {
	fastxReader.seqBuffer.Reset()
	fastxReader.qualBuffer.Reset()

	var p = fastxReader.buffer.Bytes()
	if j := bytes.IndexByte(p, '\n'); j > 0 {
		fastxReader.head = dropCR(p[0:j])
		r := j + 1

		if !fastxReader.IsFastq { // FASTA
			for {
				if k := bytes.IndexByte(p[r:], '\n'); k >= 0 {
					fastxReader.seqBuffer.Write(dropCR(p[r : r+k]))
					r += k + 1
					continue
				}
				fastxReader.seqBuffer.Write(dropCR(p[r:]))
				break
			}
			fastxReader.seq = fastxReader.seqBuffer.Bytes()
		} else { // FASTQ
			var isQual bool
			for {
				if k := bytes.IndexByte(p[r:], '\n'); k >= 0 {
					if k > 0 && p[r] == '+' && !isQual {
						isQual = true
					} else if isQual {
						fastxReader.qualBuffer.Write(dropCR(p[r : r+k]))
					} else {
						fastxReader.seqBuffer.Write(dropCR(p[r : r+k]))
					}
					r += k + 1
					continue
				}
				if isQual {
					fastxReader.qualBuffer.Write(dropCR(p[r:]))
				}
				break
			}

			// may be the case of quality line starts with "@"
			if fastxReader.seqBuffer.Len() != fastxReader.qualBuffer.Len() {
				return fastxReader.seqBuffer.Len() > fastxReader.qualBuffer.Len(), ErrUnequalSeqAndQual
			}

			fastxReader.seq = fastxReader.seqBuffer.Bytes()
			fastxReader.qual = fastxReader.qualBuffer.Bytes()
		}

	} else {
		fastxReader.head = dropCR(dropLF(p))
		fastxReader.seq = []byte{}
		fastxReader.qual = []byte{}
	}

	// guess alphabet
	if fastxReader.firstseq {
		if fastxReader.t == nil {
			fastxReader.t = seq.GuessAlphabetLessConservatively(fastxReader.seq)
		}
		fastxReader.firstseq = false
	}

	if len(fastxReader.head) == 0 && len(fastxReader.seq) == 0 {
		return false, io.EOF
	}
	// new record
	if fastxReader.IsFastq {
		fastxReader.record, fastxReader.Err = NewRecordWithQual(fastxReader.t,
			fastxReader.parseHeadID(fastxReader.head), fastxReader.head,
			fastxReader.seq, fastxReader.qual)
	} else {
		fastxReader.record, fastxReader.Err = NewRecord(fastxReader.t,
			fastxReader.parseHeadID(fastxReader.head), fastxReader.head,
			fastxReader.seq)
	}

	if fastxReader.Err != nil {
		fastxReader.close()
	}

	return false, fastxReader.Err
}

func (fastxReader *Reader) parseHeadID(head []byte) []byte {
	return parseHeadID(fastxReader.IDRegexp, head)
}

// ParseHeadID parse ID from head by IDRegexp
func ParseHeadID(idRegexp *regexp.Regexp, head []byte) []byte {
	found := idRegexp.FindSubmatch(head)
	if found == nil { // not match
		return head
	}
	return found[1]
}

// ParseHeadID parse ID from head by IDRegexp
func parseHeadID(idRegexp *regexp.Regexp, head []byte) []byte {
	if isUsingDefaultIDRegexp {
		if i := bytes.IndexByte(head, ' '); i > 0 {
			return head[0:i]
		}
		return head
	}

	found := idRegexp.FindSubmatch(head)
	if found == nil { // not match
		return head
	}
	return found[1]
}

// Alphabet returns Alphabet of the file
func (fastxReader *Reader) Alphabet() *seq.Alphabet {
	return fastxReader.t
}

func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

func dropLF(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\n' {
		return data[0 : len(data)-1]
	}
	return data
}

// -------------------------------------------------

// RecordChunk is chunk for records
type RecordChunk struct {
	ID   uint64
	Data []*Record
	Err  error
}

// ChunkChan asynchronously reads FASTA/Q records, and returns a channel of
// Record Chunk, from which you can easily access the records.
// bufferSize is the number of buffered chunks, and chunkSize is the size
// of records in a chunk.
func (fastxReader *Reader) ChunkChan(bufferSize int, chunkSize int) chan RecordChunk {
	var ch chan RecordChunk
	if bufferSize <= 0 {
		ch = make(chan RecordChunk)
	} else {
		ch = make(chan RecordChunk, bufferSize)
	}
	if chunkSize < 1 {
		chunkSize = 1
	}

	go func() {
		var i int
		var id uint64
		chunkData := make([]*Record, chunkSize)

		for {
			record, err := fastxReader.Read()
			if err != nil {
				if err == io.EOF {
					break
				}
				ch <- RecordChunk{id, chunkData[0:i], err}
				close(ch)
				return
			}
			chunkData[i] = record.Clone()
			i++

			if i == chunkSize {
				ch <- RecordChunk{id, chunkData[0:i], nil}
				id++
				i = 0
				chunkData = make([]*Record, chunkSize)
			}
		}

		ch <- RecordChunk{id, chunkData[0:i], nil}
		close(ch)
		return
	}()

	return ch
}
<commit_msg>export reader.Close()<commit_after>package fastx

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"regexp"
	"syscall"

	"github.com/shenwei356/bio/seq"
	"github.com/shenwei356/xopen"
)

// ErrNotFASTXFormat means that the file is not FASTA/Q
var ErrNotFASTXFormat = errors.New("fastx: invalid FASTA/Q format")

// ErrBadFASTQFormat means bad fastq format
var ErrBadFASTQFormat = errors.New("fastx: bad fastq format")

// ErrUnequalSeqAndQual means unequal sequence and quality
var ErrUnequalSeqAndQual = errors.New("fastx: unequal sequence and quality")

var pageSize = syscall.Getpagesize()

// Reader seamlessly parse both FASTA and FASTQ formats
type Reader struct {
	fh *xopen.Reader // file handle, xopen is such a wonderful package

	buf                []byte // for store readed data from fh
	r                  int
	buffer             *bytes.Buffer // buffer of a record
	needMoreCheckOfBuf bool
	lastByte           byte
	checkSeqType       bool
	lastPart           bool
	finished           bool

	firstseq bool // for guess alphabet by the first seq
	delim    byte
	IsFastq  bool // if the file is fastq format

	t        *seq.Alphabet  // alphabet
	IDRegexp *regexp.Regexp // regexp for parsing record id

	head, seq, qual []byte
	seqBuffer       *bytes.Buffer
	qualBuffer      *bytes.Buffer
	record          *Record

	Err error // Current error
}

// regexp for checking idRegexp string.
// The regular expression must contain "(" and ")" to capture matched ID
var reCheckIDregexpStr = regexp.MustCompile(`\(.+\)`)

// DefaultIDRegexp is the default ID parsing regular expression
var DefaultIDRegexp = `^([^\s]+)\s?`
var isUsingDefaultIDRegexp bool

// NewDefaultReader automaticlly recognizes sequence type and parses id with default manner
func NewDefaultReader(file string) (*Reader, error) {
	return NewReader(nil, file, "")
}

// NewReader is constructor of FASTX Reader.
//
// Parameters:
//
//        t            sequence alphabet
//                     if nil is given, it will guess alphabet by the first record
//        file         file name, "-" for stdin
//        idRegexp     id parsing regular expression string, must contains "(" and ")" to capture matched ID
//                     "" for default value: `^([^\s]+)\s?`
//                     if record head does not match the idRegxp, whole name will be the id
//
func NewReader(t *seq.Alphabet, file string, idRegexp string) (*Reader, error) {
	var r *regexp.Regexp
	if idRegexp == "" {
		r = regexp.MustCompile(DefaultIDRegexp)
		isUsingDefaultIDRegexp = true
	} else {
		if !reCheckIDregexpStr.MatchString(idRegexp) {
			return nil, fmt.Errorf(`fastx: regular expression must contain "(" and ")" to capture matched ID. default: %s`, DefaultIDRegexp)
		}
		var err error
		r, err = regexp.Compile(idRegexp)
		if err != nil {
			return nil, fmt.Errorf("fastx: fail to compile regexp: %s", idRegexp)
		}
		if idRegexp == DefaultIDRegexp {
			isUsingDefaultIDRegexp = true
		}
	}

	fh, err := xopen.Ropen(file)
	if err != nil {
		return nil, fmt.Errorf("fastx: %s", err)
	}

	fastxReader := &Reader{
		fh:           fh,
		buf:          make([]byte, pageSize),
		t:            t,
		IDRegexp:     r,
		firstseq:     true,
		checkSeqType: true,
	}
	fastxReader.buffer = bytes.NewBuffer(make([]byte, 0, defaultBytesBufferSize))
	fastxReader.seqBuffer = bytes.NewBuffer(make([]byte, 0, defaultBytesBufferSize))
	fastxReader.qualBuffer = bytes.NewBuffer(make([]byte, 0, defaultBytesBufferSize))

	return fastxReader, nil
}

// Close closes the reader
func (fastxReader *Reader) Close() {
	fastxReader.fh.Close()
}

// Read reads and return one FASTA/Q record.
// Note that, similar to bytes.Buffer.Bytes() method,
// the current record will change after your another call of this method.
// So, you could use record.Clone() to make a copy.
func (fastxReader *Reader) Read() (*Record, error) {
	fastxReader.read()
	return fastxReader.record, fastxReader.Err
}

func (fastxReader *Reader) read() {
	if fastxReader.lastPart && fastxReader.finished {
		fastxReader.Err = io.EOF
		return
	}
	if fastxReader.Err != nil {
		return
	}

	var n int
	var err error
	var p []byte

	for {
		if !fastxReader.needMoreCheckOfBuf && !fastxReader.lastPart {
			n, err = fastxReader.fh.Read(fastxReader.buf)
			if err != nil {
				if err == io.EOF {
					fastxReader.lastPart = true
				} else {
					fastxReader.Err = err
					fastxReader.Close()
					return
				}
			}

			// last part of file OR just because reader not fulfill the buf,
			// like reading from stdin
			if n < len(fastxReader.buf) {
				// fastxReader.lastPart = true
				fastxReader.buf = fastxReader.buf[0:n] // very important!
			}
			fastxReader.r = 0 /// TO CHECK
		}

		if fastxReader.checkSeqType {
			pn := 0
		FORCHECK:
			for i := range fastxReader.buf {
				switch fastxReader.buf[i] {
				case '>':
					fastxReader.checkSeqType = false
					fastxReader.IsFastq = false
					fastxReader.delim = '>'
					fastxReader.r = i + 1
					break FORCHECK
				case '@':
					fastxReader.IsFastq = true
					fastxReader.delim = '@'
					fastxReader.r = i + 1
					break FORCHECK
				case '\n': // allow some lines
					pn++
					if pn > 100 {
						if i > 10240 { // ErrNotFASTXFormat
							fastxReader.Err = ErrNotFASTXFormat
							fastxReader.Close()
							return
						}
					}
					// break FORCHECK
				default: // not typical FASTA/Q
					// if i > 10240 || fastxReader.lastPart { // ErrNotFASTXFormat
					fastxReader.Err = ErrNotFASTXFormat
					fastxReader.Close()
					return
					// }
				}
			}
			fastxReader.checkSeqType = false
		}

		var shorterQual bool
	FORSEARCH:
		for {
			if i := bytes.IndexByte(fastxReader.buf[fastxReader.r:], fastxReader.delim); i >= 0 {
				if i > 0 {
					fastxReader.lastByte = fastxReader.buf[fastxReader.r+i-1]
				} else {
					p = fastxReader.buffer.Bytes()
					fastxReader.lastByte = p[len(p)-1]
				}
				if fastxReader.lastByte == '\n' { // yes!
					if i > 0 {
						fastxReader.buffer.Write(dropCR(fastxReader.buf[fastxReader.r : fastxReader.r+i-1]))
					} else {
						fastxReader.buffer.WriteByte('\n')
					}

					// we have to avoid the case of quality line starts with "@"
					shorterQual, err = fastxReader.parseRecord()
					if fastxReader.IsFastq && err != nil && err == ErrUnequalSeqAndQual {
						if shorterQual {
							fastxReader.buffer.WriteByte('\n')
							fastxReader.buffer.WriteByte(fastxReader.delim)
							fastxReader.needMoreCheckOfBuf = true
							fastxReader.r += i + 1
							continue FORSEARCH
						}
						fastxReader.Err = ErrBadFASTQFormat
						return
					}
					fastxReader.buffer.Reset()
					fastxReader.needMoreCheckOfBuf = true
					fastxReader.r += i + 1
					return
				}
				// inline >/@
				fastxReader.buffer.Write(fastxReader.buf[fastxReader.r : fastxReader.r+i+1])
				fastxReader.r += i + 1
				fastxReader.needMoreCheckOfBuf = true
				continue FORSEARCH
			}

			fastxReader.buffer.Write(fastxReader.buf[fastxReader.r:])
			if fastxReader.lastPart {
				_, err = fastxReader.parseRecord()
				if err != nil { // no any chance
					fastxReader.Err = err
					fastxReader.Close()
					return
				}
				fastxReader.buffer.Reset()
				fastxReader.Close()
				fastxReader.finished = true
				return
			}
			fastxReader.needMoreCheckOfBuf = false
			break FORSEARCH
		}
	}
}

// parseRecord parse a FASTA/Q record from the fastxReader.buffer
func (fastxReader *Reader) parseRecord() (bool, error) {
	fastxReader.seqBuffer.Reset()
	fastxReader.qualBuffer.Reset()

	var p = fastxReader.buffer.Bytes()
	if j := bytes.IndexByte(p, '\n'); j > 0 {
		fastxReader.head = dropCR(p[0:j])
		r := j + 1

		if !fastxReader.IsFastq { // FASTA
			for {
				if k := bytes.IndexByte(p[r:], '\n'); k >= 0 {
					fastxReader.seqBuffer.Write(dropCR(p[r : r+k]))
					r += k + 1
					continue
				}
				fastxReader.seqBuffer.Write(dropCR(p[r:]))
				break
			}
			fastxReader.seq = fastxReader.seqBuffer.Bytes()
		} else { // FASTQ
			var isQual bool
			for {
				if k := bytes.IndexByte(p[r:], '\n'); k >= 0 {
					if k > 0 && p[r] == '+' && !isQual {
						isQual = true
					} else if isQual {
						fastxReader.qualBuffer.Write(dropCR(p[r : r+k]))
					} else {
						fastxReader.seqBuffer.Write(dropCR(p[r : r+k]))
					}
					r += k + 1
					continue
				}
				if isQual {
					fastxReader.qualBuffer.Write(dropCR(p[r:]))
				}
				break
			}

			// may be the case of quality line starts with "@"
			if fastxReader.seqBuffer.Len() != fastxReader.qualBuffer.Len() {
				return fastxReader.seqBuffer.Len() > fastxReader.qualBuffer.Len(), ErrUnequalSeqAndQual
			}

			fastxReader.seq = fastxReader.seqBuffer.Bytes()
			fastxReader.qual = fastxReader.qualBuffer.Bytes()
		}

	} else {
		fastxReader.head = dropCR(dropLF(p))
		fastxReader.seq = []byte{}
		fastxReader.qual = []byte{}
	}

	// guess alphabet
	if fastxReader.firstseq {
		if fastxReader.t == nil {
			fastxReader.t = seq.GuessAlphabetLessConservatively(fastxReader.seq)
		}
		fastxReader.firstseq = false
	}

	if len(fastxReader.head) == 0 && len(fastxReader.seq) == 0 {
		return false, io.EOF
	}
	// new record
	if fastxReader.IsFastq {
		fastxReader.record, fastxReader.Err = NewRecordWithQual(fastxReader.t,
			fastxReader.parseHeadID(fastxReader.head), fastxReader.head,
			fastxReader.seq, fastxReader.qual)
	} else {
		fastxReader.record, fastxReader.Err = NewRecord(fastxReader.t,
			fastxReader.parseHeadID(fastxReader.head), fastxReader.head,
			fastxReader.seq)
	}

	if fastxReader.Err != nil {
		fastxReader.Close()
	}

	return false, fastxReader.Err
}

func (fastxReader *Reader) parseHeadID(head []byte) []byte {
	return parseHeadID(fastxReader.IDRegexp, head)
}

// ParseHeadID parse ID from head by IDRegexp
func ParseHeadID(idRegexp *regexp.Regexp, head []byte) []byte {
	found := idRegexp.FindSubmatch(head)
	if found == nil { // not match
		return head
	}
	return found[1]
}

// ParseHeadID parse ID from head by IDRegexp
func parseHeadID(idRegexp *regexp.Regexp, head []byte) []byte {
	if isUsingDefaultIDRegexp {
		if i := bytes.IndexByte(head, ' '); i > 0 {
			return head[0:i]
		}
		return head
	}

	found := idRegexp.FindSubmatch(head)
	if found == nil { // not match
		return head
	}
	return found[1]
}

// Alphabet returns Alphabet of the file
func (fastxReader *Reader) Alphabet() *seq.Alphabet {
	return fastxReader.t
}

func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

func dropLF(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\n' {
		return data[0 : len(data)-1]
	}
	return data
}

// -------------------------------------------------

// RecordChunk is chunk for records
type RecordChunk struct {
	ID   uint64
	Data []*Record
	Err  error
}

// ChunkChan asynchronously reads FASTA/Q records, and returns a channel of
// Record Chunk, from which you can easily access the records.
// bufferSize is the number of buffered chunks, and chunkSize is the size
// of records in a chunk.
func (fastxReader *Reader) ChunkChan(bufferSize int, chunkSize int) chan RecordChunk {
	var ch chan RecordChunk
	if bufferSize <= 0 {
		ch = make(chan RecordChunk)
	} else {
		ch = make(chan RecordChunk, bufferSize)
	}
	if chunkSize < 1 {
		chunkSize = 1
	}

	go func() {
		var i int
		var id uint64
		chunkData := make([]*Record, chunkSize)

		for {
			record, err := fastxReader.Read()
			if err != nil {
				if err == io.EOF {
					break
				}
				ch <- RecordChunk{id, chunkData[0:i], err}
				close(ch)
				return
			}
			chunkData[i] = record.Clone()
			i++

			if i == chunkSize {
				ch <- RecordChunk{id, chunkData[0:i], nil}
				id++
				i = 0
				chunkData = make([]*Record, chunkSize)
			}
		}

		ch <- RecordChunk{id, chunkData[0:i], nil}
		close(ch)
		return
	}()

	return ch
}
<|endoftext|>


<commit_before>package serf

import (
	"io/ioutil"
	"log"
	"os"
	"reflect"
	"testing"
	"time"
)

func TestSnapshoter(t *testing.T) {
	td, err := ioutil.TempDir("", "serf")
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(td)

	clock := new(LamportClock)
	outCh := make(chan Event, 64)
	stopCh := make(chan struct{})
	logger := log.New(os.Stderr, "", log.LstdFlags)
	inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, outCh, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Write some user events
	ue := UserEvent{
		LTime: 42,
		Name:  "bar",
	}
	inCh <- ue

	// Write some member events
	clock.Witness(100)
	meJoin := MemberEvent{
		Type: EventMemberJoin,
		Members: []Member{
			Member{
				Name: "foo",
				Addr: []byte{127, 0, 0, 1},
				Port: 5000,
			},
		},
	}
	meFail := MemberEvent{
		Type: EventMemberFailed,
		Members: []Member{
			Member{
				Name: "foo",
				Addr: []byte{127, 0, 0, 1},
				Port: 5000,
			},
		},
	}
	inCh <- meJoin
	inCh <- meFail
	inCh <- meJoin

	// Check these get passed through
	select {
	case e := <-outCh:
		if !reflect.DeepEqual(e, ue) {
			t.Fatalf("expected user event: %#v", e)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatalf("timeout")
	}

	select {
	case e := <-outCh:
		if !reflect.DeepEqual(e, meJoin) {
			t.Fatalf("expected member event: %#v", e)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatalf("timeout")
	}

	select {
	case e := <-outCh:
		if !reflect.DeepEqual(e, meFail) {
			t.Fatalf("expected member event: %#v", e)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatalf("timeout")
	}

	select {
	case e := <-outCh:
		if !reflect.DeepEqual(e, meJoin) {
			t.Fatalf("expected member event: %#v", e)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatalf("timeout")
	}

	// Close the snapshoter
	close(stopCh)
	snap.Wait()

	// Open the snapshoter
	stopCh = make(chan struct{})
	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, outCh, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the values
	if snap.LastClock() != 100 {
		t.Fatalf("bad clock %d", snap.LastClock())
	}
	if snap.LastEventClock() != 42 {
		t.Fatalf("bad clock %d", snap.LastEventClock())
	}

	prev := snap.AliveNodes()
	if len(prev) != 1 {
		t.Fatalf("expected alive: %#v", prev)
	}
	if prev[0].Name != "foo" {
		t.Fatalf("bad name: %#v", prev[0])
	}
	if prev[0].Addr != "127.0.0.1:5000" {
		t.Fatalf("bad addr: %#v", prev[0])
	}
}

func TestSnapshoter_forceCompact(t *testing.T) {
	td, err := ioutil.TempDir("", "serf")
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(td)

	clock := new(LamportClock)
	stopCh := make(chan struct{})
	logger := log.New(os.Stderr, "", log.LstdFlags)

	// Create a very low limit
	inCh, snap, err := NewSnapshotter(td+"snap", 1024,
		logger, clock, nil, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Write lots of user events
	for i := 0; i < 1024; i++ {
		ue := UserEvent{
			LTime: LamportTime(i),
		}
		inCh <- ue
	}

	// Wait for drain
	for len(inCh) > 0 {
		time.Sleep(20 * time.Millisecond)
	}

	// Close the snapshoter
	close(stopCh)
	snap.Wait()

	// Open the snapshoter
	stopCh = make(chan struct{})
	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, nil, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the values
	if snap.LastEventClock() != 1023 {
		t.Fatalf("bad clock %d", snap.LastEventClock())
	}

	close(stopCh)
	snap.Wait()
}

func TestSnapshoter_leave(t *testing.T) {
	td, err := ioutil.TempDir("", "serf")
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(td)

	clock := new(LamportClock)
	stopCh := make(chan struct{})
	logger := log.New(os.Stderr, "", log.LstdFlags)
	inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, nil, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Write a user event
	ue := UserEvent{
		LTime: 42,
		Name:  "bar",
	}
	inCh <- ue

	// Write some member events
	clock.Witness(100)
	meJoin := MemberEvent{
		Type: EventMemberJoin,
		Members: []Member{
			Member{
				Name: "foo",
				Addr: []byte{127, 0, 0, 1},
				Port: 5000,
			},
		},
	}
	inCh <- meJoin

	// Leave the cluster!
	snap.Leave()

	// Close the snapshoter
	close(stopCh)
	snap.Wait()

	// Open the snapshoter
	stopCh = make(chan struct{})
	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, nil, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the values
	if snap.LastClock() != 0 {
		t.Fatalf("bad clock %d", snap.LastClock())
	}
	if snap.LastEventClock() != 0 {
		t.Fatalf("bad clock %d", snap.LastEventClock())
	}

	prev := snap.AliveNodes()
	if len(prev) != 0 {
		t.Fatalf("expected none alive: %#v", prev)
	}
}
<commit_msg>serf: Increase test timeouts for Travis CI<commit_after>package serf

import (
	"io/ioutil"
	"log"
	"os"
	"reflect"
	"testing"
	"time"
)

func TestSnapshoter(t *testing.T) {
	td, err := ioutil.TempDir("", "serf")
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(td)

	clock := new(LamportClock)
	outCh := make(chan Event, 64)
	stopCh := make(chan struct{})
	logger := log.New(os.Stderr, "", log.LstdFlags)
	inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, outCh, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Write some user events
	ue := UserEvent{
		LTime: 42,
		Name:  "bar",
	}
	inCh <- ue

	// Write some member events
	clock.Witness(100)
	meJoin := MemberEvent{
		Type: EventMemberJoin,
		Members: []Member{
			Member{
				Name: "foo",
				Addr: []byte{127, 0, 0, 1},
				Port: 5000,
			},
		},
	}
	meFail := MemberEvent{
		Type: EventMemberFailed,
		Members: []Member{
			Member{
				Name: "foo",
				Addr: []byte{127, 0, 0, 1},
				Port: 5000,
			},
		},
	}
	inCh <- meJoin
	inCh <- meFail
	inCh <- meJoin

	// Check these get passed through
	select {
	case e := <-outCh:
		if !reflect.DeepEqual(e, ue) {
			t.Fatalf("expected user event: %#v", e)
		}
	case <-time.After(200 * time.Millisecond):
		t.Fatalf("timeout")
	}

	select {
	case e := <-outCh:
		if !reflect.DeepEqual(e, meJoin) {
			t.Fatalf("expected member event: %#v", e)
		}
	case <-time.After(200 * time.Millisecond):
		t.Fatalf("timeout")
	}

	select {
	case e := <-outCh:
		if !reflect.DeepEqual(e, meFail) {
			t.Fatalf("expected member event: %#v", e)
		}
	case <-time.After(200 * time.Millisecond):
		t.Fatalf("timeout")
	}

	select {
	case e := <-outCh:
		if !reflect.DeepEqual(e, meJoin) {
			t.Fatalf("expected member event: %#v", e)
		}
	case <-time.After(200 * time.Millisecond):
		t.Fatalf("timeout")
	}

	// Close the snapshoter
	close(stopCh)
	snap.Wait()

	// Open the snapshoter
	stopCh = make(chan struct{})
	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, outCh, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the values
	if snap.LastClock() != 100 {
		t.Fatalf("bad clock %d", snap.LastClock())
	}
	if snap.LastEventClock() != 42 {
		t.Fatalf("bad clock %d", snap.LastEventClock())
	}

	prev := snap.AliveNodes()
	if len(prev) != 1 {
		t.Fatalf("expected alive: %#v", prev)
	}
	if prev[0].Name != "foo" {
		t.Fatalf("bad name: %#v", prev[0])
	}
	if prev[0].Addr != "127.0.0.1:5000" {
		t.Fatalf("bad addr: %#v", prev[0])
	}
}

func TestSnapshoter_forceCompact(t *testing.T) {
	td, err := ioutil.TempDir("", "serf")
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(td)

	clock := new(LamportClock)
	stopCh := make(chan struct{})
	logger := log.New(os.Stderr, "", log.LstdFlags)

	// Create a very low limit
	inCh, snap, err := NewSnapshotter(td+"snap", 1024,
		logger, clock, nil, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Write lots of user events
	for i := 0; i < 1024; i++ {
		ue := UserEvent{
			LTime: LamportTime(i),
		}
		inCh <- ue
	}

	// Wait for drain
	for len(inCh) > 0 {
		time.Sleep(20 * time.Millisecond)
	}

	// Close the snapshoter
	close(stopCh)
	snap.Wait()

	// Open the snapshoter
	stopCh = make(chan struct{})
	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, nil, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the values
	if snap.LastEventClock() != 1023 {
		t.Fatalf("bad clock %d", snap.LastEventClock())
	}

	close(stopCh)
	snap.Wait()
}

func TestSnapshoter_leave(t *testing.T) {
	td, err := ioutil.TempDir("", "serf")
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(td)

	clock := new(LamportClock)
	stopCh := make(chan struct{})
	logger := log.New(os.Stderr, "", log.LstdFlags)
	inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, nil, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Write a user event
	ue := UserEvent{
		LTime: 42,
		Name:  "bar",
	}
	inCh <- ue

	// Write some member events
	clock.Witness(100)
	meJoin := MemberEvent{
		Type: EventMemberJoin,
		Members: []Member{
			Member{
				Name: "foo",
				Addr: []byte{127, 0, 0, 1},
				Port: 5000,
			},
		},
	}
	inCh <- meJoin

	// Leave the cluster!
	snap.Leave()

	// Close the snapshoter
	close(stopCh)
	snap.Wait()

	// Open the snapshoter
	stopCh = make(chan struct{})
	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit,
		logger, clock, nil, stopCh)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the values
	if snap.LastClock() != 0 {
		t.Fatalf("bad clock %d", snap.LastClock())
	}
	if snap.LastEventClock() != 0 {
		t.Fatalf("bad clock %d", snap.LastEventClock())
	}

	prev := snap.AliveNodes()
	if len(prev) != 0 {
		t.Fatalf("expected none alive: %#v", prev)
	}
}
<|endoftext|>