<commit_before>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package tail

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/hpcloud/tail/ratelimiter"
	"github.com/hpcloud/tail/util"
	"github.com/hpcloud/tail/watch"
	"gopkg.in/tomb.v1"
)

var (
	ErrStop = fmt.Errorf("tail should now stop")
)

type Line struct {
	Text string
	Time time.Time
	Err  error // Error from tail
}

// NewLine returns a Line with present time.
func NewLine(text string) *Line {
	return &Line{text, time.Now(), nil}
}

// SeekInfo represents arguments to `os.Seek`
type SeekInfo struct {
	Offset int64
	Whence int // os.SEEK_*
}

type logger interface {
	Fatal(v ...interface{})
	Fatalf(format string, v ...interface{})
	Fatalln(v ...interface{})
	Panic(v ...interface{})
	Panicf(format string, v ...interface{})
	Panicln(v ...interface{})
	Print(v ...interface{})
	Printf(format string, v ...interface{})
	Println(v ...interface{})
}

// Config is used to specify how a file must be tailed.
type Config struct {
	// File-specifc
	Location    *SeekInfo // Seek to this location before tailing
	ReOpen      bool      // Reopen recreated files (tail -F)
	MustExist   bool      // Fail early if the file does not exist
	Poll        bool      // Poll for file changes instead of using inotify
	Pipe        bool      // Is a named pipe (mkfifo)
	RateLimiter *ratelimiter.LeakyBucket

	// Generic IO
	Follow      bool // Continue looking for new lines (tail -f)
	MaxLineSize int  // If non-zero, split longer lines into multiple lines

	// Logger, when nil, is set to tail.DefaultLogger
	// To disable logging: set field to tail.DiscardingLogger
	Logger logger
}

type Tail struct {
	Filename string
	Lines    chan *Line
	Config

	file   *os.File
	reader *bufio.Reader

	watcher watch.FileWatcher
	changes *watch.FileChanges

	tomb.Tomb // provides: Done, Kill, Dying

	lk sync.Mutex
}

var (
	// DefaultLogger is used when Config.Logger == nil
	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
	// DiscardingLogger can be used to disable logging output
	DiscardingLogger = log.New(ioutil.Discard, "", 0)
)

// TailFile begins tailing the file. Output stream is made available
// via the `Tail.Lines` channel. To handle errors during tailing,
// invoke the `Wait` or `Err` method after finishing reading from the
// `Lines` channel.
func TailFile(filename string, config Config) (*Tail, error) {
	if config.ReOpen && !config.Follow {
		util.Fatal("cannot set ReOpen without Follow.")
	}

	t := &Tail{
		Filename: filename,
		Lines:    make(chan *Line),
		Config:   config,
	}

	// when Logger was not specified in config, use default logger
	if t.Logger == nil {
		t.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	if t.Poll {
		t.watcher = watch.NewPollingFileWatcher(filename)
	} else {
		t.watcher = watch.NewInotifyFileWatcher(filename)
	}

	if t.MustExist {
		var err error
		t.file, err = OpenFile(t.Filename)
		if err != nil {
			return nil, err
		}
	}

	go t.tailFileSync()

	return t, nil
}

// Return the file's current position, like stdio's ftell().
// But this value is not very accurate.
// it may readed one line in the chan(tail.Lines),
// so it may lost one line.
func (tail *Tail) Tell() (offset int64, err error) {
	if tail.file == nil {
		return
	}
	offset, err = tail.file.Seek(0, os.SEEK_CUR)
	if err == nil {
		tail.lk.Lock()
		offset -= int64(tail.reader.Buffered())
		tail.lk.Unlock()
	}
	return
}

// Stop stops the tailing activity.
func (tail *Tail) Stop() error {
	tail.Kill(nil)
	return tail.Wait()
}

// StopAtEOF stops tailing as soon as the end of the file is reached.
func (tail *Tail) StopAtEOF() error {
	tail.Kill(errStopAtEOF)
	return tail.Wait()
}

var errStopAtEOF = errors.New("tail: stop at eof")

func (tail *Tail) close() {
	close(tail.Lines)
	tail.closeFile()
}

func (tail *Tail) closeFile() {
	if tail.file != nil {
		tail.file.Close()
		tail.file = nil
	}
}

func (tail *Tail) reopen() error {
	tail.closeFile()
	for {
		var err error
		tail.file, err = OpenFile(tail.Filename)
		if err != nil {
			if os.IsNotExist(err) {
				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
					if err == tomb.ErrDying {
						return err
					}
					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
				}
				continue
			}
			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
		}
		break
	}
	return nil
}

func (tail *Tail) readLine() (string, error) {
	tail.lk.Lock()
	line, err := tail.reader.ReadString('\n')
	tail.lk.Unlock()
	if err != nil {
		// Note ReadString "returns the data read before the error" in
		// case of an error, including EOF, so we return it as is. The
		// caller is expected to process it if err is EOF.
		return line, err
	}

	line = strings.TrimRight(line, "\n")

	return line, err
}

func (tail *Tail) tailFileSync() {
	defer tail.Done()
	defer tail.close()

	if !tail.MustExist {
		// deferred first open.
		err := tail.reopen()
		if err != nil {
			if err != tomb.ErrDying {
				tail.Kill(err)
			}
			return
		}
	}

	// Seek to requested location on first open of the file.
	if tail.Location != nil {
		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
		tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location)
		if err != nil {
			tail.Killf("Seek error on %s: %s", tail.Filename, err)
			return
		}
	}

	tail.openReader()

	var offset int64 = 0
	var err error

	// Read line by line.
	for {
		// do not seek in named pipes
		if !tail.Pipe {
			// grab the position in case we need to back up in the event of a half-line
			offset, err = tail.Tell()
			if err != nil {
				tail.Kill(err)
				return
			}
		}

		line, err := tail.readLine()

		// Process `line` even if err is EOF.
		if err == nil {
			cooloff := !tail.sendLine(line)
			if cooloff {
				// Wait a second before seeking till the end of
				// file when rate limit is reached.
				msg := fmt.Sprintf(
					"Too much log activity; waiting a second " +
						"before resuming tailing")
				tail.Lines <- &Line{msg, time.Now(), fmt.Errorf(msg)}
				select {
				case <-time.After(time.Second):
				case <-tail.Dying():
					return
				}
				if err := tail.seekEnd(); err != nil {
					tail.Kill(err)
					return
				}
			}
		} else if err == io.EOF {
			if !tail.Follow {
				if line != "" {
					tail.sendLine(line)
				}
				return
			}

			if tail.Follow && line != "" {
				// this has the potential to never return the last line if
				// it's not followed by a newline; seems a fair trade here
				err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0})
				if err != nil {
					tail.Kill(err)
					return
				}
			}

			// When EOF is reached, wait for more data to become
			// available. Wait strategy is based on the `tail.watcher`
			// implementation (inotify or polling).
			err := tail.waitForChanges()
			if err != nil {
				if err != ErrStop {
					tail.Kill(err)
				}
				return
			}
		} else {
			// non-EOF error
			tail.Killf("Error reading %s: %s", tail.Filename, err)
			return
		}

		select {
		case <-tail.Dying():
			if tail.Err() == errStopAtEOF {
				continue
			}
			return
		default:
		}
	}
}

// waitForChanges waits until the file has been appended, deleted,
// moved or truncated. When moved or deleted - the file will be
// reopened if ReOpen is true. Truncated files are always reopened.
func (tail *Tail) waitForChanges() error {
	if tail.changes == nil {
		pos, err := tail.file.Seek(0, os.SEEK_CUR)
		if err != nil {
			return err
		}
		tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos)
		if err != nil {
			return err
		}
	}

	select {
	case <-tail.changes.Modified:
		return nil
	case <-tail.changes.Deleted:
		tail.changes = nil
		if tail.ReOpen {
			// XXX: we must not log from a library.
			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
			if err := tail.reopen(); err != nil {
				return err
			}
			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
			tail.openReader()
			return nil
		} else {
			tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
			return ErrStop
		}
	case <-tail.changes.Truncated:
		// Always reopen truncated files (Follow is true)
		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
		if err := tail.reopen(); err != nil {
			return err
		}
		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
		tail.openReader()
		return nil
	case <-tail.Dying():
		return ErrStop
	}
	panic("unreachable")
}

func (tail *Tail) openReader() {
	if tail.MaxLineSize > 0 {
		// add 2 to account for newline characters
		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
	} else {
		tail.reader = bufio.NewReader(tail.file)
	}
}

func (tail *Tail) seekEnd() error {
	return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END})
}

func (tail *Tail) seekTo(pos SeekInfo) error {
	_, err := tail.file.Seek(pos.Offset, pos.Whence)
	if err != nil {
		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
	}
	// Reset the read buffer whenever the file is re-seek'ed
	tail.reader.Reset(tail.file)
	return nil
}

// sendLine sends the line(s) to Lines channel, splitting longer lines
// if necessary. Return false if rate limit is reached.
func (tail *Tail) sendLine(line string) bool {
	now := time.Now()
	lines := []string{line}

	// Split longer lines
	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
		lines = util.PartitionString(line, tail.MaxLineSize)
	}

	for _, line := range lines {
		tail.Lines <- &Line{line, now, nil}
	}

	if tail.Config.RateLimiter != nil {
		ok := tail.Config.RateLimiter.Pour(uint16(len(lines)))
		if !ok {
			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n",
				tail.Filename)
			return false
		}
	}

	return true
}

// Cleanup removes inotify watches added by the tail package. This function is
// meant to be invoked from a process's exit handler. Linux kernel may not
// automatically remove inotify watches after the process exits.
func (tail *Tail) Cleanup() {
	watch.Cleanup(tail.Filename)
}
<commit_msg>fix reader nil pointer<commit_after>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package tail

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/hpcloud/tail/ratelimiter"
	"github.com/hpcloud/tail/util"
	"github.com/hpcloud/tail/watch"
	"gopkg.in/tomb.v1"
)

var (
	ErrStop = fmt.Errorf("tail should now stop")
)

type Line struct {
	Text string
	Time time.Time
	Err  error // Error from tail
}

// NewLine returns a Line with present time.
func NewLine(text string) *Line {
	return &Line{text, time.Now(), nil}
}

// SeekInfo represents arguments to `os.Seek`
type SeekInfo struct {
	Offset int64
	Whence int // os.SEEK_*
}

type logger interface {
	Fatal(v ...interface{})
	Fatalf(format string, v ...interface{})
	Fatalln(v ...interface{})
	Panic(v ...interface{})
	Panicf(format string, v ...interface{})
	Panicln(v ...interface{})
	Print(v ...interface{})
	Printf(format string, v ...interface{})
	Println(v ...interface{})
}

// Config is used to specify how a file must be tailed.
type Config struct {
	// File-specifc
	Location    *SeekInfo // Seek to this location before tailing
	ReOpen      bool      // Reopen recreated files (tail -F)
	MustExist   bool      // Fail early if the file does not exist
	Poll        bool      // Poll for file changes instead of using inotify
	Pipe        bool      // Is a named pipe (mkfifo)
	RateLimiter *ratelimiter.LeakyBucket

	// Generic IO
	Follow      bool // Continue looking for new lines (tail -f)
	MaxLineSize int  // If non-zero, split longer lines into multiple lines

	// Logger, when nil, is set to tail.DefaultLogger
	// To disable logging: set field to tail.DiscardingLogger
	Logger logger
}

type Tail struct {
	Filename string
	Lines    chan *Line
	Config

	file   *os.File
	reader *bufio.Reader

	watcher watch.FileWatcher
	changes *watch.FileChanges

	tomb.Tomb // provides: Done, Kill, Dying

	lk sync.Mutex
}

var (
	// DefaultLogger is used when Config.Logger == nil
	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
	// DiscardingLogger can be used to disable logging output
	DiscardingLogger = log.New(ioutil.Discard, "", 0)
)

// TailFile begins tailing the file. Output stream is made available
// via the `Tail.Lines` channel. To handle errors during tailing,
// invoke the `Wait` or `Err` method after finishing reading from the
// `Lines` channel.
func TailFile(filename string, config Config) (*Tail, error) {
	if config.ReOpen && !config.Follow {
		util.Fatal("cannot set ReOpen without Follow.")
	}

	t := &Tail{
		Filename: filename,
		Lines:    make(chan *Line),
		Config:   config,
	}

	// when Logger was not specified in config, use default logger
	if t.Logger == nil {
		t.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	if t.Poll {
		t.watcher = watch.NewPollingFileWatcher(filename)
	} else {
		t.watcher = watch.NewInotifyFileWatcher(filename)
	}

	if t.MustExist {
		var err error
		t.file, err = OpenFile(t.Filename)
		if err != nil {
			return nil, err
		}
	}

	go t.tailFileSync()

	return t, nil
}

// Return the file's current position, like stdio's ftell().
// But this value is not very accurate.
// it may readed one line in the chan(tail.Lines),
// so it may lost one line.
func (tail *Tail) Tell() (offset int64, err error) {
	if tail.file == nil {
		return
	}
	offset, err = tail.file.Seek(0, os.SEEK_CUR)
	if err != nil {
		return
	}

	tail.lk.Lock()
	defer tail.lk.Unlock()
	if tail.reader == nil {
		return
	}

	offset -= int64(tail.reader.Buffered())
	return
}

// Stop stops the tailing activity.
func (tail *Tail) Stop() error {
	tail.Kill(nil)
	return tail.Wait()
}

// StopAtEOF stops tailing as soon as the end of the file is reached.
func (tail *Tail) StopAtEOF() error {
	tail.Kill(errStopAtEOF)
	return tail.Wait()
}

var errStopAtEOF = errors.New("tail: stop at eof")

func (tail *Tail) close() {
	close(tail.Lines)
	tail.closeFile()
}

func (tail *Tail) closeFile() {
	if tail.file != nil {
		tail.file.Close()
		tail.file = nil
	}
}

func (tail *Tail) reopen() error {
	tail.closeFile()
	for {
		var err error
		tail.file, err = OpenFile(tail.Filename)
		if err != nil {
			if os.IsNotExist(err) {
				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
					if err == tomb.ErrDying {
						return err
					}
					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
				}
				continue
			}
			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
		}
		break
	}
	return nil
}

func (tail *Tail) readLine() (string, error) {
	tail.lk.Lock()
	line, err := tail.reader.ReadString('\n')
	tail.lk.Unlock()
	if err != nil {
		// Note ReadString "returns the data read before the error" in
		// case of an error, including EOF, so we return it as is. The
		// caller is expected to process it if err is EOF.
		return line, err
	}

	line = strings.TrimRight(line, "\n")

	return line, err
}

func (tail *Tail) tailFileSync() {
	defer tail.Done()
	defer tail.close()

	if !tail.MustExist {
		// deferred first open.
		err := tail.reopen()
		if err != nil {
			if err != tomb.ErrDying {
				tail.Kill(err)
			}
			return
		}
	}

	// Seek to requested location on first open of the file.
	if tail.Location != nil {
		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
		tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location)
		if err != nil {
			tail.Killf("Seek error on %s: %s", tail.Filename, err)
			return
		}
	}

	tail.openReader()

	var offset int64 = 0
	var err error

	// Read line by line.
	for {
		// do not seek in named pipes
		if !tail.Pipe {
			// grab the position in case we need to back up in the event of a half-line
			offset, err = tail.Tell()
			if err != nil {
				tail.Kill(err)
				return
			}
		}

		line, err := tail.readLine()

		// Process `line` even if err is EOF.
		if err == nil {
			cooloff := !tail.sendLine(line)
			if cooloff {
				// Wait a second before seeking till the end of
				// file when rate limit is reached.
				msg := fmt.Sprintf(
					"Too much log activity; waiting a second " +
						"before resuming tailing")
				tail.Lines <- &Line{msg, time.Now(), fmt.Errorf(msg)}
				select {
				case <-time.After(time.Second):
				case <-tail.Dying():
					return
				}
				if err := tail.seekEnd(); err != nil {
					tail.Kill(err)
					return
				}
			}
		} else if err == io.EOF {
			if !tail.Follow {
				if line != "" {
					tail.sendLine(line)
				}
				return
			}

			if tail.Follow && line != "" {
				// this has the potential to never return the last line if
				// it's not followed by a newline; seems a fair trade here
				err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0})
				if err != nil {
					tail.Kill(err)
					return
				}
			}

			// When EOF is reached, wait for more data to become
			// available. Wait strategy is based on the `tail.watcher`
			// implementation (inotify or polling).
			err := tail.waitForChanges()
			if err != nil {
				if err != ErrStop {
					tail.Kill(err)
				}
				return
			}
		} else {
			// non-EOF error
			tail.Killf("Error reading %s: %s", tail.Filename, err)
			return
		}

		select {
		case <-tail.Dying():
			if tail.Err() == errStopAtEOF {
				continue
			}
			return
		default:
		}
	}
}

// waitForChanges waits until the file has been appended, deleted,
// moved or truncated. When moved or deleted - the file will be
// reopened if ReOpen is true. Truncated files are always reopened.
func (tail *Tail) waitForChanges() error {
	if tail.changes == nil {
		pos, err := tail.file.Seek(0, os.SEEK_CUR)
		if err != nil {
			return err
		}
		tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos)
		if err != nil {
			return err
		}
	}

	select {
	case <-tail.changes.Modified:
		return nil
	case <-tail.changes.Deleted:
		tail.changes = nil
		if tail.ReOpen {
			// XXX: we must not log from a library.
			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
			if err := tail.reopen(); err != nil {
				return err
			}
			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
			tail.openReader()
			return nil
		} else {
			tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
			return ErrStop
		}
	case <-tail.changes.Truncated:
		// Always reopen truncated files (Follow is true)
		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
		if err := tail.reopen(); err != nil {
			return err
		}
		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
		tail.openReader()
		return nil
	case <-tail.Dying():
		return ErrStop
	}
	panic("unreachable")
}

func (tail *Tail) openReader() {
	if tail.MaxLineSize > 0 {
		// add 2 to account for newline characters
		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
	} else {
		tail.reader = bufio.NewReader(tail.file)
	}
}

func (tail *Tail) seekEnd() error {
	return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END})
}

func (tail *Tail) seekTo(pos SeekInfo) error {
	_, err := tail.file.Seek(pos.Offset, pos.Whence)
	if err != nil {
		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
	}
	// Reset the read buffer whenever the file is re-seek'ed
	tail.reader.Reset(tail.file)
	return nil
}

// sendLine sends the line(s) to Lines channel, splitting longer lines
// if necessary. Return false if rate limit is reached.
func (tail *Tail) sendLine(line string) bool {
	now := time.Now()
	lines := []string{line}

	// Split longer lines
	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
		lines = util.PartitionString(line, tail.MaxLineSize)
	}

	for _, line := range lines {
		tail.Lines <- &Line{line, now, nil}
	}

	if tail.Config.RateLimiter != nil {
		ok := tail.Config.RateLimiter.Pour(uint16(len(lines)))
		if !ok {
			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n",
				tail.Filename)
			return false
		}
	}

	return true
}

// Cleanup removes inotify watches added by the tail package. This function is
// meant to be invoked from a process's exit handler. Linux kernel may not
// automatically remove inotify watches after the process exits.
func (tail *Tail) Cleanup() {
	watch.Cleanup(tail.Filename)
}
<|endoftext|>


<commit_before>package fscache

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

type Test struct {
	t   *testing.T
	dir string
}

func Wrap(t *testing.T, dir string) *Test {
	test := &Test{
		t: t,
	}

	dir, err := ioutil.TempDir("", dir)
	test.AssertNoError(err)
	test.dir = dir
	return test
}

func (t *Test) AssertError(err error) {
	if err == nil {
		t.t.Fatal(errors.New(fmt.Sprintf("expected error: %s", err)))
	}
}

func (t *Test) AssertNoError(err error) {
	if err != nil {
		t.t.Fatal(errors.New(fmt.Sprintf("expected no error: %s", err)))
	}
}

func (t *Test) Assert(cond bool, msg ...string) {
	if !cond {
		t.t.Fatal(errors.New(fmt.Sprintf("expected true: got %t %s", cond,
			handleMsg(msg))))
	}
}

func (t *Test) AssertByteEqual(p, q []byte) {
	t.Assert(bytes.Equal(p, q),
		fmt.Sprintf("expected %s, got %s", string(p), string(q)))
}

func handleMsg(msg []string) string {
	if msg == nil || len(msg) == 0 {
		return ""
	} else {
		return msg[0]
	}
}

func (t *Test) Dir() string {
	return t.dir
}

func (t *Test) CreateFile(name string) *os.File {
	f, err := os.OpenFile(filepath.Join(t.dir, name),
		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	t.AssertNoError(err)
	return f
}

func (t *Test) Close() {
	err := os.RemoveAll(t.dir)
	t.AssertNoError(err)
}
<commit_msg>add stack traces to test errors<commit_after>package fscache

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/spacemonkeygo/errors"
)

var (
	Error = errors.NewClass("test error")
)

type Test struct {
	t   *testing.T
	dir string
}

func Wrap(t *testing.T, dir string) *Test {
	test := &Test{
		t: t,
	}

	dir, err := ioutil.TempDir("", dir)
	test.AssertNoError(err)
	test.dir = dir
	return test
}

func (t *Test) AssertError(err error) {
	if err == nil {
		t.t.Fatal(Error.New("expected error: %s", err))
	}
}

func (t *Test) AssertNoError(err error) {
	if err != nil {
		t.t.Fatal(Error.New("expected no error: %s", err))
	}
}

func (t *Test) Assert(cond bool, msg ...string) {
	if !cond {
		t.t.Fatal(Error.New("expected true: got %t %s", cond,
			handleMsg(msg)))
	}
}

func (t *Test) AssertByteEqual(p, q []byte) {
	t.Assert(bytes.Equal(p, q),
		fmt.Sprintf("expected %s, got %s", string(p), string(q)))
}

func handleMsg(msg []string) string {
	if msg == nil || len(msg) == 0 {
		return ""
	} else {
		return msg[0]
	}
}

func (t *Test) Dir() string {
	return t.dir
}

func (t *Test) CreateFile(name string) *os.File {
	f, err := os.OpenFile(filepath.Join(t.dir, name),
		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	t.AssertNoError(err)
	return f
}

func (t *Test) Close() {
	err := os.RemoveAll(t.dir)
	t.AssertNoError(err)
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"fmt"
	"github.com/cognusion/go-cache-lru"
	"time"
	"crypto/rand"
	"math/big"
	"math"
	"strings"
	"net/http"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"
	"github.com/cognusion/tinysum"
)

var VERSION = "go-tiny 1.0.5"
var C *cache.Cache
var GLOBALOFFSET uint32

// Simply return the version
func version(c web.C, w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "%s", VERSION)
}

// Simply return the number of cached urls
func count(c web.C, w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "%v", C.ItemCount())
}

// Grab an item from the cache and redirect to the stored URL
func fetch(c web.C, w http.ResponseWriter, r *http.Request) {

	tiny := c.URLParams["tiny"]
	fmt.Printf("Getting %v\n", tiny)

	turl, found := C.Get(tiny)
	if ! found {
		http.Error(w, http.StatusText(404), 404)
		return
	}

	url := turl.(string)
	if strings.HasPrefix(url,"http:") || strings.HasPrefix(url,"https:") {
		url = strings.Replace(url, ":/", "://", 1)
	} else {
		url = "http://" + url
	}
	fmt.Printf("Got %v as %v\n", tiny, turl)

	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}

// Generate the offset crc32, and set it in the cache
func set(c web.C, w http.ResponseWriter, r *http.Request) {

	url := r.RequestURI
	url = url[5:len(url)] // get rid of leading "/set"

	fmt.Printf("Setting %v\n", url)

	// We take the crc32 of the URL add a random offset
	v := tinysum.OffsetStringSum(url, GLOBALOFFSET)

	C.Set(v, url, cache.DefaultExpiration)

	fmt.Printf("Set %v to %v\n", url,v)

	msg := `

	%v

	`
	msg = fmt.Sprintf(msg, "/" + v, "/" + v)

	io.WriteString(w, msg)
}

func main() {

	// So the URLs aren't pure crc32s of the URI
	thirtyTwo := math.Pow(2,32)-1
	max := *big.NewInt(int64(thirtyTwo))
	roff,_ := rand.Int(rand.Reader,&max)
	GLOBALOFFSET = uint32(roff.Uint64())
	fmt.Printf("Offset is %v\n",GLOBALOFFSET)

	// Keep items for 24hours, clean every 30s, cap at 50k items.
	C = cache.New(24*time.Hour, 30*time.Second, 50000)

	// Set the URI handlers, and go!
	goji.Get("/version", version)
	goji.Get("/count", count)
	goji.Get("/set/*", set)
	goji.Get("/:tiny", fetch)
	goji.Serve()

}
<commit_msg>Formats source appropriately<commit_after>package main

import (
	"crypto/rand"
	"fmt"
	"github.com/cognusion/go-cache-lru"
	"github.com/cognusion/tinysum"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"
	"io"
	"math"
	"math/big"
	"net/http"
	"strings"
	"time"
)

var VERSION = "go-tiny 1.0.5"
var C *cache.Cache
var GLOBALOFFSET uint32

// Simply return the version
func version(c web.C, w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "%s", VERSION)
}

// Simply return the number of cached urls
func count(c web.C, w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "%v", C.ItemCount())
}

// Grab an item from the cache and redirect to the stored URL
func fetch(c web.C, w http.ResponseWriter, r *http.Request) {

	tiny := c.URLParams["tiny"]
	fmt.Printf("Getting %v\n", tiny)

	turl, found := C.Get(tiny)
	if !found {
		http.Error(w, http.StatusText(404), 404)
		return
	}

	url := turl.(string)
	if strings.HasPrefix(url, "http:") || strings.HasPrefix(url, "https:") {
		url = strings.Replace(url, ":/", "://", 1)
	} else {
		url = "http://" + url
	}
	fmt.Printf("Got %v as %v\n", tiny, turl)

	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}

// Generate the offset crc32, and set it in the cache
func set(c web.C, w http.ResponseWriter, r *http.Request) {

	url := r.RequestURI
	url = url[5:len(url)] // get rid of leading "/set"

	fmt.Printf("Setting %v\n", url)

	// We take the crc32 of the URL add a random offset
	v := tinysum.OffsetStringSum(url, GLOBALOFFSET)

	C.Set(v, url, cache.DefaultExpiration)

	fmt.Printf("Set %v to %v\n", url, v)

	msg := `

	%v

	`
	msg = fmt.Sprintf(msg, "/"+v, "/"+v)

	io.WriteString(w, msg)
}

func main() {

	// So the URLs aren't pure crc32s of the URI
	thirtyTwo := math.Pow(2, 32) - 1
	max := *big.NewInt(int64(thirtyTwo))
	roff, _ := rand.Int(rand.Reader, &max)
	GLOBALOFFSET = uint32(roff.Uint64())
	fmt.Printf("Offset is %v\n", GLOBALOFFSET)

	// Keep items for 24hours, clean every 30s, cap at 50k items.
	C = cache.New(24*time.Hour, 30*time.Second, 50000)

	// Set the URI handlers, and go!
	goji.Get("/version", version)
	goji.Get("/count", count)
	goji.Get("/set/*", set)
	goji.Get("/:tiny", fetch)
	goji.Serve()

}
<|endoftext|>


<commit_before>/* Copyright (c) 2014-2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import "html/template"

var tmpl *template.Template

func loadTemplates() {
	for name, s := range templates {
		var t *template.Template
		if tmpl == nil {
			tmpl = template.New(name)
		}
		if name == tmpl.Name() {
			t = tmpl
		} else {
			t = tmpl.New(name)
		}
		if _, err := t.Parse(s); err != nil {
			panic("could not load templates")
		}
	}
}

var templates = map[string]string{
	"/": `

Set up an alias:

    $ alias pcat='curl -F "{{.FieldName}}=<-" {{.SiteURL}}'

Upload a new paste:

    $ echo foo | pcat
    {{.SiteURL}}/a63d03b9

Fetch it:

    $ curl {{.SiteURL}}/a63d03b9
    foo

You can also use the web form.
{{if gt .MaxSize 0.0}}
Maximum size per paste is {{.MaxSize}}
{{end}}{{if gt .LifeTime 0}}
Pastes will be deleted after {{.LifeTime}}
{{end}}
github.com/mvdan/pastecat

`,
	"/form": `

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<textarea cols=80 rows=24 name="{{.FieldName}}"></textarea>

		<button type="submit">Paste text</button>
	</form>
	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<input type="file" name="{{.FieldName}}"></input>

		<button type="submit">Paste file</button>
	</form>

`,
}
<commit_msg>Better limit wording on the main page<commit_after>/* Copyright (c) 2014-2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import "html/template"

var tmpl *template.Template

func loadTemplates() {
	for name, s := range templates {
		var t *template.Template
		if tmpl == nil {
			tmpl = template.New(name)
		}
		if name == tmpl.Name() {
			t = tmpl
		} else {
			t = tmpl.New(name)
		}
		if _, err := t.Parse(s); err != nil {
			panic("could not load templates")
		}
	}
}

var templates = map[string]string{
	"/": `

Set up an alias:

    $ alias pcat='curl -F "{{.FieldName}}=<-" {{.SiteURL}}'

Upload a new paste:

    $ echo foo | pcat
    {{.SiteURL}}/a63d03b9

Fetch it:

    $ curl {{.SiteURL}}/a63d03b9
    foo

You can also use the web form.
{{if gt .MaxSize 0.0}}
The maximum size per paste is {{.MaxSize}}.
{{end}}{{if gt .LifeTime 0}}
Each paste will be deleted after {{.LifeTime}}.
{{end}}
github.com/mvdan/pastecat

`,
	"/form": `

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<textarea cols=80 rows=24 name="{{.FieldName}}"></textarea>

		<button type="submit">Paste text</button>
	</form>
	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<input type="file" name="{{.FieldName}}"></input>

		<button type="submit">Paste file</button>
	</form>

`,
}
<|endoftext|>


<commit_before>package buildah

import (
	"os/user"
	"strconv"

	"github.com/opencontainers/runtime-spec/specs-go"
)

// TODO: we should doing these lookups using data that's actually in the container.
func getUser(username string) (specs.User, error) {
	if username == "" {
		return specs.User{}, nil
	}
	runuser, err := user.Lookup(username)
	if err != nil {
		return specs.User{}, err
	}
	uid, err := strconv.ParseUint(runuser.Uid, 10, 32)
	if err != nil {
		return specs.User{}, nil
	}
	gid, err := strconv.ParseUint(runuser.Gid, 10, 32)
	if err != nil {
		return specs.User{}, nil
	}
	groups, err := runuser.GroupIds()
	if err != nil {
		return specs.User{}, err
	}
	gids := []uint32{}
	for _, group := range groups {
		if g, err := user.LookupGroup(group); err == nil {
			if gid, err := strconv.ParseUint(g.Gid, 10, 32); err == nil {
				gids = append(gids, uint32(gid))
			}
		}
	}
	u := specs.User{
		UID:            uint32(uid),
		GID:            uint32(gid),
		AdditionalGids: gids,
		Username:       username,
	}
	return u, nil
}
<commit_msg>Drop supplemental groups for "run"<commit_after>package buildah

import (
	"os/user"
	"strconv"

	"github.com/opencontainers/runtime-spec/specs-go"
)

// TODO: we should doing these lookups using data that's actually in the container.
func getUser(username string) (specs.User, error) {
	if username == "" {
		return specs.User{}, nil
	}
	runuser, err := user.Lookup(username)
	if err != nil {
		return specs.User{}, err
	}
	uid, err := strconv.ParseUint(runuser.Uid, 10, 32)
	if err != nil {
		return specs.User{}, nil
	}
	gid, err := strconv.ParseUint(runuser.Gid, 10, 32)
	if err != nil {
		return specs.User{}, nil
	}
	u := specs.User{
		UID:      uint32(uid),
		GID:      uint32(gid),
		Username: username,
	}
	return u, nil
}
<|endoftext|>


<commit_before>package ztex

import (
	"fmt"

	"github.com/google/gousb"
)

const (
	// VendorID is the ZTEX USB vendor ID (VID).
	VendorID = gousb.ID(0x221A)

	// ProductID is the standard ZTEX USB product ID (PID)
	ProductID = gousb.ID(0x0100)
)

func binaryPrefix(n uint64, unit string) string {
	switch {
	case n != 0 && n&(1<<30-1) == 0:
		return fmt.Sprintf("%vGi%v [%v%v]", n>>30, unit, n, unit)
	case n != 0 && n&(1<<20-1) == 0:
		return fmt.Sprintf("%vMi%v [%v%v]", n>>20, unit, n, unit)
	case n != 0 && n&(1<<10-1) == 0:
		return fmt.Sprintf("%vki%v [%v%v]", n>>10, unit, n, unit)
	default:
		return fmt.Sprintf("%v%v", n, unit)
	}
}
<commit_msg>Reformat binary prefix.<commit_after>package ztex

import (
	"fmt"

	"github.com/google/gousb"
)

const (
	// VendorID is the ZTEX USB vendor ID (VID).
	VendorID = gousb.ID(0x221A)

	// ProductID is the standard ZTEX USB product ID (PID)
	ProductID = gousb.ID(0x0100)
)

func binaryPrefix(n uint64, unit string) string {
	switch {
	case n != 0 && n&(1<<30-1) == 0:
		return fmt.Sprintf("%v%v [%vGi%v]", n, unit, n>>30, unit)
	case n != 0 && n&(1<<20-1) == 0:
		return fmt.Sprintf("%v%v [%vMi%v]", n, unit, n>>20, unit)
	case n != 0 && n&(1<<10-1) == 0:
		return fmt.Sprintf("%v%v [%vki%v]", n, unit, n>>10, unit)
	default:
		return fmt.Sprintf("%v%v", n, unit)
	}
}
<|endoftext|>


<commit_before>package gophercloud

import (
	"errors"
	"strings"
	"time"
)

// WaitFor polls a predicate function, once per second, up to a timeout limit.
// It usually does this to wait for a resource to transition to a certain state.
// Resource packages will wrap this in a more convenient function that's
// specific to a certain resource, but it can also be useful on its own.
func WaitFor(timeout int, predicate func() (bool, error)) error {
	start := time.Now().Second()
	for {
		// Force a 1s sleep
		time.Sleep(1 * time.Second)

		// If a timeout is set, and that's been exceeded, shut it down
		if timeout >= 0 && time.Now().Second()-start >= timeout {
			return errors.New("A timeout occurred")
		}

		// Execute the function
		satisfied, err := predicate()
		if err != nil {
			return err
		}
		if satisfied {
			return nil
		}
	}
}

// NormalizeURL ensures that each endpoint URL has a closing `/`, as expected
// by ServiceClient.
func NormalizeURL(url string) string {
	if !strings.HasSuffix(url, "/") {
		return url + "/"
	}
	return url
}
<commit_msg>Right, NormalizeURL is internal, too.<commit_after>package gophercloud

import (
	"errors"
	"strings"
	"time"
)

// WaitFor polls a predicate function, once per second, up to a timeout limit.
// It usually does this to wait for a resource to transition to a certain state.
// Resource packages will wrap this in a more convenient function that's
// specific to a certain resource, but it can also be useful on its own.
func WaitFor(timeout int, predicate func() (bool, error)) error {
	start := time.Now().Second()
	for {
		// Force a 1s sleep
		time.Sleep(1 * time.Second)

		// If a timeout is set, and that's been exceeded, shut it down
		if timeout >= 0 && time.Now().Second()-start >= timeout {
			return errors.New("A timeout occurred")
		}

		// Execute the function
		satisfied, err := predicate()
		if err != nil {
			return err
		}
		if satisfied {
			return nil
		}
	}
}

// NormalizeURL is an internal function to be used by provider clients.
//
// It ensures that each endpoint URL has a closing `/`, as expected by
// ServiceClient's methods.
func NormalizeURL(url string) string {
	if !strings.HasSuffix(url, "/") {
		return url + "/"
	}
	return url
}
<|endoftext|>


<commit_before>package mgoauth

import (
	"github.com/kidstuff/auth/model"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"time"
)

type LoginState struct {
	ExpiredOn time.Time     `bson:"ExpiredOn"`
	UserId    bson.ObjectId `bson:"UserId"`
	Token     string        `bson:"_id"`
}

// getId returns bson.ObjectId form given id.
// id must be a valid bson.ObjectId or a valid ObjectIdHex
func getId(id string) (bson.ObjectId, error) {
	if !bson.IsObjectIdHex(id) {
		return "", model.ErrInvalidId
	}

	return bson.ObjectIdHex(id), nil
}

// EnsureIndex builds the index for users data and login state collection.
func EnsureIndex(db *mgo.Database) error {
	groupColl := db.C("mgoauth_group")
	userColl := db.C("mgoauth_user")
	loginColl := db.C("mgoauth_login")

	err := userColl.EnsureIndex(mgo.Index{
		Key:    []string{"Email"},
		Unique: true,
	})
	if err != nil {
		return err
	}

	err = userColl.EnsureIndexKey("LastActivity")
	if err != nil {
		return err
	}

	err = userColl.EnsureIndexKey("BriefGroups._id")
	if err != nil {
		return err
	}

	err = loginColl.EnsureIndex(mgo.Index{
		Key:      []string{"UserId"},
		DropDups: true,
	})
	if err != nil {
		return err
	}

	err = loginColl.EnsureIndex(mgo.Index{
		Key:         []string{"ExpiredOn"},
		ExpireAfter: time.Minute,
	})

	err = groupColl.EnsureIndex(mgo.Index{
		Key:    []string{"Name"},
		Unique: true,
	})

	return nil
}
<commit_msg>fix index name<commit_after>package mgoauth

import (
	"github.com/kidstuff/auth/model"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"time"
)

type LoginState struct {
	ExpiredOn time.Time     `bson:"ExpiredOn"`
	UserId    bson.ObjectId `bson:"UserId"`
	Token     string        `bson:"_id"`
}

// getId returns bson.ObjectId form given id.
// id must be a valid bson.ObjectId or a valid ObjectIdHex
func getId(id string) (bson.ObjectId, error) {
	if !bson.IsObjectIdHex(id) {
		return "", model.ErrInvalidId
	}

	return bson.ObjectIdHex(id), nil
}

// EnsureIndex builds the index for users data and login state collection.
func EnsureIndex(db *mgo.Database) error {
	groupColl := db.C("mgoauth_group")
	userColl := db.C("mgoauth_user")
	loginColl := db.C("mgoauth_login")

	err := userColl.EnsureIndex(mgo.Index{
		Key:    []string{"Email"},
		Unique: true,
	})
	if err != nil {
		return err
	}

	err = userColl.EnsureIndexKey("LastActivity")
	if err != nil {
		return err
	}

	err = userColl.EnsureIndexKey("Groups._id")
	if err != nil {
		return err
	}

	err = loginColl.EnsureIndex(mgo.Index{
		Key:      []string{"UserId"},
		DropDups: true,
	})
	if err != nil {
		return err
	}

	err = loginColl.EnsureIndex(mgo.Index{
		Key:         []string{"ExpiredOn"},
		ExpireAfter: time.Minute,
	})

	err = groupColl.EnsureIndex(mgo.Index{
		Key:    []string{"Name"},
		Unique: true,
	})

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

var (
	parseSizeRegex *regexp.Regexp
)

// Utility method to determine if a file/dir exists
func FileOrDirExists(path string) (exists bool, isDir bool) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, false
	} else {
		return true, fi.IsDir()
	}
}

// Utility method to determine if a file (NOT dir) exists
func FileExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && !isDir
}

// Utility method to determine if a dir (NOT file) exists
func DirExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && isDir
}

// Utility method to determine if a file/dir exists and is of a specific size
func FileExistsAndIsOfSize(path string, sz int64) bool {
	fi, err := os.Stat(path)

	if err != nil && os.IsNotExist(err) {
		return false
	}

	return fi.Size() == sz
}

// Parse a string representing a size into a number of bytes
// supports m/mb = megabytes, g/gb = gigabytes etc (case insensitive)
func ParseSize(str string) (int64, error) {
	if parseSizeRegex == nil {
		parseSizeRegex = regexp.MustCompile(`(?i)^\s*([\d\.]+)\s*([KMGTP]?B?)\s*$`)
	}

	if match := parseSizeRegex.FindStringSubmatch(str); match != nil {
		value, err := strconv.ParseFloat(match[1], 32)
		if err != nil {
			return 0, err
		}
		strUnits := strings.ToUpper(match[2])
		switch strUnits {
		case "KB", "K":
			return int64(value * (1 << 10)), nil
		case "MB", "M":
			return int64(value * (1 << 20)), nil
		case "GB", "G":
			return int64(value * (1 << 30)), nil
		case "TB", "T":
			return int64(value * (1 << 40)), nil
		case "PB", "P":
			return int64(value * (1 << 50)), nil
		default:
			return int64(value), nil

		}

	} else {
		return 0, errors.New(fmt.Sprintf("Invalid size: %v", str))
	}

}

// Format a number of bytes into a display format
func FormatSize(sz int64) string {

	switch {
	case sz >= (1 << 50):
		return fmt.Sprintf("%.3gPB", float32(sz)/float32(1<<50))
	case sz >= (1 << 40):
		return fmt.Sprintf("%.3gTB", float32(sz)/float32(1<<40))
	case sz >= (1 << 30):
		return fmt.Sprintf("%.3gGB", float32(sz)/float32(1<<30))
	case sz >= (1 << 20):
		return fmt.Sprintf("%.3gMB", float32(sz)/float32(1<<20))
	case sz >= (1 << 10):
		return fmt.Sprintf("%.3gKB", float32(sz)/float32(1<<10))
	default:
		return fmt.Sprintf("%d", sz)
	}
}

// Format a bytes per second transfer rate into a display format
func FormatTransferRate(bytesPerSecond int64) string {
	switch {
	case bytesPerSecond >= (1 << 50): // yeah, right ;)
		return fmt.Sprintf("%.3gPB/s", float32(bytesPerSecond)/float32(1<<50))
	case bytesPerSecond >= (1 << 40):
		return fmt.Sprintf("%.3gTB/s", float32(bytesPerSecond)/float32(1<<40))
	case bytesPerSecond >= (1 << 30):
		return fmt.Sprintf("%.3gGB/s", float32(bytesPerSecond)/float32(1<<30))
	case bytesPerSecond >= (1 << 20):
		return fmt.Sprintf("%.3gMB/s", float32(bytesPerSecond)/float32(1<<20))
	case bytesPerSecond >= (1 << 10):
		return fmt.Sprintf("%.3gKB/s", float32(bytesPerSecond)/float32(1<<10))
	default:
		return fmt.Sprintf("%dBytes/s", bytesPerSecond)
	}
}

// Calculates transfer rates by averaging over n samples
type TransferRateCalculator struct {
	numSamples      int
	samples         []int64 // bytesPerSecond samples
	sampleInsertIdx int
}

func NewTransferRateCalculator(numSamples int) *TransferRateCalculator {
	return &TransferRateCalculator{numSamples, make([]int64, numSamples), 0}
}
func (t *TransferRateCalculator) AddSample(bytesPerSecond int64) {
	t.samples[t.sampleInsertIdx] = bytesPerSecond
	t.sampleInsertIdx = (t.sampleInsertIdx + 1) % t.numSamples
}
func (t *TransferRateCalculator) Average() int64 {
	var sum int64
	for _, s := range t.samples {
		sum += s
	}
	return sum / int64(t.numSamples)
}

// Search a sorted slice of strings for a specific string
// Returns boolean for if found, and either location or insertion point
func StringBinarySearch(sortedSlice []string, searchTerm string) (bool, int) {
	// Convenience method to easily provide boolean of whether to insert or not
	idx := sort.SearchStrings(sortedSlice, searchTerm)
	found := idx < len(sortedSlice) && sortedSlice[idx] == searchTerm
	return found, idx
}

// Overwrite the current line in the console (e.g. for progressive update)
// Requires the previous line length so that it can clear it with spaces
func OverwriteConsoleLine(newString string, lastLineLength int, iobuf io.Writer) {
	if len(newString) < lastLineLength {
		fmt.Fprintf(iobuf, "\r%v%v", newString, strings.Repeat(" ", lastLineLength-len(newString)))
	} else {
		fmt.Fprintf(iobuf, "\r%v", newString)
	}

}

// Remove duplicates from a slice of strings (in place)
// Linear to logarithmic time, doesn't change the ordering of the slice
// allocates/frees a new map of up to the size of the slice though
func StringRemoveDuplicates(s *[]string) {
	if s == nil || *s == nil {
		return
	}
	uniques := NewStringSet()
	insertidx := 0
	for _, x := range *s {
		if !uniques.Contains(x) {
			uniques.Add(x)
			(*s)[insertidx] = x // could do this only when x != insertidx but prob wasteful compare
			insertidx++
		}
	}
	// If any were eliminated it will now be shorter
	*s = (*s)[:insertidx]
}

// Return whether a given filename passes the include / exclude path filters
// Only paths that are in includePaths and outside excludePaths are passed
// If includePaths is empty that filter always passes and the same with excludePaths
// Both path lists support wildcard matches
func FilenamePassesIncludeExcludeFilter(filename string, includePaths, excludePaths []string) bool {
	if len(includePaths) == 0 && len(excludePaths) == 0 {
		return true
	}

	if len(includePaths) > 0 {
		matched := false
		for _, inc := range includePaths {
			matched, _ = filepath.Match(inc, filename)
			if !matched {
				// Also support matching a parent directory without a wildcard
				if strings.HasPrefix(filename, inc+string(filepath.Separator)) {
					matched = true
				}
			}
			if matched {
				break
			}

		}
		if !matched {
			return false
		}
	}

	if len(excludePaths) > 0 {
		for _, ex := range includePaths {
			matched, _ := filepath.Match(ex, filename)
			if matched {
				return false
			}
			// Also support matching a parent directory without a wildcard
			if strings.HasPrefix(filename, ex+string(filepath.Separator)) {
				return false
			}

		}
	}

	return true

}
<commit_msg>Fix exclude paths, was iterating include paths instead (copy/paste error)<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

var (
	parseSizeRegex *regexp.Regexp
)

// Utility method to determine if a file/dir exists
func FileOrDirExists(path string) (exists bool, isDir bool) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, false
	} else {
		return true, fi.IsDir()
	}
}

// Utility method to determine if a file (NOT dir) exists
func FileExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && !isDir
}

// Utility method to determine if a dir (NOT file) exists
func DirExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && isDir
}

// Utility method to determine if a file/dir exists and is of a specific size
func FileExistsAndIsOfSize(path string, sz int64) bool {
	fi, err := os.Stat(path)

	if err != nil && os.IsNotExist(err) {
		return false
	}

	return fi.Size() == sz
}

// Parse a string representing a size into a number of bytes
// supports m/mb = megabytes, g/gb = gigabytes etc (case insensitive)
func ParseSize(str string) (int64, error) {
	if parseSizeRegex == nil {
		parseSizeRegex = regexp.MustCompile(`(?i)^\s*([\d\.]+)\s*([KMGTP]?B?)\s*$`)
	}

	if match := parseSizeRegex.FindStringSubmatch(str); match != nil {
		value, err := strconv.ParseFloat(match[1], 32)
		if err != nil {
			return 0, err
		}
		strUnits := strings.ToUpper(match[2])
		switch strUnits {
		case "KB", "K":
			return int64(value * (1 << 10)), nil
		case "MB", "M":
			return int64(value * (1 << 20)), nil
		case "GB", "G":
			return int64(value * (1 << 30)), nil
		case "TB", "T":
			return int64(value * (1 << 40)), nil
		case "PB", "P":
			return int64(value * (1 << 50)), nil
		default:
			return int64(value), nil

		}

	} else {
		return 0, errors.New(fmt.Sprintf("Invalid size: %v", str))
	}

}

// Format a number of bytes into a display format
func FormatSize(sz int64) string {

	switch {
	case sz >= (1 << 50):
		return fmt.Sprintf("%.3gPB", float32(sz)/float32(1<<50))
	case sz >= (1 << 40):
		return fmt.Sprintf("%.3gTB", float32(sz)/float32(1<<40))
	case sz >= (1 << 30):
		return fmt.Sprintf("%.3gGB", float32(sz)/float32(1<<30))
	case sz >= (1 << 20):
		return fmt.Sprintf("%.3gMB", float32(sz)/float32(1<<20))
	case sz >= (1 << 10):
		return fmt.Sprintf("%.3gKB", float32(sz)/float32(1<<10))
	default:
		return fmt.Sprintf("%d", sz)
	}
}

// Format a bytes per second transfer rate into a display format
func FormatTransferRate(bytesPerSecond int64) string {
	switch {
	case bytesPerSecond >= (1 << 50): // yeah, right ;)
		return fmt.Sprintf("%.3gPB/s", float32(bytesPerSecond)/float32(1<<50))
	case bytesPerSecond >= (1 << 40):
		return fmt.Sprintf("%.3gTB/s", float32(bytesPerSecond)/float32(1<<40))
	case bytesPerSecond >= (1 << 30):
		return fmt.Sprintf("%.3gGB/s", float32(bytesPerSecond)/float32(1<<30))
	case bytesPerSecond >= (1 << 20):
		return fmt.Sprintf("%.3gMB/s", float32(bytesPerSecond)/float32(1<<20))
	case bytesPerSecond >= (1 << 10):
		return fmt.Sprintf("%.3gKB/s", float32(bytesPerSecond)/float32(1<<10))
	default:
		return fmt.Sprintf("%dBytes/s", bytesPerSecond)
	}
}

// Calculates transfer rates by averaging over n samples
type TransferRateCalculator struct {
	numSamples      int
	samples         []int64 // bytesPerSecond samples
	sampleInsertIdx int
}

func NewTransferRateCalculator(numSamples int) *TransferRateCalculator {
	return &TransferRateCalculator{numSamples, make([]int64, numSamples), 0}
}
func (t *TransferRateCalculator) AddSample(bytesPerSecond int64) {
	t.samples[t.sampleInsertIdx] = bytesPerSecond
	t.sampleInsertIdx = (t.sampleInsertIdx + 1) % t.numSamples
}
func (t *TransferRateCalculator) Average() int64 {
	var sum int64
	for _, s := range t.samples {
		sum += s
	}
	return sum / int64(t.numSamples)
}

// Search a sorted slice of strings for a specific string
// Returns boolean for if found, and either location or insertion point
func StringBinarySearch(sortedSlice []string, searchTerm string) (bool, int) {
	// Convenience method to easily provide boolean of whether to insert or not
	idx := sort.SearchStrings(sortedSlice, searchTerm)
	found := idx < len(sortedSlice) && sortedSlice[idx] == searchTerm
	return found, idx
}

// Overwrite the current line in the console (e.g. for progressive update)
// Requires the previous line length so that it can clear it with spaces
func OverwriteConsoleLine(newString string, lastLineLength int, iobuf io.Writer) {
	if len(newString) < lastLineLength {
		fmt.Fprintf(iobuf, "\r%v%v", newString, strings.Repeat(" ", lastLineLength-len(newString)))
	} else {
		fmt.Fprintf(iobuf, "\r%v", newString)
	}

}

// Remove duplicates from a slice of strings (in place)
// Linear to logarithmic time, doesn't change the ordering of the slice
// allocates/frees a new map of up to the size of the slice though
func StringRemoveDuplicates(s *[]string) {
	if s == nil || *s == nil {
		return
	}
	uniques := NewStringSet()
	insertidx := 0
	for _, x := range *s {
		if !uniques.Contains(x) {
			uniques.Add(x)
			(*s)[insertidx] = x // could do this only when x != insertidx but prob wasteful compare
			insertidx++
		}
	}
	// If any were eliminated it will now be shorter
	*s = (*s)[:insertidx]
}

// Return whether a given filename passes the include / exclude path filters
// Only paths that are in includePaths and outside excludePaths are passed
// If includePaths is empty that filter always passes and the same with excludePaths
// Both path lists support wildcard matches
func FilenamePassesIncludeExcludeFilter(filename string, includePaths, excludePaths []string) bool {
	if len(includePaths) == 0 && len(excludePaths) == 0 {
		return true
	}

	if len(includePaths) > 0 {
		matched := false
		for _, inc := range includePaths {
			matched, _ = filepath.Match(inc, filename)
			if !matched {
				// Also support matching a parent directory without a wildcard
				if strings.HasPrefix(filename, inc+string(filepath.Separator)) {
					matched = true
				}
			}
			if matched {
				break
			}

		}
		if !matched {
			return false
		}
	}

	if len(excludePaths) > 0 {
		for _, ex := range excludePaths {
			matched, _ := filepath.Match(ex, filename)
			if matched {
				return false
			}
			// Also support matching a parent directory without a wildcard
			if strings.HasPrefix(filename, ex+string(filepath.Separator)) {
				return false
			}

		}
	}

	return true

}
<|endoftext|>


<commit_before>/*
Copyright (c) 2012 Matthias S. Benkmann

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA  02110-1301, USA.
*/

// Various re-usable utility functions.
package util

import (
         "io"
         "fmt"
         "net"
         "time"
         "bytes"
         "crypto/md5"
         "runtime/debug"
       )

// Returns the md5sum of its argument as a string of hex digits.
func Md5sum(s string) string {
  md5 := md5.New()
  io.WriteString(md5, s)
  return fmt.Sprintf("%x",md5.Sum(nil))
}

// Number of consecutive short writes before WriteAll() will give up
// ATTENTION! The wait time between tries increases exponetially, so don't
// blindly increase this number.
const write_all_max_tries = 8

// Calls g wrapped in a panic handler that logs the panic and recovers from it.
// Example:
//   go util.WithPanicHandler(foobar)
//   go util.WithPanicHandler(func(){ Send_foreign_job_updates(server, jobs) })
func WithPanicHandler(g func()) {
  defer func() {
    if x := recover(); x != nil {
      Log(0, "PANIC! %v\n%v", x, string(debug.Stack()))
    }
  }()
  g()
}

// Returns a new channel that will return start as first item and then
// always the next number.
func Counter(start uint64) chan uint64 {
  c := make(chan uint64)
  go func() {
    for {
      c<-start
      start++
    }
  }()
  return c
}

// Writes data to w, with automatic handling of short writes.
// A short write error will only be returned if multiple attempts
// failed in a row.
func WriteAll(w io.Writer, data []byte) (n int, err error) {
  // Yeah, I know. Other people just ignore the issue of
  // short writes. That's why their code fails more often than mine :-P
  tries := write_all_max_tries
  var bytes_written int
  for n = 0; n < len(data); {
    bytes_written, err = w.Write(data[n:])
    n += bytes_written

    if err != nil && err != io.ErrShortWrite {
      return n, err
    }

    if bytes_written == 0 {
      tries--
      if tries <= 0 {
        if err == nil {
          err = io.ErrShortWrite
        }
        return n, err
      }

      // The first time we don't sleep. The 2nd time we sleep 1ms. The 3rd time 2ms.
      // The 4th time 4ms. Then 8ms, 16ms, 32ms, 64ms,...
      var wait time.Duration = (1 << (write_all_max_tries-2)) >> uint(tries)
      time.Sleep(wait * time.Millisecond)

    } else {
      tries = write_all_max_tries  // every time we succeed at writing we start tries again
    }
  }

  return n, nil
}

// Opens a connection to target (e.g. "foo.example.com:20081"), sends msg followed
// by \r\n and then closes the connection.
// If timeout >= 0, then the connection will be terminated after at most this duration.
func SendLnTo(target string, msg string, timeout time.Duration) {
  conn, err := net.Dial("tcp", target)
  if err != nil {
    Log(0, "ERROR! Dial: %v", err)
    return
  }
  defer conn.Close()
  SendLn(conn, msg, timeout)
}

// Sends strings via connection conn, followed by "\r\n"
// If timeout >= 0, then the connection will be terminated after at most this duration.
func SendLn(conn net.Conn, s string, timeout time.Duration) {
  sendbuf := make([]byte, len(s)+2)
  copy(sendbuf, s)
  sendbuf[len(s)]='\r'
  sendbuf[len(s)+1]='\n'

  if timeout >= 0 {
    conn.SetWriteDeadline(time.Now().Add(timeout))
  }
  _, err := WriteAll(conn, sendbuf)
  if err != nil {
    Log(0, "ERROR! WriteAll: %v", err)
  }
}

// Reads from the connection until \n is seen (or timeout or error) and
// returns the first line with trailing \n and \r removed.
// If timeout >= 0, then the connection will be terminated after at most this duration.
func ReadLn(conn net.Conn, timeout time.Duration) string {
  var buf = make([]byte, 65536)
  i := 0
  n := 1

  if timeout >= 0 {
    conn.SetReadDeadline(time.Now().Add(timeout))
  }
  var err error
  for n != 0 {
    n, err = conn.Read(buf[i:])
    if err != nil && err != io.EOF {
      Log(0, "ERROR! Read: %v", err)
    }
    if err == io.EOF && i != 0 {
      Log(0, "ERROR! Incomplete message (i.e. not terminated by \"\\n\") of %v bytes", i)
    }

    i += n

    if i == len(buf) {
      buf_new := make([]byte, len(buf)+65536)
      copy(buf_new, buf)
      buf = buf_new
    }

    // Find complete line terminated by '\n' and return it
    eol := bytes.IndexByte(buf[0:i], '\n')

    if eol >= 0 {
      for ; eol >= 0 && (buf[eol] == '\n' || buf[eol] == '\r') ; { eol-- }
      return string(buf[0:eol+1])
    }
  }

  return ""
}
<commit_msg>SendLn returns error; old deadlines are reset in SendLn and ReadLn<commit_after>/*
Copyright (c) 2012 Matthias S. Benkmann

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA  02110-1301, USA.
*/

// Various re-usable utility functions.
package util

import (
         "io"
         "fmt"
         "net"
         "time"
         "bytes"
         "crypto/md5"
         "runtime/debug"
       )

// Returns the md5sum of its argument as a string of hex digits.
func Md5sum(s string) string {
  md5 := md5.New()
  io.WriteString(md5, s)
  return fmt.Sprintf("%x",md5.Sum(nil))
}

// Number of consecutive short writes before WriteAll() will give up
// ATTENTION! The wait time between tries increases exponetially, so don't
// blindly increase this number.
const write_all_max_tries = 8

// Calls g wrapped in a panic handler that logs the panic and recovers from it.
// Example:
//   go util.WithPanicHandler(foobar)
//   go util.WithPanicHandler(func(){ Send_foreign_job_updates(server, jobs) })
func WithPanicHandler(g func()) {
  defer func() {
    if x := recover(); x != nil {
      Log(0, "PANIC! %v\n%v", x, string(debug.Stack()))
    }
  }()
  g()
}

// Returns a new channel that will return start as first item and then
// always the next number.
func Counter(start uint64) chan uint64 {
  c := make(chan uint64)
  go func() {
    for {
      c<-start
      start++
    }
  }()
  return c
}

// Writes data to w, with automatic handling of short writes.
// A short write error will only be returned if multiple attempts
// failed in a row.
func WriteAll(w io.Writer, data []byte) (n int, err error) {
  // Yeah, I know. Other people just ignore the issue of
  // short writes. That's why their code fails more often than mine :-P
  tries := write_all_max_tries
  var bytes_written int
  for n = 0; n < len(data); {
    bytes_written, err = w.Write(data[n:])
    n += bytes_written

    if err != nil && err != io.ErrShortWrite {
      return n, err
    }

    if bytes_written == 0 {
      tries--
      if tries <= 0 {
        if err == nil {
          err = io.ErrShortWrite
        }
        return n, err
      }

      // The first time we don't sleep. The 2nd time we sleep 1ms. The 3rd time 2ms.
      // The 4th time 4ms. Then 8ms, 16ms, 32ms, 64ms,...
      var wait time.Duration = (1 << (write_all_max_tries-2)) >> uint(tries)
      time.Sleep(wait * time.Millisecond)

    } else {
      tries = write_all_max_tries  // every time we succeed at writing we start tries again
    }
  }

  return n, nil
}

// Opens a connection to target (e.g. "foo.example.com:20081"), sends msg followed
// by \r\n and then closes the connection.
// If timeout >= 0, then the connection will be terminated after at most this duration.
func SendLnTo(target string, msg string, timeout time.Duration) {
  conn, err := net.Dial("tcp", target)
  if err != nil {
    Log(0, "ERROR! Dial: %v", err)
    return
  }
  defer conn.Close()
  SendLn(conn, msg, timeout)
}

// Sends strings via connection conn, followed by "\r\n"
// If timeout >= 0, then the send attempt will be aborted after at most this duration.
// Returns nil if sending was successful, an error otherwise. The error will also
// be sent to the log, so the caller does not have to log it.
func SendLn(conn net.Conn, s string, timeout time.Duration) error {
  sendbuf := make([]byte, len(s)+2)
  copy(sendbuf, s)
  sendbuf[len(s)]='\r'
  sendbuf[len(s)+1]='\n'

  var deadline time.Time // zero value means "no deadline"
  if timeout >= 0 { deadline = time.Now().Add(timeout) }
  conn.SetWriteDeadline(deadline)

  _, err := WriteAll(conn, sendbuf)
  if err != nil {
    Log(0, "ERROR! WriteAll: %v", err)
  }
  return err
}

// Reads from the connection until \n is seen (or timeout or error) and
// returns the first line with trailing \n and \r removed.
// If timeout >= 0, then the connection will be terminated after at most this duration.
func ReadLn(conn net.Conn, timeout time.Duration) string {
  var buf = make([]byte, 65536)
  i := 0
  n := 1

  var deadline time.Time // zero value means "no deadline"
  if timeout >= 0 { deadline = time.Now().Add(timeout) }
  conn.SetReadDeadline(deadline)

  var err error
  for n != 0 {
    n, err = conn.Read(buf[i:])
    if err != nil && err != io.EOF {
      Log(0, "ERROR! Read: %v", err)
    }
    if err == io.EOF && i != 0 {
      Log(0, "ERROR! Incomplete message (i.e. not terminated by \"\\n\") of %v bytes", i)
    }

    i += n

    if i == len(buf) {
      buf_new := make([]byte, len(buf)+65536)
      copy(buf_new, buf)
      buf = buf_new
    }

    // Find complete line terminated by '\n' and return it
    eol := bytes.IndexByte(buf[0:i], '\n')

    if eol >= 0 {
      for ; eol >= 0 && (buf[eol] == '\n' || buf[eol] == '\r') ; { eol-- }
      return string(buf[0:eol+1])
    }
  }

  return ""
}
<|endoftext|>


<commit_before>package sofa

import (
	"bytes"
	"encoding/json"
	"fmt"
)

// BooleanParameter is a special type of boolean created to have a zero value
// where it is not included in URL parameter output. This is useful for taking
// the default values of a parameter.
type BooleanParameter string

func (b BooleanParameter) String() string {
	return string(b)
}

const (
	// Empty is the zero value for the BooleanParameter type. It is the default
	// type and values of this type are not included in the URL parameters.
	Empty BooleanParameter = ""
	// True is the BooleanParameter equivalent of true. It will always be
	// included in a query string.
	True BooleanParameter = "true"
	// False is the BooleanParameter equivalent of true. It will always be
	// included in a query string.
	False BooleanParameter = "false"
)

type InterfaceParameter struct {
	innerVal interface{}
}

func (i InterfaceParameter) String() string {
	out, err := json.Marshal(i.innerVal)
	if err != nil {
		panic(err)
	}

	return string(out)
}

// ViewParams provides a type-safe implementation of the paramaters which may
// be passed to an execution of a CouchDB view function:
//  - conflicts (boolean) – Includes conflicts information in response.
//    Ignored if include_docs isn’t true. Default is false
//  - descending (boolean) – Return the documents in descending by key
//    order. Default is false
//  - endkey (json) – Stop returning records when the specified key is
//    reached. Optional
//  - end_key (json) – Alias for endkey param
//  - endkey_docid (string) – Stop returning records when the specified
//    document ID is reached. Requires endkey to be specified for this to
//    have any effect. Optional
//  - end_key_doc_id (string) – Alias for endkey_docid param
//  - group (boolean) – Group the results using the reduce function to a
//    group or single row. Default is false
//  - group_level (number) – Specify the group level to be used. Optional
//  - include_docs (boolean) – Include the associated document with each row.
//    Default is false.
//  - attachments (boolean) – Include the Base64-encoded content of
//    attachments in the documents that are included if include_docs is true.
//    Ignored if include_docs isn’t true. Default is false.
//  - att_encoding_info (boolean) – Include encoding information in
//    attachment stubs if include_docs is true and the particular attachment
//    is compressed. Ignored if include_docs isn’t true. Default is false.
//  - inclusive_end (boolean) – Specifies whether the specified end key
//    should be included in the result. Default is true
//  - key (json) – Return only documents that match the specified key.
//    Optional
//  - keys (json-array) – Return only documents where the key matches one of
//    the keys specified in the array. Optional
//  - limit (number) – Limit the number of the returned documents to the
//    specified number. Optional
//  - reduce (boolean) – Use the reduction function. Default is true
//  - skip (number) – Skip this number of records before starting to return
//    the results. Default is 0
//  - sorted (boolean) – Sort returned rows (see Sorting Returned Rows).
//    Setting this to false offers a performance boost. The total_rows and
//    offset fields are not available when this is set to false.
//    Default is true
//  - stale (string) – Allow the results from a stale view to be used.
//    Supported values: ok and update_after. Optional
//  - startkey (json) – Return records starting with the specified key.
//    Optional
//  - start_key (json) – Alias for startkey param
//  - startkey_docid (string) – Return records starting with the specified
//    document ID. Requires startkey to be specified for this to have any
//    effect. Optional
//  - start_key_doc_id (string) – Alias for startkey_docid param
//  - update_seq (boolean) – Response includes an update_seq value
//    indicating which sequence id of the database the view reflects.
//    Default is false
type ViewParams struct {
	Conflicts              BooleanParameter    `url:"conflicts,omitempty"`
	Descending             BooleanParameter    `url:"descending,omitempty"`
	EndKey                 *InterfaceParameter `url:"endkey,omitempty"`
	EndKeyDocID            string              `url:"endkey_docid,omitempty"`
	Group                  BooleanParameter    `url:"group,omitempty"`
	GroupLevel             float64             `url:"group_level,omitempty"`
	IncludeDocs            BooleanParameter    `url:"include_docs,omitempty"`
	Attachments            BooleanParameter    `url:"attachments,omitempty"`
	AttachmentEncodingInfo BooleanParameter    `url:"att_encoding_info,omitempty"`
	InclusiveEnd           BooleanParameter    `url:"inclusive_end,omitempty"`
	Key                    *InterfaceParameter `url:"key,omitempty"`
	Keys                   []interface{}       `url:"keys,omitempty"`
	Limit                  float64             `url:"limit,omitempty"`
	Reduce                 BooleanParameter    `url:"reduce,omitempty"`
	Skip                   float64             `url:"skip,omitempty"`
	Sorted                 BooleanParameter    `url:"sorted,omitempty"`
	Stale                  string              `url:"stale,omitempty"`
	StartKey               *InterfaceParameter `url:"startkey,omitempty"`
	StartKeyDocID          string              `url:"startkey_docid,omitempty"`
	UpdateSeq              BooleanParameter    `url:"update_seq,omitempty"`
}

func (v *ViewParams) URLOptions() (*URLOptions, error) {
	u := NewURLOptions()

	// Process booleans
	if v.Conflicts != Empty {
		if err := u.Set("conflicts", v.Conflicts); err != nil {
			return nil, err
		}
	}
	if v.Descending != Empty {
		if err := u.Set("descending", v.Descending); err != nil {
			return nil, err
		}
	}
	if v.Group != Empty {
		if err := u.Set("group", v.Group); err != nil {
			return nil, err
		}
	}
	if v.IncludeDocs != Empty {
		if err := u.Set("include_docs", v.IncludeDocs); err != nil {
			return nil, err
		}
	}
	if v.Attachments != Empty {
		if err := u.Set("attachments", v.Attachments); err != nil {
			return nil, err
		}
	}
	if v.AttachmentEncodingInfo != Empty {
		if err := u.Set("att_encoding_info", v.AttachmentEncodingInfo); err != nil {
			return nil, err
		}
	}
	if v.InclusiveEnd != Empty {
		if err := u.Set("inclusive_end", v.InclusiveEnd); err != nil {
			return nil, err
		}
	}
	if v.Reduce != Empty {
		if err := u.Set("reduce", v.Reduce); err != nil {
			return nil, err
		}
	}
	if v.Sorted != Empty {
		if err := u.Set("sorted", v.Sorted); err != nil {
			return nil, err
		}
	}
	if v.UpdateSeq != Empty {
		if err := u.Set("update_seq", v.UpdateSeq); err != nil {
			return nil, err
		}
	}

	// Process interfaces
	if v.StartKey != nil {
		if err := u.Set("startkey", v.StartKey); err != nil {
			return nil, err
		}
	}
	if v.EndKey != nil {
		if err := u.Set("endkey", v.EndKey); err != nil {
			return nil, err
		}
	}
	if v.Key != nil {
		if err := u.Set("key", v.Key); err != nil {
			return nil, err
		}
	}

	// Process lists
	if v.Keys != nil {
		if err := u.Set("keys", v.Keys); err != nil {
			return nil, err
		}
	}

	// Process strings
	if v.EndKeyDocID != "" {
		if err := u.Set("endkey_docid", v.EndKeyDocID); err != nil {
			return nil, err
		}
	}
	if v.StartKeyDocID != "" {
		if err := u.Set("startkey_docid", v.StartKeyDocID); err != nil {
			return nil, err
		}
	}
	if v.Stale != "" {
		if err := u.Set("stale", v.Stale); err != nil {
			return nil, err
		}
	}

	// Process floats
	// TODO: Can something better be done that checking for zero?
	if v.GroupLevel != 0 {
		if err := u.Set("group_level", v.GroupLevel); err != nil {
			return nil, err
		}
	}
	if v.Limit != 0 {
		if err := u.Set("limit", v.Limit); err != nil {
			return nil, err
		}
	}
	if v.Skip != 0 {
		if err := u.Set("skip", v.Skip); err != nil {
			return nil, err
		}
	}

	return &u, nil
}

// View is an interface representing the way that views are executed and
// their results returned.
type View interface {
	Execute(Options) (DocumentList, error)
}

// TemporaryView is a type of view which can be created & accessed on the fly.
// Temporary views are good for debugging purposed but should never be used in
// production as they are slow for any large number of documents.
type TemporaryView struct {
	Map    string `json:"map,omitempty"`
	Reduce string `json:"reduce,omitempty"`

	db *Database
}

// TemporaryView creates a temporary view for this database. Only the map function is
// required but other parameters canbe added to the resulting TemporaryView if
// required.
func (d *Database) TemporaryView(mapFunc string) TemporaryView {
	return TemporaryView{
		Map: mapFunc,

		db: d,
	}
}

// Execute implements View for TemporaryView.
func (v TemporaryView) Execute(params ViewParams) (DocumentList, error) {
	jsString, err := json.Marshal(v)
	if err != nil {
		return DocumentList{}, err
	}

	opts, err := params.URLOptions()
	if err != nil {
		return DocumentList{}, err
	}

	var docs DocumentList
	_, err = v.db.con.unmarshalRequest("POST", v.db.ViewPath("_temp_view"), opts, bytes.NewBuffer(jsString), &docs)
	if err != nil {
		return DocumentList{}, err
	}

	return docs, nil
}

// NamedView represents a view stored on a design document in the database.
// It must be accessed with both the name of the design document and the
// name of the view.
type NamedView struct {
	DesignDoc string
	Name      string

	db *Database
}

// NamedView creates a new NamedView for this database. This can then be used
// to access the current results of the permanent view on the design document.
func (d *Database) NamedView(design, name string) NamedView {
	return NamedView{
		DesignDoc: design,
		Name:      name,

		db: d,
	}
}

// Execute implements View for NamedView.
func (v NamedView) Execute(params ViewParams) (DocumentList, error) {
	opts, err := params.URLOptions()
	if err != nil {
		return DocumentList{}, err
	}

	var docs DocumentList
	if _, err := v.db.con.unmarshalRequest("GET", v.db.ViewPath(v.Path()), opts, nil, &docs); err != nil {
		return DocumentList{}, err
	}

	return docs, nil
}

// Path gets the path of the NamedView relative to the database root.
func (v NamedView) Path() string {
	return fmt.Sprintf("_design/%s/_view/%s", v.DesignDoc, v.Name)
}

// FullPath gets the path of the NamedView relative to the server root.
func (v NamedView) FullPath() string {
	return urlConcat(v.db.Name(), v.Path())
}
<commit_msg>Remove unused String function & add way to create an InterfaceParameter<commit_after>package sofa

import (
	"bytes"
	"encoding/json"
	"fmt"
)

// BooleanParameter is a special type of boolean created to have a zero value
// where it is not included in URL parameter output. This is useful for taking
// the default values of a parameter.
type BooleanParameter string

func (b BooleanParameter) String() string {
	return string(b)
}

const (
	// Empty is the zero value for the BooleanParameter type. It is the default
	// type and values of this type are not included in the URL parameters.
	Empty BooleanParameter = ""
	// True is the BooleanParameter equivalent of true. It will always be
	// included in a query string.
	True BooleanParameter = "true"
	// False is the BooleanParameter equivalent of true. It will always be
	// included in a query string.
	False BooleanParameter = "false"
)

// InterfaceParameter is a wrapper for an empty interface which ensures that it is correctly formatted when passed
// as a query parameter to CouchDB. The reason for this is that strings passed are usually required not to be quoted
// but in these fields which can also take JSON objects of other types the quotes seem required
type InterfaceParameter struct {
	innerVal interface{}
}

// NewInterfaceParameter returns a pointer to an InterfaceParameter wrapping the provided value. All new
// InterfaceParameters must be created through this function.
func NewInterfaceParameter(iface interface{}) *InterfaceParameter {
	return &InterfaceParameter{
		innerVal: iface,
	}
}

// MarshalJSON simply marshals the internal value, to ensure these objects are always included using the
// JSON-formatted representation of just the inner value.
func (i InterfaceParameter) MarshalJSON() ([]byte, error) {
	return json.Marshal(i.innerVal)
}

// ViewParams provides a type-safe implementation of the paramaters which may
// be passed to an execution of a CouchDB view function:
//  - conflicts (boolean) – Includes conflicts information in response.
//    Ignored if include_docs isn’t true. Default is false
//  - descending (boolean) – Return the documents in descending by key
//    order. Default is false
//  - endkey (json) – Stop returning records when the specified key is
//    reached. Optional
//  - end_key (json) – Alias for endkey param
//  - endkey_docid (string) – Stop returning records when the specified
//    document ID is reached. Requires endkey to be specified for this to
//    have any effect. Optional
//  - end_key_doc_id (string) – Alias for endkey_docid param
//  - group (boolean) – Group the results using the reduce function to a
//    group or single row. Default is false
//  - group_level (number) – Specify the group level to be used. Optional
//  - include_docs (boolean) – Include the associated document with each row.
//    Default is false.
//  - attachments (boolean) – Include the Base64-encoded content of
//    attachments in the documents that are included if include_docs is true.
//    Ignored if include_docs isn’t true. Default is false.
//  - att_encoding_info (boolean) – Include encoding information in
//    attachment stubs if include_docs is true and the particular attachment
//    is compressed. Ignored if include_docs isn’t true. Default is false.
//  - inclusive_end (boolean) – Specifies whether the specified end key
//    should be included in the result. Default is true
//  - key (json) – Return only documents that match the specified key.
//    Optional
//  - keys (json-array) – Return only documents where the key matches one of
//    the keys specified in the array. Optional
//  - limit (number) – Limit the number of the returned documents to the
//    specified number. Optional
//  - reduce (boolean) – Use the reduction function. Default is true
//  - skip (number) – Skip this number of records before starting to return
//    the results. Default is 0
//  - sorted (boolean) – Sort returned rows (see Sorting Returned Rows).
//    Setting this to false offers a performance boost. The total_rows and
//    offset fields are not available when this is set to false.
//    Default is true
//  - stale (string) – Allow the results from a stale view to be used.
//    Supported values: ok and update_after. Optional
//  - startkey (json) – Return records starting with the specified key.
//    Optional
//  - start_key (json) – Alias for startkey param
//  - startkey_docid (string) – Return records starting with the specified
//    document ID. Requires startkey to be specified for this to have any
//    effect. Optional
//  - start_key_doc_id (string) – Alias for startkey_docid param
//  - update_seq (boolean) – Response includes an update_seq value
//    indicating which sequence id of the database the view reflects.
//    Default is false
type ViewParams struct {
	Conflicts              BooleanParameter    `url:"conflicts,omitempty"`
	Descending             BooleanParameter    `url:"descending,omitempty"`
	EndKey                 *InterfaceParameter `url:"endkey,omitempty"`
	EndKeyDocID            string              `url:"endkey_docid,omitempty"`
	Group                  BooleanParameter    `url:"group,omitempty"`
	GroupLevel             float64             `url:"group_level,omitempty"`
	IncludeDocs            BooleanParameter    `url:"include_docs,omitempty"`
	Attachments            BooleanParameter    `url:"attachments,omitempty"`
	AttachmentEncodingInfo BooleanParameter    `url:"att_encoding_info,omitempty"`
	InclusiveEnd           BooleanParameter    `url:"inclusive_end,omitempty"`
	Key                    *InterfaceParameter `url:"key,omitempty"`
	Keys                   []interface{}       `url:"keys,omitempty"`
	Limit                  float64             `url:"limit,omitempty"`
	Reduce                 BooleanParameter    `url:"reduce,omitempty"`
	Skip                   float64             `url:"skip,omitempty"`
	Sorted                 BooleanParameter    `url:"sorted,omitempty"`
	Stale                  string              `url:"stale,omitempty"`
	StartKey               *InterfaceParameter `url:"startkey,omitempty"`
	StartKeyDocID          string              `url:"startkey_docid,omitempty"`
	UpdateSeq              BooleanParameter    `url:"update_seq,omitempty"`
}

// URLOptions creates a URLOptions instance containing all of the currently-set values for this ViewParams
// instance.
func (v *ViewParams) URLOptions() (*URLOptions, error) {
	u := NewURLOptions()

	// Process booleans
	if v.Conflicts != Empty {
		if err := u.Set("conflicts", v.Conflicts); err != nil {
			return nil, err
		}
	}
	if v.Descending != Empty {
		if err := u.Set("descending", v.Descending); err != nil {
			return nil, err
		}
	}
	if v.Group != Empty {
		if err := u.Set("group", v.Group); err != nil {
			return nil, err
		}
	}
	if v.IncludeDocs != Empty {
		if err := u.Set("include_docs", v.IncludeDocs); err != nil {
			return nil, err
		}
	}
	if v.Attachments != Empty {
		if err := u.Set("attachments", v.Attachments); err != nil {
			return nil, err
		}
	}
	if v.AttachmentEncodingInfo != Empty {
		if err := u.Set("att_encoding_info", v.AttachmentEncodingInfo); err != nil {
			return nil, err
		}
	}
	if v.InclusiveEnd != Empty {
		if err := u.Set("inclusive_end", v.InclusiveEnd); err != nil {
			return nil, err
		}
	}
	if v.Reduce != Empty {
		if err := u.Set("reduce", v.Reduce); err != nil {
			return nil, err
		}
	}
	if v.Sorted != Empty {
		if err := u.Set("sorted", v.Sorted); err != nil {
			return nil, err
		}
	}
	if v.UpdateSeq != Empty {
		if err := u.Set("update_seq", v.UpdateSeq); err != nil {
			return nil, err
		}
	}

	// Process interfaces
	if v.StartKey != nil {
		if err := u.Set("startkey", v.StartKey); err != nil {
			return nil, err
		}
	}
	if v.EndKey != nil {
		if err := u.Set("endkey", v.EndKey); err != nil {
			return nil, err
		}
	}
	if v.Key != nil {
		if err := u.Set("key", v.Key); err != nil {
			return nil, err
		}
	}

	// Process lists
	if v.Keys != nil {
		if err := u.Set("keys", v.Keys); err != nil {
			return nil, err
		}
	}

	// Process strings
	if v.EndKeyDocID != "" {
		if err := u.Set("endkey_docid", v.EndKeyDocID); err != nil {
			return nil, err
		}
	}
	if v.StartKeyDocID != "" {
		if err := u.Set("startkey_docid", v.StartKeyDocID); err != nil {
			return nil, err
		}
	}
	if v.Stale != "" {
		if err := u.Set("stale", v.Stale); err != nil {
			return nil, err
		}
	}

	// Process floats
	// TODO: Can something better be done that checking for zero?
	if v.GroupLevel != 0 {
		if err := u.Set("group_level", v.GroupLevel); err != nil {
			return nil, err
		}
	}
	if v.Limit != 0 {
		if err := u.Set("limit", v.Limit); err != nil {
			return nil, err
		}
	}
	if v.Skip != 0 {
		if err := u.Set("skip", v.Skip); err != nil {
			return nil, err
		}
	}

	return &u, nil
}

// View is an interface representing the way that views are executed and
// their results returned.
type View interface {
	Execute(Options) (DocumentList, error)
}

// TemporaryView is a type of view which can be created & accessed on the fly.
// Temporary views are good for debugging purposed but should never be used in
// production as they are slow for any large number of documents.
type TemporaryView struct {
	Map    string `json:"map,omitempty"`
	Reduce string `json:"reduce,omitempty"`

	db *Database
}

// TemporaryView creates a temporary view for this database. Only the map function is
// required but other parameters canbe added to the resulting TemporaryView if
// required.
func (d *Database) TemporaryView(mapFunc string) TemporaryView {
	return TemporaryView{
		Map: mapFunc,

		db: d,
	}
}

// Execute implements View for TemporaryView.
func (v TemporaryView) Execute(params ViewParams) (DocumentList, error) {
	jsString, err := json.Marshal(v)
	if err != nil {
		return DocumentList{}, err
	}

	opts, err := params.URLOptions()
	if err != nil {
		return DocumentList{}, err
	}

	var docs DocumentList
	_, err = v.db.con.unmarshalRequest("POST", v.db.ViewPath("_temp_view"), opts, bytes.NewBuffer(jsString), &docs)
	if err != nil {
		return DocumentList{}, err
	}

	return docs, nil
}

// NamedView represents a view stored on a design document in the database.
// It must be accessed with both the name of the design document and the
// name of the view.
type NamedView struct {
	DesignDoc string
	Name      string

	db *Database
}

// NamedView creates a new NamedView for this database. This can then be used
// to access the current results of the permanent view on the design document.
func (d *Database) NamedView(design, name string) NamedView {
	return NamedView{
		DesignDoc: design,
		Name:      name,

		db: d,
	}
}

// Execute implements View for NamedView.
func (v NamedView) Execute(params ViewParams) (DocumentList, error) {
	opts, err := params.URLOptions()
	if err != nil {
		return DocumentList{}, err
	}

	var docs DocumentList
	if _, err := v.db.con.unmarshalRequest("GET", v.db.ViewPath(v.Path()), opts, nil, &docs); err != nil {
		return DocumentList{}, err
	}

	return docs, nil
}

// Path gets the path of the NamedView relative to the database root.
func (v NamedView) Path() string {
	return fmt.Sprintf("_design/%s/_view/%s", v.DesignDoc, v.Name)
}

// FullPath gets the path of the NamedView relative to the server root.
func (v NamedView) FullPath() string {
	return urlConcat(v.db.Name(), v.Path())
}
<|endoftext|>


<commit_before>package hdfs

import (
	"os"
	"path/filepath"
	"sort"
)

// Walk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root. All errors that arise visiting files
// and directories are filtered by walkFn. The files are walked in lexical
// order, which makes the output deterministic but means that for very large
// directories Walk can be inefficient. Walk does not follow symbolic links.
func (c *Client) Walk(root string, walkFn filepath.WalkFunc) error {
	return c.walk(root, walkFn)
}

func (c *Client) walk(path string, walkFn filepath.WalkFunc) error {
	file, err := c.Open(path)
	var info os.FileInfo
	if file != nil {
		info = file.Stat()
	}

	err = walkFn(path, info, err)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}

		return err
	}

	if info == nil || !info.IsDir() {
		return nil
	}

	names, err := file.Readdirnames(0)
	if err != nil {
		return walkFn(path, info, err)
	}

	sort.Strings(names)
	for _, name := range names {
		err = c.walk(filepath.Join(path, name), walkFn)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Never use filepath.Join on the client<commit_after>package hdfs

import (
	"os"
	"path"
	"path/filepath"
	"sort"
)

// Walk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root. All errors that arise visiting files
// and directories are filtered by walkFn. The files are walked in lexical
// order, which makes the output deterministic but means that for very large
// directories Walk can be inefficient. Walk does not follow symbolic links.
func (c *Client) Walk(root string, walkFn filepath.WalkFunc) error {
	return c.walk(root, walkFn)
}

func (c *Client) walk(path string, walkFn filepath.WalkFunc) error {
	file, err := c.Open(path)
	var info os.FileInfo
	if file != nil {
		info = file.Stat()
	}

	err = walkFn(path, info, err)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}

		return err
	}

	if info == nil || !info.IsDir() {
		return nil
	}

	names, err := file.Readdirnames(0)
	if err != nil {
		return walkFn(path, info, err)
	}

	sort.Strings(names)
	for _, name := range names {
		err = c.walk(path.Join(path, name), walkFn)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>