<commit_before>package condor

import (
	"fmt"
	"funnel/config"
	"funnel/logger"
	pbf "funnel/proto/funnel"
	"funnel/proto/tes"
	sched "funnel/scheduler"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"text/template"
)

var log = logger.New("condor")

// prefix is a string prefixed to condor worker IDs, so that condor
// workers can be identified by ShouldStartWorker() below.
// TODO move to worker metadata to be consistent with GCE
const prefix = "condor-worker-"

// NewScheduler returns a new HTCondor Scheduler instance.
func NewScheduler(conf config.Config) sched.Scheduler {
	return &scheduler{conf}
}

type scheduler struct {
	conf config.Config
}

// Schedule schedules a job on the HTCondor queue and returns a corresponding Offer.
func (s *scheduler) Schedule(j *tes.Job) *sched.Offer {
	log.Debug("Running condor scheduler")

	disk := s.conf.Worker.Resources.Disk
	if disk == 0.0 {
		for _, v := range j.Task.GetResources().GetVolumes() {
			disk += v.GetSizeGb()
		}
	}

	cpus := s.conf.Worker.Resources.Cpus
	if cpus == 0 {
		cpus = j.Task.GetResources().GetMinimumCpuCores()
	}

	ram := s.conf.Worker.Resources.Ram
	if ram == 0.0 {
		ram = j.Task.GetResources().GetMinimumRamGb()
	}

	// TODO could we call condor_submit --dry-run to test if a job would succeed?
	w := &pbf.Worker{
		Id: prefix + j.JobID,
		Resources: &pbf.Resources{
			Cpus: cpus,
			Ram:  ram,
			Disk: disk,
		},
	}
	return sched.NewOffer(w, j, sched.Scores{})
}

func (s *scheduler) ShouldStartWorker(w *pbf.Worker) bool {
	return strings.HasPrefix(w.Id, prefix) &&
		w.State == pbf.WorkerState_Uninitialized
}

// StartWorker submits a job via "condor_submit" to start a new worker.
func (s *scheduler) StartWorker(w *pbf.Worker) error {
	log.Debug("Starting condor worker")
	var err error

	// TODO document that these working dirs need manual cleanup
	workdir := path.Join(s.conf.WorkDir, w.Id)
	workdir, _ = filepath.Abs(workdir)
	err = os.MkdirAll(workdir, 0755)
	if err != nil {
		return err
	}

	c := s.conf.Worker
	c.ID = w.Id
	// 5 second timeout
	c.Timeout = 5000000000
	c.Resources.Cpus = w.Resources.Cpus
	c.Resources.Ram = w.Resources.Ram
	c.Resources.Disk = w.Resources.Disk

	confPath := path.Join(workdir, "worker.conf.yml")
	c.ToYamlFile(confPath)

	workerPath := sched.DetectWorkerPath()

	submitPath := path.Join(workdir, "condor.submit")
	f, err := os.Create(submitPath)
	if err != nil {
		return err
	}

	submitTpl, err := template.New("condor.submit").Parse(`
		universe       = vanilla
		executable     = {{.Executable}}
		arguments      = worker --config worker.conf.yml
		environment    = "PATH=/usr/bin"
		log            = {{.WorkDir}}/condor-event-log
		error          = {{.WorkDir}}/tes-worker-stderr
		output         = {{.WorkDir}}/tes-worker-stdout
		input          = {{.Config}}
		{{.Resources}}
		should_transfer_files   = YES
		when_to_transfer_output = ON_EXIT
		queue
	`)
	if err != nil {
		return err
	}

	err = submitTpl.Execute(f, map[string]string{
		"Executable": workerPath,
		"WorkDir":    workdir,
		"Config":     confPath,
		"Resources":  resolveCondorResourceRequest(int(w.Resources.Cpus), w.Resources.Ram, w.Resources.Disk),
	})
	if err != nil {
		return err
	}
	f.Close()

	cmd := exec.Command("condor_submit")
	stdin, err := os.Open(submitPath)
	if err != nil {
		return err
	}
	cmd.Stdin = stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return err
	}

	return nil
}

func resolveCondorResourceRequest(cpus int, ram float64, disk float64) string {
	var resources = []string{}
	if cpus != 0 {
		resources = append(resources, fmt.Sprintf("request_cpus = %d", cpus))
	}
	if ram != 0.0 {
		resources = append(resources, fmt.Sprintf("request_memory = %f GB", ram))
	}
	if disk != 0.0 {
		// Convert GB to KiB
		resources = append(resources, fmt.Sprintf("request_disk = %f", disk*976562))
	}
	return strings.Join(resources, "\n")
}
<commit_msg>fixed condor submit formatting<commit_after>package condor

import (
	"fmt"
	"funnel/config"
	"funnel/logger"
	pbf "funnel/proto/funnel"
	"funnel/proto/tes"
	sched "funnel/scheduler"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"text/template"
)

var log = logger.New("condor")

// prefix is a string prefixed to condor worker IDs, so that condor
// workers can be identified by ShouldStartWorker() below.
// TODO move to worker metadata to be consistent with GCE
const prefix = "condor-worker-"

// NewScheduler returns a new HTCondor Scheduler instance.
func NewScheduler(conf config.Config) sched.Scheduler {
	return &scheduler{conf}
}

type scheduler struct {
	conf config.Config
}

// Schedule schedules a job on the HTCondor queue and returns a corresponding Offer.
func (s *scheduler) Schedule(j *tes.Job) *sched.Offer {
	log.Debug("Running condor scheduler")

	disk := s.conf.Worker.Resources.Disk
	if disk == 0.0 {
		for _, v := range j.Task.GetResources().GetVolumes() {
			disk += v.GetSizeGb()
		}
	}

	cpus := s.conf.Worker.Resources.Cpus
	if cpus == 0 {
		cpus = j.Task.GetResources().GetMinimumCpuCores()
	}

	ram := s.conf.Worker.Resources.Ram
	if ram == 0.0 {
		ram = j.Task.GetResources().GetMinimumRamGb()
	}

	// TODO could we call condor_submit --dry-run to test if a job would succeed?
	w := &pbf.Worker{
		Id: prefix + j.JobID,
		Resources: &pbf.Resources{
			Cpus: cpus,
			Ram:  ram,
			Disk: disk,
		},
	}
	return sched.NewOffer(w, j, sched.Scores{})
}

func (s *scheduler) ShouldStartWorker(w *pbf.Worker) bool {
	return strings.HasPrefix(w.Id, prefix) &&
		w.State == pbf.WorkerState_Uninitialized
}

// StartWorker submits a job via "condor_submit" to start a new worker.
func (s *scheduler) StartWorker(w *pbf.Worker) error {
	log.Debug("Starting condor worker")
	var err error

	// TODO document that these working dirs need manual cleanup
	workdir := path.Join(s.conf.WorkDir, w.Id)
	workdir, _ = filepath.Abs(workdir)
	err = os.MkdirAll(workdir, 0755)
	if err != nil {
		return err
	}

	c := s.conf.Worker
	c.ID = w.Id
	// 5 second timeout
	c.Timeout = 5000000000
	c.Resources.Cpus = w.Resources.Cpus
	c.Resources.Ram = w.Resources.Ram
	c.Resources.Disk = w.Resources.Disk

	confPath := path.Join(workdir, "worker.conf.yml")
	c.ToYamlFile(confPath)

	workerPath := sched.DetectWorkerPath()

	submitPath := path.Join(workdir, "condor.submit")
	f, err := os.Create(submitPath)
	if err != nil {
		return err
	}

	submitTpl, err := template.New("condor.submit").Parse(`
universe       = vanilla
executable     = {{.Executable}}
arguments      = worker --config worker.conf.yml
environment    = "PATH=/usr/bin"
log            = {{.WorkDir}}/condor-event-log
error          = {{.WorkDir}}/tes-worker-stderr
output         = {{.WorkDir}}/tes-worker-stdout
input          = {{.Config}}
{{.Resources}}
should_transfer_files   = YES
when_to_transfer_output = ON_EXIT
queue
`)
	if err != nil {
		return err
	}

	err = submitTpl.Execute(f, map[string]string{
		"Executable": workerPath,
		"WorkDir":    workdir,
		"Config":     confPath,
		"Resources":  resolveCondorResourceRequest(int(w.Resources.Cpus), w.Resources.Ram, w.Resources.Disk),
	})
	if err != nil {
		return err
	}
	f.Close()

	cmd := exec.Command("condor_submit")
	stdin, err := os.Open(submitPath)
	if err != nil {
		return err
	}
	cmd.Stdin = stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return err
	}

	return nil
}

func resolveCondorResourceRequest(cpus int, ram float64, disk float64) string {
	var resources = []string{}
	if cpus != 0 {
		resources = append(resources, fmt.Sprintf("request_cpus = %d", cpus))
	}
	if ram != 0.0 {
		resources = append(resources, fmt.Sprintf("request_memory = %f GB", ram))
	}
	if disk != 0.0 {
		// Convert GB to KiB
		resources = append(resources, fmt.Sprintf("request_disk = %f", disk*976562))
	}
	return strings.Join(resources, "\n")
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 Timo Savola. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package facile

import (
	"bytes"
	"errors"

	"github.com/tsavola/gate/build"
	"github.com/tsavola/gate/entry"
	"github.com/tsavola/gate/image"
	"github.com/tsavola/gate/snapshot"
	"github.com/tsavola/wag/compile"
	"github.com/tsavola/wag/object"
	"github.com/tsavola/wag/wa"
)

type Filesystem struct {
	fs *image.Filesystem
}

func NewFilesystem(root string) (filesystem *Filesystem, err error) {
	fs, err := image.NewFilesystem(root)
	if err != nil {
		return
	}

	filesystem = &Filesystem{fs}
	return
}

func (filesystem *Filesystem) Close() error {
	return filesystem.fs.Close()
}

type ProgramImage struct {
	image   *image.Program
	buffers snapshot.Buffers
}

func NewProgramImage(programStorage *Filesystem, wasm []byte) (prog *ProgramImage, err error) {
	storage := image.CombinedStorage(programStorage.fs, image.Memory)

	var codeMap object.CallMap

	b, err := build.New(storage, len(wasm), compile.DefaultMaxTextSize, &codeMap, false)
	if err != nil {
		return
	}
	defer b.Close()

	reader := bytes.NewReader(wasm)

	b.InstallPrematureSnapshotSectionLoaders(errors.New)

	b.Module, err = compile.LoadInitialSections(b.ModuleConfig(), reader)
	if err != nil {
		return
	}

	b.StackSize = wa.PageSize
	b.MaxMemorySize = b.Module.MemorySizeLimit()

	err = b.BindFunctions("")
	if err != nil {
		return
	}

	err = compile.LoadCodeSection(b.CodeConfig(), reader, b.Module)
	if err != nil {
		return
	}

	b.InstallSnapshotSectionLoaders(errors.New)

	err = compile.LoadCustomSections(&b.Config, reader)
	if err != nil {
		return
	}

	err = b.FinishImageText()
	if err != nil {
		return
	}

	b.InstallLateSnapshotSectionLoaders(errors.New)

	err = compile.LoadDataSection(b.DataConfig(), reader, b.Module)
	if err != nil {
		return
	}

	err = compile.LoadCustomSections(&b.Config, reader)
	if err != nil {
		return
	}

	progImage, err := b.FinishProgramImage()
	if err != nil {
		return
	}

	prog = &ProgramImage{progImage, b.Buffers}
	return
}

func (prog *ProgramImage) Close() error {
	return prog.image.Close()
}

type InstanceImage struct {
	image   *image.Instance
	buffers snapshot.Buffers
}

func NewInstanceImage(prog *ProgramImage, entryFunction string) (inst *InstanceImage, err error) {
	var entryIndex uint32
	var entryAddr uint32

	if entryFunction != "" {
		entryIndex, err = entry.MapFuncIndex(prog.image.Manifest().EntryIndexes, entryFunction)
		if err != nil {
			return
		}

		entryAddr = entry.MapFuncAddr(prog.image.Manifest().EntryAddrs, entryIndex)
	}

	stackSize := wa.PageSize

	instImage, err := image.NewInstance(prog.image, stackSize, entryIndex, entryAddr)
	if err != nil {
		return
	}

	inst = &InstanceImage{instImage, prog.buffers}
	return
}

func (inst *InstanceImage) Close() error {
	return inst.image.Close()
}
<commit_msg>facile: update CodeConfig call<commit_after>// Copyright (c) 2019 Timo Savola. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package facile

import (
	"bytes"
	"errors"

	"github.com/tsavola/gate/build"
	"github.com/tsavola/gate/entry"
	"github.com/tsavola/gate/image"
	"github.com/tsavola/gate/snapshot"
	"github.com/tsavola/wag/compile"
	"github.com/tsavola/wag/object"
	"github.com/tsavola/wag/wa"
)

type Filesystem struct {
	fs *image.Filesystem
}

func NewFilesystem(root string) (filesystem *Filesystem, err error) {
	fs, err := image.NewFilesystem(root)
	if err != nil {
		return
	}

	filesystem = &Filesystem{fs}
	return
}

func (filesystem *Filesystem) Close() error {
	return filesystem.fs.Close()
}

type ProgramImage struct {
	image   *image.Program
	buffers snapshot.Buffers
}

func NewProgramImage(programStorage *Filesystem, wasm []byte) (prog *ProgramImage, err error) {
	storage := image.CombinedStorage(programStorage.fs, image.Memory)

	var codeMap object.CallMap

	b, err := build.New(storage, len(wasm), compile.DefaultMaxTextSize, &codeMap, false)
	if err != nil {
		return
	}
	defer b.Close()

	reader := bytes.NewReader(wasm)

	b.InstallPrematureSnapshotSectionLoaders(errors.New)

	b.Module, err = compile.LoadInitialSections(b.ModuleConfig(), reader)
	if err != nil {
		return
	}

	b.StackSize = wa.PageSize
	b.MaxMemorySize = b.Module.MemorySizeLimit()

	err = b.BindFunctions("")
	if err != nil {
		return
	}

	err = compile.LoadCodeSection(b.CodeConfig(&codeMap), reader, b.Module)
	if err != nil {
		return
	}

	b.InstallSnapshotSectionLoaders(errors.New)

	err = compile.LoadCustomSections(&b.Config, reader)
	if err != nil {
		return
	}

	err = b.FinishImageText()
	if err != nil {
		return
	}

	b.InstallLateSnapshotSectionLoaders(errors.New)

	err = compile.LoadDataSection(b.DataConfig(), reader, b.Module)
	if err != nil {
		return
	}

	err = compile.LoadCustomSections(&b.Config, reader)
	if err != nil {
		return
	}

	progImage, err := b.FinishProgramImage()
	if err != nil {
		return
	}

	prog = &ProgramImage{progImage, b.Buffers}
	return
}

func (prog *ProgramImage) Close() error {
	return prog.image.Close()
}

type InstanceImage struct {
	image   *image.Instance
	buffers snapshot.Buffers
}

func NewInstanceImage(prog *ProgramImage, entryFunction string) (inst *InstanceImage, err error) {
	var entryIndex uint32
	var entryAddr uint32

	if entryFunction != "" {
		entryIndex, err = entry.MapFuncIndex(prog.image.Manifest().EntryIndexes, entryFunction)
		if err != nil {
			return
		}

		entryAddr = entry.MapFuncAddr(prog.image.Manifest().EntryAddrs, entryIndex)
	}

	stackSize := wa.PageSize

	instImage, err := image.NewInstance(prog.image, stackSize, entryIndex, entryAddr)
	if err != nil {
		return
	}

	inst = &InstanceImage{instImage, prog.buffers}
	return
}

func (inst *InstanceImage) Close() error {
	return inst.image.Close()
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package suffixarray implements substring search in logarithmic time using
// an in-memory suffix array.
//
// Example use:
//
//	// create index for some data
//	index := suffixarray.New(data)
//
//	// lookup byte slice s
//	offsets1 := index.Lookup(s, -1) // the list of all indices where s occurs in data
//	offsets2 := index.Lookup(s, 3)  // the list of at most 3 indices where s occurs in data
//
package suffixarray

import (
	"bytes"
	"encoding/binary"
	"io"
	"regexp"
	"sort"
)

// Index implements a suffix array for fast substring search.
type Index struct {
	data []byte
	sa   []int // suffix array for data; len(sa) == len(data)
}

// New creates a new Index for data.
// Index creation time is O(N*log(N)) for N = len(data).
func New(data []byte) *Index {
	return &Index{data, qsufsort(data)}
}

// writeInt writes an int x to w using buf to buffer the write.
func writeInt(w io.Writer, buf []byte, x int) error {
	binary.PutVarint(buf, int64(x))
	_, err := w.Write(buf[0:binary.MaxVarintLen64])
	return err
}

// readInt reads an int x from r using buf to buffer the read and returns x.
func readInt(r io.Reader, buf []byte) (int, error) {
	_, err := io.ReadFull(r, buf[0:binary.MaxVarintLen64]) // ok to continue with error
	x, _ := binary.Varint(buf)
	return int(x), err
}

// writeSlice writes data[:n] to w and returns n.
// It uses buf to buffer the write.
func writeSlice(w io.Writer, buf []byte, data []int) (n int, err error) {
	// encode as many elements as fit into buf
	p := binary.MaxVarintLen64
	for ; n < len(data) && p+binary.MaxVarintLen64 <= len(buf); n++ {
		p += binary.PutUvarint(buf[p:], uint64(data[n]))
	}

	// update buffer size
	binary.PutVarint(buf, int64(p))

	// write buffer
	_, err = w.Write(buf[0:p])
	return
}

// readSlice reads data[:n] from r and returns n.
// It uses buf to buffer the read.
func readSlice(r io.Reader, buf []byte, data []int) (n int, err error) {
	// read buffer size
	var size int
	size, err = readInt(r, buf)
	if err != nil {
		return
	}

	// read buffer w/o the size
	if _, err = io.ReadFull(r, buf[binary.MaxVarintLen64:size]); err != nil {
		return
	}

	// decode as many elements as present in buf
	for p := binary.MaxVarintLen64; p < size; n++ {
		x, w := binary.Uvarint(buf[p:])
		data[n] = int(x)
		p += w
	}

	return
}

const bufSize = 16 << 10 // reasonable for BenchmarkSaveRestore

// Read reads the index from r into x; x must not be nil.
func (x *Index) Read(r io.Reader) error {
	// buffer for all reads
	buf := make([]byte, bufSize)

	// read length
	n, err := readInt(r, buf)
	if err != nil {
		return err
	}

	// allocate space
	if 2*n < cap(x.data) || cap(x.data) < n {
		// new data is significantly smaller or larger then
		// existing buffers - allocate new ones
		x.data = make([]byte, n)
		x.sa = make([]int, n)
	} else {
		// re-use existing buffers
		x.data = x.data[0:n]
		x.sa = x.sa[0:n]
	}

	// read data
	if _, err := io.ReadFull(r, x.data); err != nil {
		return err
	}

	// read index
	for sa := x.sa; len(sa) > 0; {
		n, err := readSlice(r, buf, sa)
		if err != nil {
			return err
		}
		sa = sa[n:]
	}
	return nil
}

// Write writes the index x to w.
func (x *Index) Write(w io.Writer) error {
	// buffer for all writes
	buf := make([]byte, bufSize)

	// write length
	if err := writeInt(w, buf, len(x.data)); err != nil {
		return err
	}

	// write data
	if _, err := w.Write(x.data); err != nil {
		return err
	}

	// write index
	for sa := x.sa; len(sa) > 0; {
		n, err := writeSlice(w, buf, sa)
		if err != nil {
			return err
		}
		sa = sa[n:]
	}
	return nil
}

// Bytes returns the data over which the index was created.
// It must not be modified.
//
func (x *Index) Bytes() []byte {
	return x.data
}

func (x *Index) at(i int) []byte {
	return x.data[x.sa[i]:]
}

// lookupAll returns a slice into the matching region of the index.
// The runtime is O(log(N)*len(s)).
func (x *Index) lookupAll(s []byte) []int {
	// find matching suffix index range [i:j]
	// find the first index where s would be the prefix
	i := sort.Search(len(x.sa), func(i int) bool { return bytes.Compare(x.at(i), s) >= 0 })
	// starting at i, find the first index at which s is not a prefix
	j := i + sort.Search(len(x.sa)-i, func(j int) bool { return !bytes.HasPrefix(x.at(j+i), s) })
	return x.sa[i:j]
}

// Lookup returns an unsorted list of at most n indices where the byte string s
// occurs in the indexed data. If n < 0, all occurrences are returned.
// The result is nil if s is empty, s is not found, or n == 0.
// Lookup time is O(log(N)*len(s) + len(result)) where N is the
// size of the indexed data.
//
func (x *Index) Lookup(s []byte, n int) (result []int) {
	if len(s) > 0 && n != 0 {
		matches := x.lookupAll(s)
		if n < 0 || len(matches) < n {
			n = len(matches)
		}
		// 0 <= n <= len(matches)
		if n > 0 {
			result = make([]int, n)
			copy(result, matches)
		}
	}
	return
}

// FindAllIndex returns a sorted list of non-overlapping matches of the
// regular expression r, where a match is a pair of indices specifying
// the matched slice of x.Bytes(). If n < 0, all matches are returned
// in successive order. Otherwise, at most n matches are returned and
// they may not be successive. The result is nil if there are no matches,
// or if n == 0.
//
func (x *Index) FindAllIndex(r *regexp.Regexp, n int) (result [][]int) {
	// a non-empty literal prefix is used to determine possible
	// match start indices with Lookup
	prefix, complete := r.LiteralPrefix()
	lit := []byte(prefix)

	// worst-case scenario: no literal prefix
	if prefix == "" {
		return r.FindAllIndex(x.data, n)
	}

	// if regexp is a literal just use Lookup and convert its
	// result into match pairs
	if complete {
		// Lookup returns indices that may belong to overlapping matches.
		// After eliminating them, we may end up with fewer than n matches.
		// If we don't have enough at the end, redo the search with an
		// increased value n1, but only if Lookup returned all the requested
		// indices in the first place (if it returned fewer than that then
		// there cannot be more).
		for n1 := n; ; n1 += 2 * (n - len(result)) /* overflow ok */ {
			indices := x.Lookup(lit, n1)
			if len(indices) == 0 {
				return
			}
			sort.Ints(indices)
			pairs := make([]int, 2*len(indices))
			result = make([][]int, len(indices))
			count := 0
			prev := 0
			for _, i := range indices {
				if count == n {
					break
				}
				// ignore indices leading to overlapping matches
				if prev <= i {
					j := 2 * count
					pairs[j+0] = i
					pairs[j+1] = i + len(lit)
					result[count] = pairs[j : j+2]
					count++
					prev = i + len(lit)
				}
			}
			result = result[0:count]
			if len(result) >= n || len(indices) != n1 {
				// found all matches or there's no chance to find more
				// (n and n1 can be negative)
				break
			}
		}
		if len(result) == 0 {
			result = nil
		}
		return
	}

	// regexp has a non-empty literal prefix; Lookup(lit) computes
	// the indices of possible complete matches; use these as starting
	// points for anchored searches
	// (regexp "^" matches beginning of input, not beginning of line)
	r = regexp.MustCompile("^" + r.String()) // compiles because r compiled

	// same comment about Lookup applies here as in the loop above
	for n1 := n; ; n1 += 2 * (n - len(result)) /* overflow ok */ {
		indices := x.Lookup(lit, n1)
		if len(indices) == 0 {
			return
		}
		sort.Ints(indices)
		result = result[0:0]
		prev := 0
		for _, i := range indices {
			if len(result) == n {
				break
			}
			m := r.FindIndex(x.data[i:]) // anchored search - will not run off
			// ignore indices leading to overlapping matches
			if m != nil && prev <= i {
				m[0] = i // correct m
				m[1] += i
				result = append(result, m)
				prev = m[1]
			}
		}
		if len(result) >= n || len(indices) != n1 {
			// found all matches or there's no chance to find more
			// (n and n1 can be negative)
			break
		}
	}
	if len(result) == 0 {
		result = nil
	}
	return
}
<commit_msg>index/suffixarray: fix a typo mistake in comments<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package suffixarray implements substring search in logarithmic time using
// an in-memory suffix array.
//
// Example use:
//
//	// create index for some data
//	index := suffixarray.New(data)
//
//	// lookup byte slice s
//	offsets1 := index.Lookup(s, -1) // the list of all indices where s occurs in data
//	offsets2 := index.Lookup(s, 3)  // the list of at most 3 indices where s occurs in data
//
package suffixarray

import (
	"bytes"
	"encoding/binary"
	"io"
	"regexp"
	"sort"
)

// Index implements a suffix array for fast substring search.
type Index struct {
	data []byte
	sa   []int // suffix array for data; len(sa) == len(data)
}

// New creates a new Index for data.
// Index creation time is O(N*log(N)) for N = len(data).
func New(data []byte) *Index {
	return &Index{data, qsufsort(data)}
}

// writeInt writes an int x to w using buf to buffer the write.
func writeInt(w io.Writer, buf []byte, x int) error {
	binary.PutVarint(buf, int64(x))
	_, err := w.Write(buf[0:binary.MaxVarintLen64])
	return err
}

// readInt reads an int x from r using buf to buffer the read and returns x.
func readInt(r io.Reader, buf []byte) (int, error) {
	_, err := io.ReadFull(r, buf[0:binary.MaxVarintLen64]) // ok to continue with error
	x, _ := binary.Varint(buf)
	return int(x), err
}

// writeSlice writes data[:n] to w and returns n.
// It uses buf to buffer the write.
func writeSlice(w io.Writer, buf []byte, data []int) (n int, err error) {
	// encode as many elements as fit into buf
	p := binary.MaxVarintLen64
	for ; n < len(data) && p+binary.MaxVarintLen64 <= len(buf); n++ {
		p += binary.PutUvarint(buf[p:], uint64(data[n]))
	}

	// update buffer size
	binary.PutVarint(buf, int64(p))

	// write buffer
	_, err = w.Write(buf[0:p])
	return
}

// readSlice reads data[:n] from r and returns n.
// It uses buf to buffer the read.
func readSlice(r io.Reader, buf []byte, data []int) (n int, err error) {
	// read buffer size
	var size int
	size, err = readInt(r, buf)
	if err != nil {
		return
	}

	// read buffer w/o the size
	if _, err = io.ReadFull(r, buf[binary.MaxVarintLen64:size]); err != nil {
		return
	}

	// decode as many elements as present in buf
	for p := binary.MaxVarintLen64; p < size; n++ {
		x, w := binary.Uvarint(buf[p:])
		data[n] = int(x)
		p += w
	}

	return
}

const bufSize = 16 << 10 // reasonable for BenchmarkSaveRestore

// Read reads the index from r into x; x must not be nil.
func (x *Index) Read(r io.Reader) error {
	// buffer for all reads
	buf := make([]byte, bufSize)

	// read length
	n, err := readInt(r, buf)
	if err != nil {
		return err
	}

	// allocate space
	if 2*n < cap(x.data) || cap(x.data) < n {
		// new data is significantly smaller or larger than
		// existing buffers - allocate new ones
		x.data = make([]byte, n)
		x.sa = make([]int, n)
	} else {
		// re-use existing buffers
		x.data = x.data[0:n]
		x.sa = x.sa[0:n]
	}

	// read data
	if _, err := io.ReadFull(r, x.data); err != nil {
		return err
	}

	// read index
	for sa := x.sa; len(sa) > 0; {
		n, err := readSlice(r, buf, sa)
		if err != nil {
			return err
		}
		sa = sa[n:]
	}
	return nil
}

// Write writes the index x to w.
func (x *Index) Write(w io.Writer) error {
	// buffer for all writes
	buf := make([]byte, bufSize)

	// write length
	if err := writeInt(w, buf, len(x.data)); err != nil {
		return err
	}

	// write data
	if _, err := w.Write(x.data); err != nil {
		return err
	}

	// write index
	for sa := x.sa; len(sa) > 0; {
		n, err := writeSlice(w, buf, sa)
		if err != nil {
			return err
		}
		sa = sa[n:]
	}
	return nil
}

// Bytes returns the data over which the index was created.
// It must not be modified.
//
func (x *Index) Bytes() []byte {
	return x.data
}

func (x *Index) at(i int) []byte {
	return x.data[x.sa[i]:]
}

// lookupAll returns a slice into the matching region of the index.
// The runtime is O(log(N)*len(s)).
func (x *Index) lookupAll(s []byte) []int {
	// find matching suffix index range [i:j]
	// find the first index where s would be the prefix
	i := sort.Search(len(x.sa), func(i int) bool { return bytes.Compare(x.at(i), s) >= 0 })
	// starting at i, find the first index at which s is not a prefix
	j := i + sort.Search(len(x.sa)-i, func(j int) bool { return !bytes.HasPrefix(x.at(j+i), s) })
	return x.sa[i:j]
}

// Lookup returns an unsorted list of at most n indices where the byte string s
// occurs in the indexed data. If n < 0, all occurrences are returned.
// The result is nil if s is empty, s is not found, or n == 0.
// Lookup time is O(log(N)*len(s) + len(result)) where N is the
// size of the indexed data.
//
func (x *Index) Lookup(s []byte, n int) (result []int) {
	if len(s) > 0 && n != 0 {
		matches := x.lookupAll(s)
		if n < 0 || len(matches) < n {
			n = len(matches)
		}
		// 0 <= n <= len(matches)
		if n > 0 {
			result = make([]int, n)
			copy(result, matches)
		}
	}
	return
}

// FindAllIndex returns a sorted list of non-overlapping matches of the
// regular expression r, where a match is a pair of indices specifying
// the matched slice of x.Bytes(). If n < 0, all matches are returned
// in successive order. Otherwise, at most n matches are returned and
// they may not be successive. The result is nil if there are no matches,
// or if n == 0.
//
func (x *Index) FindAllIndex(r *regexp.Regexp, n int) (result [][]int) {
	// a non-empty literal prefix is used to determine possible
	// match start indices with Lookup
	prefix, complete := r.LiteralPrefix()
	lit := []byte(prefix)

	// worst-case scenario: no literal prefix
	if prefix == "" {
		return r.FindAllIndex(x.data, n)
	}

	// if regexp is a literal just use Lookup and convert its
	// result into match pairs
	if complete {
		// Lookup returns indices that may belong to overlapping matches.
		// After eliminating them, we may end up with fewer than n matches.
		// If we don't have enough at the end, redo the search with an
		// increased value n1, but only if Lookup returned all the requested
		// indices in the first place (if it returned fewer than that then
		// there cannot be more).
		for n1 := n; ; n1 += 2 * (n - len(result)) /* overflow ok */ {
			indices := x.Lookup(lit, n1)
			if len(indices) == 0 {
				return
			}
			sort.Ints(indices)
			pairs := make([]int, 2*len(indices))
			result = make([][]int, len(indices))
			count := 0
			prev := 0
			for _, i := range indices {
				if count == n {
					break
				}
				// ignore indices leading to overlapping matches
				if prev <= i {
					j := 2 * count
					pairs[j+0] = i
					pairs[j+1] = i + len(lit)
					result[count] = pairs[j : j+2]
					count++
					prev = i + len(lit)
				}
			}
			result = result[0:count]
			if len(result) >= n || len(indices) != n1 {
				// found all matches or there's no chance to find more
				// (n and n1 can be negative)
				break
			}
		}
		if len(result) == 0 {
			result = nil
		}
		return
	}

	// regexp has a non-empty literal prefix; Lookup(lit) computes
	// the indices of possible complete matches; use these as starting
	// points for anchored searches
	// (regexp "^" matches beginning of input, not beginning of line)
	r = regexp.MustCompile("^" + r.String()) // compiles because r compiled

	// same comment about Lookup applies here as in the loop above
	for n1 := n; ; n1 += 2 * (n - len(result)) /* overflow ok */ {
		indices := x.Lookup(lit, n1)
		if len(indices) == 0 {
			return
		}
		sort.Ints(indices)
		result = result[0:0]
		prev := 0
		for _, i := range indices {
			if len(result) == n {
				break
			}
			m := r.FindIndex(x.data[i:]) // anchored search - will not run off
			// ignore indices leading to overlapping matches
			if m != nil && prev <= i {
				m[0] = i // correct m
				m[1] += i
				result = append(result, m)
				prev = m[1]
			}
		}
		if len(result) >= n || len(indices) != n1 {
			// found all matches or there's no chance to find more
			// (n and n1 can be negative)
			break
		}
	}
	if len(result) == 0 {
		result = nil
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
)

var (
	buf    bytes.Buffer
	logger *log.Logger
)

func debugPrintOpts(opts *Opts) {
	debugString := ""
	debugString = fmt.Sprintf("%sopts:\n", debugString)
	debugString = fmt.Sprintf("%sdebugFlag: %t\n", debugString, opts.debugFlag)
	debugString = fmt.Sprintf("%shelpFlag: %t\n", debugString, opts.helpFlag)
	debugString = fmt.Sprintf("%sloadFlag: %t\n", debugString, opts.loadFlag)
	debugString = fmt.Sprintf("%sprettyFlag: %t\n", debugString, opts.prettyFlag)
	debugString = fmt.Sprintf("%sscriptFlag: %t\n", debugString, opts.scriptFlag)
	debugString = fmt.Sprintf("%stableFlag: %t\n", debugString, opts.tableFlag)
	debugString = fmt.Sprintf("%sversionFlag: %t\n", debugString, opts.versionFlag)
	debugString = fmt.Sprintf("%splanetsCount: %d\n", debugString, opts.planetsCount)
	debugString = fmt.Sprintf("%scommand: %s\n", debugString, opts.command)
	debugString = fmt.Sprintf("%scurrentDBDet: %s\n", debugString, opts.currentDBDet)
	debugString = fmt.Sprintf("%scurrentDet: %s\n", debugString, opts.currentDet)
	debugString = fmt.Sprintf("%sscriptName: %s\n", debugString, opts.scriptName)
	debugString = fmt.Sprintf("%splanets: %v\n", debugString, opts.planets)
	fmt.Print(debugString)
	log.Output(1, debugString)
}

func debugPrintStructuredOutput(strucOut *StructuredOuput) {
	debugString := ""
	//debugString = fmt.Sprintf("%sstrucOut: %v\n", debugString, strucOut)
	debugString = fmt.Sprintf("%splanet: %s\n", debugString, strucOut.planet)
	//debugString = fmt.Sprintf("%sout: %s\n", debugString, strucOut.output)
	debugString = fmt.Sprintf("%smaxLineLength: %d\n", debugString, strucOut.maxOutLength)
	fmt.Print(debugString)
	log.Output(1, debugString)

}

func debugPrintPlanets(planets []Planet) {
	debugString := ""
	for _, planet := range planets {
		debugString = fmt.Sprintf("%s%s\n", debugString, planet)
	}
	fmt.Print(debugString)
	log.Output(1, debugString)
}

func debugPrintString(message string) {
	debugString := ""
	debugString = fmt.Sprintf("%s", message)
	fmt.Print(debugString)
	log.Output(1, debugString)
}

func printDebugStart() {
	debugString := ""
	debugString = fmt.Sprintf("###################################### Program Start ######################################\n")
	debugString = fmt.Sprintf("%s%v\n", debugString, os.Args)
	fmt.Print(debugString)
	log.Output(1, debugString)
}

func printDebugEnd() {
	debugString := ""
	debugString = fmt.Sprintf("###################################### Program End ######################################\n")
	fmt.Print(debugString)
	log.Output(1, debugString)
}
<commit_msg>Debughandler gone.<commit_after><|endoftext|>


<commit_before>package felica

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"os"
	"regexp"
	"strings"
)

// 正規表現とアクション
type re_action struct {
	_regexpes []*regexp.Regexp
	regexpes  []string
	action    func(match []string)
}

// FeliCaダンプファイルを読込む
func Read(path string) CardInfo {
	cardinfo := CardInfo{}

	file, err := os.Open(path)
	if err != nil {
		// エラー処理をする
		return nil
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)

	var svccode string
	orphan := empty_sysinfo()
	currsys := orphan

	actions := [](*re_action){
		// IDm
		{
			regexpes: []string{
				"(?i)IDm = *([0-9A-F]+)",
				"(?i)IDm :(( [0-9A-F]+)+)",
			},
			action: func(match []string) {
				currsys.IDm = strings.Replace(match[1], " ", "", -1)
			},
		},

		// PMm
		{
			regexpes: []string{
				"(?i)PMm = *([0-9A-F]+)",
				"(?i)PMm :(( [0-9A-F]+)+)",
			},
			action: func(match []string) {
				currsys.PMm = strings.Replace(match[1], " ", "", -1)
			},
		},

		// システムコード
		{
			regexpes: []string{
				"(?i)^# FELICA SYSTEM_CODE = *([0-9A-F]+)",
				"(?i)^# System code: ([0-9A-F]+)",
			},
			action: func(match []string) {
				syscode := match[1]
				currsys = empty_sysinfo()
				cardinfo[syscode] = currsys
			},
		},

		// サービスコード
		{
			regexpes: []string{
				"(?i)^# [0-9A-F]+:[0-9A-F]+:([0-9A-F]+) #[0-9A-F]+",
				"(?i)# Serivce code = *([0-9A-F]+)",
			},
			action: func(match []string) {
				svccode = match[1]
				currsys.ServiceCodes = append(currsys.ServiceCodes, svccode)
				currsys.Services[svccode] = [][]byte{}
			},
		},

		// データ
		{
			regexpes: []string{
				"(?i)^ *[0-9A-F]+:[0-9A-F]+:([0-9A-F]+):[0-9A-F]+:([0-9A-F]{32})",
				"(?i)^ *([0-9A-F]+):[0-9A-F]+(( [0-9A-F]+){16})",
			},
			action: func(match []string) {
				data := match[2]
				data = strings.Replace(data, " ", "", -1)
				buf, _ := hex.DecodeString(data)
				currsys.Services[svccode] = append(currsys.Services[svccode], buf)
			},
		},
	}

	// 正規表現のコンパイル
	re_action_compile(actions)

	for scanner.Scan() {
		line := scanner.Text()
		re_match_action(line, actions, true)
	}

	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	if len(orphan.IDm) != 0 || len(orphan.PMm) != 0 {
		for _, currsys := range cardinfo {
			currsys.IDm = orphan.IDm
			currsys.PMm = orphan.PMm
		}
	}

	return cardinfo
}

// 空の SystemInfo を作成する
func empty_sysinfo() *SystemInfo {
	return &SystemInfo{ServiceCodes: []string{}, Services: make(ServiceInfo)}
}

// 正規表現をコンパイルする
func re_action_compile(actions [](*re_action)) {
	for _, a := range actions {
		a._regexpes = make([]*regexp.Regexp, len(a.regexpes))

		for i, s := range a.regexpes {
			a._regexpes[i] = regexp.MustCompile(s)
		}

	}
}

// 正規表現に一致したら対応するアクションを実行する
func re_match_action(text string, actions [](*re_action), is_break bool) {
	for _, a := range actions {
		for _, re := range a._regexpes {
			match := re.FindStringSubmatch(text)
			if match != nil {
				a.action(match)

				if is_break {
					// 残りは実行しない
					return
				}
			}
		}
	}
}
<commit_msg>felica_dump(libpafe) のダンプファイルに対応<commit_after>package felica

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// 正規表現とアクション
type re_action struct {
	_regexpes []*regexp.Regexp
	regexpes  []string
	action    func(match []string)
}

// FeliCaダンプファイルを読込む
func Read(path string) CardInfo {
	cardinfo := CardInfo{}

	file, err := os.Open(path)
	if err != nil {
		// エラー処理をする
		return nil
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)

	var svccode string
	orphan := empty_sysinfo()
	currsys := orphan

	actions := [](*re_action){
		// IDm
		{
			regexpes: []string{
				"(?i)IDm = *([0-9A-F]+)",
				"(?i)IDm :(( [0-9A-F]+)+)",
			},
			action: func(match []string) {
				currsys.IDm = strings.Replace(match[1], " ", "", -1)
			},
		},

		// PMm
		{
			regexpes: []string{
				"(?i)PMm = *([0-9A-F]+)",
				"(?i)PMm :(( [0-9A-F]+)+)",
			},
			action: func(match []string) {
				currsys.PMm = strings.Replace(match[1], " ", "", -1)
			},
		},

		// システムコード
		{
			regexpes: []string{
				"(?i)^# FELICA SYSTEM_CODE = *([0-9A-F]+)",
				"(?i)^# System code: ([0-9A-F]+)",
			},
			action: func(match []string) {
				syscode := match[1]
				currsys = empty_sysinfo()
				cardinfo[syscode] = currsys
			},
		},

		// サービスコード
		{
			regexpes: []string{
				"(?i)^# [0-9A-F]+:[0-9A-F]+:([0-9A-F]+) #[0-9A-F]+",
				"(?i)# Serivce code = *([0-9A-F]+)",
			},
			action: func(match []string) {
				svccode = match[1]
				currsys.ServiceCodes = append(currsys.ServiceCodes, svccode)
				currsys.Services[svccode] = [][]byte{}
			},
		},

		// felica_dump サービスコード
		{
			regexpes: []string{
				"(?i)^# ([0-9A-F]{4}):([0-9A-F]{4}) ",
			},
			action: func(match []string) {
				code, _ := strconv.ParseInt(match[1], 16, 0)
				attr, _ := strconv.ParseInt(match[2], 16, 0)
				svccode = fmt.Sprintf("%04X", code<<6+attr)
				currsys.ServiceCodes = append(currsys.ServiceCodes, svccode)
				currsys.Services[svccode] = [][]byte{}
			},
		},

		// データ
		{
			regexpes: []string{
				"(?i)^ *[0-9A-F]+:[0-9A-F]+:([0-9A-F]+):[0-9A-F]+:([0-9A-F]{32})",
				"(?i)^ *([0-9A-F]+):[0-9A-F]+(( [0-9A-F]+){16})",
			},
			action: func(match []string) {
				data := match[2]
				data = strings.Replace(data, " ", "", -1)
				buf, _ := hex.DecodeString(data)
				currsys.Services[svccode] = append(currsys.Services[svccode], buf)
			},
		},

		// felica_dump データ
		{
			regexpes: []string{
				"(?i)^  [0-9A-F]{4}:[0-9A-F]{4}:([0-9A-F]{32})",
			},
			action: func(match []string) {
				data := match[1]
				data = strings.Replace(data, " ", "", -1)
				buf, _ := hex.DecodeString(data)
				currsys.Services[svccode] = append(currsys.Services[svccode], buf)
			},
		},
	}

	// 正規表現のコンパイル
	re_action_compile(actions)

	for scanner.Scan() {
		line := scanner.Text()
		re_match_action(line, actions, true)
	}

	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	if len(orphan.IDm) != 0 || len(orphan.PMm) != 0 {
		for _, currsys := range cardinfo {
			currsys.IDm = orphan.IDm
			currsys.PMm = orphan.PMm
		}
	}

	return cardinfo
}

// 空の SystemInfo を作成する
func empty_sysinfo() *SystemInfo {
	return &SystemInfo{ServiceCodes: []string{}, Services: make(ServiceInfo)}
}

// 正規表現をコンパイルする
func re_action_compile(actions [](*re_action)) {
	for _, a := range actions {
		a._regexpes = make([]*regexp.Regexp, len(a.regexpes))

		for i, s := range a.regexpes {
			a._regexpes[i] = regexp.MustCompile(s)
		}

	}
}

// 正規表現に一致したら対応するアクションを実行する
func re_match_action(text string, actions [](*re_action), is_break bool) {
	for _, a := range actions {
		for _, re := range a._regexpes {
			match := re.FindStringSubmatch(text)
			if match != nil {
				a.action(match)

				if is_break {
					// 残りは実行しない
					return
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package goldb

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

type Storage struct {
	Context
	dir string
	db  *leveldb.DB
	op  *opt.Options
	seq map[Entity]uint64
	mx  sync.Mutex
}

func NewStorage(dir string, op *opt.Options) (s *Storage) {
	dir = strings.TrimSuffix(dir, "/")

	s = &Storage{
		dir: dir,
		op:  op,
		seq: map[Entity]uint64{},
	}

	if err := s.Open(); err == nil {
		return

	} else if !errors.IsCorrupted(err) {
		panic(err)
	}

	// try to recover files
	if err := s.Recover(); err != nil {
		log.Println("!!! db.Storage.Recover-ERROR: ", err)
	}
	if err := s.Open(); err != nil {
		panic(err)
	}

	return
}

func (s *Storage) Open() error {
	// TODO: RecoverFile ???

	db, err := leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return err
	}
	s.db = db
	s.Context.qCtx = db
	return nil
}

func (s *Storage) Recover() error {
	if db, err := leveldb.RecoverFile(s.dir, nil); err != nil {
		return err
	} else {
		return db.Close()
	}
}

func (s *Storage) Close() error {
	if s.db != nil {
		if err := s.db.Close(); err != leveldb.ErrClosed {
			return err
		}
	}
	return nil
}

func (s *Storage) Drop() error {
	if err := s.Close(); err != nil {
		return err
	}
	return os.RemoveAll(s.dir)
}

func (s *Storage) Size() (size uint64) {
	filepath.Walk(s.dir, func(_ string, info os.FileInfo, err error) error {
		if info != nil && !info.IsDir() {
			size += uint64(info.Size())
		}
		return err
	})
	return
}

func (s *Storage) Truncate() error {
	s.mx.Lock()
	defer s.mx.Unlock()

	if err := s.Drop(); err != nil {
		return err
	}
	return s.Open()
}

// Exec executes transaction.
// The executing transaction can be discard by methods tx.Fail(err) or by panic(err)
func (s *Storage) Exec(fn func(tx *Transaction)) (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	t := newTransaction(s)
	defer func() {
		if e, _ := recover().(error); e != nil {
			t.Discard()
			err = e
		}
	}()
	if t.err != nil {
		return t.err
	}
	fn(t)
	if t.err == nil {
		t.Commit()
	} else {
		t.Discard()
	}
	return t.err
}

func (s *Storage) Reindex() (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	os.RemoveAll(s.dir + ".reindex")
	os.RemoveAll(s.dir + ".old")

	dbOld := s.db

	// lock db
	trLock, err := dbOld.OpenTransaction()
	if err != nil {
		return
	}
	defer trLock.Discard()

	dbNew, err := leveldb.OpenFile(s.dir+".reindex", s.op)
	if err != nil {
		return
	}

	iterator := dbOld.NewIterator(&util.Range{}, s.ReadOptions)

	var tr *leveldb.Transaction
	defer func() {
		iterator.Release()
		if err == nil {
			err = iterator.Error()
		}
		if tr != nil {
			tr.Discard()
		}
	}()
	for i := 0; iterator.Next(); i++ {
		if err = iterator.Error(); err != nil {
			return
		}
		if i%10000 == 0 {
			if tr != nil {
				if err = tr.Commit(); err != nil {
					return
				}
			}
			if tr, err = dbNew.OpenTransaction(); err != nil {
				return
			}
		}
		// put values to new DB
		key := iterator.Key()
		val := iterator.Value()
		if err = tr.Put(key, val, s.WriteOptions); err != nil {
			return
		}
	}
	if tr != nil {
		if err = tr.Commit(); err != nil {
			return
		}
		tr = nil
	}

	if err = dbNew.Close(); err != nil {
		return
	}

	if err = os.Rename(s.dir, s.dir+".old"); err != nil {
		return
	}
	if err = os.Rename(s.dir+".reindex", s.dir); err != nil {
		return
	}

	// reopen db
	dbNew, err = leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return
	}
	s.Context.qCtx = dbNew
	s.db = dbNew
	dbOld.Close()

	os.RemoveAll(s.dir + ".old")

	return
}
<commit_msg>refactored reindex method<commit_after>package goldb

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

type Storage struct {
	Context
	dir string
	db  *leveldb.DB
	op  *opt.Options
	seq map[Entity]uint64
	mx  sync.Mutex
}

func NewStorage(dir string, op *opt.Options) (s *Storage) {
	dir = strings.TrimSuffix(dir, "/")

	s = &Storage{
		dir: dir,
		op:  op,
		seq: map[Entity]uint64{},
	}

	if err := s.Open(); err == nil {
		return

	} else if !errors.IsCorrupted(err) {
		panic(err)
	}

	// try to recover files
	if err := s.Recover(); err != nil {
		log.Println("!!! db.Storage.Recover-ERROR: ", err)
	}
	if err := s.Open(); err != nil {
		panic(err)
	}

	return
}

func (s *Storage) Open() error {
	db, err := leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return err
	}
	s.db = db
	s.Context.qCtx = db
	return nil
}

func (s *Storage) Recover() error {
	if db, err := leveldb.RecoverFile(s.dir, nil); err != nil {
		return err
	} else {
		return db.Close()
	}
}

func (s *Storage) Close() error {
	if s.db != nil {
		if err := s.db.Close(); err != leveldb.ErrClosed {
			return err
		}
	}
	return nil
}

func (s *Storage) Drop() error {
	if err := s.Close(); err != nil {
		return err
	}
	return os.RemoveAll(s.dir)
}

func (s *Storage) Size() (size uint64) {
	filepath.Walk(s.dir, func(_ string, info os.FileInfo, err error) error {
		if info != nil && !info.IsDir() {
			size += uint64(info.Size())
		}
		return err
	})
	return
}

func (s *Storage) Truncate() error {
	s.mx.Lock()
	defer s.mx.Unlock()

	if err := s.Drop(); err != nil {
		return err
	}
	return s.Open()
}

// Exec executes transaction.
// The executing transaction can be discard by methods tx.Fail(err) or by panic(err)
func (s *Storage) Exec(fn func(tx *Transaction)) (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	t := newTransaction(s)
	defer func() {
		if e, _ := recover().(error); e != nil {
			t.Discard()
			err = e
		}
	}()
	if t.err != nil {
		return t.err
	}
	fn(t)
	if t.err == nil {
		t.Commit()
	} else {
		t.Discard()
	}
	return t.err
}

func (s *Storage) Reindex() (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	tmpDir := s.dir + ".reindex"
	oldDir := s.dir + ".old"

	defer os.RemoveAll(tmpDir)
	os.RemoveAll(tmpDir)
	os.RemoveAll(oldDir)

	dbOld := s.db
	dbNew, err := leveldb.OpenFile(tmpDir, s.op)
	if err != nil {
		return
	}

	iterator := dbOld.NewIterator(&util.Range{}, s.ReadOptions)

	var tr *leveldb.Transaction
	defer func() {
		iterator.Release()
		if err == nil {
			err = iterator.Error()
		}
		if tr != nil {
			tr.Discard()
		}
	}()
	for i := 0; iterator.Next(); i++ {
		if err = iterator.Error(); err != nil {
			return
		}
		if i%10000 == 0 {
			if tr != nil {
				if err = tr.Commit(); err != nil {
					return
				}
			}
			if tr, err = dbNew.OpenTransaction(); err != nil {
				return
			}
		}
		// put values to new DB
		key := iterator.Key()
		val := iterator.Value()
		if err = tr.Put(key, val, s.WriteOptions); err != nil {
			return
		}
	}
	if tr != nil {
		if err = tr.Commit(); err != nil {
			return
		}
		tr = nil
	}

	if err = dbNew.Close(); err != nil {
		return
	}

	if err = os.Rename(s.dir, oldDir); err != nil {
		return
	}
	if err = os.Rename(tmpDir, s.dir); err != nil {
		return
	}

	// reopen db
	dbNew, err = leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return
	}
	s.Context.qCtx = dbNew
	s.db = dbNew
	dbOld.Close()

	os.RemoveAll(oldDir)

	return
}
<|endoftext|>


<commit_before>package aptcacher

import (
	"container/heap"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/cybozu-go/log"
	"github.com/pkg/errors"
)

var (
	// ErrNotFound is returned by Storage.Lookup for non-existing items.
	ErrNotFound = errors.New("not found")

	// ErrBadPath is returned by Storage.Insert if path is bad
	ErrBadPath = errors.New("bad path")
)

// entry represents an item in the cache.
type entry struct {
	*FileInfo

	// for container/heap.
	// atime is used as priorities.
	atime uint64
	index int
}

// Storage stores cache items in local file system.
//
// Cached items will be removed in LRU fashion when the total size of
// items exceeds the capacity.
type Storage struct {
	dir      string // directory for cache items
	capacity uint64

	mu     sync.Mutex
	used   uint64
	cache  map[string]*entry
	lru    []*entry // for container/heap
	lclock uint64   // ditto
}

// NewStorage creates a Storage.
//
// dir is the directory for cached items.
// capacity is the maximum total size (bytes) of items in the cache.
// If capacity is zero, items will not be evicted.
func NewStorage(dir string, capacity uint64) *Storage {
	if !filepath.IsAbs(dir) {
		panic("dir must be an absolute path")
	}
	return &Storage{
		dir:      dir,
		cache:    make(map[string]*entry),
		capacity: capacity,
	}
}

// Len implements heap.Interface.
func (cm *Storage) Len() int {
	return len(cm.lru)
}

// Less implements heap.Interface.
func (cm *Storage) Less(i, j int) bool {
	return cm.lru[i].atime < cm.lru[j].atime
}

// Swap implements heap.Interface.
func (cm *Storage) Swap(i, j int) {
	cm.lru[i], cm.lru[j] = cm.lru[j], cm.lru[i]
	cm.lru[i].index = i
	cm.lru[j].index = j
}

// Push implements heap.Interface.
func (cm *Storage) Push(x interface{}) {
	e, ok := x.(*entry)
	if !ok {
		panic("Storage.Push: wrong type")
	}
	n := len(cm.lru)
	e.index = n
	cm.lru = append(cm.lru, e)
}

// Pop implements heap.Interface.
func (cm *Storage) Pop() interface{} {
	n := len(cm.lru)
	e := cm.lru[n-1]
	e.index = -1 // for safety
	cm.lru = cm.lru[0 : n-1]
	return e
}

// maint removes unused items from cache until used < capacity.
// cm.mu lock must be acquired beforehand.
func (cm *Storage) maint() {
	for cm.capacity > 0 && cm.used > cm.capacity {
		e := heap.Pop(cm).(*entry)
		delete(cm.cache, e.Path())
		cm.used -= e.Size()
		if err := os.Remove(filepath.Join(cm.dir, e.Path())); err != nil {
			log.Warn("Storage.maint", map[string]interface{}{
				"_err": err.Error(),
			})
		}
		log.Info("removed", map[string]interface{}{
			"_path": e.Path(),
		})
	}
}

func readData(path string) ([]byte, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return ioutil.ReadAll(f)
}

// Load loads existing items in filesystem.
func (cm *Storage) Load() error {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	wf := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.Mode().IsRegular() {
			return nil
		}
		subpath, err := filepath.Rel(cm.dir, path)
		if err != nil {
			return err
		}
		if _, ok := cm.cache[subpath]; ok {
			return nil
		}

		size := uint64(info.Size())
		e := &entry{
			// delay calculation of checksums.
			FileInfo: &FileInfo{
				path: subpath,
				size: size,
			},
			atime: cm.lclock,
			index: len(cm.lru),
		}
		cm.used += size
		cm.lclock++
		cm.lru = append(cm.lru, e)
		cm.cache[subpath] = e
		log.Debug("Storage.Load", map[string]interface{}{
			"_path": subpath,
		})
		return nil
	}

	if err := filepath.Walk(cm.dir, wf); err != nil {
		return err
	}
	heap.Init(cm)

	cm.maint()

	return nil
}

// Insert inserts or updates a cache item.
//
// fi.Path() must be as clean as filepath.Clean() and
// must not be filepath.IsAbs().
func (cm *Storage) Insert(data []byte, fi *FileInfo) error {
	switch {
	case fi.path != filepath.Clean(fi.path):
		return ErrBadPath
	case filepath.IsAbs(fi.path):
		return ErrBadPath
	case fi.path == ".":
		return ErrBadPath
	}

	f, err := ioutil.TempFile(cm.dir, "_tmp")
	if err != nil {
		return err
	}
	defer func() {
		f.Close()
		os.Remove(f.Name())
	}()

	_, err = f.Write(data)
	if err != nil {
		return err
	}
	err = f.Sync()
	if err != nil {
		return err
	}

	p := fi.path
	destpath := filepath.Join(cm.dir, p)
	dirpath := filepath.Dir(destpath)

	_, err = os.Stat(dirpath)
	switch {
	case os.IsNotExist(err):
		err = os.MkdirAll(dirpath, 0755)
		if err != nil {
			return err
		}
	case err != nil:
		return err
	}

	cm.mu.Lock()
	defer cm.mu.Unlock()

	if existing, ok := cm.cache[p]; ok {
		err = os.Remove(destpath)
		if err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			log.Warn("cache file was removed already", map[string]interface{}{
				"_path": p,
			})
		}
		cm.used -= existing.Size()
		heap.Remove(cm, existing.index)
		delete(cm.cache, p)
		if log.Enabled(log.LvDebug) {
			log.Debug("deleted existing item", map[string]interface{}{
				"_path": p,
			})
		}
	}

	err = os.Rename(f.Name(), destpath)
	if err != nil {
		return err
	}

	e := &entry{
		FileInfo: fi,
		atime:    cm.lclock,
	}
	cm.used += fi.size
	cm.lclock++
	heap.Push(cm, e)
	cm.cache[p] = e

	cm.maint()

	return nil
}

func calcChecksum(dir string, e *entry) error {
	if e.FileInfo.md5sum != nil {
		return nil
	}

	data, err := readData(filepath.Join(dir, e.Path()))
	if err != nil {
		return err
	}
	md5sum := md5.Sum(data)
	sha1sum := sha1.Sum(data)
	sha256sum := sha256.Sum256(data)
	e.FileInfo.md5sum = md5sum[:]
	e.FileInfo.sha1sum = sha1sum[:]
	e.FileInfo.sha256sum = sha256sum[:]
	return nil
}

// Lookup looks up an item in the cache.
// If no item matching fi is found, ErrNotFound is returned.
//
// The caller is responsible to close the retured os.File.
func (cm *Storage) Lookup(fi *FileInfo) (*os.File, error) {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	e, ok := cm.cache[fi.path]
	if !ok {
		return nil, ErrNotFound
	}

	// delayed checksum calculation
	err := calcChecksum(cm.dir, e)
	if err != nil {
		return nil, err
	}

	if !fi.Same(e.FileInfo) {
		// checksum mismatch
		return nil, ErrNotFound
	}

	e.atime = cm.lclock
	cm.lclock++
	heap.Fix(cm, e.index)
	return os.Open(filepath.Join(cm.dir, fi.path))
}

// ListAll returns a list of FileInfo for all cached items.
func (cm *Storage) ListAll() []*FileInfo {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	l := make([]*FileInfo, cm.Len())
	for i, e := range cm.lru {
		l[i] = e.FileInfo
	}
	return l
}

// Delete deletes an item from the cache.
func (cm *Storage) Delete(p string) error {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	e, ok := cm.cache[p]
	if !ok {
		return nil
	}

	err := os.Remove(filepath.Join(cm.dir, p))
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		log.Warn("cached file was already removed", map[string]interface{}{
			"_path": p,
		})
	}

	cm.used -= e.size
	heap.Remove(cm, e.index)
	delete(cm.cache, p)
	log.Info("deleted item", map[string]interface{}{
		"_path": p,
	})
	return nil
}
<commit_msg>fix a typo.<commit_after>package aptcacher

import (
	"container/heap"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/cybozu-go/log"
	"github.com/pkg/errors"
)

var (
	// ErrNotFound is returned by Storage.Lookup for non-existing items.
	ErrNotFound = errors.New("not found")

	// ErrBadPath is returned by Storage.Insert if path is bad
	ErrBadPath = errors.New("bad path")
)

// entry represents an item in the cache.
type entry struct {
	*FileInfo

	// for container/heap.
	// atime is used as priorities.
	atime uint64
	index int
}

// Storage stores cache items in local file system.
//
// Cached items will be removed in LRU fashion when the total size of
// items exceeds the capacity.
type Storage struct {
	dir      string // directory for cache items
	capacity uint64

	mu     sync.Mutex
	used   uint64
	cache  map[string]*entry
	lru    []*entry // for container/heap
	lclock uint64   // ditto
}

// NewStorage creates a Storage.
//
// dir is the directory for cached items.
// capacity is the maximum total size (bytes) of items in the cache.
// If capacity is zero, items will not be evicted.
func NewStorage(dir string, capacity uint64) *Storage {
	if !filepath.IsAbs(dir) {
		panic("dir must be an absolute path")
	}
	return &Storage{
		dir:      dir,
		cache:    make(map[string]*entry),
		capacity: capacity,
	}
}

// Len implements heap.Interface.
func (cm *Storage) Len() int {
	return len(cm.lru)
}

// Less implements heap.Interface.
func (cm *Storage) Less(i, j int) bool {
	return cm.lru[i].atime < cm.lru[j].atime
}

// Swap implements heap.Interface.
func (cm *Storage) Swap(i, j int) {
	cm.lru[i], cm.lru[j] = cm.lru[j], cm.lru[i]
	cm.lru[i].index = i
	cm.lru[j].index = j
}

// Push implements heap.Interface.
func (cm *Storage) Push(x interface{}) {
	e, ok := x.(*entry)
	if !ok {
		panic("Storage.Push: wrong type")
	}
	n := len(cm.lru)
	e.index = n
	cm.lru = append(cm.lru, e)
}

// Pop implements heap.Interface.
func (cm *Storage) Pop() interface{} {
	n := len(cm.lru)
	e := cm.lru[n-1]
	e.index = -1 // for safety
	cm.lru = cm.lru[0 : n-1]
	return e
}

// maint removes unused items from cache until used < capacity.
// cm.mu lock must be acquired beforehand.
func (cm *Storage) maint() {
	for cm.capacity > 0 && cm.used > cm.capacity {
		e := heap.Pop(cm).(*entry)
		delete(cm.cache, e.Path())
		cm.used -= e.Size()
		if err := os.Remove(filepath.Join(cm.dir, e.Path())); err != nil {
			log.Warn("Storage.maint", map[string]interface{}{
				"_err": err.Error(),
			})
		}
		log.Info("removed", map[string]interface{}{
			"_path": e.Path(),
		})
	}
}

func readData(path string) ([]byte, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return ioutil.ReadAll(f)
}

// Load loads existing items in filesystem.
func (cm *Storage) Load() error {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	wf := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.Mode().IsRegular() {
			return nil
		}
		subpath, err := filepath.Rel(cm.dir, path)
		if err != nil {
			return err
		}
		if _, ok := cm.cache[subpath]; ok {
			return nil
		}

		size := uint64(info.Size())
		e := &entry{
			// delay calculation of checksums.
			FileInfo: &FileInfo{
				path: subpath,
				size: size,
			},
			atime: cm.lclock,
			index: len(cm.lru),
		}
		cm.used += size
		cm.lclock++
		cm.lru = append(cm.lru, e)
		cm.cache[subpath] = e
		log.Debug("Storage.Load", map[string]interface{}{
			"_path": subpath,
		})
		return nil
	}

	if err := filepath.Walk(cm.dir, wf); err != nil {
		return err
	}
	heap.Init(cm)

	cm.maint()

	return nil
}

// Insert inserts or updates a cache item.
//
// fi.Path() must be as clean as filepath.Clean() and
// must not be filepath.IsAbs().
func (cm *Storage) Insert(data []byte, fi *FileInfo) error {
	switch {
	case fi.path != filepath.Clean(fi.path):
		return ErrBadPath
	case filepath.IsAbs(fi.path):
		return ErrBadPath
	case fi.path == ".":
		return ErrBadPath
	}

	f, err := ioutil.TempFile(cm.dir, "_tmp")
	if err != nil {
		return err
	}
	defer func() {
		f.Close()
		os.Remove(f.Name())
	}()

	_, err = f.Write(data)
	if err != nil {
		return err
	}
	err = f.Sync()
	if err != nil {
		return err
	}

	p := fi.path
	destpath := filepath.Join(cm.dir, p)
	dirpath := filepath.Dir(destpath)

	_, err = os.Stat(dirpath)
	switch {
	case os.IsNotExist(err):
		err = os.MkdirAll(dirpath, 0755)
		if err != nil {
			return err
		}
	case err != nil:
		return err
	}

	cm.mu.Lock()
	defer cm.mu.Unlock()

	if existing, ok := cm.cache[p]; ok {
		err = os.Remove(destpath)
		if err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			log.Warn("cache file was removed already", map[string]interface{}{
				"_path": p,
			})
		}
		cm.used -= existing.Size()
		heap.Remove(cm, existing.index)
		delete(cm.cache, p)
		if log.Enabled(log.LvDebug) {
			log.Debug("deleted existing item", map[string]interface{}{
				"_path": p,
			})
		}
	}

	err = os.Rename(f.Name(), destpath)
	if err != nil {
		return err
	}

	e := &entry{
		FileInfo: fi,
		atime:    cm.lclock,
	}
	cm.used += fi.size
	cm.lclock++
	heap.Push(cm, e)
	cm.cache[p] = e

	cm.maint()

	return nil
}

func calcChecksum(dir string, e *entry) error {
	if e.FileInfo.md5sum != nil {
		return nil
	}

	data, err := readData(filepath.Join(dir, e.Path()))
	if err != nil {
		return err
	}
	md5sum := md5.Sum(data)
	sha1sum := sha1.Sum(data)
	sha256sum := sha256.Sum256(data)
	e.FileInfo.md5sum = md5sum[:]
	e.FileInfo.sha1sum = sha1sum[:]
	e.FileInfo.sha256sum = sha256sum[:]
	return nil
}

// Lookup looks up an item in the cache.
// If no item matching fi is found, ErrNotFound is returned.
//
// The caller is responsible to close the returned os.File.
func (cm *Storage) Lookup(fi *FileInfo) (*os.File, error) {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	e, ok := cm.cache[fi.path]
	if !ok {
		return nil, ErrNotFound
	}

	// delayed checksum calculation
	err := calcChecksum(cm.dir, e)
	if err != nil {
		return nil, err
	}

	if !fi.Same(e.FileInfo) {
		// checksum mismatch
		return nil, ErrNotFound
	}

	e.atime = cm.lclock
	cm.lclock++
	heap.Fix(cm, e.index)
	return os.Open(filepath.Join(cm.dir, fi.path))
}

// ListAll returns a list of FileInfo for all cached items.
func (cm *Storage) ListAll() []*FileInfo {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	l := make([]*FileInfo, cm.Len())
	for i, e := range cm.lru {
		l[i] = e.FileInfo
	}
	return l
}

// Delete deletes an item from the cache.
func (cm *Storage) Delete(p string) error {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	e, ok := cm.cache[p]
	if !ok {
		return nil
	}

	err := os.Remove(filepath.Join(cm.dir, p))
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		log.Warn("cached file was already removed", map[string]interface{}{
			"_path": p,
		})
	}

	cm.used -= e.size
	heap.Remove(cm, e.index)
	delete(cm.cache, p)
	log.Info("deleted item", map[string]interface{}{
		"_path": p,
	})
	return nil
}
<|endoftext|>


<commit_before>// Package sub provides a very simple event subscriber, using the file system
// as an event store, and the file name for any particular event as the event
// ID. It's designed to be used with the pub package that is also included in
// the fspubsub repository.
package sub

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sync"

	"github.com/rjeczalik/notify"
)

// defaultNotifyBufferSize describes the default buffer size. This is used for
// both the filesystem and event buffer at this point in time.
//
// This needs to be adequately tuned to the needs of the application - the
// filesystem notifier does not block sending events, so the subscriber will
// miss events if there is an overrun.
const defaultBufferSize = 10

// Event represents a single event.
type Event struct {
	// The ID of the event. This normally translates to the file name from the
	// store.
	ID string

	// The event data.
	Data interface{}
}

// Subscriber is a simple event subscriber, designed to read events from the
// file system.
//
// The stream location on the filesystem is composed of a base directory and
// the name of the type that you are watching, without the package name
// included. As an example, if you set the directory to be ./, and the type you
// were watching was main.TestEvent, the stream path would be ./TestEvent. The
// directory is created if it does not exist.
//
// Note that the directory the event store is in must only contain events -
// functions will fail if they encounter non-event data (ie: JSON that it
// cannot parse into the event type).
type Subscriber struct {
	// The event channel. This is buffered to the size of the file system
	// notification buffer.
	Queue chan Event

	// The done channel. This should be watched to determine if the event stream
	// has been shut down.
	Done chan struct{}

	// The directory the event publisher will read events from. This is composed
	// of a base directory supplied upon creation of the publisher, and the
	// package-local name of the type used for the event.
	dir string

	// The type for the event that this publisher processes. Events passed to the
	// publisher need to match this type.
	eventType reflect.Type

	// A mutex for blocking access to the watcher.
	m sync.Mutex

	// An internal channel for signaling that we are done watching FS events.
	fsDone chan struct{}
}

// NewSubscriber creates a subscriber to a directory-based event stream, being
// a mix of the path supplied and the event type passed to event.
//
// Any data in event is ignored - it just serves to infer the type of event
// this subscriber is locked to.
func NewSubscriber(dir string, event interface{}) (*Subscriber, error) {
	if event == nil {
		return nil, errors.New("event cannot be nil")
	}
	s := &Subscriber{
		Queue:     make(chan Event, defaultBufferSize),
		Done:      make(chan struct{}, 1),
		dir:       filepath.Clean(dir) + "/" + reflect.TypeOf(event).Name(),
		eventType: reflect.TypeOf(event),
		fsDone:    make(chan struct{}, 1),
	}

	stat, err := os.Stat(s.dir)
	switch {
	case err == nil:
		if !stat.Mode().IsDir() {
			return nil, fmt.Errorf("%s exists and is not a directory", s.dir)
		}
	case err != nil && os.IsNotExist(err):
		if err := os.Mkdir(s.dir, 0777); err != nil {
			return nil, fmt.Errorf("cannot create directory %s: %s", s.dir, err)
		}
	case err != nil:
		return nil, fmt.Errorf("Could not stat dir %s: %s", s.dir, err)
	}

	return s, nil
}

// Subscribe starts watching the directory for events, and sends the events
// over the Queue channel. Only one subscription can be open at any point in
// time - this function blocks if the subscriber is currently open.
//
// When the stream shuts down, a message will be sent over the Done channel to
// signal that the consumer should stop reading from the Queue.
//
// If the stream is interrupted for any other reason than the subscriber being
// closed with Close, this function will return an error. This includes bad
// event data, which will shut down the subscriber.
func (s *Subscriber) Subscribe() error {
	s.m.Lock()
	defer s.m.Unlock()
	c := make(chan notify.EventInfo, defaultBufferSize)
	if err := notify.Watch(s.dir, c, notify.InCloseWrite); err != nil {
		return fmt.Errorf("error watching directory %s: %s", s.dir, err)
	}
	defer notify.Stop(c)
	for {
		select {
		case ei := <-c:
			d := reflect.New(s.eventType)
			b, err := ioutil.ReadFile(ei.Path())
			if err != nil {
				return fmt.Errorf("error reading event data at %s: %s", ei.Path(), err)
			}
			if err := json.Unmarshal(b, d.Interface()); err != nil {
				return fmt.Errorf("error unmarshaling event data from %s: %s", ei.Path(), err)
			}
			s.Queue <- Event{
				ID:   filepath.Base(ei.Path()),
				Data: d.Elem().Interface(),
			}
		case <-s.fsDone:
			goto done
		}
	}
done:
	s.Done <- struct{}{}
	return nil
}

// SubscribeCallback is a helper that provides a very simple event loop around
// Subscribe. Events are passed to the callback function supplied by cb, with
// the ID and data.
//
// This function does not take responsbility for handling event processing
// errors. It's up to the callback to hand errors as it sees fit.
func (s *Subscriber) SubscribeCallback(cb func(string, interface{})) error {
	go func() {
		for {
			select {
			case event := <-s.Queue:
				cb(event.ID, event.Data)
			case <-s.Done:
				return
			}
		}
	}()
	return s.Subscribe()
}

// Close signals to Subscribe that we are done and that the subscription is no
// longer needed. This initiates shutdown in Subscribe and will make it return
// without error, as long as there is none.
func (s *Subscriber) Close() {
	s.fsDone <- struct{}{}
}

// Dump dumps all of the events in the store for this stream. Technically, it's
// just dumping all of the events in the directory that the stream has been
// configured to watch. This is returned as an Event slice.
//
// The order of the returned events is not deterministic. If it is up to the
// consumer to structure the data or the handling of the data in a way that
// facilitates proper hydration.
func (s *Subscriber) Dump() ([]Event, error) {
	entries, err := ioutil.ReadDir(s.dir)
	var es []Event
	if err != nil {
		return nil, fmt.Errorf("error reading event directory %s: %s", s.dir, err)
	}
	for _, f := range entries {
		if !f.Mode().IsRegular() {
			continue
		}
		id := f.Name()
		ep := s.dir + "/" + id
		d := reflect.New(s.eventType)
		b, err := ioutil.ReadFile(ep)
		if err != nil {
			return nil, fmt.Errorf("error reading event data at %s: %s", ep, err)
		}
		if err := json.Unmarshal(b, d.Interface()); err != nil {
			return nil, fmt.Errorf("error unmarshaling event data from %s: %s", ep, err)
		}
		es = append(es, Event{
			ID:   id,
			Data: d.Elem().Interface(),
		})
	}
	return es, nil
}

// Fetch reads a single event from the store. The supplied ID is essentially the
// file name.
func (s *Subscriber) Fetch(id string) (Event, error) {
	var e Event
	ep := s.dir + "/" + id
	d := reflect.New(s.eventType)
	b, err := ioutil.ReadFile(ep)
	if err != nil {
		return e, fmt.Errorf("error reading event data at %s: %s", ep, err)
	}
	if err := json.Unmarshal(b, d.Interface()); err != nil {
		return e, fmt.Errorf("error unmarshaling event data from %s: %s", ep, err)
	}
	e.ID = id
	e.Data = d.Elem().Interface()
	return e, nil
}
<commit_msg>hand -> handle<commit_after>// Package sub provides a very simple event subscriber, using the file system
// as an event store, and the file name for any particular event as the event
// ID. It's designed to be used with the pub package that is also included in
// the fspubsub repository.
package sub

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sync"

	"github.com/rjeczalik/notify"
)

// defaultNotifyBufferSize describes the default buffer size. This is used for
// both the filesystem and event buffer at this point in time.
//
// This needs to be adequately tuned to the needs of the application - the
// filesystem notifier does not block sending events, so the subscriber will
// miss events if there is an overrun.
const defaultBufferSize = 10

// Event represents a single event.
type Event struct {
	// The ID of the event. This normally translates to the file name from the
	// store.
	ID string

	// The event data.
	Data interface{}
}

// Subscriber is a simple event subscriber, designed to read events from the
// file system.
//
// The stream location on the filesystem is composed of a base directory and
// the name of the type that you are watching, without the package name
// included. As an example, if you set the directory to be ./, and the type you
// were watching was main.TestEvent, the stream path would be ./TestEvent. The
// directory is created if it does not exist.
//
// Note that the directory the event store is in must only contain events -
// functions will fail if they encounter non-event data (ie: JSON that it
// cannot parse into the event type).
type Subscriber struct {
	// The event channel. This is buffered to the size of the file system
	// notification buffer.
	Queue chan Event

	// The done channel. This should be watched to determine if the event stream
	// has been shut down.
	Done chan struct{}

	// The directory the event publisher will read events from. This is composed
	// of a base directory supplied upon creation of the publisher, and the
	// package-local name of the type used for the event.
	dir string

	// The type for the event that this publisher processes. Events passed to the
	// publisher need to match this type.
	eventType reflect.Type

	// A mutex for blocking access to the watcher.
	m sync.Mutex

	// An internal channel for signaling that we are done watching FS events.
	fsDone chan struct{}
}

// NewSubscriber creates a subscriber to a directory-based event stream, being
// a mix of the path supplied and the event type passed to event.
//
// Any data in event is ignored - it just serves to infer the type of event
// this subscriber is locked to.
func NewSubscriber(dir string, event interface{}) (*Subscriber, error) {
	if event == nil {
		return nil, errors.New("event cannot be nil")
	}
	s := &Subscriber{
		Queue:     make(chan Event, defaultBufferSize),
		Done:      make(chan struct{}, 1),
		dir:       filepath.Clean(dir) + "/" + reflect.TypeOf(event).Name(),
		eventType: reflect.TypeOf(event),
		fsDone:    make(chan struct{}, 1),
	}

	stat, err := os.Stat(s.dir)
	switch {
	case err == nil:
		if !stat.Mode().IsDir() {
			return nil, fmt.Errorf("%s exists and is not a directory", s.dir)
		}
	case err != nil && os.IsNotExist(err):
		if err := os.Mkdir(s.dir, 0777); err != nil {
			return nil, fmt.Errorf("cannot create directory %s: %s", s.dir, err)
		}
	case err != nil:
		return nil, fmt.Errorf("Could not stat dir %s: %s", s.dir, err)
	}

	return s, nil
}

// Subscribe starts watching the directory for events, and sends the events
// over the Queue channel. Only one subscription can be open at any point in
// time - this function blocks if the subscriber is currently open.
//
// When the stream shuts down, a message will be sent over the Done channel to
// signal that the consumer should stop reading from the Queue.
//
// If the stream is interrupted for any other reason than the subscriber being
// closed with Close, this function will return an error. This includes bad
// event data, which will shut down the subscriber.
func (s *Subscriber) Subscribe() error {
	s.m.Lock()
	defer s.m.Unlock()
	c := make(chan notify.EventInfo, defaultBufferSize)
	if err := notify.Watch(s.dir, c, notify.InCloseWrite); err != nil {
		return fmt.Errorf("error watching directory %s: %s", s.dir, err)
	}
	defer notify.Stop(c)
	for {
		select {
		case ei := <-c:
			d := reflect.New(s.eventType)
			b, err := ioutil.ReadFile(ei.Path())
			if err != nil {
				return fmt.Errorf("error reading event data at %s: %s", ei.Path(), err)
			}
			if err := json.Unmarshal(b, d.Interface()); err != nil {
				return fmt.Errorf("error unmarshaling event data from %s: %s", ei.Path(), err)
			}
			s.Queue <- Event{
				ID:   filepath.Base(ei.Path()),
				Data: d.Elem().Interface(),
			}
		case <-s.fsDone:
			goto done
		}
	}
done:
	s.Done <- struct{}{}
	return nil
}

// SubscribeCallback is a helper that provides a very simple event loop around
// Subscribe. Events are passed to the callback function supplied by cb, with
// the ID and data.
//
// This function does not take responsbility for handling event processing
// errors. It's up to the callback to handle errors as it sees fit.
func (s *Subscriber) SubscribeCallback(cb func(string, interface{})) error {
	go func() {
		for {
			select {
			case event := <-s.Queue:
				cb(event.ID, event.Data)
			case <-s.Done:
				return
			}
		}
	}()
	return s.Subscribe()
}

// Close signals to Subscribe that we are done and that the subscription is no
// longer needed. This initiates shutdown in Subscribe and will make it return
// without error, as long as there is none.
func (s *Subscriber) Close() {
	s.fsDone <- struct{}{}
}

// Dump dumps all of the events in the store for this stream. Technically, it's
// just dumping all of the events in the directory that the stream has been
// configured to watch. This is returned as an Event slice.
//
// The order of the returned events is not deterministic. If it is up to the
// consumer to structure the data or the handling of the data in a way that
// facilitates proper hydration.
func (s *Subscriber) Dump() ([]Event, error) {
	entries, err := ioutil.ReadDir(s.dir)
	var es []Event
	if err != nil {
		return nil, fmt.Errorf("error reading event directory %s: %s", s.dir, err)
	}
	for _, f := range entries {
		if !f.Mode().IsRegular() {
			continue
		}
		id := f.Name()
		ep := s.dir + "/" + id
		d := reflect.New(s.eventType)
		b, err := ioutil.ReadFile(ep)
		if err != nil {
			return nil, fmt.Errorf("error reading event data at %s: %s", ep, err)
		}
		if err := json.Unmarshal(b, d.Interface()); err != nil {
			return nil, fmt.Errorf("error unmarshaling event data from %s: %s", ep, err)
		}
		es = append(es, Event{
			ID:   id,
			Data: d.Elem().Interface(),
		})
	}
	return es, nil
}

// Fetch reads a single event from the store. The supplied ID is essentially the
// file name.
func (s *Subscriber) Fetch(id string) (Event, error) {
	var e Event
	ep := s.dir + "/" + id
	d := reflect.New(s.eventType)
	b, err := ioutil.ReadFile(ep)
	if err != nil {
		return e, fmt.Errorf("error reading event data at %s: %s", ep, err)
	}
	if err := json.Unmarshal(b, d.Interface()); err != nil {
		return e, fmt.Errorf("error unmarshaling event data from %s: %s", ep, err)
	}
	e.ID = id
	e.Data = d.Elem().Interface()
	return e, nil
}
<|endoftext|>


<commit_before>// 11 february 2014
package main

import (
	"runtime"
)

// The sysData type contains all system data. It provides the system-specific underlying implementation. It is guaranteed to have the following by embedding:
type cSysData struct {
	ctype			int
	event			chan struct{}
}
func (c *cSysData) make(initText string, initWidth int, initHeight int, window *sysData) error {
	panic(runtime.GOOS + " sysData does not define make()")
}
func (c *cSysData) show() error {
	panic(runtime.GOOS + " sysData does not define show()")
}
func (c *cSysData) hide() error {
	panic(runtime.GOOS + " sysData does not define hide()")
}
func (c *cSysData) setText(text string) error {
	panic(runtime.GOOS + " sysData does not define setText()")
}

const (
	c_window = iota
	c_button
	nctypes
)
<commit_msg>Formatting change for cSysData itself.<commit_after>// 11 february 2014
package main

import (
	"runtime"
)

// The sysData type contains all system data. It provides the system-specific underlying implementation. It is guaranteed to have the following by embedding:
type cSysData struct {
	ctype	int
	event	chan struct{}
}
func (c *cSysData) make(initText string, initWidth int, initHeight int, window *sysData) error {
	panic(runtime.GOOS + " sysData does not define make()")
}
func (c *cSysData) show() error {
	panic(runtime.GOOS + " sysData does not define show()")
}
func (c *cSysData) hide() error {
	panic(runtime.GOOS + " sysData does not define hide()")
}
func (c *cSysData) setText(text string) error {
	panic(runtime.GOOS + " sysData does not define setText()")
}

const (
	c_window = iota
	c_button
	nctypes
)
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net"
	"strings"
	"time"
)

type handler struct {
	s string
	f func(net.Conn, []byte, time.Duration, bool)
}

var verbose bool

var responses = map[string]handler{
	"EHLO": {"250-Pleased to meet you!\r\n250-PIPELINING\r\n250 CHUNKING\r\n", nil},
	"HELO": {"250 Pleased to meet you!\r\n", nil},
	"MAIL": {"250 OK\r\n", nil},
	"RCPT": {"250 OK\r\n", nil},
	"DATA": {"354 End data with <CR><LF>.<CR><LF>\r\n", handleData}, // Need to read data until \r\n.\r\n is received.
	"BDAT": {"250 OK\r\n", handleBdat},                              // Should be sent once the data has been reveived
	"RSET": {"250 OK\r\n", nil},
	"QUIT": {"221 Goodbye\r\n", nil}}

func sendResponse(c net.Conn, s string, verbose bool) {
			c.Write([]byte(s))
			if verbose {
				log.Printf("<- %s", s)
			}
}

func handleConnection(c net.Conn, latency time.Duration, verbose bool) {
	// Print banner
	sendResponse(c, "220 Welcome to Blackhole SMTP!\r\n", verbose)
	for {
		readBuf := make([]byte, 4096)
		l, e := c.Read(readBuf)
		if e != nil {
			_ = c.Close()
			return
		}
		time.Sleep(latency * time.Millisecond)
		if verbose {
			log.Printf("-> [%s]", strings.Trim(string(readBuf[0:l]), "\r\n "))
		}
		h, ok := responses[string(readBuf[0:4])]
		if ok {
			sendResponse(c, h.s, verbose)
			if h.f != nil {
				h.f(c, readBuf, latency, verbose)
			}
		} else {
			sendResponse(c, "500 Command unrecognized\r\n", verbose)
		}
	}
}

func handleData(c net.Conn, b []byte, latency time.Duration, verbose bool) {
	for {
		l, e := c.Read(b)
		if e != nil || l == 0 {
			break
		}
		if bytes.Contains(b, []byte("\r\n.\r\n")) {
			time.Sleep(latency * time.Millisecond)
			sendResponse(c, "250 OK\r\n", verbose)
			break
		}
	}
}

func handleBdat(c net.Conn, b []byte, latency time.Duration, verbose bool) {
}

func main() {
	var port int
	var latency int
	var verbose bool

	flag.IntVar(&port, "port", 25, "TCP port")
	flag.IntVar(&latency, "latency", 0, "Latency in milliseconds")
	flag.BoolVar(&verbose, "verbose", false, "Show the SMTP traffic")

	flag.Parse()

	// Get address:port
	a, e := net.ResolveTCPAddr("tcp4", fmt.Sprintf(":%d", port))
	if e != nil {
		// Error!
		log.Panic(e)
		return
	}

	// Start listening for incoming connections
	l, e := net.ListenTCP("tcp", a)
	if e != nil {
		// Error!
		log.Panic(e)
		return
	}

	// Accept connections then handle each one in a dedicated goroutine
	for {
		c, e := l.Accept()
		if e != nil {
			continue
		}
		go handleConnection(c, time.Duration(latency), verbose)
	}
}
<commit_msg>go fmt<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net"
	"strings"
	"time"
)

type handler struct {
	s string
	f func(net.Conn, []byte, time.Duration, bool)
}

var verbose bool

var responses = map[string]handler{
	"EHLO": {"250-Pleased to meet you!\r\n250-PIPELINING\r\n250 CHUNKING\r\n", nil},
	"HELO": {"250 Pleased to meet you!\r\n", nil},
	"MAIL": {"250 OK\r\n", nil},
	"RCPT": {"250 OK\r\n", nil},
	"DATA": {"354 End data with <CR><LF>.<CR><LF>\r\n", handleData}, // Need to read data until \r\n.\r\n is received.
	"BDAT": {"250 OK\r\n", handleBdat},                              // Should be sent once the data has been reveived
	"RSET": {"250 OK\r\n", nil},
	"QUIT": {"221 Goodbye\r\n", nil}}

func sendResponse(c net.Conn, s string, verbose bool) {
	c.Write([]byte(s))
	if verbose {
		log.Printf("<- %s", s)
	}
}

func handleConnection(c net.Conn, latency time.Duration, verbose bool) {
	// Print banner
	sendResponse(c, "220 Welcome to Blackhole SMTP!\r\n", verbose)
	for {
		readBuf := make([]byte, 4096)
		l, e := c.Read(readBuf)
		if e != nil {
			_ = c.Close()
			return
		}
		time.Sleep(latency * time.Millisecond)
		if verbose {
			log.Printf("-> [%s]", strings.Trim(string(readBuf[0:l]), "\r\n "))
		}
		h, ok := responses[string(readBuf[0:4])]
		if ok {
			sendResponse(c, h.s, verbose)
			if h.f != nil {
				h.f(c, readBuf, latency, verbose)
			}
		} else {
			sendResponse(c, "500 Command unrecognized\r\n", verbose)
		}
	}
}

func handleData(c net.Conn, b []byte, latency time.Duration, verbose bool) {
	for {
		l, e := c.Read(b)
		if e != nil || l == 0 {
			break
		}
		if bytes.Contains(b, []byte("\r\n.\r\n")) {
			time.Sleep(latency * time.Millisecond)
			sendResponse(c, "250 OK\r\n", verbose)
			break
		}
	}
}

func handleBdat(c net.Conn, b []byte, latency time.Duration, verbose bool) {
}

func main() {
	var port int
	var latency int
	var verbose bool

	flag.IntVar(&port, "port", 25, "TCP port")
	flag.IntVar(&latency, "latency", 0, "Latency in milliseconds")
	flag.BoolVar(&verbose, "verbose", false, "Show the SMTP traffic")

	flag.Parse()

	// Get address:port
	a, e := net.ResolveTCPAddr("tcp4", fmt.Sprintf(":%d", port))
	if e != nil {
		// Error!
		log.Panic(e)
		return
	}

	// Start listening for incoming connections
	l, e := net.ListenTCP("tcp", a)
	if e != nil {
		// Error!
		log.Panic(e)
		return
	}

	// Accept connections then handle each one in a dedicated goroutine
	for {
		c, e := l.Accept()
		if e != nil {
			continue
		}
		go handleConnection(c, time.Duration(latency), verbose)
	}
}
<|endoftext|>


<commit_before>package sortedmap

import (
	"testing"
	"time"

	"github.com/umpc/go-sortedmap/asc"
)

const (
	notFoundErr = "key not found"
	keyExistsErr = "a key already exists in the collection!"
	unsortedErr = "SortedMap is not sorted!"
	invalidDelete = "invalid delete status!"
)

func TestNew(t *testing.T) {
	sm := New(nil)

	if sm.idx == nil {
		t.Fatal("TestNew failed: idx was nil!")
	}
	if sm.sorted == nil {
		t.Fatal("TestNew failed: sorted was nil!")
	}
	if sm.lessFn == nil {
		t.Fatal("TestNew failed: lessFn was nil!")
	}
}

func TestFalseLessFunc(t *testing.T) {
	if New(nil).lessFn() {
		t.Fatal("TestFalseLessFunc failed: lessFn returned true!")
	}
}

func TestInsert(t *testing.T) {
	records := randRecords(3)
	sm := New(asc.Time)

	for i := range records {
		if !sm.Insert(records[i].Key, records[i].Val) {
			t.Fatalf("Insert failed: %v", keyExistsErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}

	for i := range records {
		if sm.Insert(records[i].Key, records[i].Val) {
			t.Fatalf("Insert failed: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestReplace(t *testing.T) {
	records := randRecords(3)
	sm := New(asc.Time)

	for i := 0; i < 5; i++ {
		for ii := range records {
			sm.Replace(records[ii].Key, records[ii].Val)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestHas(t *testing.T) {
	records := randRecords(3)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	for i := range records {
		if !sm.Has(records[i].Key) {
			t.Fatalf("Has failed: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestGet(t *testing.T) {
	records := randRecords(3)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	for i := range records {
		if val, ok := sm.Get(records[i].Key); val == nil || !ok {
			t.Fatalf("Get failed: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestDelete(t *testing.T) {
	records := randRecords(300)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}

	if sm.Delete("") {
		t.Fatalf("Delete: %v", invalidDelete)
	}

	for _, rec := range records {
		sm.Delete(rec.Key)
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestLen(t *testing.T) {
	count := 100
	sm := newSortedMapFromRandRecords(count)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}

	if sm.Len() != count {
		t.Fatalf("Len: invalid SortedMap length. Expected: %v, Had: %v.", count, sm.Len())
	}
}

func TestBatchInsert(t *testing.T) {
	records := randRecords(1000)
	sm := New(asc.Time)

	for _, ok := range sm.BatchInsert(records...) {
		if !ok {
			t.Fatalf("BatchInsert failed: %v", keyExistsErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestBatchReplace(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestBatchHas(t *testing.T) {
	records := randRecords(1000)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	keys := make([]string, len(records))
	for i := range records {
		keys[i] = records[i].Key
	}

	for _, ok := range sm.BatchHas(keys...) {
		if !ok {
			t.Fatalf("BatchHas: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestBatchGet(t *testing.T) {
	records := randRecords(1000)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	keys := make([]string, len(records))
	for i := range records {
		keys[i] = records[i].Key
	}

	values, results := sm.BatchGet(keys...)
	for i, ok := range results {
		if values[i] == nil || !ok {
			t.Fatalf("BatchGet: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestBatchDelete(t *testing.T) {
	records := randRecords(300)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}

	keys := make([]string, 0)
	for i, rec := range records {
		if i == 50 {
			break
		}
		keys = append(keys, rec.Key)
	}

	for _, ok := range sm.BatchDelete(keys...) {
		if !ok {
			t.Fatalf("BatchDelete: %v", invalidDelete)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestIter(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestIterUntil(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.IterUntil(time.Now())); err != nil {
		t.Fatal(err)
	}
}

func TestIterAfter(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.IterAfter(time.Now())); err != nil {
		t.Fatal(err)
	}
}

func TestBufferedIter(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.BufferedIter(256)); err != nil {
		t.Fatal(err)
	}
}

func TestBufferedIterUntil(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.BufferedIterUntil(256, time.Now())); err != nil {
		t.Fatal(err)
	}
}

func TestBufferedIterAfter(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.BufferedIterAfter(256, time.Now())); err != nil {
		t.Fatal(err)
	}
}<commit_msg>add nil values as lessFn arguments<commit_after>package sortedmap

import (
	"testing"
	"time"

	"github.com/umpc/go-sortedmap/asc"
)

const (
	notFoundErr = "key not found"
	keyExistsErr = "a key already exists in the collection!"
	unsortedErr = "SortedMap is not sorted!"
	invalidDelete = "invalid delete status!"
)

func TestNew(t *testing.T) {
	sm := New(nil)

	if sm.idx == nil {
		t.Fatal("TestNew failed: idx was nil!")
	}
	if sm.sorted == nil {
		t.Fatal("TestNew failed: sorted was nil!")
	}
	if sm.lessFn == nil {
		t.Fatal("TestNew failed: lessFn was nil!")
	}
}

func TestFalseLessFunc(t *testing.T) {
	if New(nil).lessFn(nil, nil) {
		t.Fatal("TestFalseLessFunc failed: lessFn returned true!")
	}
}

func TestInsert(t *testing.T) {
	records := randRecords(3)
	sm := New(asc.Time)

	for i := range records {
		if !sm.Insert(records[i].Key, records[i].Val) {
			t.Fatalf("Insert failed: %v", keyExistsErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}

	for i := range records {
		if sm.Insert(records[i].Key, records[i].Val) {
			t.Fatalf("Insert failed: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestReplace(t *testing.T) {
	records := randRecords(3)
	sm := New(asc.Time)

	for i := 0; i < 5; i++ {
		for ii := range records {
			sm.Replace(records[ii].Key, records[ii].Val)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestHas(t *testing.T) {
	records := randRecords(3)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	for i := range records {
		if !sm.Has(records[i].Key) {
			t.Fatalf("Has failed: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestGet(t *testing.T) {
	records := randRecords(3)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	for i := range records {
		if val, ok := sm.Get(records[i].Key); val == nil || !ok {
			t.Fatalf("Get failed: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestDelete(t *testing.T) {
	records := randRecords(300)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}

	if sm.Delete("") {
		t.Fatalf("Delete: %v", invalidDelete)
	}

	for _, rec := range records {
		sm.Delete(rec.Key)
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestLen(t *testing.T) {
	count := 100
	sm := newSortedMapFromRandRecords(count)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}

	if sm.Len() != count {
		t.Fatalf("Len: invalid SortedMap length. Expected: %v, Had: %v.", count, sm.Len())
	}
}

func TestBatchInsert(t *testing.T) {
	records := randRecords(1000)
	sm := New(asc.Time)

	for _, ok := range sm.BatchInsert(records...) {
		if !ok {
			t.Fatalf("BatchInsert failed: %v", keyExistsErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestBatchReplace(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestBatchHas(t *testing.T) {
	records := randRecords(1000)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	keys := make([]string, len(records))
	for i := range records {
		keys[i] = records[i].Key
	}

	for _, ok := range sm.BatchHas(keys...) {
		if !ok {
			t.Fatalf("BatchHas: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestBatchGet(t *testing.T) {
	records := randRecords(1000)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	keys := make([]string, len(records))
	for i := range records {
		keys[i] = records[i].Key
	}

	values, results := sm.BatchGet(keys...)
	for i, ok := range results {
		if values[i] == nil || !ok {
			t.Fatalf("BatchGet: %v", notFoundErr)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestBatchDelete(t *testing.T) {
	records := randRecords(300)
	sm := New(asc.Time)
	sm.BatchReplace(records...)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}

	keys := make([]string, 0)
	for i, rec := range records {
		if i == 50 {
			break
		}
		keys = append(keys, rec.Key)
	}

	for _, ok := range sm.BatchDelete(keys...) {
		if !ok {
			t.Fatalf("BatchDelete: %v", invalidDelete)
		}
	}

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestIter(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.Iter()); err != nil {
		t.Fatal(err)
	}
}

func TestIterUntil(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.IterUntil(time.Now())); err != nil {
		t.Fatal(err)
	}
}

func TestIterAfter(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.IterAfter(time.Now())); err != nil {
		t.Fatal(err)
	}
}

func TestBufferedIter(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.BufferedIter(256)); err != nil {
		t.Fatal(err)
	}
}

func TestBufferedIterUntil(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.BufferedIterUntil(256, time.Now())); err != nil {
		t.Fatal(err)
	}
}

func TestBufferedIterAfter(t *testing.T) {
	sm := newSortedMapFromRandRecords(1000)

	if err := verifyRecords(sm.BufferedIterAfter(256, time.Now())); err != nil {
		t.Fatal(err)
	}
}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	taskies "github.com/dimerica-industries/taskies/src"
	"io/ioutil"
	"os"
	"runtime/debug"
)

const (
	DEFAULT_FILE = "Taskies"
)

type ArrayOpts []string

func (arr *ArrayOpts) String() string {
	return fmt.Sprint(*arr)
}

func (arr *ArrayOpts) Set(str string) error {
	*arr = append(*arr, str)
	return nil
}

func main() {
	defer func() {
		if err := recover(); err != nil {
			fmt.Printf("Error: %s\n", err)
			taskies.Debug(func() {
				debug.PrintStack()
			})
			os.Exit(1)
		}
	}()

	files := &ArrayOpts{}
	flag.Var(files, "f", "")
	help := flag.Bool("h", false, "Show help")
	list := flag.Bool("l", false, "List all available tasks")

	flag.Parse()

	tasks := flag.Args()

	if len(*files) == 0 {
		*files = ArrayOpts{DEFAULT_FILE}
	}

	if *help {
		flag.Usage()
		os.Exit(0)
	}

	ts := taskies.NewTaskSet()

	for _, path := range *files {
		contents, err := ioutil.ReadFile(path)

		if err != nil {
			panic("Cannot read " + path)
		}

		err = taskies.DecodeYAML(contents, ts)

		if err != nil {
			panic("YAML decode error: " + err.Error())
		}
	}

	if *list {
		fmt.Printf("Available Tasks:\n")
		for name, t := range ts.ExportedTasks {
			fmt.Printf("  %s - %s\n", name, t.Description())
		}

		os.Exit(0)
	}

	if len(tasks) == 0 {
		flag.Usage()
		os.Exit(1)
	}

	runner := taskies.NewRunner(ts, ts.Env, os.Stdin, os.Stdout, os.Stderr)
	err := runner.Run(tasks...)

	if err != nil {
		panic(err)
	}
}
<commit_msg>format the tasks better<commit_after>package main

import (
	"flag"
	"fmt"
	taskies "github.com/dimerica-industries/taskies/src"
	"io/ioutil"
	"os"
	"runtime/debug"
    "text/tabwriter"
)

const (
	DEFAULT_FILE = "./Taskies"
)

func main() {
	defer func() {
		if err := recover(); err != nil {
			fmt.Printf("Error: %s\n", err)
			taskies.Debug(func() {
				debug.PrintStack()
			})
			os.Exit(1)
		}
	}()

    file := flag.String("f", DEFAULT_FILE, "Location of the taskie file")
	help := flag.Bool("h", false, "Show help")
	list := flag.Bool("l", false, "List all available tasks")

	flag.Parse()

	tasks := flag.Args()

	if *help {
		flag.Usage()
		os.Exit(0)
	}

	ts := taskies.NewTaskSet()

    contents, err := ioutil.ReadFile(*file)

    if err != nil {
        panic("Cannot read " + *file)
    }

    err = taskies.DecodeYAML(contents, ts)

    if err != nil {
        panic("YAML decode error: " + err.Error())
    }

    l := func() {
		fmt.Printf("Available Tasks:\n")
        w := new(tabwriter.Writer)
        w.Init(os.Stdout, 0, 8, 0, '\t', 0)

		for name, t := range ts.ExportedTasks {
			fmt.Fprintf(w, "   %s\t%s\n", name, t.Description())
		}

        w.Flush()
    }

	if *list {
        l()
		os.Exit(0)
	}

	if len(tasks) == 0 {
		flag.Usage()
        fmt.Println()
        l()
		os.Exit(1)
	}

	runner := taskies.NewRunner(ts, ts.Env, os.Stdin, os.Stdout, os.Stderr)
	err = runner.Run(tasks...)

	if err != nil {
		panic(err)
	}
}
<|endoftext|>