<commit_before>package emulator

import (
	"fmt"
	"log"
	"syscall"

	"github.com/twtiger/gosecco/data"

	"golang.org/x/sys/unix"
)

func init() {
	log.SetFlags(0)
}

// Emulate will execute a seccomp filter program against the given working memory.
func Emulate(d data.SeccompWorkingMemory, filters []unix.SockFilter) uint32 {
	e := &emulator{data: d, filters: filters, pointer: 0}
	for {
		val, finished := e.next()
		if finished {
			return val
		}
	}
}

type emulator struct {
	data    data.SeccompWorkingMemory
	filters []unix.SockFilter
	pointer uint32

	X uint32
	A uint32
	M [16]uint32
}

func bpfClass(code uint16) uint16 {
	return code & 0x07
}

func bpfSize(code uint16) uint16 {
	return code & 0x18
}

func bpfMode(code uint16) uint16 {
	return code & 0xe0
}

func bpfOp(code uint16) uint16 {
	return code & 0xf0
}
func bpfMiscOp(code uint16) uint16 {
	return code & 0xf8
}

func bpfSrc(code uint16) uint16 {
	return code & 0x08
}

func (e *emulator) execRet(current unix.SockFilter) (uint32, bool) {
	switch bpfSrc(current.Code) {
	case syscall.BPF_K:
		return current.K, true
	case syscall.BPF_X:
		return e.X, true
	default:
		panic(fmt.Sprintf("Invalid ret source: %d", bpfSrc(current.Code)))
	}
	return 0, true
}

func (e *emulator) getFromWorkingMemory(ix uint32) uint32 {
	switch ix {
	case 0:
		return uint32(e.data.NR)
	case 4:
		return e.data.Arch
	case 8:
		return uint32(e.data.InstructionPointer >> 32)
	case 12:
		return uint32(e.data.InstructionPointer & 0xFFFF)
	case 16:
		return uint32(e.data.Args[0] >> 32)
	case 20:
		return uint32(e.data.Args[0] & 0xFFFFFFFF)
	case 24:
		return uint32(e.data.Args[1] >> 32)
	case 28:
		return uint32(e.data.Args[1] & 0xFFFFFFFF)
	case 32:
		return uint32(e.data.Args[2] >> 32)
	case 36:
		return uint32(e.data.Args[2] & 0xFFFFFFFF)
	case 40:
		return uint32(e.data.Args[3] >> 32)
	case 44:
		return uint32(e.data.Args[3] & 0xFFFFFFFF)
	case 48:
		return uint32(e.data.Args[4] >> 32)
	case 52:
		return uint32(e.data.Args[4] & 0xFFFFFFFF)
	case 56:
		return uint32(e.data.Args[5] >> 32)
	case 60:
		return uint32(e.data.Args[5] & 0xFFFFFFFF)
	default:
		return 0
	}
}

func (e *emulator) loadFromWorkingMemory(ix uint32) {
	e.A = e.getFromWorkingMemory(ix)
}

func (e *emulator) execLd(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	if bpfSize(cd) != syscall.BPF_W {
		panic("Invalid code, we can't load smaller values than wide ones")
	}

	switch bpfMode(cd) {
	case syscall.BPF_ABS:
		e.loadFromWorkingMemory(current.K)
	case syscall.BPF_IND:
		e.loadFromWorkingMemory(e.X + current.K)
	case syscall.BPF_LEN:
		e.A = uint32(64)
	case syscall.BPF_IMM:
		e.A = current.K
	case syscall.BPF_MEM:
		e.A = e.M[current.K] // TODO handle out of index error?
	default:
		panic(fmt.Sprintf("Invalid mode: %d", bpfMode(cd)))
	}
	return 0, false
}

// add support for working mem

// load something into x
func (e *emulator) execLdx(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	if bpfSize(cd) != syscall.BPF_W {
		panic("Invalid code, we can't load smaller values than wide ones")
	}

	switch bpfMode(cd) {
	case syscall.BPF_LEN:
		e.X = uint32(64)
	case syscall.BPF_IMM:
		e.X = current.K
	case syscall.BPF_MEM:
		e.X = e.M[current.K]
	default:
		panic(fmt.Sprintf("Invalid mode: %d", bpfMode(cd)))
	}
	return 0, false
}

// BPF_MOD is BPF_MOD - it is supported in Linux from v3.7+, but not in go's syscall...
const BPF_MOD = 0x90

// BPF_XOR is BPF_XOR - it is supported in Linux from v3.7+, but not in go's syscall...
const BPF_XOR = 0xa0

func (e *emulator) execAlu(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	right := uint32(0)

	switch bpfSrc(cd) {
	case syscall.BPF_K:
		right = current.K
	case syscall.BPF_X:
		right = e.X
	default:
		panic(fmt.Sprintf("Invalid source for right hand side of operation: %d", bpfSrc(cd)))
	}

	switch bpfOp(cd) {
	case syscall.BPF_ADD:
		e.A += right
	case syscall.BPF_SUB:
		e.A -= right
	case syscall.BPF_MUL:
		e.A *= right
	case syscall.BPF_DIV:
		e.A /= right
	case syscall.BPF_AND:
		e.A &= right
	case syscall.BPF_OR:
		e.A |= right
	case BPF_XOR:
		e.A ^= right
	case syscall.BPF_LSH:
		e.A <<= right
	case syscall.BPF_RSH:
		e.A >>= right
	case BPF_MOD:
		e.A %= right
	case syscall.BPF_NEG:
		e.A = -e.A
	default:
		panic(fmt.Sprintf("Invalid op: %d", bpfOp(cd)))
	}
	return 0, false
}

func (e *emulator) execMisc(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	switch bpfMiscOp(cd) {
	case syscall.BPF_TAX:
		e.X = e.A
	case syscall.BPF_TXA:
		e.A = e.X
	default:
		panic(fmt.Sprintf("Invalid op: %d", bpfMiscOp(cd)))
	}
	return 0, false
}

func (e *emulator) execJmp(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	right := uint32(0)
	switch bpfSrc(cd) {
	case syscall.BPF_K:
		right = current.K
	case syscall.BPF_X:
		right = e.X
	default:
		panic(fmt.Sprintf("Invalid source for right hand side of operation: %d", bpfSrc(cd)))
	}

	switch bpfOp(cd) {
	case syscall.BPF_JA:
		e.pointer += current.K
	case syscall.BPF_JGT:
		if e.A > right {
			e.pointer += uint32(current.Jt)
		} else {
			e.pointer += uint32(current.Jf)
		}
	case syscall.BPF_JGE:
		if e.A >= right {
			e.pointer += uint32(current.Jt)
		} else {
			e.pointer += uint32(current.Jf)
		}
	case syscall.BPF_JEQ:
		if e.A == right {
			e.pointer += uint32(current.Jt)
		} else {
			e.pointer += uint32(current.Jf)
		}
	case syscall.BPF_JSET:
		if e.A&right != 0 {
			e.pointer += uint32(current.Jt)
		} else {
			e.pointer += uint32(current.Jf)
		}
	default:
		panic(fmt.Sprintf("Invalid op: %d", bpfOp(cd)))
	}
	return 0, false
}

func (e *emulator) execStore(current unix.SockFilter) (uint32, bool) {
	switch bpfClass(current.Code) {
	case syscall.BPF_ST:
		e.M[current.K] = e.A
		return 0, false
	case syscall.BPF_STX:
		e.M[current.K] = e.X
		return 0, false
	default:
		panic(fmt.Sprintf("Invalid op: %d", bpfClass(current.Code)))
	}
	return 0, false // not sure what the error case should be here
}

func (e *emulator) next() (uint32, bool) {
	if e.pointer >= uint32(len(e.filters)) {
		return 0, true
	}

	current := e.filters[e.pointer]
	e.pointer++
	switch bpfClass(current.Code) {
	case syscall.BPF_RET:
		return e.execRet(current)
	case syscall.BPF_LD:
		return e.execLd(current)
	case syscall.BPF_LDX:
		return e.execLdx(current)
	case syscall.BPF_ALU:
		return e.execAlu(current)
	case syscall.BPF_MISC:
		return e.execMisc(current)
	case syscall.BPF_JMP:
		return e.execJmp(current)
	case syscall.BPF_ST:
		return e.execStore(current)
	case syscall.BPF_STX:
		return e.execStore(current)
	}

	return 0, true
}
<commit_msg>adding out of index errors<commit_after>package emulator

import (
	"fmt"
	"log"
	"syscall"

	"github.com/twtiger/gosecco/data"

	"golang.org/x/sys/unix"
)

func init() {
	log.SetFlags(0)
}

// Emulate will execute a seccomp filter program against the given working memory.
func Emulate(d data.SeccompWorkingMemory, filters []unix.SockFilter) uint32 {
	e := &emulator{data: d, filters: filters, pointer: 0}
	for {
		val, finished := e.next()
		if finished {
			return val
		}
	}
}

const MEMWORDS = 16

type emulator struct {
	data    data.SeccompWorkingMemory
	filters []unix.SockFilter
	pointer uint32

	X uint32
	A uint32
	M [MEMWORDS]uint32
}

func bpfClass(code uint16) uint16 {
	return code & 0x07
}

func bpfSize(code uint16) uint16 {
	return code & 0x18
}

func bpfMode(code uint16) uint16 {
	return code & 0xe0
}

func bpfOp(code uint16) uint16 {
	return code & 0xf0
}
func bpfMiscOp(code uint16) uint16 {
	return code & 0xf8
}

func bpfSrc(code uint16) uint16 {
	return code & 0x08
}

func (e *emulator) execRet(current unix.SockFilter) (uint32, bool) {
	switch bpfSrc(current.Code) {
	case syscall.BPF_K:
		return current.K, true
	case syscall.BPF_X:
		return e.X, true
	default:
		panic(fmt.Sprintf("Invalid ret source: %d", bpfSrc(current.Code)))
	}
	return 0, true
}

func (e *emulator) getFromWorkingMemory(ix uint32) uint32 {
	switch ix {
	case 0:
		return uint32(e.data.NR)
	case 4:
		return e.data.Arch
	case 8:
		return uint32(e.data.InstructionPointer >> 32)
	case 12:
		return uint32(e.data.InstructionPointer & 0xFFFF)
	case 16:
		return uint32(e.data.Args[0] >> 32)
	case 20:
		return uint32(e.data.Args[0] & 0xFFFFFFFF)
	case 24:
		return uint32(e.data.Args[1] >> 32)
	case 28:
		return uint32(e.data.Args[1] & 0xFFFFFFFF)
	case 32:
		return uint32(e.data.Args[2] >> 32)
	case 36:
		return uint32(e.data.Args[2] & 0xFFFFFFFF)
	case 40:
		return uint32(e.data.Args[3] >> 32)
	case 44:
		return uint32(e.data.Args[3] & 0xFFFFFFFF)
	case 48:
		return uint32(e.data.Args[4] >> 32)
	case 52:
		return uint32(e.data.Args[4] & 0xFFFFFFFF)
	case 56:
		return uint32(e.data.Args[5] >> 32)
	case 60:
		return uint32(e.data.Args[5] & 0xFFFFFFFF)
	default:
		return 0
	}
}

func (e *emulator) loadFromWorkingMemory(ix uint32) {
	e.A = e.getFromWorkingMemory(ix)
}

func (e *emulator) execLd(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	if bpfSize(cd) != syscall.BPF_W {
		panic("Invalid code, we can't load smaller values than wide ones")
	}

	switch bpfMode(cd) {
	case syscall.BPF_ABS:
		e.loadFromWorkingMemory(current.K)
	case syscall.BPF_IND:
		e.loadFromWorkingMemory(e.X + current.K)
	case syscall.BPF_LEN:
		e.A = uint32(64)
	case syscall.BPF_IMM:
		e.A = current.K
	case syscall.BPF_MEM:
		if current.K < MEMWORDS {
			e.A = e.M[current.K]
		} else {
			panic(fmt.Sprintf("Index out of range: %d greater than MEMWORDS %d", current.K, MEMWORDS))
		}
	default:
		panic(fmt.Sprintf("Invalid mode: %d", bpfMode(cd)))
	}
	return 0, false
}

func (e *emulator) execLdx(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	if bpfSize(cd) != syscall.BPF_W {
		panic("Invalid code, we can't load smaller values than wide ones")
	}

	switch bpfMode(cd) {
	case syscall.BPF_LEN:
		e.X = uint32(64)
	case syscall.BPF_IMM:
		e.X = current.K
	case syscall.BPF_MEM:
		if current.K < MEMWORDS {
			e.X = e.M[current.K]
		} else {
			panic(fmt.Sprintf("Index out of range: %d greater than MEMWORDS", current.K, MEMWORDS))
		}
	default:
		panic(fmt.Sprintf("Invalid mode: %d", bpfMode(cd)))
	}
	return 0, false
}

// BPF_MOD is BPF_MOD - it is supported in Linux from v3.7+, but not in go's syscall...
const BPF_MOD = 0x90

// BPF_XOR is BPF_XOR - it is supported in Linux from v3.7+, but not in go's syscall...
const BPF_XOR = 0xa0

func (e *emulator) execAlu(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	right := uint32(0)

	switch bpfSrc(cd) {
	case syscall.BPF_K:
		right = current.K
	case syscall.BPF_X:
		right = e.X
	default:
		panic(fmt.Sprintf("Invalid source for right hand side of operation: %d", bpfSrc(cd)))
	}

	switch bpfOp(cd) {
	case syscall.BPF_ADD:
		e.A += right
	case syscall.BPF_SUB:
		e.A -= right
	case syscall.BPF_MUL:
		e.A *= right
	case syscall.BPF_DIV:
		e.A /= right
	case syscall.BPF_AND:
		e.A &= right
	case syscall.BPF_OR:
		e.A |= right
	case BPF_XOR:
		e.A ^= right
	case syscall.BPF_LSH:
		e.A <<= right
	case syscall.BPF_RSH:
		e.A >>= right
	case BPF_MOD:
		e.A %= right
	case syscall.BPF_NEG:
		e.A = -e.A
	default:
		panic(fmt.Sprintf("Invalid op: %d", bpfOp(cd)))
	}
	return 0, false
}

func (e *emulator) execMisc(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	switch bpfMiscOp(cd) {
	case syscall.BPF_TAX:
		e.X = e.A
	case syscall.BPF_TXA:
		e.A = e.X
	default:
		panic(fmt.Sprintf("Invalid op: %d", bpfMiscOp(cd)))
	}
	return 0, false
}

func (e *emulator) execJmp(current unix.SockFilter) (uint32, bool) {
	cd := current.Code

	right := uint32(0)
	switch bpfSrc(cd) {
	case syscall.BPF_K:
		right = current.K
	case syscall.BPF_X:
		right = e.X
	default:
		panic(fmt.Sprintf("Invalid source for right hand side of operation: %d", bpfSrc(cd)))
	}

	switch bpfOp(cd) {
	case syscall.BPF_JA:
		e.pointer += current.K
	case syscall.BPF_JGT:
		if e.A > right {
			e.pointer += uint32(current.Jt)
		} else {
			e.pointer += uint32(current.Jf)
		}
	case syscall.BPF_JGE:
		if e.A >= right {
			e.pointer += uint32(current.Jt)
		} else {
			e.pointer += uint32(current.Jf)
		}
	case syscall.BPF_JEQ:
		if e.A == right {
			e.pointer += uint32(current.Jt)
		} else {
			e.pointer += uint32(current.Jf)
		}
	case syscall.BPF_JSET:
		if e.A&right != 0 {
			e.pointer += uint32(current.Jt)
		} else {
			e.pointer += uint32(current.Jf)
		}
	default:
		panic(fmt.Sprintf("Invalid op: %d", bpfOp(cd)))
	}
	return 0, false
}

func (e *emulator) execStore(current unix.SockFilter) (uint32, bool) {
	switch bpfClass(current.Code) {
	case syscall.BPF_ST:
		e.M[current.K] = e.A
		return 0, false
	case syscall.BPF_STX:
		e.M[current.K] = e.X
		return 0, false
	default:
		panic(fmt.Sprintf("Invalid op: %d", bpfClass(current.Code)))
	}
	return 0, false
}

func (e *emulator) next() (uint32, bool) {
	if e.pointer >= uint32(len(e.filters)) {
		return 0, true
	}

	current := e.filters[e.pointer]
	e.pointer++
	switch bpfClass(current.Code) {
	case syscall.BPF_RET:
		return e.execRet(current)
	case syscall.BPF_LD:
		return e.execLd(current)
	case syscall.BPF_LDX:
		return e.execLdx(current)
	case syscall.BPF_ALU:
		return e.execAlu(current)
	case syscall.BPF_MISC:
		return e.execMisc(current)
	case syscall.BPF_JMP:
		return e.execJmp(current)
	case syscall.BPF_ST:
		return e.execStore(current)
	case syscall.BPF_STX:
		return e.execStore(current)
	}

	return 0, true
}
<|endoftext|>


<commit_before>// srclib-docco is a docco-like static documentation generator.
// TODO: write this in a literal style.
package srclib_docco

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"text/template"

	"github.com/jessevdk/go-flags"
	"github.com/sourcegraph/annotate"
	"github.com/sourcegraph/syntaxhighlight"
)

var CLI = flags.NewNamedParser("src-docco", flags.Default)

var GlobalOpt struct {
	Verbose bool `short:"v" description:"show verbose output"`
}

var vLogger = log.New(os.Stderr, "", 0)

func vLogf(format string, v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Printf(format, v...)
}

func vLog(v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Println(v...)
}

func init() {
	CLI.LongDescription = "TODO"
	CLI.AddGroup("Global options", "", &GlobalOpt)

	_, err := CLI.AddCommand("gen",
		"generate documentation",
		"Generate docco-like documentation for a thing",
		&genCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO: add limitations help output.

type GenCmd struct {
	Dir string `long:"dir" description:"The root directory for the project"`
	// TODO: must be relative to Dir.
	SiteDirName string `long:"site-dir" description:"The directory name for the output files" default:"site"`
}

var genCmd GenCmd

// Source units have lots of information associated with them, but we
// only care about the files.
type unit struct {
	Files []string
}

type units []unit

func (us units) collateFiles() []string {
	var fs []string
	for _, u := range us {
		for _, f := range u.Files {
			fs = append(fs, f)
		}
	}
	return fs
}

type failedCmd struct {
	cmd interface{}
	err interface{}
}

// TODO: Is this correct?
func (f failedCmd) Error() string {
	return fmt.Sprintf("command %v failed: %s", f.cmd, f.err)
}

func ensureSrclibExists() error {
	cmd := exec.Command("src", "version")
	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf(
			"error with srclib: %v, %s, %s",
			err,
			stdout.String(),
			stderr.String(),
		)
	}
	return nil
}

func command(argv []string) (cmd *exec.Cmd, stdout *bytes.Buffer, stderr *bytes.Buffer) {
	if len(argv) == 0 {
		panic("command: argv must have at least one item")
	}
	cmd = exec.Command(argv[0], argv[1:]...)
	stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return cmd, stdout, stderr
}

func (c *GenCmd) Execute(args []string) error {
	if err := ensureSrclibExists(); err != nil {
		log.Fatal(err)
	}

	// First, we need to get a list of all of the files that we
	// want to generate.
	//
	// We could import sourcegraph.com/sourcegraph/srclib, but I
	// want to demonstrate how to use its command line interface.
	var dir string
	if c.Dir == "" {
		d, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	} else {
		d, err := filepath.Abs(c.Dir)
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	}
	argv := []string{"src", "api", "units", dir}
	cmd, stdout, stderr := command(argv)
	vLogf("Running %v", argv)
	// TODO: remove failedCmd
	if err := cmd.Run(); err != nil {
		log.Fatal(failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}})
	}
	if stdout.Len() == 0 {
		log.Fatal(failedCmd{argv, "no output"})
	}
	var us units
	if err := json.Unmarshal(stdout.Bytes(), &us); err != nil {
		log.Fatal(err)
	}
	return genSite(dir, c.SiteDirName, us.collateFiles())
}

type doc struct {
	Format string
	Data   string
	Start  uint32
	End    uint32
}

type docs []doc

func (d docs) Len() int      { return len(d) }
func (d docs) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d docs) Less(i, j int) bool {
	return d[i].Start < d[j].Start || (d[i].Start == d[j].Start && d[i].End < d[j].End)
}

var _ sort.Interface = docs{}

type ref struct {
	DefUnit string
	DefPath string
	Start   uint32
}

type refs []ref

func (r refs) Len() int           { return len(r) }
func (r refs) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
func (r refs) Less(i, j int) bool { return r[i].Start < r[j].Start }

var _ sort.Interface = refs{}

func genSite(root, siteName string, files []string) error {
	vLog("Generating Site")
	sitePath := filepath.Join(root, siteName)
	if err := os.MkdirAll(sitePath, 0755); err != nil {
		log.Fatal(err)
	}
	for _, f := range files {
		vLog("Processing", f)
		src, err := ioutil.ReadFile(filepath.Join(root, f))
		if err != nil {
			return err
		}
		argv := []string{"src", "api", "list", "--file", filepath.Join(root, f), "--no-defs"}
		cmd, stdout, stderr := command(argv)
		vLog("Running", argv)
		if err := cmd.Run(); err != nil {
			return failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}}
		}
		out := struct {
			Refs []ref
			Docs []doc
		}{}
		if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
			log.Fatal(err)
		}

		seenHTMLDoc := make(map[struct{ start, end uint32 }]struct{})
		var htmlDocs []doc
		for _, d := range out.Docs {
			if d.Format == "text/html" {
				if _, seen := seenHTMLDoc[struct{ start, end uint32 }{d.Start, d.End}]; !seen {
					htmlDocs = append(htmlDocs, d)
					seenHTMLDoc[struct{ start, end uint32 }{d.Start, d.End}] = struct{}{}
				}
			}
		}
		sort.Sort(docs(htmlDocs))
		htmlFile := "/" + f + ".html"
		sort.Sort(refs(out.Refs))
		anns, err := ann(src, out.Refs, htmlFile)
		if err != nil {
			return err
		}
		vLogf("Creating dir %s", filepath.Dir(filepath.Join(sitePath, htmlFile)))
		if err := os.MkdirAll(filepath.Dir(filepath.Join(sitePath, htmlFile)), 0755); err != nil {
			log.Fatal(err)
		}
		s, err := createSegments(src, anns, htmlDocs)
		if err != nil {
			return err
		}
		vLogf("Creating file %s", filepath.Join(sitePath, htmlFile))
		w, err := os.Create(filepath.Join(sitePath, htmlFile))
		if err != nil {
			return err
		}
		if err := codeTemplate.Execute(w, HTMLOutput{f, s}); err != nil {
			return err
		}
	}
	return nil
}

type HTMLOutput struct {
	Title    string
	Segments []segment
}

var codeTemplate = template.Must(template.New("code").Parse(codeText))

var codeText = `
<!DOCTYPE html>

    <title>{{.Title}}</title>
    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css">

      {{ range .Segments}}

{{.DocHTML}}

{{.CodeHTML}}

      {{ end }}

`

type annotation struct {
	annotate.Annotation
	Comment bool
}

type htmlAnnotator syntaxhighlight.HTMLConfig

func (c htmlAnnotator) class(kind syntaxhighlight.Kind) string {
	switch kind {
	case syntaxhighlight.String:
		return c.String
	case syntaxhighlight.Keyword:
		return c.Keyword
	case syntaxhighlight.Comment:
		return c.Comment
	case syntaxhighlight.Type:
		return c.Type
	case syntaxhighlight.Literal:
		return c.Literal
	case syntaxhighlight.Punctuation:
		return c.Punctuation
	case syntaxhighlight.Plaintext:
		return c.Plaintext
	case syntaxhighlight.Tag:
		return c.Tag
	case syntaxhighlight.HTMLTag:
		return c.HTMLTag
	case syntaxhighlight.HTMLAttrName:
		return c.HTMLAttrName
	case syntaxhighlight.HTMLAttrValue:
		return c.HTMLAttrValue
	case syntaxhighlight.Decimal:
		return c.Decimal
	}
	return ""
}

func (a htmlAnnotator) Annotate(start int, kind syntaxhighlight.Kind, tokText string) (*annotate.Annotation, error) {
	class := a.class(kind)
	if class == "" {
		return nil, nil
	}
	return &annotate.Annotation{
		Start: start,
		End:   start + len(tokText),
		Left:  []byte(class),
		Right: nil,
	}, nil
}

func ann(src []byte, refs []ref, htmlFile string) ([]annotation, error) {
	vLog("Annotating", htmlFile)
	annotations, err := syntaxhighlight.Annotate(src, htmlAnnotator(syntaxhighlight.DefaultHTMLConfig))
	if err != nil {
		return nil, err
	}
	sort.Sort(annotations)

	var refAtIndex int
	refAt := func(start uint32) (r ref, found bool) {
		for refAtIndex < len(refs) {
			if refs[refAtIndex].Start == start {
				defer func() { refAtIndex++ }()
				return refs[refAtIndex], true
			} else if refs[refAtIndex].Start < start {
				refAtIndex++
			} else { // refs[refAtIndex].Start > start
				return ref{}, false
			}
		}
		return ref{}, false
	}

	anns := make([]annotation, 0, len(annotations))
	for _, a := range annotations {
		r, found := refAt(uint32(a.Start))
		if !found {
			a.Left = []byte(fmt.Sprintf(`<span class="%s">`, string(a.Left)))
			a.Right = []byte(`</span>`)
			anns = append(anns, annotation{*a, false})
			continue
		}
		a.Left = []byte(fmt.Sprintf(
			`<span class="%s" id="%s"><a href="%s">`,
			string(a.Left),
			filepath.Join(r.DefUnit, r.DefPath),
			htmlFile,
		))
		a.Right = []byte(`</span></a>`)
		anns = append(anns, annotation{*a, false})
	}

	return anns, nil
}

type segment struct {
	DocHTML  string
	CodeHTML string
}

// anns and docs must be sorted.
func createSegments(src []byte, anns []annotation, docs []doc) ([]segment, error) {
	vLog("Creating segments")
	var segments []segment
	var s segment
	for i := 0; i < len(src); {
		for len(docs) != 0 && docs[0].Start == uint32(i) {
			// Add doc
			s.DocHTML = docs[0].Data
			i = int(docs[0].End)
			docs = docs[1:]
		}
		var runTo int
		if len(docs) != 0 {
			runTo = int(docs[0].Start)
		} else {
			runTo = len(src)
		}
		for len(anns) != 0 && i > anns[0].Start {
			anns = anns[1:]
		}
		for src[i] == '\n' {
			i++
		}
		for i < runTo {
			if len(anns) == 0 {
				s.CodeHTML += template.HTMLEscapeString(string(src[i:runTo]))
				i = runTo
				break
			}
			a := anns[0]
			if i  runTo {
				log.Fatal("createSegment: illegal state: a.End > runTo")
			}
			s.CodeHTML += string(a.Left) +
				template.HTMLEscapeString(string(src[a.Start:a.End])) +
				string(a.Right)
			i = a.End
			anns = anns[1:]
		}
		segments = append(segments, s)
		s = segment{}
	}
	return segments, nil
}

func Main() error {
	log.SetFlags(log.Lshortfile)
	_, err := CLI.Parse()
	return err
}
<commit_msg>add filemap (wip)<commit_after>// srclib-docco is a docco-like static documentation generator.
// TODO: write this in a literal style.
package srclib_docco

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"text/template"

	"github.com/jessevdk/go-flags"
	"github.com/sourcegraph/annotate"
	"github.com/sourcegraph/syntaxhighlight"
)

var CLI = flags.NewNamedParser("src-docco", flags.Default)

var GlobalOpt struct {
	Verbose bool `short:"v" description:"show verbose output"`
}

var vLogger = log.New(os.Stderr, "", 0)

func vLogf(format string, v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Printf(format, v...)
}

func vLog(v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Println(v...)
}

func init() {
	CLI.LongDescription = "TODO"
	CLI.AddGroup("Global options", "", &GlobalOpt)

	_, err := CLI.AddCommand("gen",
		"generate documentation",
		"Generate docco-like documentation for a thing",
		&genCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO: add limitations help output.

type GenCmd struct {
	Dir string `long:"dir" description:"The root directory for the project"`
	// TODO: must be relative to Dir.
	SiteDirName string `long:"site-dir" description:"The directory name for the output files" default:"site"`
}

var genCmd GenCmd

// Source units have lots of information associated with them, but we
// only care about the files.
type unit struct {
	Files []string
}

type units []unit

func (us units) collateFiles() []string {
	var fs []string
	for _, u := range us {
		for _, f := range u.Files {
			fs = append(fs, f)
		}
	}
	return fs
}

type failedCmd struct {
	cmd interface{}
	err interface{}
}

// TODO: Is this correct?
func (f failedCmd) Error() string {
	return fmt.Sprintf("command %v failed: %s", f.cmd, f.err)
}

func ensureSrclibExists() error {
	cmd := exec.Command("src", "version")
	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf(
			"error with srclib: %v, %s, %s",
			err,
			stdout.String(),
			stderr.String(),
		)
	}
	return nil
}

func command(argv []string) (cmd *exec.Cmd, stdout *bytes.Buffer, stderr *bytes.Buffer) {
	if len(argv) == 0 {
		panic("command: argv must have at least one item")
	}
	cmd = exec.Command(argv[0], argv[1:]...)
	stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return cmd, stdout, stderr
}

func (c *GenCmd) Execute(args []string) error {
	if err := ensureSrclibExists(); err != nil {
		log.Fatal(err)
	}

	// First, we need to get a list of all of the files that we
	// want to generate.
	//
	// We could import sourcegraph.com/sourcegraph/srclib, but I
	// want to demonstrate how to use its command line interface.
	var dir string
	if c.Dir == "" {
		d, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	} else {
		d, err := filepath.Abs(c.Dir)
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	}
	argv := []string{"src", "api", "units", dir}
	cmd, stdout, stderr := command(argv)
	vLogf("Running %v", argv)
	if err := cmd.Run(); err != nil {
		log.Fatal(failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}})
	}
	if stdout.Len() == 0 {
		log.Fatal(failedCmd{argv, "no output"})
	}
	var us units
	if err := json.Unmarshal(stdout.Bytes(), &us); err != nil {
		log.Fatal(err)
	}
	return genSite(dir, c.SiteDirName, us.collateFiles())
}

type doc struct {
	Format string
	Data   string
	Start  uint32
	End    uint32
}

type docs []doc

func (d docs) Len() int      { return len(d) }
func (d docs) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d docs) Less(i, j int) bool {
	return d[i].Start < d[j].Start || (d[i].Start == d[j].Start && d[i].End < d[j].End)
}

var _ sort.Interface = docs{}

type ref struct {
	DefUnit string
	DefPath string
	File    string
	Start   uint32
}

type refs []ref

func (r refs) Len() int           { return len(r) }
func (r refs) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
func (r refs) Less(i, j int) bool { return r[i].Start < r[j].Start }

var _ sort.Interface = refs{}

func genSite(root, siteName string, files []string) error {
	vLog("Generating Site")
	sitePath := filepath.Join(root, siteName)
	if err := os.MkdirAll(sitePath, 0755); err != nil {
		log.Fatal(err)
	}
	filesMap := map[string]struct{}{}
	for _, f := range files {
		filesMap[f] = struct{}{}
	}
	for _, f := range files {
		vLog("Processing", f)
		src, err := ioutil.ReadFile(filepath.Join(root, f))
		if err != nil {
			return err
		}
		argv := []string{"src", "api", "list", "--file", filepath.Join(root, f), "--no-defs"}
		cmd, stdout, stderr := command(argv)
		vLog("Running", argv)
		if err := cmd.Run(); err != nil {
			return failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}}
		}
		out := struct {
			Refs []ref
			Docs []doc
		}{}
		if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
			log.Fatal(err)
		}

		seenHTMLDoc := make(map[struct{ start, end uint32 }]struct{})
		var htmlDocs []doc
		for _, d := range out.Docs {
			if d.Format == "text/html" {
				if _, seen := seenHTMLDoc[struct{ start, end uint32 }{d.Start, d.End}]; !seen {
					htmlDocs = append(htmlDocs, d)
					seenHTMLDoc[struct{ start, end uint32 }{d.Start, d.End}] = struct{}{}
				}
			}
		}
		sort.Sort(docs(htmlDocs))
		sort.Sort(refs(out.Refs))
		anns, err := ann(src, out.Refs, f, filesMap)
		if err != nil {
			return err
		}
		htmlFile := htmlFilename(f)
		vLogf("Creating dir %s", filepath.Dir(filepath.Join(sitePath, htmlFile)))
		if err := os.MkdirAll(filepath.Dir(filepath.Join(sitePath, htmlFile)), 0755); err != nil {
			log.Fatal(err)
		}
		s, err := createSegments(src, anns, htmlDocs)
		if err != nil {
			return err
		}
		vLogf("Creating file %s", filepath.Join(sitePath, htmlFile))
		w, err := os.Create(filepath.Join(sitePath, htmlFile))
		if err != nil {
			return err
		}
		if err := codeTemplate.Execute(w, HTMLOutput{f, s}); err != nil {
			return err
		}
	}
	return nil
}

type HTMLOutput struct {
	Title    string
	Segments []segment
}

var codeTemplate = template.Must(template.New("code").Parse(codeText))

var codeText = `
<!DOCTYPE html>

    <title>{{.Title}}</title>
    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css">

      {{ range .Segments}}

{{.DocHTML}}

{{.CodeHTML}}

      {{ end }}

`

type annotation struct {
	annotate.Annotation
	Comment bool
}

type htmlAnnotator syntaxhighlight.HTMLConfig

func (c htmlAnnotator) class(kind syntaxhighlight.Kind) string {
	switch kind {
	case syntaxhighlight.String:
		return c.String
	case syntaxhighlight.Keyword:
		return c.Keyword
	case syntaxhighlight.Comment:
		return c.Comment
	case syntaxhighlight.Type:
		return c.Type
	case syntaxhighlight.Literal:
		return c.Literal
	case syntaxhighlight.Punctuation:
		return c.Punctuation
	case syntaxhighlight.Plaintext:
		return c.Plaintext
	case syntaxhighlight.Tag:
		return c.Tag
	case syntaxhighlight.HTMLTag:
		return c.HTMLTag
	case syntaxhighlight.HTMLAttrName:
		return c.HTMLAttrName
	case syntaxhighlight.HTMLAttrValue:
		return c.HTMLAttrValue
	case syntaxhighlight.Decimal:
		return c.Decimal
	}
	return ""
}

func (a htmlAnnotator) Annotate(start int, kind syntaxhighlight.Kind, tokText string) (*annotate.Annotation, error) {
	class := a.class(kind)
	if class == "" {
		return nil, nil
	}
	return &annotate.Annotation{
		Start: start,
		End:   start + len(tokText),
		Left:  []byte(class),
		Right: nil,
	}, nil
}

func htmlFilename(filename string) string {
	return filepath.Join("/", filename+".html")
}

func ann(src []byte, refs []ref, filename string, filesMap map[string]struct{}) ([]annotation, error) {
	vLog("Annotating", filename)
	annotations, err := syntaxhighlight.Annotate(src, htmlAnnotator(syntaxhighlight.DefaultHTMLConfig))
	if err != nil {
		return nil, err
	}
	sort.Sort(annotations)

	var refAtIndex int
	refAt := func(start uint32) (r ref, found bool) {
		for refAtIndex < len(refs) {
			if refs[refAtIndex].Start == start {
				defer func() { refAtIndex++ }()
				return refs[refAtIndex], true
			} else if refs[refAtIndex].Start < start {
				refAtIndex++
			} else { // refs[refAtIndex].Start > start
				return ref{}, false
			}
		}
		return ref{}, false
	}

	anns := make([]annotation, 0, len(annotations))
	for _, a := range annotations {
		r, found := refAt(uint32(a.Start))
		if !found {
			a.Left = []byte(fmt.Sprintf(`<span class="%s">`, string(a.Left)))
			a.Right = []byte(`</span>`)
			anns = append(anns, annotation{*a, false})
			continue
		}
		id := filepath.Join(r.DefUnit, r.DefPath)
		var href string
		if _, in := filesMap[r.File]; in {
			href = htmlFilename(r.File) + "#" + id
			a.Left = []byte(fmt.Sprintf(
				`<span class="%s" id="%s"><a href="%s">`,
				string(a.Left),
				id,
				href,
			))
			a.Right = []byte(`</span></a>`)
		} else {
			a.Left = []byte(fmt.Sprintf(
				`<span class="%s" id="%s">`,
				string(a.Left),
				id,
			))
			a.Right = []byte(`</span>`)
		}
		anns = append(anns, annotation{*a, false})
	}

	return anns, nil
}

type segment struct {
	DocHTML  string
	CodeHTML string
}

// anns and docs must be sorted.
func createSegments(src []byte, anns []annotation, docs []doc) ([]segment, error) {
	vLog("Creating segments")
	var segments []segment
	var s segment
	for i := 0; i < len(src); {
		for len(docs) != 0 && docs[0].Start == uint32(i) {
			// Add doc
			s.DocHTML = docs[0].Data
			i = int(docs[0].End)
			docs = docs[1:]
		}
		var runTo int
		if len(docs) != 0 {
			runTo = int(docs[0].Start)
		} else {
			runTo = len(src)
		}
		for len(anns) != 0 && i > anns[0].Start {
			anns = anns[1:]
		}
		for src[i] == '\n' {
			i++
		}
		for i < runTo {
			if len(anns) == 0 {
				s.CodeHTML += template.HTMLEscapeString(string(src[i:runTo]))
				i = runTo
				break
			}
			a := anns[0]
			if i  runTo {
				log.Fatal("createSegment: illegal state: a.End > runTo")
			}
			s.CodeHTML += string(a.Left) +
				template.HTMLEscapeString(string(src[a.Start:a.End])) +
				string(a.Right)
			i = a.End
			anns = anns[1:]
		}
		segments = append(segments, s)
		s = segment{}
	}
	return segments, nil
}

func Main() error {
	log.SetFlags(log.Lshortfile)
	_, err := CLI.Parse()
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/slantview/doctl/commands"
)

const AppVersion = "0.0.9"

func init() {
	commands.APIKey = os.Getenv("DIGITAL_OCEAN_API_KEY")
}

func main() {
	app := cli.NewApp()
	app.Name = "doctl"
	app.Version = AppVersion
	app.Usage = "Digital Ocean Control TooL."
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "api-key,k", Value: "", Usage: "API Key for DO APIv2."},
		cli.StringFlag{Name: "format,f", Value: "yaml", Usage: "Format for output."},
		cli.BoolFlag{Name: "debug,d", Usage: "Turn on debug output."},
	}
	app.Before = func(ctx *cli.Context) error {
		if ctx.String("api-key") != "" {
			commands.APIKey = ctx.String("api-key")
		}

		switch ctx.String("format") {
		case "json":
			commands.OutputFormat = ctx.String("format")
		case "yaml":
			commands.OutputFormat = ctx.String("format")
		default:
			fmt.Printf("Invalid output format: %s. Available output options: json, yaml.\n", ctx.String("format"))
			os.Exit(64)
		}

		if commands.APIKey == "" {
			cli.ShowAppHelp(ctx)
			fmt.Println("Must provide API Key via DIGITAL_OCEAN_API_KEY environment variable or via CLI argument.")
			os.Exit(1)
		}

		return nil
	}
	app.Commands = []cli.Command{
		commands.ActionCommand,
		commands.DomainCommand,
		commands.DropletCommand,
		commands.RegionCommand,
		commands.SizeCommand,
		commands.SSHCommand,
	}

	app.Run(os.Args)
}
<commit_msg>update version<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/slantview/doctl/commands"
)

const AppVersion = "0.0.10"

func init() {
	commands.APIKey = os.Getenv("DIGITAL_OCEAN_API_KEY")
}

func main() {
	app := cli.NewApp()
	app.Name = "doctl"
	app.Version = AppVersion
	app.Usage = "Digital Ocean Control TooL."
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "api-key,k", Value: "", Usage: "API Key for DO APIv2."},
		cli.StringFlag{Name: "format,f", Value: "yaml", Usage: "Format for output."},
		cli.BoolFlag{Name: "debug,d", Usage: "Turn on debug output."},
	}
	app.Before = func(ctx *cli.Context) error {
		if ctx.String("api-key") != "" {
			commands.APIKey = ctx.String("api-key")
		}

		switch ctx.String("format") {
		case "json":
			commands.OutputFormat = ctx.String("format")
		case "yaml":
			commands.OutputFormat = ctx.String("format")
		default:
			fmt.Printf("Invalid output format: %s. Available output options: json, yaml.\n", ctx.String("format"))
			os.Exit(64)
		}

		if commands.APIKey == "" {
			cli.ShowAppHelp(ctx)
			fmt.Println("Must provide API Key via DIGITAL_OCEAN_API_KEY environment variable or via CLI argument.")
			os.Exit(1)
		}

		return nil
	}
	app.Commands = []cli.Command{
		commands.ActionCommand,
		commands.DomainCommand,
		commands.DropletCommand,
		commands.RegionCommand,
		commands.SizeCommand,
		commands.SSHCommand,
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package logging

import (
	"encoding/json"
	"io"
	"os"
	"time"

	"github.com/gin-gonic/gin"
)

// recoverLoggingFailure is a recover when failed to logging
var recoverLoggingFailure func()

// SetRecoverLoggingFailure is a set recoverLoggingFailure
func SetRecoverLoggingFailure(f func()) {
	recoverLoggingFailure = f
}

// AccessLogger is a middleware for logging access info
func AccessLogger(out io.Writer) gin.HandlerFunc {

	if out == nil {
		out = os.Stdout
	}

	return func(c *gin.Context) {

		if recoverLoggingFailure != nil {
			defer recoverLoggingFailure()
		}

		start := time.Now()

		c.Next()

		al := AccessLog{
			LogInfo: GenerateLogInfo(c, start),
		}

		if err := c.LastError(); err != nil {
			al.Error = err
		}

		bytes, err := json.Marshal(al)
		if err != nil {
			panic(err)
		}

		out.Write(append(bytes, 10))
	}
}

// ActivityLogger is a middleware for logging user action info
func ActivityLogger(out io.Writer, getExtra func(c *gin.Context) (interface{}, error)) gin.HandlerFunc {

	if out == nil {
		out = os.Stdout
	}

	return func(c *gin.Context) {

		if recoverLoggingFailure != nil {
			defer recoverLoggingFailure()
		}

		// check a request method
		if c.Request.Method == "GET" {
			return
		}

		start := time.Now()
		b, err := ConvertToMapFromBody(c)
		if err != nil {
			panic(err)
		}

		c.Next()

		// check a response status
		if c.Writer.Status() < 200 || c.Writer.Status() > 299 {
			return
		}

		al := ActivityLog{
			LogInfo:     GenerateLogInfo(c, start),
			RequestBody: b,
		}

		// get to Extra
		if getExtra != nil {
			al.Extra, err = getExtra(c)
			if err != nil {
				panic(err)
			}
		}

		bytes, err := json.Marshal(al)
		if err != nil {
			panic(err)
		}

		out.Write(append(bytes, 10))
	}
}
<commit_msg>Bugfix<commit_after>package logging

import (
	"encoding/json"
	"io"
	"os"
	"time"

	"github.com/gin-gonic/gin"
)

// recoverLoggingFailure is a recover when failed to logging
var recoverLoggingFailure func()

// SetRecoverLoggingFailure is a set recoverLoggingFailure
func SetRecoverLoggingFailure(f func()) {
	recoverLoggingFailure = f
}

// AccessLogger is a middleware for logging access info
func AccessLogger(out io.Writer) gin.HandlerFunc {

	if out == nil {
		out = os.Stdout
	}

	return func(c *gin.Context) {

		start := time.Now()

		c.Next()

		if recoverLoggingFailure != nil {
			defer recoverLoggingFailure()
		}

		al := AccessLog{
			LogInfo: GenerateLogInfo(c, start),
		}

		if err := c.LastError(); err != nil {
			al.Error = err
		}

		bytes, err := json.Marshal(al)
		if err != nil {
			panic(err)
		}

		out.Write(append(bytes, 10))
	}
}

// ActivityLogger is a middleware for logging user action info
func ActivityLogger(out io.Writer, getExtra func(c *gin.Context) (interface{}, error)) gin.HandlerFunc {

	if out == nil {
		out = os.Stdout
	}

	return func(c *gin.Context) {

		// check a request method
		if c.Request.Method == "GET" {
			return
		}

		start := time.Now()
		b, err := ConvertToMapFromBody(c)
		if err != nil {
			panic(err)
		}

		c.Next()

		if recoverLoggingFailure != nil {
			defer recoverLoggingFailure()
		}

		// check a response status
		if c.Writer.Status() < 200 || c.Writer.Status() > 299 {
			return
		}

		al := ActivityLog{
			LogInfo:     GenerateLogInfo(c, start),
			RequestBody: b,
		}

		// get to Extra
		if getExtra != nil {
			al.Extra, err = getExtra(c)
			if err != nil {
				panic(err)
			}
		}

		bytes, err := json.Marshal(al)
		if err != nil {
			panic(err)
		}

		out.Write(append(bytes, 10))
	}
}
<|endoftext|>


<commit_before>package driver

import (
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/baggageclaim/volume/copy"
)

var mountOpts string

func init() {
	metacopyEnabled()
}

// Metacopy is an overlayfs feature. If all you're doing is chown/chmod'ing a
// file then it will not create a copy of the file. Files will only be copied
// when they are written to.
func metacopyEnabled() {
	if _, err := os.Stat("/sys/module/overlay/parameters/metacopy"); os.IsNotExist(err) {
		mountOpts = "lowerdir=%s,upperdir=%s,workdir=%s"
		return
	}

	mountOpts = "lowerdir=%s,upperdir=%s,workdir=%s,metacopy=on"
}

type OverlayDriver struct {
	OverlaysDir string
}

func NewOverlayDriver(overlaysDir string) volume.Driver {
	return &OverlayDriver{
		OverlaysDir: overlaysDir,
	}
}

func (driver *OverlayDriver) CreateVolume(vol volume.FilesystemInitVolume) error {
	path := vol.DataPath()
	err := os.Mkdir(path, 0755)
	if err != nil {
		return err
	}

	return driver.bindMount(vol)
}

func (driver *OverlayDriver) DestroyVolume(vol volume.FilesystemVolume) error {
	path := vol.DataPath()

	err := syscall.Unmount(path, 0)
	// when a path is already unmounted, and unmount is called
	// on it, syscall.EINVAL is returned as an error
	// ignore this error and continue to clean up
	if err != nil && err != os.ErrInvalid {
		return err
	}

	err = os.RemoveAll(driver.workDir(vol))
	if err != nil {
		return err
	}

	err = os.RemoveAll(driver.layerDir(vol))
	if err != nil {
		return err
	}

	return os.RemoveAll(path)
}

func (driver *OverlayDriver) CreateCopyOnWriteLayer(
	child volume.FilesystemInitVolume,
	parent volume.FilesystemLiveVolume,
) error {
	path := child.DataPath()
	err := os.MkdirAll(path, 0755)
	if err != nil {
		return err
	}

	rootParent, err := driver.findRootParent(child, parent)
	if err != nil{
		return err
	}

	return driver.overlayMount(child, rootParent)
}

func (driver *OverlayDriver) Recover(fs volume.Filesystem) error {
	vols, err := fs.ListVolumes()
	if err != nil {
		return err
	}

	type cow struct {
		parent volume.FilesystemLiveVolume
		child  volume.FilesystemLiveVolume
	}

	cows := []cow{}
	for _, vol := range vols {
		parentVol, hasParent, err := vol.Parent()
		if err != nil {
			return fmt.Errorf("get parent: %w", err)
		}

		if hasParent {
			cows = append(cows, cow{
				parent: parentVol,
				child:  vol,
			})
			continue
		}

		err = driver.bindMount(vol)
		if err != nil {
			return fmt.Errorf("recover bind mount: %w", err)
		}
	}

	for _, cow := range cows {
		rootParent, err := driver.findRootParent(cow.child, cow.parent)
		if err != nil{
			return err
		}

		err = driver.overlayMount(cow.child, rootParent)
		if err != nil {
			return fmt.Errorf("recover overlay mount: %w", err)
		}
	}

	return nil
}

func (driver *OverlayDriver) findRootParent(child volume.FilesystemVolume,
	parent volume.FilesystemLiveVolume) (volume.FilesystemLiveVolume, error) {
	rootParent := parent
	grandparent, hasGrandparent, err := parent.Parent()
	if err != nil {
		return nil, err
	}

	if hasGrandparent {
		childDir := driver.layerDir(child)
		parentDir := driver.layerDir(parent)
		err := copy.Cp(false, parentDir, childDir)
		if err != nil {
			return nil, fmt.Errorf("copy parent data to child: %w", err)
		}

		rootParent = grandparent

		// resolve to root volume
		for {
			grandparent, hasGrandparent, err := rootParent.Parent()
			if err != nil {
				return nil, err
			}

			if !hasGrandparent {
				break
			}

			rootParent = grandparent
		}
	}

	return rootParent, nil
}

func (driver *OverlayDriver) bindMount(vol volume.FilesystemVolume) error {
	layerDir := driver.layerDir(vol)
	err := os.MkdirAll(layerDir, 0755)
	if err != nil {
		return err
	}

	err = syscall.Mount(layerDir, vol.DataPath(), "", syscall.MS_BIND, "")
	if err != nil {
		return err
	}

	return nil
}

func (driver *OverlayDriver) overlayMount(child volume.FilesystemVolume, parent volume.FilesystemLiveVolume) error {
	childDir := driver.layerDir(child)
	err := os.MkdirAll(childDir, 0755)
	if err != nil {
		return err
	}

	workDir := driver.workDir(child)
	err = os.MkdirAll(workDir, 0755)
	if err != nil {
		return err
	}

	opts := fmt.Sprintf(
		mountOpts,
		parent.DataPath(), //lowerdir
		childDir,          //upperdir
		workDir,           //workdir
	)

	err = syscall.Mount("overlay", child.DataPath(), "overlay", 0, opts)
	if err != nil {
		return err
	}

	return nil
}

func (driver *OverlayDriver) layerDir(vol volume.FilesystemVolume) string {
	return filepath.Join(driver.OverlaysDir, vol.Handle())
}

func (driver *OverlayDriver) workDir(vol volume.FilesystemVolume) string {
	return filepath.Join(driver.OverlaysDir, "work", vol.Handle())
}
<commit_msg>Rename metacopyEnabled -> metacopySupported<commit_after>package driver

import (
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/baggageclaim/volume/copy"
)

var mountOpts string

func init() {
	if metacopySupported() {
		mountOpts = "lowerdir=%s,upperdir=%s,workdir=%s,metacopy=on"
	} else {
		mountOpts = "lowerdir=%s,upperdir=%s,workdir=%s"
	}
}

// Metacopy is an overlayfs feature. If all you're doing is chown/chmod'ing a
// file then it will not create a copy of the file. Files will only be copied
// when they are written to.
func metacopySupported() bool {
	_, err := os.Stat("/sys/module/overlay/parameters/metacopy")
	return !os.IsNotExist(err)
}

type OverlayDriver struct {
	OverlaysDir string
}

func NewOverlayDriver(overlaysDir string) volume.Driver {
	return &OverlayDriver{
		OverlaysDir: overlaysDir,
	}
}

func (driver *OverlayDriver) CreateVolume(vol volume.FilesystemInitVolume) error {
	path := vol.DataPath()
	err := os.Mkdir(path, 0755)
	if err != nil {
		return err
	}

	return driver.bindMount(vol)
}

func (driver *OverlayDriver) DestroyVolume(vol volume.FilesystemVolume) error {
	path := vol.DataPath()

	err := syscall.Unmount(path, 0)
	// when a path is already unmounted, and unmount is called
	// on it, syscall.EINVAL is returned as an error
	// ignore this error and continue to clean up
	if err != nil && err != os.ErrInvalid {
		return err
	}

	err = os.RemoveAll(driver.workDir(vol))
	if err != nil {
		return err
	}

	err = os.RemoveAll(driver.layerDir(vol))
	if err != nil {
		return err
	}

	return os.RemoveAll(path)
}

func (driver *OverlayDriver) CreateCopyOnWriteLayer(
	child volume.FilesystemInitVolume,
	parent volume.FilesystemLiveVolume,
) error {
	path := child.DataPath()
	err := os.MkdirAll(path, 0755)
	if err != nil {
		return err
	}

	rootParent, err := driver.findRootParent(child, parent)
	if err != nil {
		return err
	}

	return driver.overlayMount(child, rootParent)
}

func (driver *OverlayDriver) Recover(fs volume.Filesystem) error {
	vols, err := fs.ListVolumes()
	if err != nil {
		return err
	}

	type cow struct {
		parent volume.FilesystemLiveVolume
		child  volume.FilesystemLiveVolume
	}

	cows := []cow{}
	for _, vol := range vols {
		parentVol, hasParent, err := vol.Parent()
		if err != nil {
			return fmt.Errorf("get parent: %w", err)
		}

		if hasParent {
			cows = append(cows, cow{
				parent: parentVol,
				child:  vol,
			})
			continue
		}

		err = driver.bindMount(vol)
		if err != nil {
			return fmt.Errorf("recover bind mount: %w", err)
		}
	}

	for _, cow := range cows {
		rootParent, err := driver.findRootParent(cow.child, cow.parent)
		if err != nil {
			return err
		}

		err = driver.overlayMount(cow.child, rootParent)
		if err != nil {
			return fmt.Errorf("recover overlay mount: %w", err)
		}
	}

	return nil
}

func (driver *OverlayDriver) findRootParent(child volume.FilesystemVolume,
	parent volume.FilesystemLiveVolume) (volume.FilesystemLiveVolume, error) {
	rootParent := parent
	grandparent, hasGrandparent, err := parent.Parent()
	if err != nil {
		return nil, err
	}

	if hasGrandparent {
		childDir := driver.layerDir(child)
		parentDir := driver.layerDir(parent)
		err := copy.Cp(false, parentDir, childDir)
		if err != nil {
			return nil, fmt.Errorf("copy parent data to child: %w", err)
		}

		rootParent = grandparent

		// resolve to root volume
		for {
			grandparent, hasGrandparent, err := rootParent.Parent()
			if err != nil {
				return nil, err
			}

			if !hasGrandparent {
				break
			}

			rootParent = grandparent
		}
	}

	return rootParent, nil
}

func (driver *OverlayDriver) bindMount(vol volume.FilesystemVolume) error {
	layerDir := driver.layerDir(vol)
	err := os.MkdirAll(layerDir, 0755)
	if err != nil {
		return err
	}

	err = syscall.Mount(layerDir, vol.DataPath(), "", syscall.MS_BIND, "")
	if err != nil {
		return err
	}

	return nil
}

func (driver *OverlayDriver) overlayMount(child volume.FilesystemVolume, parent volume.FilesystemLiveVolume) error {
	childDir := driver.layerDir(child)
	err := os.MkdirAll(childDir, 0755)
	if err != nil {
		return err
	}

	workDir := driver.workDir(child)
	err = os.MkdirAll(workDir, 0755)
	if err != nil {
		return err
	}

	opts := fmt.Sprintf(
		mountOpts,
		parent.DataPath(), //lowerdir
		childDir,          //upperdir
		workDir,           //workdir
	)

	err = syscall.Mount("overlay", child.DataPath(), "overlay", 0, opts)
	if err != nil {
		return err
	}

	return nil
}

func (driver *OverlayDriver) layerDir(vol volume.FilesystemVolume) string {
	return filepath.Join(driver.OverlaysDir, vol.Handle())
}

func (driver *OverlayDriver) workDir(vol volume.FilesystemVolume) string {
	return filepath.Join(driver.OverlaysDir, "work", vol.Handle())
}
<|endoftext|>


<commit_before>package dos

import (
	"golang.org/x/sys/windows"
)

const enableVirtualTerminalProcessing uint32 = 0x0004

func doEnableVirtualTerminalProcessing(console windows.Handle) (func(), error) {
	var mode uint32
	err := windows.GetConsoleMode(console, &mode)
	if err != nil {
		return func() {}, err
	}
	deferFunc := func() { windows.SetConsoleMode(console, mode) }
	err = windows.SetConsoleMode(console, mode|enableVirtualTerminalProcessing)
	return deferFunc, err
}

// EnableStdoutVirtualTerminalProcessing enables Windows10's native ESCAPE SEQUENCE support on STDOUT
func EnableStdoutVirtualTerminalProcessing() (func(), error) {
	return doEnableVirtualTerminalProcessing(windows.Stdout)
}

// EnableStderrVirtualTerminalProcessing enables Windows10's native ESCAPE SEQUENCE support on STDERR
func EnableStderrVirtualTerminalProcessing() (func(), error) {
	return doEnableVirtualTerminalProcessing(windows.Stderr)
}
<commit_msg>( Add dos.ChangeConsoleMode )<commit_after>package dos

import (
	"golang.org/x/sys/windows"
)

type ModeOp interface {
	Op(mode uint32) uint32
}

type ModeReset uint32

func (this ModeReset) Op(mode uint32) uint32 {
	return mode &^ uint32(this)
}

type ModeSet uint32

func (this ModeSet) Op(mode uint32) uint32 {
	return mode | uint32(this)
}

func ChangeConsoleMode(console windows.Handle, ops ...ModeOp) (func(), error) {
	var mode uint32
	err := windows.GetConsoleMode(console, &mode)
	if err != nil {
		return func() {}, err
	}
	restore := func() { windows.SetConsoleMode(console, mode) }

	newMode := mode
	for _, op1 := range ops {
		newMode = op1.Op(newMode)
	}
	err = windows.SetConsoleMode(console, newMode)
	return restore, err
}

const enableVirtualTerminalProcessing uint32 = 0x0004

// EnableStdoutVirtualTerminalProcessing enables Windows10's native ESCAPE SEQUENCE support on STDOUT
func EnableStdoutVirtualTerminalProcessing() (func(), error) {
	return ChangeConsoleMode(windows.Stdout, ModeSet(enableVirtualTerminalProcessing))
}

// EnableStderrVirtualTerminalProcessing enables Windows10's native ESCAPE SEQUENCE support on STDERR
func EnableStderrVirtualTerminalProcessing() (func(), error) {
	return ChangeConsoleMode(windows.Stderr, ModeSet(enableVirtualTerminalProcessing))
}
<|endoftext|>


<commit_before>package drain

import (
	"fmt"
	"net"
	"time"

  "github.com/DataDog/datadog-agent/pkg/logs/config"
	"github.com/DataDog/datadog-agent/pkg/logs/sender"

	"github.com/nanopack/logvac/core"
)

// Datadog drain implements the publisher interface for publishing logs to datadog.
type Datadog struct {
  ID        string // app ID or name
  Key       string // datadog api key
	manager   *sender.ConnectionManager
	conn      net.Conn
}

// NewDatadogClient creates a new mist publisher
func NewDatadogClient(id, key string) (*Datadog, error) {
  // emulate the server config
  config := config.NewServerConfig("intake.logs.datadoghq.com", 10514, true)

  // initialize a connection manager
	manager := sender.NewConnectionManager(config, "")

  // establish a connection
	conn := manager.NewConnection()

	return &Datadog{
    ID:       id,
    Key:      key,
    manager:  manager,
    conn:     conn,
  }, nil
}

// Init initializes a connection to mist
func (p Datadog) Init() error {

	// add drain
	logvac.AddDrain("datadog", p.Publish)

	return nil
}

// Publish utilizes mist's Publish to "drain" a log message
func (p *Datadog) Publish(msg logvac.Message) {
  // keep track of the attempts
	msg.PubTries++

  // re-establish the connection if it's been closed
	if p.conn == nil {
		p.conn = p.manager.NewConnection() // doesn't block (don't goroutine call to Publish)
	}

  // generate the payload for this entry
  payload := formatDataDogMessage(msg, p.ID, p.Key)

  // send the payload
	_, err := p.conn.Write(payload)
	if err != nil {
    // it's possible the connection is bad, so let's close it
		p.manager.CloseConnection(p.conn)
		p.conn = nil

    // let's try to send it again (at least 3 times)
		if msg.PubTries <= 3 {
      // give it a sec, networks are fickle
			time.Sleep(2 * time.Second)
      // retry!
			p.Publish(msg)
		}
	}
}

// Close closes the connection to datadog.
func (p *Datadog) Close() error {
	p.manager.CloseConnection(p.conn)
	return nil
}

// format the syslog message, prefixed with the datadog api key
func formatDataDogMessage(msg logvac.Message, id, key string) []byte {
  // format the date in the proper syslog format
  date := fmt.Sprintf("%s %02d %02d:%02d:%02d",
    msg.Time.Month().String()[:3],
    msg.Time.Day(),
    msg.Time.Hour(),
    msg.Time.Minute(),
    msg.Time.Second())

  // prefix the app id to the hostname identifier
  hostname := fmt.Sprintf("%s.%s", id, msg.Id)

  // extract the first tag as the tag (ie: nginx[access])
  tag := msg.Tag[0]

  // the final message
  message := fmt.Sprintf("%s <%d>%s %s %s: %s\n",
    key, msg.Priority, date, hostname, tag, msg.Content)

  // return the message as a byte array
  return []byte(message)
}
<commit_msg>Attempt to dial datadog on port 443<commit_after>package drain

import (
	"fmt"
	"net"
	"time"

  "github.com/DataDog/datadog-agent/pkg/logs/config"
	"github.com/DataDog/datadog-agent/pkg/logs/sender"

	"github.com/nanopack/logvac/core"
)

// Datadog drain implements the publisher interface for publishing logs to datadog.
type Datadog struct {
  ID        string // app ID or name
  Key       string // datadog api key
	manager   *sender.ConnectionManager
	conn      net.Conn
}

// NewDatadogClient creates a new mist publisher
func NewDatadogClient(id, key string) (*Datadog, error) {
  // emulate the server config
  config := config.NewServerConfig("agent-443-intake.logs.datadoghq.com", 443, true)

  // initialize a connection manager
	manager := sender.NewConnectionManager(config, "")

  // establish a connection
	conn := manager.NewConnection()

	return &Datadog{
    ID:       id,
    Key:      key,
    manager:  manager,
    conn:     conn,
  }, nil
}

// Init initializes a connection to mist
func (p Datadog) Init() error {

	// add drain
	logvac.AddDrain("datadog", p.Publish)

	return nil
}

// Publish utilizes mist's Publish to "drain" a log message
func (p *Datadog) Publish(msg logvac.Message) {
  // keep track of the attempts
	msg.PubTries++

  // re-establish the connection if it's been closed
	if p.conn == nil {
		p.conn = p.manager.NewConnection() // doesn't block (don't goroutine call to Publish)
	}

  // generate the payload for this entry
  payload := formatDataDogMessage(msg, p.ID, p.Key)

  // send the payload
	_, err := p.conn.Write(payload)
	if err != nil {
    // it's possible the connection is bad, so let's close it
		p.manager.CloseConnection(p.conn)
		p.conn = nil

    // let's try to send it again (at least 3 times)
		if msg.PubTries <= 3 {
      // give it a sec, networks are fickle
			time.Sleep(2 * time.Second)
      // retry!
			p.Publish(msg)
		}
	}
}

// Close closes the connection to datadog.
func (p *Datadog) Close() error {
	p.manager.CloseConnection(p.conn)
	return nil
}

// format the syslog message, prefixed with the datadog api key
func formatDataDogMessage(msg logvac.Message, id, key string) []byte {
  // format the date in the proper syslog format
  date := fmt.Sprintf("%s %02d %02d:%02d:%02d",
    msg.Time.Month().String()[:3],
    msg.Time.Day(),
    msg.Time.Hour(),
    msg.Time.Minute(),
    msg.Time.Second())

  // prefix the app id to the hostname identifier
  hostname := fmt.Sprintf("%s.%s", id, msg.Id)

  // extract the first tag as the tag (ie: nginx[access])
  tag := msg.Tag[0]

  // the final message
  message := fmt.Sprintf("%s <%d>%s %s %s: %s\n",
    key, msg.Priority, date, hostname, tag, msg.Content)

  // return the message as a byte array
  return []byte(message)
}
<|endoftext|>


<commit_before>package wally

import (
	"fmt"
	"io/ioutil"
	"strings"
	"testing"

	rdb "github.com/dancannon/gorethink"
	"github.com/stretchr/testify/assert"
)

var (
	session *rdb.Session
)

func init() {
	Database = "testing"
	DocumentTable = "documents"
	IndexTable = "indexes"
}

func DbConnect() {
	var err error
	session, err = rdb.Connect(rdb.ConnectOpts{
		Address:  "localhost:28015",
		Database: Database,
	})

	if err != nil {
		fmt.Errorf(err.Error())
	}
}

func DbBootstrap() {
	DbConnect()
	// Reset database
	rdb.DbDrop(Database).Exec(session)
	rdb.DbCreate(Database).Exec(session)

	DbConnect()

	// Drop tables
	rdb.Db(Database).TableDrop(DocumentTable).Run(session)
	rdb.Db(Database).TableDrop(IndexTable).Run(session)

	// Create tables
	rdb.Db(Database).TableCreate(DocumentTable).Run(session)
	rdb.Db(Database).TableCreate(IndexTable).Run(session)
}

func TestIndexer_Stopper(t *testing.T) {
	tests := []struct {
		Input  string
		Output string
	}{
		{
			"computer",
			"computer",
		},

		{
			"the",
			"",
		},

		{
			"technology",
			"technology",
		},

		{
			"wasn't",
			"",
		},
	}

	for _, test := range tests {
		assert.Equal(t, test.Output, Stopper(test.Input))
	}
}

func TestIndexer_SplitTextIntoWords(t *testing.T) {
	tests := []struct {
		Input  interface{}
		Output []string
	}{
		{
			"I am a block of text",
			[]string{"I", "am", "a", "block", "of", "text"},
		},

		{
			"        superfluous    whitespace ",
			[]string{"superfluous", "whitespace"},
		},

		{
			"              ",
			[]string{},
		},

		{
			[]byte("fancy a byte?"),
			[]string{"fancy", "a", "byte?"},
		},

		{
			32,
			[]string{},
		},
	}

	for _, test := range tests {
		assert.Equal(t, test.Output, SplitTextIntoWords(test.Input))
	}
}

func TestIndexer_Parse(t *testing.T) {
	docId := "12345-67890-ABCDE"

	tests := []struct {
		Input  interface{}
		Output []Index
	}{
		{
			"I am a block of text and I am going to be parsed",
			[]Index{
				Index{Id: "block", DocumentId: docId, Count: 0},
				Index{Id: "text", DocumentId: docId, Count: 0},
				Index{Id: "going", DocumentId: docId, Count: 0},
				Index{Id: "parsed", DocumentId: docId, Count: 0},
			},
		},

		{
			[]byte("I am another block of text but now I am in bytes"),
			[]Index{
				Index{Id: "another", DocumentId: docId, Count: 0},
				Index{Id: "block", DocumentId: docId, Count: 0},
				Index{Id: "text", DocumentId: docId, Count: 0},
				Index{Id: "now", DocumentId: docId, Count: 0},
				Index{Id: "bytes", DocumentId: docId, Count: 0},
			},
		},
	}

	for _, test := range tests {
		assert.Equal(t, test.Output, Parse(test.Input, docId))
	}
}

func TestIndexer_IndexString(t *testing.T) {
	indexId := "world"

	index := Index{Id: indexId}

	assert.Equal(t, "Index#world", index.String())
}

func TestIndexer_IndexPut(t *testing.T) {
	DbBootstrap()

	index := Index{Id: "hello", Count: 5, DocumentId: "12345-67890-ABCDE"}

	err := index.Put(session)
	assert.Nil(t, err)

	res, err := rdb.Db(Database).Table(IndexTable).Get("hello").Run(session)
	assert.Nil(t, err)

	var i Index
	err = res.One(&i)
	assert.Nil(t, err)

	assert.Equal(t, i.Id, "hello")
	assert.Equal(t, i.Count, 5)
	assert.Equal(t, i.DocumentId, "12345-67890-ABCDE")
}

func TestIndexer_DocumentString(t *testing.T) {
	docId := "12345-67890-ABCDE"

	doc := Document{Id: docId}

	assert.Equal(t, "Document#12345-67890-ABCDE", doc.String())
}

func TestIndexer_DocumentPut(t *testing.T) {
	DbBootstrap()

	doc := Document{
		Source: "www.google.com",
		Content: "Lorem ipsum dolor sit amet.",
	}

	err := doc.Put(session)
	assert.Nil(t, err)

	res, err := rdb.Db(Database).Table(DocumentTable).Run(session)
	assert.Nil(t, err)

	var d Document
	err = res.One(&d)
	assert.Nil(t, err)

	assert.NotEqual(t, d.Id, "")
	assert.Equal(t, d.Source, "www.google.com")
	assert.Equal(t, d.Content, "Lorem ipsum dolor sit amet.")
}

func BenchmarkSplitTextIntoWords(b *testing.B) {
	file, err := ioutil.ReadFile("test_data/test.txt") // 30654 words
	if err != nil {
		b.Error("Could not load test data")
	}

	for n := 0; n < b.N; n++ {
		SplitTextIntoWords(file)
	}
}

func BenchmarkSplitTextIntoWords_two(b *testing.B) {
	file, err := ioutil.ReadFile("test_data/test_2.txt") // 1891 words
	if err != nil {
		b.Error("Could not load test data")
	}

	for n := 0; n < b.N; n++ {
		SplitTextIntoWords(file)
	}
}

func BenchmarkStopper(b *testing.B) {
	file, err := ioutil.ReadFile("test_data/test.txt") // 30654 words
	if err != nil {
		b.Error("Could not load test data")
	}

	data := strings.Fields(string(file))

	for n := 0; n < b.N; n++ {
		for _, word := range data {
			Stopper(word)
		}
	}
}

func BenchmarkStopper_two(b *testing.B) {
	file, err := ioutil.ReadFile("test_data/test_2.txt") // 1891 words
	if err != nil {
		b.Error("Could not load test data")
	}

	data := strings.Fields(string(file))

	for n := 0; n < b.N; n++ {
		for _, word := range data {
			Stopper(word)
		}
	}
}

func BenchmarkParse(b *testing.B) {
	docId := "12345-67890-ABCDE"
	file, err := ioutil.ReadFile("test_data/test.txt") // 30654 words
	if err != nil {
		b.Error("Could not load test data")
	}

	for n := 0; n < b.N; n++ {
		Parse(file, docId)
	}
}

func BenchmarkParse_two(b *testing.B) {
	docId := "12345-67890-ABCDE"
	file, err := ioutil.ReadFile("test_data/test_2.txt") // 30654 words
	if err != nil {
		b.Error("Could not load test data")
	}

	for n := 0; n < b.N; n++ {
		Parse(file, docId)
	}
}
<commit_msg>fix #2 Connect to a database that we know will always exist, 'test'<commit_after>package wally

import (
	"fmt"
	"io/ioutil"
	"strings"
	"testing"

	rdb "github.com/dancannon/gorethink"
	"github.com/stretchr/testify/assert"
)

var (
	session *rdb.Session
)

func init() {
	Database = "testing"
	DocumentTable = "documents"
	IndexTable = "indexes"

	var err error
	session, err = rdb.Connect(rdb.ConnectOpts{
		Address:  "localhost:28015",
		Database: "test",
	})

	if err != nil {
		fmt.Errorf(err.Error())
	}
}

func DbBootstrap() {
	rdb.DbCreate(Database).Exec(session)

	// Drop tables
	rdb.Db(Database).TableDrop(DocumentTable).Run(session)
	rdb.Db(Database).TableDrop(IndexTable).Run(session)

	// Create tables
	rdb.Db(Database).TableCreate(DocumentTable).Run(session)
	rdb.Db(Database).TableCreate(IndexTable).Run(session)
}

func TestIndexer_Stopper(t *testing.T) {
	tests := []struct {
		Input  string
		Output string
	}{
		{
			"computer",
			"computer",
		},

		{
			"the",
			"",
		},

		{
			"technology",
			"technology",
		},

		{
			"wasn't",
			"",
		},
	}

	for _, test := range tests {
		assert.Equal(t, test.Output, Stopper(test.Input))
	}
}

func TestIndexer_SplitTextIntoWords(t *testing.T) {
	tests := []struct {
		Input  interface{}
		Output []string
	}{
		{
			"I am a block of text",
			[]string{"I", "am", "a", "block", "of", "text"},
		},

		{
			"        superfluous    whitespace ",
			[]string{"superfluous", "whitespace"},
		},

		{
			"              ",
			[]string{},
		},

		{
			[]byte("fancy a byte?"),
			[]string{"fancy", "a", "byte?"},
		},

		{
			32,
			[]string{},
		},
	}

	for _, test := range tests {
		assert.Equal(t, test.Output, SplitTextIntoWords(test.Input))
	}
}

func TestIndexer_Parse(t *testing.T) {
	docId := "12345-67890-ABCDE"

	tests := []struct {
		Input  interface{}
		Output []Index
	}{
		{
			"I am a block of text and I am going to be parsed",
			[]Index{
				Index{Id: "block", DocumentId: docId, Count: 0},
				Index{Id: "text", DocumentId: docId, Count: 0},
				Index{Id: "going", DocumentId: docId, Count: 0},
				Index{Id: "parsed", DocumentId: docId, Count: 0},
			},
		},

		{
			[]byte("I am another block of text but now I am in bytes"),
			[]Index{
				Index{Id: "another", DocumentId: docId, Count: 0},
				Index{Id: "block", DocumentId: docId, Count: 0},
				Index{Id: "text", DocumentId: docId, Count: 0},
				Index{Id: "now", DocumentId: docId, Count: 0},
				Index{Id: "bytes", DocumentId: docId, Count: 0},
			},
		},
	}

	for _, test := range tests {
		assert.Equal(t, test.Output, Parse(test.Input, docId))
	}
}

func TestIndexer_IndexString(t *testing.T) {
	indexId := "world"

	index := Index{Id: indexId}

	assert.Equal(t, "Index#world", index.String())
}

func TestIndexer_IndexPut(t *testing.T) {
	DbBootstrap()

	index := Index{Id: "hello", Count: 5, DocumentId: "12345-67890-ABCDE"}

	err := index.Put(session)
	assert.Nil(t, err)

	res, err := rdb.Db(Database).Table(IndexTable).Get("hello").Run(session)
	assert.Nil(t, err)

	var i Index
	err = res.One(&i)
	assert.Nil(t, err)

	assert.Equal(t, i.Id, "hello")
	assert.Equal(t, i.Count, 5)
	assert.Equal(t, i.DocumentId, "12345-67890-ABCDE")
}

func TestIndexer_DocumentString(t *testing.T) {
	docId := "12345-67890-ABCDE"

	doc := Document{Id: docId}

	assert.Equal(t, "Document#12345-67890-ABCDE", doc.String())
}

func TestIndexer_DocumentPut(t *testing.T) {
	DbBootstrap()

	doc := Document{
		Source: "www.google.com",
		Content: "Lorem ipsum dolor sit amet.",
	}

	err := doc.Put(session)
	assert.Nil(t, err)

	res, err := rdb.Db(Database).Table(DocumentTable).Run(session)
	assert.Nil(t, err)

	var d Document
	err = res.One(&d)
	assert.Nil(t, err)

	assert.NotEqual(t, d.Id, "")
	assert.Equal(t, d.Source, "www.google.com")
	assert.Equal(t, d.Content, "Lorem ipsum dolor sit amet.")
}

func BenchmarkSplitTextIntoWords(b *testing.B) {
	file, err := ioutil.ReadFile("test_data/test.txt") // 30654 words
	if err != nil {
		b.Error("Could not load test data")
	}

	for n := 0; n < b.N; n++ {
		SplitTextIntoWords(file)
	}
}

func BenchmarkSplitTextIntoWords_two(b *testing.B) {
	file, err := ioutil.ReadFile("test_data/test_2.txt") // 1891 words
	if err != nil {
		b.Error("Could not load test data")
	}

	for n := 0; n < b.N; n++ {
		SplitTextIntoWords(file)
	}
}

func BenchmarkStopper(b *testing.B) {
	file, err := ioutil.ReadFile("test_data/test.txt") // 30654 words
	if err != nil {
		b.Error("Could not load test data")
	}

	data := strings.Fields(string(file))

	for n := 0; n < b.N; n++ {
		for _, word := range data {
			Stopper(word)
		}
	}
}

func BenchmarkStopper_two(b *testing.B) {
	file, err := ioutil.ReadFile("test_data/test_2.txt") // 1891 words
	if err != nil {
		b.Error("Could not load test data")
	}

	data := strings.Fields(string(file))

	for n := 0; n < b.N; n++ {
		for _, word := range data {
			Stopper(word)
		}
	}
}

func BenchmarkParse(b *testing.B) {
	docId := "12345-67890-ABCDE"
	file, err := ioutil.ReadFile("test_data/test.txt") // 30654 words
	if err != nil {
		b.Error("Could not load test data")
	}

	for n := 0; n < b.N; n++ {
		Parse(file, docId)
	}
}

func BenchmarkParse_two(b *testing.B) {
	docId := "12345-67890-ABCDE"
	file, err := ioutil.ReadFile("test_data/test_2.txt") // 30654 words
	if err != nil {
		b.Error("Could not load test data")
	}

	for n := 0; n < b.N; n++ {
		Parse(file, docId)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package e2e

import (
	"flag"
	"os"
	"runtime"
	"testing"

	"github.com/coreos/etcd/pkg/testutil"
)

var binDir string
var certDir string

func TestMain(m *testing.M) {
	os.Setenv("ETCD_UNSUPPORTED_ARCH", runtime.GOARCH)

	flag.StringVar(&binDir, "bin-dir", "../bin", "The directory for store etcd and etcdctl binaries.")
	flag.StringVar(&certDir, "cert-dir", "../integration/fixtures", "The directory for store certificate files.")
	flag.Parse()

	v := m.Run()
	if v == 0 && testutil.CheckLeakedGoroutine() {
		os.Exit(1)
	}
	os.Exit(v)
}
<commit_msg>e2e: unset ETCDCTL_API env var before running u2e tests<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package e2e

import (
	"flag"
	"os"
	"runtime"
	"testing"

	"github.com/coreos/etcd/pkg/testutil"
)

var binDir string
var certDir string

func TestMain(m *testing.M) {
	os.Setenv("ETCD_UNSUPPORTED_ARCH", runtime.GOARCH)
	os.Unsetenv("ETCDCTL_API")

	flag.StringVar(&binDir, "bin-dir", "../bin", "The directory for store etcd and etcdctl binaries.")
	flag.StringVar(&certDir, "cert-dir", "../integration/fixtures", "The directory for store certificate files.")
	flag.Parse()

	v := m.Run()
	if v == 0 && testutil.CheckLeakedGoroutine() {
		os.Exit(1)
	}
	os.Exit(v)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flushfs

import (
	"fmt"
	"os"
	"sync"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
)

// Create a file system whose sole contents are a file named "foo" and a
// directory named "bar".
//
// The file may be opened for reading and/or writing. Its initial contents are
// empty. Whenever a flush or fsync is received, the supplied function will be
// called with the current contents of the file and its status returned.
//
// The directory cannot be modified.
func NewFileSystem(
	reportFlush func(string) error,
	reportFsync func(string) error) (server fuse.Server, err error) {
	fs := &flushFS{
		reportFlush: reportFlush,
		reportFsync: reportFsync,
	}

	server = fuseutil.NewFileSystemServer(fs)
	return
}

const (
	fooID = fuseops.RootInodeID + 1 + iota
	barID
)

type flushFS struct {
	fuseutil.NotImplementedFileSystem

	reportFlush func(string) error
	reportFsync func(string) error

	mu          sync.Mutex
	fooContents []byte // GUARDED_BY(mu)
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// LOCKS_REQUIRED(fs.mu)
func (fs *flushFS) rootAttributes() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Mode:  0777 | os.ModeDir,
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *flushFS) fooAttributes() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Mode:  0777,
		Size:  uint64(len(fs.fooContents)),
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *flushFS) barAttributes() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Mode:  0777 | os.ModeDir,
	}
}

////////////////////////////////////////////////////////////////////////
// FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *flushFS) Init(
	op *fuseops.InitOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	return
}

func (fs *flushFS) LookUpInode(
	op *fuseops.LookUpInodeOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check.
	if op.Parent != fuseops.RootInodeID {
		err = fuse.ENOENT
		return
	}

	// Set up the entry.
	switch op.Name {
	case "foo":
		op.Entry = fuseops.ChildInodeEntry{
			Child:      fooID,
			Attributes: fs.fooAttributes(),
		}

	case "bar":
		op.Entry = fuseops.ChildInodeEntry{
			Child:      barID,
			Attributes: fs.barAttributes(),
		}

	default:
		err = fuse.ENOENT
		return
	}

	return
}

func (fs *flushFS) GetInodeAttributes(
	op *fuseops.GetInodeAttributesOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	switch op.Inode {
	case fuseops.RootInodeID:
		op.Attributes = fs.rootAttributes()
		return

	case fooID:
		op.Attributes = fs.fooAttributes()
		return

	case barID:
		op.Attributes = fs.barAttributes()
		return

	default:
		err = fuse.ENOENT
		return
	}
}

func (fs *flushFS) OpenFile(
	op *fuseops.OpenFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check.
	if op.Inode != fooID {
		err = fuse.ENOSYS
		return
	}

	return
}

func (fs *flushFS) ReadFile(
	op *fuseops.ReadFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Ensure the offset is in range.
	if op.Offset > int64(len(fs.fooContents)) {
		return
	}

	// Read what we can.
	op.Data = make([]byte, op.Size)
	copy(op.Data, fs.fooContents[op.Offset:])

	return
}

func (fs *flushFS) WriteFile(
	op *fuseops.WriteFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Ensure that the contents slice is long enough.
	newLen := int(op.Offset) + len(op.Data)
	if len(fs.fooContents) < newLen {
		padding := make([]byte, newLen-len(fs.fooContents))
		fs.fooContents = append(fs.fooContents, padding...)
	}

	// Copy in the data.
	n := copy(fs.fooContents[op.Offset:], op.Data)

	// Sanity check.
	if n != len(op.Data) {
		panic(fmt.Sprintf("Unexpected short copy: %v", n))
	}

	return
}

func (fs *flushFS) SyncFile(
	op *fuseops.SyncFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	err = fs.reportFsync(string(fs.fooContents))
	return
}

func (fs *flushFS) FlushFile(
	op *fuseops.FlushFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	err = fs.reportFlush(string(fs.fooContents))
	return
}

func (fs *flushFS) OpenDir(
	op *fuseops.OpenDirOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check.
	switch op.Inode {
	case fuseops.RootInodeID:
	case barID:

	default:
		err = fuse.ENOENT
		return
	}

	return
}
<commit_msg>flushFS.ReadDir<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flushfs

import (
	"fmt"
	"os"
	"sync"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
)

// Create a file system whose sole contents are a file named "foo" and a
// directory named "bar".
//
// The file may be opened for reading and/or writing. Its initial contents are
// empty. Whenever a flush or fsync is received, the supplied function will be
// called with the current contents of the file and its status returned.
//
// The directory cannot be modified.
func NewFileSystem(
	reportFlush func(string) error,
	reportFsync func(string) error) (server fuse.Server, err error) {
	fs := &flushFS{
		reportFlush: reportFlush,
		reportFsync: reportFsync,
	}

	server = fuseutil.NewFileSystemServer(fs)
	return
}

const (
	fooID = fuseops.RootInodeID + 1 + iota
	barID
)

type flushFS struct {
	fuseutil.NotImplementedFileSystem

	reportFlush func(string) error
	reportFsync func(string) error

	mu          sync.Mutex
	fooContents []byte // GUARDED_BY(mu)
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// LOCKS_REQUIRED(fs.mu)
func (fs *flushFS) rootAttributes() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Mode:  0777 | os.ModeDir,
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *flushFS) fooAttributes() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Mode:  0777,
		Size:  uint64(len(fs.fooContents)),
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *flushFS) barAttributes() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Mode:  0777 | os.ModeDir,
	}
}

////////////////////////////////////////////////////////////////////////
// FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *flushFS) Init(
	op *fuseops.InitOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	return
}

func (fs *flushFS) LookUpInode(
	op *fuseops.LookUpInodeOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check.
	if op.Parent != fuseops.RootInodeID {
		err = fuse.ENOENT
		return
	}

	// Set up the entry.
	switch op.Name {
	case "foo":
		op.Entry = fuseops.ChildInodeEntry{
			Child:      fooID,
			Attributes: fs.fooAttributes(),
		}

	case "bar":
		op.Entry = fuseops.ChildInodeEntry{
			Child:      barID,
			Attributes: fs.barAttributes(),
		}

	default:
		err = fuse.ENOENT
		return
	}

	return
}

func (fs *flushFS) GetInodeAttributes(
	op *fuseops.GetInodeAttributesOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	switch op.Inode {
	case fuseops.RootInodeID:
		op.Attributes = fs.rootAttributes()
		return

	case fooID:
		op.Attributes = fs.fooAttributes()
		return

	case barID:
		op.Attributes = fs.barAttributes()
		return

	default:
		err = fuse.ENOENT
		return
	}
}

func (fs *flushFS) OpenFile(
	op *fuseops.OpenFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check.
	if op.Inode != fooID {
		err = fuse.ENOSYS
		return
	}

	return
}

func (fs *flushFS) ReadFile(
	op *fuseops.ReadFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Ensure the offset is in range.
	if op.Offset > int64(len(fs.fooContents)) {
		return
	}

	// Read what we can.
	op.Data = make([]byte, op.Size)
	copy(op.Data, fs.fooContents[op.Offset:])

	return
}

func (fs *flushFS) WriteFile(
	op *fuseops.WriteFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Ensure that the contents slice is long enough.
	newLen := int(op.Offset) + len(op.Data)
	if len(fs.fooContents) < newLen {
		padding := make([]byte, newLen-len(fs.fooContents))
		fs.fooContents = append(fs.fooContents, padding...)
	}

	// Copy in the data.
	n := copy(fs.fooContents[op.Offset:], op.Data)

	// Sanity check.
	if n != len(op.Data) {
		panic(fmt.Sprintf("Unexpected short copy: %v", n))
	}

	return
}

func (fs *flushFS) SyncFile(
	op *fuseops.SyncFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	err = fs.reportFsync(string(fs.fooContents))
	return
}

func (fs *flushFS) FlushFile(
	op *fuseops.FlushFileOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	err = fs.reportFlush(string(fs.fooContents))
	return
}

func (fs *flushFS) OpenDir(
	op *fuseops.OpenDirOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check.
	switch op.Inode {
	case fuseops.RootInodeID:
	case barID:

	default:
		err = fuse.ENOENT
		return
	}

	return
}

func (fs *flushFS) ReadDir(
	op *fuseops.ReadDirOp) {
	var err error
	defer fuseutil.RespondToOp(op, &err)

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Create the appropriate listing.
	var dirents []fuseutil.Dirent

	switch op.Inode {
	case fuseops.RootInodeID:
		dirents = []fuseutil.Dirent{
			fuseutil.Dirent{
				Offset: 1,
				Inode:  fooID,
				Name:   "foo",
				Type:   fuseutil.DT_File,
			},

			fuseutil.Dirent{
				Offset: 2,
				Inode:  barID,
				Name:   "bar",
				Type:   fuseutil.DT_Directory,
			},
		}

	case barID:

	default:
		err = fmt.Errorf("Unexpected inode: %v", op.Inode)
		return
	}

	// If the offset is for the end of the listing, we're done. Otherwise we
	// expect it to be for the start.
	switch op.Offset {
	case fuseops.DirOffset(len(dirents)):
		return

	case 0:

	default:
		err = fmt.Errorf("Unexpected offset: %v", op.Offset)
		return
	}

	// Fill in the listing.
	for _, de := range dirents {
		op.Data = fuseutil.AppendDirent(op.Data, de)
	}

	// We don't support doing this in anything more than one shot.
	if len(op.Data) > op.Size {
		err = fmt.Errorf("Couldn't fit listing in %v bytes", op.Size)
		return
	}

	return
}
<|endoftext|>