<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

// git runs git with args and returns its output.
func git(args ...string) string {
	cmd := exec.Command("git", args...)
	out, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			fmt.Fprintf(os.Stderr, "%s\n", string(err.Stderr))
		}
		log.Fatalf("git %s failed: %s", shellEscapeList(args), err)
	}
	return strings.TrimSuffix(string(out), "\n")
}

// tryGit runs git with args and returns its output and a non-nil
// error if the command exits with a non-zero status.
func tryGit(args ...string) (string, error) {
	cmd := exec.Command("git", args...)
	out, err := cmd.CombinedOutput()
	if _, ok := err.(*exec.ExitError); err != nil && !ok {
		log.Fatalf("git %s failed: %s", shellEscapeList(args), err)
	}
	return strings.TrimSuffix(string(out), "\n"), err
}

func lines(s string) []string {
	lines := strings.Split(s, "\n")
	if len(lines) > 0 && lines[len(lines)-1] == "" {
		lines = lines[:len(lines)-1]
	}
	return lines
}

// getRemote returns the remote name for the given remote URL.
func getRemote(url string) (string, error) {
	for _, line := range lines(git("remote", "-v")) {
		fs := strings.Fields(line)
		if len(fs) >= 2 && fs[1] == url {
			return fs[0], nil
		}
	}
	return "", fmt.Errorf("no remote found for %s", url)
}

// upstreamOf returns the full upstream ref name of the given ref, or
// "".
func upstreamOf(ref string) string {
	// This fails with code 128 and "fatal: no upstream configured
	// for branch 'xxx'" if there's no upstream. It also fails
	// with 128 and "fatal: HEAD does not point to a branch" if
	// ref is not a branch or a symbolic ref to a branch.
	out, err := tryGit("rev-parse", "--symbolic-full-name", ref+"@{u}")
	if err != nil {
		return ""
	}
	return out
}

// gitPatchID returns the git patch ID of commit, which is effectively
// a hash of that commit's diff. See man git-patch-id for details.
func gitPatchID(commit string) (string, error) {
	var err error
	// Run git diff-tree -p $commit -- | git patch-id --stable.
	diffTree := exec.Command("git", "diff-tree", "-p", commit, "--")
	patchID := exec.Command("git", "patch-id", "--stable")
	r, w, err := os.Pipe()
	if err != nil {
		log.Fatal("failed to create pipe: ", err)
	}
	patchID.Stdin, diffTree.Stdout = r, w
	if err := diffTree.Start(); err != nil {
		log.Fatal("failed to start %s: ", shellEscapeList(diffTree.Args), err)
	}
	w.Close()
	out, err := patchID.Output()
	r.Close()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			fmt.Fprintf(os.Stderr, "%s\n", string(err.Stderr))
		}
		log.Fatalf("%s failed: %s", shellEscapeList(patchID.Args), err)
	}
	if diffTree.Wait() != nil {
		return "", fmt.Errorf("bad revision %q", commit)
	}
	fs := bytes.Fields(out)
	if len(fs) != 2 {
		log.Fatal("unexpected output from %s: %s", shellEscapeList(patchID.Args), out)
	}
	return string(fs[0]), nil
}

// gitCommitMessage returns the commit message for commit.
func gitCommitMessage(commit string) (string, error) {
	// Get the commit object.
	obj, err := tryGit("cat-file", "commit", commit)
	if err != nil {
		return "", fmt.Errorf("bad revision %q", commit)
	}
	// Extract the commit message.
	if i := strings.Index(obj, "\n\n"); i >= 0 {
		return obj[i+2:], nil
	}
	return "", nil
}

// changeIds returns the full Gerrit change IDs of each commit. The
// change ID will be "" if missing.
func changeIds(project, forBranch string, commits []string) []string {
	if i := strings.LastIndexByte(forBranch, '/'); i >= 0 {
		forBranch = forBranch[i+1:]
	}

	// Construct input.
	var input bytes.Buffer
	for _, c := range commits {
		fmt.Fprintf(&input, "%s\n", c)
	}

	// Run batch cat-file command.
	args := []string{"cat-file", "--batch", "--buffer"}
	cmd := exec.Command("git", args...)
	cmd.Stdin = &input
	out, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			fmt.Fprintf(os.Stderr, "%s\n", string(err.Stderr))
		}
		log.Fatalf("git %s failed: %s", shellEscapeList(args), err)
	}

	// Parse output.
	cids := make([]string, len(commits))
	for i, commit := range commits {
		// Get "<sha1> SP <type> SP <size> LF" line.
		nl := bytes.IndexByte(out, '\n')
		if nl < 0 {
			log.Fatal("malformed git cat-file output")
		}
		fs := strings.Fields(string(out[:nl]))
		out = out[nl+1:]
		if len(fs) < 2 || fs[0] != commit {
			log.Fatal("malformed git cat-file output")
		}
		if fs[1] == "missing" {
			continue
		}
		if fs[1] != "commit" {
			log.Fatal("unexpected object type %q for %s", fs[1], fs[0])
		}

		// Get commit object.
		size, _ := strconv.Atoi(fs[2])
		if len(out) <= size || out[size] != '\n' {
			log.Fatal("git cat-file out of sync")
		}
		var obj []byte
		obj, out = out[:size], out[size+1:]

		// Find the Change-Id in the commit.
		for _, line := range bytes.Split(obj, []byte("\n")) {
			if bytes.HasPrefix(line, []byte("Change-Id: ")) {
				lfs := bytes.Fields(line)
				if len(lfs) == 2 {
					cid := string(lfs[1])
					cids[i] = project + "~" + forBranch + "~" + cid
				}
			}
		}
	}
	return cids
}
<commit_msg>git-p: fix upstreamOf<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

// git runs git with args and returns its output.
func git(args ...string) string {
	cmd := exec.Command("git", args...)
	out, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			fmt.Fprintf(os.Stderr, "%s\n", string(err.Stderr))
		}
		log.Fatalf("git %s failed: %s", shellEscapeList(args), err)
	}
	return strings.TrimSuffix(string(out), "\n")
}

// tryGit runs git with args and returns its output and a non-nil
// error if the command exits with a non-zero status.
func tryGit(args ...string) (string, error) {
	cmd := exec.Command("git", args...)
	out, err := cmd.CombinedOutput()
	if _, ok := err.(*exec.ExitError); err != nil && !ok {
		log.Fatalf("git %s failed: %s", shellEscapeList(args), err)
	}
	return strings.TrimSuffix(string(out), "\n"), err
}

func lines(s string) []string {
	lines := strings.Split(s, "\n")
	if len(lines) > 0 && lines[len(lines)-1] == "" {
		lines = lines[:len(lines)-1]
	}
	return lines
}

// getRemote returns the remote name for the given remote URL.
func getRemote(url string) (string, error) {
	for _, line := range lines(git("remote", "-v")) {
		fs := strings.Fields(line)
		if len(fs) >= 2 && fs[1] == url {
			return fs[0], nil
		}
	}
	return "", fmt.Errorf("no remote found for %s", url)
}

// upstreamOf returns the full upstream ref name of the given ref, or
// "".
func upstreamOf(ref string) string {
	// This fails with code 128 and "fatal: no upstream configured
	// for branch 'xxx'" if there's no upstream. It also fails
	// with 128 and "fatal: HEAD does not point to a branch" if
	// ref is not a branch or a symbolic ref to a branch.
	//
	// The @{u} syntax requires a branchname, not a refname, so
	// strip the ref to a branch name.
	ref = strings.TrimPrefix(ref, "refs/heads/")
	out, err := tryGit("rev-parse", "--symbolic-full-name", ref+"@{u}")
	if err != nil {
		return ""
	}
	return out
}

// gitPatchID returns the git patch ID of commit, which is effectively
// a hash of that commit's diff. See man git-patch-id for details.
func gitPatchID(commit string) (string, error) {
	var err error
	// Run git diff-tree -p $commit -- | git patch-id --stable.
	diffTree := exec.Command("git", "diff-tree", "-p", commit, "--")
	patchID := exec.Command("git", "patch-id", "--stable")
	r, w, err := os.Pipe()
	if err != nil {
		log.Fatal("failed to create pipe: ", err)
	}
	patchID.Stdin, diffTree.Stdout = r, w
	if err := diffTree.Start(); err != nil {
		log.Fatal("failed to start %s: ", shellEscapeList(diffTree.Args), err)
	}
	w.Close()
	out, err := patchID.Output()
	r.Close()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			fmt.Fprintf(os.Stderr, "%s\n", string(err.Stderr))
		}
		log.Fatalf("%s failed: %s", shellEscapeList(patchID.Args), err)
	}
	if diffTree.Wait() != nil {
		return "", fmt.Errorf("bad revision %q", commit)
	}
	fs := bytes.Fields(out)
	if len(fs) != 2 {
		log.Fatal("unexpected output from %s: %s", shellEscapeList(patchID.Args), out)
	}
	return string(fs[0]), nil
}

// gitCommitMessage returns the commit message for commit.
func gitCommitMessage(commit string) (string, error) {
	// Get the commit object.
	obj, err := tryGit("cat-file", "commit", commit)
	if err != nil {
		return "", fmt.Errorf("bad revision %q", commit)
	}
	// Extract the commit message.
	if i := strings.Index(obj, "\n\n"); i >= 0 {
		return obj[i+2:], nil
	}
	return "", nil
}

// changeIds returns the full Gerrit change IDs of each commit. The
// change ID will be "" if missing.
func changeIds(project, forBranch string, commits []string) []string {
	if i := strings.LastIndexByte(forBranch, '/'); i >= 0 {
		forBranch = forBranch[i+1:]
	}

	// Construct input.
	var input bytes.Buffer
	for _, c := range commits {
		fmt.Fprintf(&input, "%s\n", c)
	}

	// Run batch cat-file command.
	args := []string{"cat-file", "--batch", "--buffer"}
	cmd := exec.Command("git", args...)
	cmd.Stdin = &input
	out, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			fmt.Fprintf(os.Stderr, "%s\n", string(err.Stderr))
		}
		log.Fatalf("git %s failed: %s", shellEscapeList(args), err)
	}

	// Parse output.
	cids := make([]string, len(commits))
	for i, commit := range commits {
		// Get "<sha1> SP <type> SP <size> LF" line.
		nl := bytes.IndexByte(out, '\n')
		if nl < 0 {
			log.Fatal("malformed git cat-file output")
		}
		fs := strings.Fields(string(out[:nl]))
		out = out[nl+1:]
		if len(fs) < 2 || fs[0] != commit {
			log.Fatal("malformed git cat-file output")
		}
		if fs[1] == "missing" {
			continue
		}
		if fs[1] != "commit" {
			log.Fatal("unexpected object type %q for %s", fs[1], fs[0])
		}

		// Get commit object.
		size, _ := strconv.Atoi(fs[2])
		if len(out) <= size || out[size] != '\n' {
			log.Fatal("git cat-file out of sync")
		}
		var obj []byte
		obj, out = out[:size], out[size+1:]

		// Find the Change-Id in the commit.
		for _, line := range bytes.Split(obj, []byte("\n")) {
			if bytes.HasPrefix(line, []byte("Change-Id: ")) {
				lfs := bytes.Fields(line)
				if len(lfs) == 2 {
					cid := string(lfs[1])
					cids[i] = project + "~" + forBranch + "~" + cid
				}
			}
		}
	}
	return cids
}
<|endoftext|>


<commit_before>// This file contains most of the public interface of Alexandria (the library).
// The remainder is the types defined in datastructures.go.

// This file is part of Alexandria which is released under AGPLv3.
// Copyright (C) 2015-2018 Colin Benner
// See LICENSE or go to https://github.com/yzhs/alexandria/LICENSE for full
// license details.

//go:generate go run contrib/generate_assets.go

package alexandria

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/pkg/errors"
)

// Programm name and version
const (
	NAME    = "Alexandria"
	VERSION = "0.1"
)

type LatexToPngBackend struct {
	renderer latexToPngRenderer
}

func NewLatexToPngBackend() LatexToPngBackend {
	return LatexToPngBackend{renderer: &xelatexImagemagickRenderer{}}
}

func (b *LatexToPngBackend) RenderAllScrolls() (numScrolls int, errors []error) {
	ids := b.IDsForAllScrolls()
	renderedIDs, errors := b.RenderScrollsByID(ids)
	return len(renderedIDs), errors
}

func (b *LatexToPngBackend) RenderScrollsByID(ids []ID) (renderedScrollIDs []ID, errors []error) {
	renderedScrollIDs = make([]ID, len(ids))
	numScrolls := 0
	for _, id := range ids {
		err := renderScroll(id, b.renderer)
		if err != nil {
			errors = append(errors, fmt.Errorf("rendering %v failed", id))
		} else {
			renderedScrollIDs[numScrolls] = id
			numScrolls++
		}
	}

	return renderedScrollIDs[:numScrolls], errors
}

func (b *LatexToPngBackend) FindMatchingScrolls(query string) ([]ID, int, error) {
	index, err := openExistingIndex()
	if err != nil {
		return []ID{}, 0, err
	}
	defer index.Close()

	newQuery := translatePlusMinusTildePrefixes(query)
	searchResults, err := performQuery(index, newQuery)
	totalMatches := int(searchResults.Total)
	if err != nil {
		if err.Error() == "syntax error" {
			err = errors.Wrapf(err, "invalid query string: '%v'", newQuery)
		} else {
			err = errors.Wrap(err, "perform query")
		}
		return []ID{}, totalMatches, err
	}

	var ids []ID
	for _, match := range searchResults.Hits {
		id := ID(match.ID)
		ids = append(ids, id)
	}

	return ids, totalMatches, nil
}

// IDsForAllScrolls goes through the library directory and creates a list of
// IDs of all available scrolls.
func (b *LatexToPngBackend) IDsForAllScrolls() []ID {
	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		panic(err)
	}

	var ids []ID
	for _, file := range files {
		if !strings.HasSuffix(file.Name(), ".tex") {
			continue
		}
		id := ID(strings.TrimSuffix(file.Name(), ".tex"))
		ids = append(ids, id)
	}
	return ids
}

func (b *LatexToPngBackend) UpdateIndex() error {
	return updateIndex()
}

func (b *LatexToPngBackend) Statistics() (Statistics, error) {
	return computeStatistics()
}

func (b *LatexToPngBackend) LoadScrolls(ids []ID) ([]Scroll, error) {
	result := make([]Scroll, len(ids))
	for i, id := range ids {
		scroll, err := loadAndParseScrollContentByID(id)
		if err != nil {
			return result, err
		}
		result[i] = scroll
	}
	return result, nil
}

// Config holds all the configuration of Alexandria.
var Config = initConfig()

func initConfig() Configuration {
	var config Configuration

	config.Quality = 90
	config.Dpi = 160
	config.MaxResults = 1000
	config.MaxProcs = 4

	dir := os.Getenv("HOME") + "/.alexandria/"

	config.AlexandriaDirectory = dir
	config.KnowledgeDirectory = dir + "library/"
	config.CacheDirectory = dir + "cache/"
	config.TemplateDirectory = dir + "templates/"
	config.TempDirectory = dir + "tmp/"

	return config
}
<commit_msg>start implementing different scroll types<commit_after>// This file contains most of the public interface of Alexandria (the library).
// The remainder is the types defined in datastructures.go.

// This file is part of Alexandria which is released under AGPLv3.
// Copyright (C) 2015-2018 Colin Benner
// See LICENSE or go to https://github.com/yzhs/alexandria/LICENSE for full
// license details.

//go:generate go run contrib/generate_assets.go

package alexandria

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/pkg/errors"
)

// Programm name and version
const (
	NAME    = "Alexandria"
	VERSION = "0.1"
)

type LatexToPngBackend struct {
	renderer latexToPngRenderer
}

type scrollType int

const (
	latexScroll scrollType = iota
	markdownScroll
)

func NewLatexToPngBackend() LatexToPngBackend {
	return LatexToPngBackend{renderer: &xelatexImagemagickRenderer{}}
}

func (b *LatexToPngBackend) RenderAllScrolls() (numScrolls int, errors []error) {
	ids := b.IDsForAllScrolls()
	renderedIDs, errors := b.RenderScrollsByID(ids)
	return len(renderedIDs), errors
}

func (b *LatexToPngBackend) RenderScrollsByID(ids []ID) (renderedScrollIDs []ID, errors []error) {
	renderedScrollIDs = make([]ID, len(ids))
	numScrolls := 0
	for _, id := range ids {
		err := renderScroll(id, b.renderer)
		if err != nil {
			errors = append(errors, fmt.Errorf("rendering %v failed", id))
		} else {
			renderedScrollIDs[numScrolls] = id
			numScrolls++
		}
	}

	return renderedScrollIDs[:numScrolls], errors
}

func (b *LatexToPngBackend) FindMatchingScrolls(query string) ([]ID, int, error) {
	index, err := openExistingIndex()
	if err != nil {
		return []ID{}, 0, err
	}
	defer index.Close()

	newQuery := translatePlusMinusTildePrefixes(query)
	searchResults, err := performQuery(index, newQuery)
	totalMatches := int(searchResults.Total)
	if err != nil {
		if err.Error() == "syntax error" {
			err = errors.Wrapf(err, "invalid query string: '%v'", newQuery)
		} else {
			err = errors.Wrap(err, "perform query")
		}
		return []ID{}, totalMatches, err
	}

	var ids []ID
	for _, match := range searchResults.Hits {
		id := ID(match.ID)
		ids = append(ids, id)
	}

	return ids, totalMatches, nil
}

// IDsForAllScrolls goes through the library directory and creates a list of
// IDs of all available scrolls.
func (b *LatexToPngBackend) IDsForAllScrolls() []ID {
	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		panic(err)
	}

	var ids []ID
	for _, file := range files {
		if !strings.HasSuffix(file.Name(), ".tex") {
			continue
		}
		id := ID(strings.TrimSuffix(file.Name(), ".tex"))
		ids = append(ids, id)
	}
	return ids
}

func (b *LatexToPngBackend) UpdateIndex() error {
	return updateIndex()
}

func (b *LatexToPngBackend) Statistics() (Statistics, error) {
	return computeStatistics()
}

func (b *LatexToPngBackend) LoadScrolls(ids []ID) ([]Scroll, error) {
	result := make([]Scroll, len(ids))
	for i, id := range ids {
		scroll, err := loadAndParseScrollContentByID(id)
		if err != nil {
			return result, err
		}
		result[i] = scroll
	}
	return result, nil
}

// Config holds all the configuration of Alexandria.
var Config = initConfig()

func initConfig() Configuration {
	var config Configuration

	config.Quality = 90
	config.Dpi = 160
	config.MaxResults = 1000
	config.MaxProcs = 4

	dir := os.Getenv("HOME") + "/.alexandria/"

	config.AlexandriaDirectory = dir
	config.KnowledgeDirectory = dir + "library/"
	config.CacheDirectory = dir + "cache/"
	config.TemplateDirectory = dir + "templates/"
	config.TempDirectory = dir + "tmp/"

	return config
}
<|endoftext|>


<commit_before>package lm2

import (
	"bytes"
	"encoding/binary"
	"errors"
	"math/rand"
	"os"
	"sort"
)

const sentinelMagic = 0xDEAD10CC

const cacheSize = 100000

type Collection struct {
	fileHeader
	f     *os.File
	wal   *WAL
	cache *recordCache
}

type fileHeader struct {
	Head          int64
	LastCommit    int64 // Unused for now
	LastLogCommit int64 // Unused for now
}

func (h fileHeader) Bytes() []byte {
	buf := bytes.NewBuffer(nil)
	binary.Write(buf, binary.LittleEndian, h)
	return buf.Bytes()
}

type recordHeader struct {
	Next    int64
	Deleted int64
	KeyLen  uint16
	ValLen  uint32
}

func (h recordHeader) Bytes() []byte {
	buf := bytes.NewBuffer(nil)
	binary.Write(buf, binary.LittleEndian, h)
	return buf.Bytes()
}

type sentinelRecord struct {
	Magic  uint32 // some fixed pattern
	Offset int64  // this record's offset
}

type record struct {
	recordHeader
	Offset int64
	Key    string
	Value  string
}

type recordCache struct {
	cache        map[int64]*record
	dirty        map[int64]bool
	maxKeyRecord *record
	size         int
	preventPurge bool

	c *Collection
}

func newCache(size int) *recordCache {
	return &recordCache{
		cache:        map[int64]*record{},
		dirty:        map[int64]bool{},
		maxKeyRecord: nil,
		size:         size,
	}
}

func (rc *recordCache) findLastLessThan(key string) int64 {
	if rc.maxKeyRecord != nil {
		if rc.maxKeyRecord.Key < key {
			return rc.maxKeyRecord.Offset
		}
	}
	max := ""
	maxOffset := int64(0)

	for offset, record := range rc.cache {
		if record.Key >= key {
			continue
		}
		if record.Key > max {
			max = record.Key
			maxOffset = offset
		}
	}
	return maxOffset
}

func (rc *recordCache) push(rec *record) {
	if rc.maxKeyRecord == nil || rc.maxKeyRecord.Key < rec.Key {
		rc.maxKeyRecord = rec
	} else if rand.Float32() >= 0.04 {
		return
	}
	rc.cache[rec.Offset] = rec
	if len(rc.cache) > rc.size && !rc.preventPurge {
		deletedKey := int64(0)
		for k := range rc.cache {
			if k == rc.maxKeyRecord.Offset {
				continue
			}
			deletedKey = k
			return
		}
		if rc.dirty[deletedKey] {
			// This is a dirty record. Flush changes to disk.
			rc.c.updateRecordHeader(deletedKey, rc.cache[deletedKey].recordHeader)
			delete(rc.dirty, deletedKey)
		}
		delete(rc.cache, deletedKey)
	}
}

func (rc *recordCache) forcePush(rec *record) {
	if rc.maxKeyRecord == nil || rc.maxKeyRecord.Key < rec.Key {
		rc.maxKeyRecord = rec
	}
	rc.cache[rec.Offset] = rec
}

func (c *Collection) readRecord(offset int64) (*record, error) {
	if offset == 0 {
		return nil, errors.New("lm2: invalid record offset 0")
	}

	if rec := c.cache.cache[offset]; rec != nil {
		return rec, nil
	}

	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return nil, err
	}

	header := recordHeader{}
	err = binary.Read(c.f, binary.LittleEndian, &header)
	if err != nil {
		return nil, err
	}

	keyValBuf := make([]byte, int(header.KeyLen)+int(header.ValLen))
	_, err = c.f.Read(keyValBuf)
	if err != nil {
		return nil, err
	}

	key := string(keyValBuf[:int(header.KeyLen)])
	value := string(keyValBuf[int(header.KeyLen):])

	rec := &record{
		recordHeader: header,
		Offset:       offset,
		Key:          key,
		Value:        value,
	}

	c.cache.push(rec)

	return rec, nil
}

func (c *Collection) setRecordNext(offset int64, next int64) error {
	if rec := c.cache.cache[offset]; rec != nil {
		rec.recordHeader.Next = next
		c.cache.dirty[offset] = true
		return nil
	}
	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return err
	}
	return binary.Write(c.f, binary.LittleEndian, next)
}

func (c *Collection) nextRecord(rec *record) *record {
	if rec == nil {
		return nil
	}
	nextRec, err := c.readRecord(rec.Next)
	if err != nil {
		return nil
	}
	return nextRec
}

func (c *Collection) writeRecord(rec *record) (int64, error) {
	offset, err := c.f.Seek(0, 2)
	if err != nil {
		return 0, err
	}

	rec.KeyLen = uint16(len(rec.Key))
	rec.ValLen = uint32(len(rec.Value))

	err = binary.Write(c.f, binary.LittleEndian, rec.recordHeader)
	if err != nil {
		return 0, err
	}

	_, err = c.f.WriteString(rec.Key)
	if err != nil {
		return 0, err
	}
	_, err = c.f.WriteString(rec.Value)
	if err != nil {
		return 0, err
	}

	rec.Offset = offset
	return offset, nil
}

func (c *Collection) writeSentinel() (int64, error) {
	offset, err := c.f.Seek(0, 2)
	if err != nil {
		return 0, err
	}
	sentinel := sentinelRecord{
		Magic:  sentinelMagic,
		Offset: offset,
	}
	err = binary.Write(c.f, binary.LittleEndian, sentinel)
	if err != nil {
		return 0, err
	}
	return offset + 12, nil
}

func (c *Collection) updateRecordHeader(offset int64, header recordHeader) error {
	if rec := c.cache.cache[offset]; rec != nil {
		rec.recordHeader = header
		c.cache.dirty[offset] = true
		return nil
	}
	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return err
	}
	return binary.Write(c.f, binary.LittleEndian, header)
}

func (c *Collection) findLastLessThanOrEqual(key string) (int64, error) {
	offset := int64(0)

	if c.Head == 0 {
		// Empty collection.
		return 0, nil
	}

	// read the head
	rec, err := c.readRecord(c.Head)
	if err != nil {
		return 0, err
	}
	if rec.Key > key { // we have a new head
		return 0, nil
	}

	cacheResult := c.cache.findLastLessThan(key)
	if cacheResult != 0 {
		rec, err = c.readRecord(cacheResult)
		if err != nil {
			return 0, err
		}
	}

	offset = rec.Offset

	for rec != nil {
		if rec.Key > key {
			break
		}
		offset = rec.Offset
		rec = c.nextRecord(rec)
	}

	return offset, nil
}

// Update atomically and durably applies a WriteBatch (a set of updates) to the collection.
func (c *Collection) Update(wb *WriteBatch) error {
	// Clean up WriteBatch.
	wb.cleanup()

	// Find and load records that will be modified into the cache.
	recordsToLoad := []int64{}

	keys := []string{}
	for key := range wb.sets {
		keys = append(keys, key)
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		if offset > 0 {
			recordsToLoad = append(recordsToLoad, offset)
		}
	}

	// Prevent cache purges.
	c.cache.preventPurge = true
	defer func() { c.cache.preventPurge = false }()

	for _, offset := range recordsToLoad {
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		c.cache.forcePush(rec)
	}

	// Sort keys to be inserted.
	sort.Strings(keys)

	// NOTE: we shouldn't be reading any more records after this point.
	// TODO: assert it.

	walEntry := newWALEntry()
	walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))

	// Append new records with the appropriate "next" pointers.

	overwrittenRecords := []int64{}
	for _, key := range keys {
		value := wb.sets[key]

		// Find last less than.
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		if offset == 0 {
			// Head.
			rec := &record{
				recordHeader: recordHeader{
					Next: c.Head,
				},
				Key:   key,
				Value: value,
			}
			newRecordOffset, err := c.writeRecord(rec)
			if err != nil {
				return err
			}
			c.Head = newRecordOffset
			walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))
			continue
		}
		prevRec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		rec := &record{
			recordHeader: recordHeader{
				Next: prevRec.Next,
			},
			Key:   key,
			Value: value,
		}
		newRecordOffset, err := c.writeRecord(rec)
		prevRec.Next = newRecordOffset
		walEntry.Push(newWALRecord(prevRec.Offset, prevRec.recordHeader.Bytes()))
		if prevRec.Key == key {
			overwrittenRecords = append(overwrittenRecords, prevRec.Offset)
		}
		c.cache.forcePush(rec)
		c.cache.forcePush(prevRec)
	}

	// Write sentinel record.

	currentOffset, err := c.writeSentinel()
	if err != nil {
		return err
	}

	// fsync data file.
	err = c.f.Sync()
	if err != nil {
		return err
	}

	// Mark deleted and overwritten records as "deleted" at sentinel offset.
	// (This happens in memory.)

	for key := range wb.deletes {
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		if rec.Deleted == 0 {
			rec.Deleted = currentOffset
			walEntry.Push(newWALRecord(rec.Offset, rec.recordHeader.Bytes()))
		}
	}

	for _, offset := range overwrittenRecords {
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		rec.Deleted = currentOffset
		walEntry.Push(newWALRecord(rec.Offset, rec.recordHeader.Bytes()))
	}

	// ^ record changes should have been serialized + buffered. Write those entries
	// out to the WAL.
	c.LastCommit = currentOffset
	walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))
	logCommit, err := c.wal.Append(walEntry)
	if err != nil {
		return err
	}

	c.LastLogCommit = logCommit

	// Update + fsync data file header.

	for _, walRec := range walEntry.records {
		n, err := c.f.WriteAt(walRec.Data, walRec.Offset)
		if err != nil {
			return err
		}
		if int64(n) != walRec.Size {
			return errors.New("lm2: incomplete data write")
		}
	}

	return c.f.Sync()
}

func NewCollection(file string) (*Collection, error) {
	f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}
	err = f.Truncate(0)
	if err != nil {
		f.Close()
		return nil, err
	}

	wal, err := newWAL(file + ".wal")
	if err != nil {
		f.Close()
		return nil, err
	}

	c := &Collection{
		f:     f,
		wal:   wal,
		cache: newCache(cacheSize),
	}
	c.cache.c = c

	// write file header
	c.fileHeader.Head = 0
	c.f.Seek(0, 0)
	err = binary.Write(c.f, binary.LittleEndian, c.fileHeader)
	if err != nil {
		c.f.Close()
		c.wal.Close()
		return nil, err
	}
	return c, nil
}

func OpenCollection(file string) (*Collection, error) {
	f, err := os.OpenFile(file, os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}

	wal, err := newWAL(file + ".wal")
	if err != nil {
		f.Close()
		return nil, err
	}

	c := &Collection{
		f:     f,
		wal:   wal,
		cache: newCache(cacheSize),
	}
	c.cache.c = c

	// read file header
	c.f.Seek(0, 0)
	err = binary.Read(c.f, binary.LittleEndian, &c.fileHeader)
	if err != nil {
		c.f.Close()
		c.wal.Close()
		return nil, err
	}
	return c, nil
}

func (c *Collection) Close() {
	c.f.Close()
	c.wal.Close()
}
<commit_msg>disallow dirty records in cache<commit_after>package lm2

import (
	"bytes"
	"encoding/binary"
	"errors"
	"math/rand"
	"os"
	"sort"
)

const sentinelMagic = 0xDEAD10CC

const cacheSize = 100000

type Collection struct {
	fileHeader
	f     *os.File
	wal   *WAL
	cache *recordCache
}

type fileHeader struct {
	Head          int64
	LastCommit    int64 // Unused for now
	LastLogCommit int64 // Unused for now
}

func (h fileHeader) Bytes() []byte {
	buf := bytes.NewBuffer(nil)
	binary.Write(buf, binary.LittleEndian, h)
	return buf.Bytes()
}

type recordHeader struct {
	Next    int64
	Deleted int64
	KeyLen  uint16
	ValLen  uint32
}

func (h recordHeader) Bytes() []byte {
	buf := bytes.NewBuffer(nil)
	binary.Write(buf, binary.LittleEndian, h)
	return buf.Bytes()
}

type sentinelRecord struct {
	Magic  uint32 // some fixed pattern
	Offset int64  // this record's offset
}

type record struct {
	recordHeader
	Offset int64
	Key    string
	Value  string
}

type recordCache struct {
	cache        map[int64]*record
	maxKeyRecord *record
	size         int
	preventPurge bool

	c *Collection
}

func newCache(size int) *recordCache {
	return &recordCache{
		cache:        map[int64]*record{},
		maxKeyRecord: nil,
		size:         size,
	}
}

func (rc *recordCache) findLastLessThan(key string) int64 {
	if rc.maxKeyRecord != nil {
		if rc.maxKeyRecord.Key < key {
			return rc.maxKeyRecord.Offset
		}
	}
	max := ""
	maxOffset := int64(0)

	for offset, record := range rc.cache {
		if record.Key >= key {
			continue
		}
		if record.Key > max {
			max = record.Key
			maxOffset = offset
		}
	}
	return maxOffset
}

func (rc *recordCache) push(rec *record) {
	if rc.maxKeyRecord == nil || rc.maxKeyRecord.Key < rec.Key {
		rc.maxKeyRecord = rec
	} else if rand.Float32() >= 0.04 {
		return
	}
	rc.cache[rec.Offset] = rec
	if len(rc.cache) > rc.size && !rc.preventPurge {
		deletedKey := int64(0)
		for k := range rc.cache {
			if k == rc.maxKeyRecord.Offset {
				continue
			}
			deletedKey = k
			return
		}
		delete(rc.cache, deletedKey)
	}
}

func (rc *recordCache) forcePush(rec *record) {
	if rc.maxKeyRecord == nil || rc.maxKeyRecord.Key < rec.Key {
		rc.maxKeyRecord = rec
	}
	rc.cache[rec.Offset] = rec
}

func (c *Collection) readRecord(offset int64) (*record, error) {
	if offset == 0 {
		return nil, errors.New("lm2: invalid record offset 0")
	}

	if rec := c.cache.cache[offset]; rec != nil {
		return rec, nil
	}

	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return nil, err
	}

	header := recordHeader{}
	err = binary.Read(c.f, binary.LittleEndian, &header)
	if err != nil {
		return nil, err
	}

	keyValBuf := make([]byte, int(header.KeyLen)+int(header.ValLen))
	_, err = c.f.Read(keyValBuf)
	if err != nil {
		return nil, err
	}

	key := string(keyValBuf[:int(header.KeyLen)])
	value := string(keyValBuf[int(header.KeyLen):])

	rec := &record{
		recordHeader: header,
		Offset:       offset,
		Key:          key,
		Value:        value,
	}

	c.cache.push(rec)

	return rec, nil
}

func (c *Collection) setRecordNext(offset int64, next int64) error {
	if rec := c.cache.cache[offset]; rec != nil {
		rec.recordHeader.Next = next
		return nil
	}
	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return err
	}
	return binary.Write(c.f, binary.LittleEndian, next)
}

func (c *Collection) nextRecord(rec *record) *record {
	if rec == nil {
		return nil
	}
	nextRec, err := c.readRecord(rec.Next)
	if err != nil {
		return nil
	}
	return nextRec
}

func (c *Collection) writeRecord(rec *record) (int64, error) {
	offset, err := c.f.Seek(0, 2)
	if err != nil {
		return 0, err
	}

	rec.KeyLen = uint16(len(rec.Key))
	rec.ValLen = uint32(len(rec.Value))

	err = binary.Write(c.f, binary.LittleEndian, rec.recordHeader)
	if err != nil {
		return 0, err
	}

	_, err = c.f.WriteString(rec.Key)
	if err != nil {
		return 0, err
	}
	_, err = c.f.WriteString(rec.Value)
	if err != nil {
		return 0, err
	}

	rec.Offset = offset
	return offset, nil
}

func (c *Collection) writeSentinel() (int64, error) {
	offset, err := c.f.Seek(0, 2)
	if err != nil {
		return 0, err
	}
	sentinel := sentinelRecord{
		Magic:  sentinelMagic,
		Offset: offset,
	}
	err = binary.Write(c.f, binary.LittleEndian, sentinel)
	if err != nil {
		return 0, err
	}
	return offset + 12, nil
}

func (c *Collection) updateRecordHeader(offset int64, header recordHeader) error {
	if rec := c.cache.cache[offset]; rec != nil {
		rec.recordHeader = header
		return nil
	}
	_, err := c.f.Seek(offset, 0)
	if err != nil {
		return err
	}
	return binary.Write(c.f, binary.LittleEndian, header)
}

func (c *Collection) findLastLessThanOrEqual(key string) (int64, error) {
	offset := int64(0)

	if c.Head == 0 {
		// Empty collection.
		return 0, nil
	}

	// read the head
	rec, err := c.readRecord(c.Head)
	if err != nil {
		return 0, err
	}
	if rec.Key > key { // we have a new head
		return 0, nil
	}

	cacheResult := c.cache.findLastLessThan(key)
	if cacheResult != 0 {
		rec, err = c.readRecord(cacheResult)
		if err != nil {
			return 0, err
		}
	}

	offset = rec.Offset

	for rec != nil {
		if rec.Key > key {
			break
		}
		offset = rec.Offset
		rec = c.nextRecord(rec)
	}

	return offset, nil
}

// Update atomically and durably applies a WriteBatch (a set of updates) to the collection.
func (c *Collection) Update(wb *WriteBatch) error {
	// Clean up WriteBatch.
	wb.cleanup()

	// Find and load records that will be modified into the cache.
	recordsToLoad := []int64{}

	keys := []string{}
	for key := range wb.sets {
		keys = append(keys, key)
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		if offset > 0 {
			recordsToLoad = append(recordsToLoad, offset)
		}
	}

	// Prevent cache purges.
	c.cache.preventPurge = true
	defer func() { c.cache.preventPurge = false }()

	for _, offset := range recordsToLoad {
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		c.cache.forcePush(rec)
	}

	// Sort keys to be inserted.
	sort.Strings(keys)

	// NOTE: we shouldn't be reading any more records after this point.
	// TODO: assert it.

	walEntry := newWALEntry()
	walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))

	// Append new records with the appropriate "next" pointers.

	overwrittenRecords := []int64{}
	for _, key := range keys {
		value := wb.sets[key]

		// Find last less than.
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		if offset == 0 {
			// Head.
			rec := &record{
				recordHeader: recordHeader{
					Next: c.Head,
				},
				Key:   key,
				Value: value,
			}
			newRecordOffset, err := c.writeRecord(rec)
			if err != nil {
				return err
			}
			c.Head = newRecordOffset
			walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))
			continue
		}
		prevRec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		rec := &record{
			recordHeader: recordHeader{
				Next: prevRec.Next,
			},
			Key:   key,
			Value: value,
		}
		newRecordOffset, err := c.writeRecord(rec)
		prevRec.Next = newRecordOffset
		walEntry.Push(newWALRecord(prevRec.Offset, prevRec.recordHeader.Bytes()))
		if prevRec.Key == key {
			overwrittenRecords = append(overwrittenRecords, prevRec.Offset)
		}
		c.cache.forcePush(rec)
		c.cache.forcePush(prevRec)
	}

	// Write sentinel record.

	currentOffset, err := c.writeSentinel()
	if err != nil {
		return err
	}

	// fsync data file.
	err = c.f.Sync()
	if err != nil {
		return err
	}

	// Mark deleted and overwritten records as "deleted" at sentinel offset.
	// (This happens in memory.)

	for key := range wb.deletes {
		offset, err := c.findLastLessThanOrEqual(key)
		if err != nil {
			return err
		}
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		if rec.Deleted == 0 {
			rec.Deleted = currentOffset
			walEntry.Push(newWALRecord(rec.Offset, rec.recordHeader.Bytes()))
		}
	}

	for _, offset := range overwrittenRecords {
		rec, err := c.readRecord(offset)
		if err != nil {
			return err
		}
		rec.Deleted = currentOffset
		walEntry.Push(newWALRecord(rec.Offset, rec.recordHeader.Bytes()))
	}

	// ^ record changes should have been serialized + buffered. Write those entries
	// out to the WAL.
	c.LastCommit = currentOffset
	walEntry.Push(newWALRecord(0, c.fileHeader.Bytes()))
	logCommit, err := c.wal.Append(walEntry)
	if err != nil {
		return err
	}

	c.LastLogCommit = logCommit

	// Update + fsync data file header.

	for _, walRec := range walEntry.records {
		n, err := c.f.WriteAt(walRec.Data, walRec.Offset)
		if err != nil {
			return err
		}
		if int64(n) != walRec.Size {
			return errors.New("lm2: incomplete data write")
		}
	}

	return c.f.Sync()
}

func NewCollection(file string) (*Collection, error) {
	f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}
	err = f.Truncate(0)
	if err != nil {
		f.Close()
		return nil, err
	}

	wal, err := newWAL(file + ".wal")
	if err != nil {
		f.Close()
		return nil, err
	}

	c := &Collection{
		f:     f,
		wal:   wal,
		cache: newCache(cacheSize),
	}
	c.cache.c = c

	// write file header
	c.fileHeader.Head = 0
	c.f.Seek(0, 0)
	err = binary.Write(c.f, binary.LittleEndian, c.fileHeader)
	if err != nil {
		c.f.Close()
		c.wal.Close()
		return nil, err
	}
	return c, nil
}

func OpenCollection(file string) (*Collection, error) {
	f, err := os.OpenFile(file, os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}

	wal, err := newWAL(file + ".wal")
	if err != nil {
		f.Close()
		return nil, err
	}

	c := &Collection{
		f:     f,
		wal:   wal,
		cache: newCache(cacheSize),
	}
	c.cache.c = c

	// read file header
	c.f.Seek(0, 0)
	err = binary.Read(c.f, binary.LittleEndian, &c.fileHeader)
	if err != nil {
		c.f.Close()
		c.wal.Close()
		return nil, err
	}
	return c, nil
}

func (c *Collection) Close() {
	c.f.Close()
	c.wal.Close()
}
<|endoftext|>


<commit_before>package main

import "log"

type loglevel bool

const debug loglevel = true
const info loglevel = true

func (d loglevel) Printf(format string, args ...interface{}) {
	if d {
		log.Printf(format, args...)
	}
}

func (d loglevel) Println(args ...interface{}) {
	if d {
		log.Println(args...)
	}
}
<commit_msg>Colorize debug log.<commit_after>package main

// This trick is learnt from a post by Rob Pike
// https://groups.google.com/d/msg/golang-nuts/gU7oQGoCkmg/j3nNxuS2O_sJ

import (
	"log"
	"os"
)

type infoLogging bool
type debugLogging bool

const debug debugLogging = true
const info infoLogging = false

var debugLog = log.New(os.Stderr, "\033[34m[DEBUG ", log.LstdFlags)

func (d debugLogging) Printf(format string, args ...interface{}) {
	if d {
		debugLog.Printf("]\033[0m "+format, args...)
	}
}

func (d infoLogging) Printf(format string, args ...interface{}) {
	if d {
		log.Printf(format, args...)
	}
}

func (d infoLogging) Println(args ...interface{}) {
	if d {
		log.Println(args...)
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed under the MIT license that can be found in the LICENSE file.

package logger

import (
	"errors"
	"fmt"
	"runtime"
	"time"
)

const (
	defaultStackSize        = 8192
	defaultEventChannelSize = 1024
)

// EventWriter is the backend of the logger package. It takes events and writes
// them to a storage, this can for example be a file, a database or the
// terminal.
type EventWriter interface {
	// Write is called every time a Log operation is called. This call should
	// block until the event is written. Advised is to buffer the events. All
	// calls to Write are synchronous.
	//
	// If an error is returned the event is expected to NOT have been written. The
	// event will written again after HandleError is called. If the EventWriter
	// returns 5 errors in a row the EventWriter is consided to be bad and will be
	// removed from the list of event writers. HandlerError will be called with
	// ErrBadEventWriter if this happens.
	Write(Event) error

	// HandleError is called every time Write returns an error.
	HandleError(error)

	// Close is called on the EventWriter once Close() (on the package) is called.
	Close() error
}

var (
	eventChannel       chan Event
	eventChannelClosed chan struct{}
	eventWriters       []EventWriter
	started            bool
)

// Start starts the logger package and enables writing to the given
// EventWriters.
func Start(ews ...EventWriter) {
	if started {
		panic("logger: can only Start once")
	}
	started = true

	if len(ews) < 1 {
		panic("logger: need atleast a single EventWriter to write to")
	}

	eventChannel = make(chan Event, defaultEventChannelSize)
	eventChannelClosed = make(chan struct{}, 1) // Can't block.
	eventWriters = ews
	go eventsWriter()
}

var badEventWriterErr = errors.New("EventWriter is bad, more then 5 faulty writes, EventWriter will be dropped")

// Needs to be run in it's own goroutine, it blocks until eventChannel is
// closed. After eventChannel is closed it sends a signal to eventChannelClosed.
// todo: benchmark and improve performance. Perhaps by passing the event writers
// and event channels as reference to this function.
func eventsWriter() {
	// Create a copy of the eventWriters which we can modify. This way we can
	// drop writers from the slice if they return too many write errors.
	var ews = make([]EventWriter, len(eventWriters))
	if n := copy(ews, eventWriters); n != len(eventWriters) {
		panic("Couldn't copy all the EventWriters")
	}

	// Slice of event that tried to be written, but returned an error. This is a
	// per EventWriter slice.
	var badWrites = make([][]Event, len(eventWriters))

	for event := range eventChannel {
		for i, eventWriter := range ews {
			if l := len(badWrites[i]); l != 0 {
				// After 5 bad writes we drop the EventWriter from the slice of
				// EventWriters, aswell as badWrites.
				if l == 5 {
					eventWriter.HandleError(badEventWriterErr)
					ews = append(ews[:i], ews[i+1:]...)
					badWrites = append(badWrites[:i], badWrites[i+1:]...)
					continue
				}

				// Try to rewite a previously failed write.
				if err := eventWriter.Write(badWrites[i][0]); err != nil {
					eventWriter.HandleError(err)
				} else {
					badWrites[i] = badWrites[i][1:]
				}
			}

			err := eventWriter.Write(event)
			if err != nil {
				eventWriter.HandleError(err)
				badWrites[i] = append(badWrites[i], event)
			}
		}
	}

	eventChannelClosed <- struct{}{}
}

// Close stops all the Log Operations from being usable, and they will panic if
// used after close is called. It also closes all EventWriters and returns the
// first returned error. The EventWriters are closed in the order they are
// passed to Start.
func Close() error {
	close(eventChannel)
	<-eventChannelClosed

	var err error
	for _, eventWriter := range eventWriters {
		er := eventWriter.Close()
		if er != nil && err == nil {
			err = er
		}
	}
	return err
}

// Subbed for testing.
var now = func() time.Time {
	return time.Now()
}

// Debug logs a debug message.
func Debug(tags Tags, msg string) {
	eventChannel <- Event{DebugEvent, now(), tags, msg, nil}
}

// Debugf is a formatted function of Debug.
func Debugf(tags Tags, format string, v ...interface{}) {
	Debug(tags, fmt.Sprintf(format, v...))
}

// Info logs an informational message.
func Info(tags Tags, msg string) {
	eventChannel <- Event{InfoEvent, now(), tags, msg, nil}
}

// Infof is a formatted function of Info.
func Infof(tags Tags, format string, v ...interface{}) {
	Info(tags, fmt.Sprintf(format, v...))
}

// Warn logs a warning message.
func Warn(tags Tags, msg string) {
	eventChannel <- Event{WarnEvent, now(), tags, msg, nil}
}

// Warnf is a formatted function of Warn.
func Warnf(tags Tags, format string, v ...interface{}) {
	Warn(tags, fmt.Sprintf(format, v...))
}

// Error logs an error message.
func Error(tags Tags, err error) {
	eventChannel <- Event{ErrorEvent, now(), tags, err.Error(), nil}
}

// Errorf is a formatted function of Error.
func Errorf(tags Tags, format string, v ...interface{}) {
	Error(tags, fmt.Errorf(format, v...))
}

// Fatal logs a recovered error which could have killed the application. Fatal
// adds a stack trace (type []byte) as Event.Data.
func Fatal(tags Tags, recv interface{}) {
	stackTrace := make([]byte, defaultStackSize)
	n := runtime.Stack(stackTrace, false)
	stackTrace = stackTrace[:n]

	msg := interfaceToString(recv)
	eventChannel <- Event{FatalEvent, now(), tags, msg, stackTrace}
}

// Thumbstone indicates a function is still used in production. When developing
// software it's possible to introduce dead code with updates and new features.
// If a function is being suspected of being dead (not used) in production, add
// a call to Thumbstone and check the production logs to see if you're right.
//
// The caller of the (possibly) dead function will be put in the message, using
// the following format:
//	Function functionName called by callerFunctionName, from file /path/to/file on line lineNumber
// For example:
//	Function myFunction called by main.main, from file /main.go on line 20
func Thumbstone(tags Tags, functionName string) {
	var msg string
	if pc, file, line, ok := runtime.Caller(2); ok {
		fn := runtime.FuncForPC(pc)
		msg = fmt.Sprintf("Function %s called by %s, from file %s on line %d",
			functionName, fn.Name(), file, line)
	} else {
		msg = "Function " + functionName + " called from unkown location"
	}

	eventChannel <- Event{ThumbEvent, now(), tags, msg, nil}
}

// Log logs a custom created event.
//
// Note: the timestamp doesn't need to be set, because it will be set by Log.
func Log(event Event) {
	event.Timestamp = now()
	eventChannel <- event
}

// interfaceToString converts a interface{} variable to a string.
func interfaceToString(value interface{}) string {
	switch v := value.(type) {
	case string:
		return v
	case fmt.Stringer:
		return v.String()
	case []byte:
		return string(v)
	case error:
		return v.Error()
	}
	return fmt.Sprintf("%v", value)
}
<commit_msg>Create eventChannel and eventChannelClosed at startup<commit_after>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed under the MIT license that can be found in the LICENSE file.

package logger

import (
	"errors"
	"fmt"
	"runtime"
	"time"
)

const (
	defaultStackSize        = 8192
	defaultEventChannelSize = 1024
)

// EventWriter is the backend of the logger package. It takes events and writes
// them to a storage, this can for example be a file, a database or the
// terminal.
type EventWriter interface {
	// Write is called every time a Log operation is called. This call should
	// block until the event is written. Advised is to buffer the events. All
	// calls to Write are synchronous.
	//
	// If an error is returned the event is expected to NOT have been written. The
	// event will written again after HandleError is called. If the EventWriter
	// returns 5 errors in a row the EventWriter is consided to be bad and will be
	// removed from the list of event writers. HandlerError will be called with
	// ErrBadEventWriter if this happens.
	Write(Event) error

	// HandleError is called every time Write returns an error.
	HandleError(error)

	// Close is called on the EventWriter once Close() (on the package) is called.
	Close() error
}

var (
	eventChannel       = make(chan Event, defaultEventChannelSize)
	eventChannelClosed = make(chan struct{}, 1) // Can't block.
	eventWriters       []EventWriter
	started            bool
)

// Start starts the logger package and enables writing to the given
// EventWriters.
func Start(ews ...EventWriter) {
	if started {
		panic("logger: can only Start once")
	}
	started = true

	if len(ews) < 1 {
		panic("logger: need atleast a single EventWriter to write to")
	}

	eventWriters = ews
	go eventsWriter()
}

var badEventWriterErr = errors.New("EventWriter is bad, more then 5 faulty writes, EventWriter will be dropped")

// Needs to be run in it's own goroutine, it blocks until eventChannel is
// closed. After eventChannel is closed it sends a signal to eventChannelClosed.
// todo: benchmark and improve performance. Perhaps by passing the event writers
// and event channels as reference to this function.
func eventsWriter() {
	// Create a copy of the eventWriters which we can modify. This way we can
	// drop writers from the slice if they return too many write errors.
	var ews = make([]EventWriter, len(eventWriters))
	if n := copy(ews, eventWriters); n != len(eventWriters) {
		panic("Couldn't copy all the EventWriters")
	}

	// Slice of event that tried to be written, but returned an error. This is a
	// per EventWriter slice.
	var badWrites = make([][]Event, len(eventWriters))

	for event := range eventChannel {
		for i, eventWriter := range ews {
			if l := len(badWrites[i]); l != 0 {
				// After 5 bad writes we drop the EventWriter from the slice of
				// EventWriters, aswell as badWrites.
				if l == 5 {
					eventWriter.HandleError(badEventWriterErr)
					ews = append(ews[:i], ews[i+1:]...)
					badWrites = append(badWrites[:i], badWrites[i+1:]...)
					continue
				}

				// Try to rewite a previously failed write.
				if err := eventWriter.Write(badWrites[i][0]); err != nil {
					eventWriter.HandleError(err)
				} else {
					badWrites[i] = badWrites[i][1:]
				}
			}

			err := eventWriter.Write(event)
			if err != nil {
				eventWriter.HandleError(err)
				badWrites[i] = append(badWrites[i], event)
			}
		}
	}

	eventChannelClosed <- struct{}{}
}

// Close stops all the Log Operations from being usable, and they will panic if
// used after close is called. It also closes all EventWriters and returns the
// first returned error. The EventWriters are closed in the order they are
// passed to Start.
func Close() error {
	close(eventChannel)
	<-eventChannelClosed

	var err error
	for _, eventWriter := range eventWriters {
		er := eventWriter.Close()
		if er != nil && err == nil {
			err = er
		}
	}
	return err
}

// Subbed for testing.
var now = func() time.Time {
	return time.Now()
}

// Debug logs a debug message.
func Debug(tags Tags, msg string) {
	eventChannel <- Event{DebugEvent, now(), tags, msg, nil}
}

// Debugf is a formatted function of Debug.
func Debugf(tags Tags, format string, v ...interface{}) {
	Debug(tags, fmt.Sprintf(format, v...))
}

// Info logs an informational message.
func Info(tags Tags, msg string) {
	eventChannel <- Event{InfoEvent, now(), tags, msg, nil}
}

// Infof is a formatted function of Info.
func Infof(tags Tags, format string, v ...interface{}) {
	Info(tags, fmt.Sprintf(format, v...))
}

// Warn logs a warning message.
func Warn(tags Tags, msg string) {
	eventChannel <- Event{WarnEvent, now(), tags, msg, nil}
}

// Warnf is a formatted function of Warn.
func Warnf(tags Tags, format string, v ...interface{}) {
	Warn(tags, fmt.Sprintf(format, v...))
}

// Error logs an error message.
func Error(tags Tags, err error) {
	eventChannel <- Event{ErrorEvent, now(), tags, err.Error(), nil}
}

// Errorf is a formatted function of Error.
func Errorf(tags Tags, format string, v ...interface{}) {
	Error(tags, fmt.Errorf(format, v...))
}

// Fatal logs a recovered error which could have killed the application. Fatal
// adds a stack trace (type []byte) as Event.Data.
func Fatal(tags Tags, recv interface{}) {
	stackTrace := make([]byte, defaultStackSize)
	n := runtime.Stack(stackTrace, false)
	stackTrace = stackTrace[:n]

	msg := interfaceToString(recv)
	eventChannel <- Event{FatalEvent, now(), tags, msg, stackTrace}
}

// Thumbstone indicates a function is still used in production. When developing
// software it's possible to introduce dead code with updates and new features.
// If a function is being suspected of being dead (not used) in production, add
// a call to Thumbstone and check the production logs to see if you're right.
//
// The caller of the (possibly) dead function will be put in the message, using
// the following format:
//	Function functionName called by callerFunctionName, from file /path/to/file on line lineNumber
// For example:
//	Function myFunction called by main.main, from file /main.go on line 20
func Thumbstone(tags Tags, functionName string) {
	var msg string
	if pc, file, line, ok := runtime.Caller(2); ok {
		fn := runtime.FuncForPC(pc)
		msg = fmt.Sprintf("Function %s called by %s, from file %s on line %d",
			functionName, fn.Name(), file, line)
	} else {
		msg = "Function " + functionName + " called from unkown location"
	}

	eventChannel <- Event{ThumbEvent, now(), tags, msg, nil}
}

// Log logs a custom created event.
//
// Note: the timestamp doesn't need to be set, because it will be set by Log.
func Log(event Event) {
	event.Timestamp = now()
	eventChannel <- event
}

// interfaceToString converts a interface{} variable to a string.
func interfaceToString(value interface{}) string {
	switch v := value.(type) {
	case string:
		return v
	case fmt.Stringer:
		return v.String()
	case []byte:
		return string(v)
	case error:
		return v.Error()
	}
	return fmt.Sprintf("%v", value)
}
<|endoftext|>


<commit_before>package accesslog

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
	"unicode"
)

// optFunc is a type of function that can add options to the option struct during initialization
type optFunc func(*opt)

// opt is the struct that holds the options for logging.
type opt struct {
	Out  io.Writer
	Time time.Time
}

// newOpt returns a new struct to hold options, with the default output to stdout.
func newOpt() *opt {
	o := new(opt)
	o.Out = os.Stdout
	return o
}

// WithOutput sets the io.Writer output for the log file.
func WithOutput(out io.Writer) optFunc {
	return func(o *opt) {
		o.Out = out
	}
}

// alog is the internal struct that will hold the status and number of bytes written
type alog struct {
	http.ResponseWriter

	status int
	wLen   int

	start time.Time
}

// WriteHeader intercepts the http.ResponseWriter WriteHeader method so we can save the status to display later
func (a *alog) WriteHeader(i int) {
	if a.status == 0 {
		a.status = i
	}
	a.ResponseWriter.WriteHeader(i)
}

// Write intercepts the http.ResponseWriter Write method so we can capture the bytes written
func (a *alog) Write(p []byte) (n int, err error) {
	if a.status == 0 {
		a.status = http.StatusOK
	}
	n, err = a.ResponseWriter.Write(p)
	a.wLen += n
	return
}

// startTime sets the start time to calculate the elapsed time for the %D directive
func (a *alog) startTime() {
	a.start = time.Now()
}

const (
	ApacheCommonLogFormat   = "%h %l %u %t \"%r\" %>s %b"                                    // The Common Log directives
	ApacheCombinedLogFormat = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" // The Combined Log directives
)

// ApacheCommonLog will log HTTP requests using the Apache Common Log format
var ApacheCommonLog = FormatWith(ApacheCommonLogFormat)

// ApacheCombinedLog will log HTTP requests using the Apache Combined Log format
var ApacheCombinedLog = FormatWith(ApacheCombinedLogFormat)

// convertTimeFormat converts strftime formatting directives to a go time.Time format
func convertTimeFormat(t time.Time, s string) string {
	m := map[rune]string{
		'a': "Mon",
		'A': "Monday",
		'b': "Jan",
		'B': "January",
		'c': "?",
		'C': "06",
		'd': "02",
		'D': "01/02/06",
		'e': "_2",
		'E': "?",
		'F': "2006-01-02",
		'G': "%d",
		'g': "%s",
		'h': "Jan",
		'H': "15",
		'I': "3",
		'j': "%d",
		'k': "•15",
		'l': "_3",
		'm': "01",
		'M': "04",
		'n': "\n",
		'O': "?",
		'p': "PM",
		'P': "pm",
		'r': "03:04:05 PM",
		'R': "15:04",
		's': "%d",
		'S': "05",
		't': "\t",
		'T': "15:04:05",
		'u': "%d",
		'U': "?",
		'V': "%d",
		'w': "%d",
		'W': "?",
		'x': "?",
		'X': "?",
		'y': "06",
		'Y': "2006",
		'z': "-700",
		'Z': "MST",
		'+': "?",
		'%': "%%",
	}

	var i []interface{}
	var x bool
	buf := new(bytes.Buffer)
	for _, r := range s {
		if r == '%' {
			x = true
			continue
		}
		if x {
			x = false
			if val, ok := m[r]; ok {
				if val == "%s" || val == "%d" {
					switch r {
					case 'G':
						y, _ := t.ISOWeek()
						i = append(i, y)
					case 'g':
						y, _ := t.ISOWeek()
						i = append(i, strconv.Itoa(y)[2:])
					case 'j':
						i = append(i, t.YearDay())
					case 's':
						i = append(i, t.Unix())
					case 'u':
						w := t.Weekday()
						if w == 0 {
							w = 7
						}
						i = append(i, w)
					case 'V':
						_, w := t.ISOWeek()
						i = append(i, w)
					case 'w':
						i = append(i, t.Weekday())
					}
				}
				buf.WriteString(val)
				continue
			}
			buf.WriteRune('%')
		}
		buf.WriteRune(r)
	}
	f := t.Format(buf.String())
	if len(i) > 0 {
		f = fmt.Sprintf(f, i...)
	}
	buf.Reset()
	return f
}

// logfmt the type that will hold all of the runtime formating
type logfmt struct {
	ti time.Time
	rq *http.Request
	al *alog

	h, u, t, q, e, ws, wl string
}

func (lf *logfmt) withTime(o *opt) *logfmt {
	if !o.Time.IsZero() {
		lf.ti = o.Time
		return lf
	}
	lf.ti = time.Now()
	return lf
}

func (lf *logfmt) withRequest(r *http.Request) *logfmt {
	lf.rq = r
	return lf
}

func (lf *logfmt) withResponse(a *alog) *logfmt {
	lf.al = a
	return lf
}

func (lf *logfmt) timeFormatted(format string) string {
	if len(lf.t) == 0 {
		lf.t = lf.ti.Format(format)
	}
	return lf.t
}

func (lf *logfmt) remoteHostname() string {
	if len(lf.h) == 0 {
		lf.h = lf.rq.URL.Host
		if len(lf.h) == 0 {
			lf.h = "127.0.0.1"
		}
	}
	return lf.h
}

func (lf *logfmt) username() string {
	if len(lf.u) == 0 {
		lf.u = "-"
		if s := strings.SplitN(lf.rq.Header.Get("Authorization"), " ", 2); len(s) == 2 {
			if b, err := base64.StdEncoding.DecodeString(s[1]); err == nil {
				if pair := strings.SplitN(string(b), ":", 2); len(pair) == 2 {
					lf.u = pair[0]
				}
			}
		}
	}
	return lf.u
}

func (lf *logfmt) requestLine() string {
	if len(lf.q) == 0 {
		lf.q = strings.ToUpper(lf.rq.Method) + " " + lf.rq.URL.Path + " " + lf.rq.Proto
	}
	return lf.q
}

func (lf *logfmt) status() string {
	if len(lf.ws) == 0 {
		lf.ws = strconv.Itoa(lf.al.status)
	}
	return lf.ws
}

func (lf *logfmt) bytesWritten() string {
	if len(lf.wl) == 0 {
		lf.wl = strconv.Itoa(lf.al.wLen)
	}
	return lf.wl
}

func (lf *logfmt) timeElapsed() string {
	if len(lf.e) > 0 {
		lf.e = time.Now().Sub(lf.al.start).String()
	}
	return lf.e
}

// flatten takes two slices and merges them into one
func flatten(o *opt, a, b []string) func(w *alog, r *http.Request) string {
	return func(w *alog, r *http.Request) string {
		lf := new(logfmt)
		lf.withTime(o).withRequest(r).withResponse(w)

		buf := new(bytes.Buffer)
		for i, s := range a {
			switch s {
			case "":
				buf.WriteString(b[i])
			case "%h":
				buf.WriteString(lf.remoteHostname())
			case "%l":
				buf.WriteString("-")
			case "%u":
				buf.WriteString(lf.username())
			case "%t":
				buf.WriteString(lf.timeFormatted("[02/01/2006:03:04:05 -0700]"))
			case "%r":
				buf.WriteString(lf.requestLine())
			case "%s", "%>s":
				buf.WriteString(lf.status())
			case "%b":
				buf.WriteString(lf.bytesWritten())
			case "%D":
				buf.WriteString(lf.timeElapsed())
			default:
				if len(s) > 4 && s[:2] == "%{" && s[len(s)-2] == '}' {
					label := s[2 : len(s)-2]
					switch s[len(s)-1] {
					case 'i':
						buf.WriteString(r.Header.Get(label))
					case 't':
						buf.WriteString(convertTimeFormat(lf.ti, label))
					}
				}
			}
		}
		return buf.String()
	}
}

func FormatWith(format string) func(...optFunc) func(http.Handler) http.Handler {
	return func(opts ...optFunc) func(http.Handler) http.Handler {
		return Format(format, opts...)
	}
}

// Format accepts a format using Apache formatting directives with option functions and returns a function that can handle standard HTTP middleware.
func Format(format string, opts ...optFunc) func(http.Handler) http.Handler {
	options := newOpt()
	for _, opt := range opts {
		opt(options)
	}

	var directives, betweens = make([]string, 0, 50), make([]string, 0, 50)
	var mBuf *bytes.Buffer
	aBuf, bBuf := new(bytes.Buffer), new(bytes.Buffer)
	mBuf = bBuf

	var isDirective, isEnclosure bool
	for i, r := range format {
		switch r {
		case '%':
			if isDirective {
				mBuf.WriteRune(r)
				continue
			}
			isDirective = true
			if i != 0 {
				directives = append(directives, aBuf.String())
				betweens = append(betweens, bBuf.String())
				aBuf.Reset()
				bBuf.Reset()
			}
			mBuf = aBuf
		case '{':
			isEnclosure = true
		case '}':
			isEnclosure = false
		case '>':
			// do the same thing
		default:
			if isDirective && !isEnclosure && !unicode.IsLetter(r) {
				isDirective = false
				isEnclosure = false
				if i != 0 {
					directives = append(directives, aBuf.String())
					betweens = append(betweens, bBuf.String())
					aBuf.Reset()
					bBuf.Reset()
				}
				mBuf = bBuf
			}
		}
		mBuf.WriteRune(r)
	}

	directives = append(directives, aBuf.String())
	betweens = append(betweens, bBuf.String())
	aBuf.Reset()
	bBuf.Reset()

	logFunc := flatten(options, directives, betweens)

	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			a := &alog{ResponseWriter: w}
			a.startTime()
			next.ServeHTTP(a, r)
			fmt.Fprintln(options.Out, logFunc(a, r))
		})
	}
}
<commit_msg>Added a comment for the FormatWith function<commit_after>package accesslog

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
	"unicode"
)

// optFunc is a type of function that can add options to the option struct during initialization
type optFunc func(*opt)

// opt is the struct that holds the options for logging.
type opt struct {
	Out  io.Writer
	Time time.Time
}

// newOpt returns a new struct to hold options, with the default output to stdout.
func newOpt() *opt {
	o := new(opt)
	o.Out = os.Stdout
	return o
}

// WithOutput sets the io.Writer output for the log file.
func WithOutput(out io.Writer) optFunc {
	return func(o *opt) {
		o.Out = out
	}
}

// alog is the internal struct that will hold the status and number of bytes written
type alog struct {
	http.ResponseWriter

	status int
	wLen   int

	start time.Time
}

// WriteHeader intercepts the http.ResponseWriter WriteHeader method so we can save the status to display later
func (a *alog) WriteHeader(i int) {
	if a.status == 0 {
		a.status = i
	}
	a.ResponseWriter.WriteHeader(i)
}

// Write intercepts the http.ResponseWriter Write method so we can capture the bytes written
func (a *alog) Write(p []byte) (n int, err error) {
	if a.status == 0 {
		a.status = http.StatusOK
	}
	n, err = a.ResponseWriter.Write(p)
	a.wLen += n
	return
}

// startTime sets the start time to calculate the elapsed time for the %D directive
func (a *alog) startTime() {
	a.start = time.Now()
}

const (
	ApacheCommonLogFormat   = "%h %l %u %t \"%r\" %>s %b"                                    // The Common Log directives
	ApacheCombinedLogFormat = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" // The Combined Log directives
)

// ApacheCommonLog will log HTTP requests using the Apache Common Log format
var ApacheCommonLog = FormatWith(ApacheCommonLogFormat)

// ApacheCombinedLog will log HTTP requests using the Apache Combined Log format
var ApacheCombinedLog = FormatWith(ApacheCombinedLogFormat)

// convertTimeFormat converts strftime formatting directives to a go time.Time format
func convertTimeFormat(t time.Time, s string) string {
	m := map[rune]string{
		'a': "Mon",
		'A': "Monday",
		'b': "Jan",
		'B': "January",
		'c': "?",
		'C': "06",
		'd': "02",
		'D': "01/02/06",
		'e': "_2",
		'E': "?",
		'F': "2006-01-02",
		'G': "%d",
		'g': "%s",
		'h': "Jan",
		'H': "15",
		'I': "3",
		'j': "%d",
		'k': "•15",
		'l': "_3",
		'm': "01",
		'M': "04",
		'n': "\n",
		'O': "?",
		'p': "PM",
		'P': "pm",
		'r': "03:04:05 PM",
		'R': "15:04",
		's': "%d",
		'S': "05",
		't': "\t",
		'T': "15:04:05",
		'u': "%d",
		'U': "?",
		'V': "%d",
		'w': "%d",
		'W': "?",
		'x': "?",
		'X': "?",
		'y': "06",
		'Y': "2006",
		'z': "-700",
		'Z': "MST",
		'+': "?",
		'%': "%%",
	}

	var i []interface{}
	var x bool
	buf := new(bytes.Buffer)
	for _, r := range s {
		if r == '%' {
			x = true
			continue
		}
		if x {
			x = false
			if val, ok := m[r]; ok {
				if val == "%s" || val == "%d" {
					switch r {
					case 'G':
						y, _ := t.ISOWeek()
						i = append(i, y)
					case 'g':
						y, _ := t.ISOWeek()
						i = append(i, strconv.Itoa(y)[2:])
					case 'j':
						i = append(i, t.YearDay())
					case 's':
						i = append(i, t.Unix())
					case 'u':
						w := t.Weekday()
						if w == 0 {
							w = 7
						}
						i = append(i, w)
					case 'V':
						_, w := t.ISOWeek()
						i = append(i, w)
					case 'w':
						i = append(i, t.Weekday())
					}
				}
				buf.WriteString(val)
				continue
			}
			buf.WriteRune('%')
		}
		buf.WriteRune(r)
	}
	f := t.Format(buf.String())
	if len(i) > 0 {
		f = fmt.Sprintf(f, i...)
	}
	buf.Reset()
	return f
}

// logfmt the type that will hold all of the runtime formating
type logfmt struct {
	ti time.Time
	rq *http.Request
	al *alog

	h, u, t, q, e, ws, wl string
}

func (lf *logfmt) withTime(o *opt) *logfmt {
	if !o.Time.IsZero() {
		lf.ti = o.Time
		return lf
	}
	lf.ti = time.Now()
	return lf
}

func (lf *logfmt) withRequest(r *http.Request) *logfmt {
	lf.rq = r
	return lf
}

func (lf *logfmt) withResponse(a *alog) *logfmt {
	lf.al = a
	return lf
}

func (lf *logfmt) timeFormatted(format string) string {
	if len(lf.t) == 0 {
		lf.t = lf.ti.Format(format)
	}
	return lf.t
}

func (lf *logfmt) remoteHostname() string {
	if len(lf.h) == 0 {
		lf.h = lf.rq.URL.Host
		if len(lf.h) == 0 {
			lf.h = "127.0.0.1"
		}
	}
	return lf.h
}

func (lf *logfmt) username() string {
	if len(lf.u) == 0 {
		lf.u = "-"
		if s := strings.SplitN(lf.rq.Header.Get("Authorization"), " ", 2); len(s) == 2 {
			if b, err := base64.StdEncoding.DecodeString(s[1]); err == nil {
				if pair := strings.SplitN(string(b), ":", 2); len(pair) == 2 {
					lf.u = pair[0]
				}
			}
		}
	}
	return lf.u
}

func (lf *logfmt) requestLine() string {
	if len(lf.q) == 0 {
		lf.q = strings.ToUpper(lf.rq.Method) + " " + lf.rq.URL.Path + " " + lf.rq.Proto
	}
	return lf.q
}

func (lf *logfmt) status() string {
	if len(lf.ws) == 0 {
		lf.ws = strconv.Itoa(lf.al.status)
	}
	return lf.ws
}

func (lf *logfmt) bytesWritten() string {
	if len(lf.wl) == 0 {
		lf.wl = strconv.Itoa(lf.al.wLen)
	}
	return lf.wl
}

func (lf *logfmt) timeElapsed() string {
	if len(lf.e) > 0 {
		lf.e = time.Now().Sub(lf.al.start).String()
	}
	return lf.e
}

// flatten takes two slices and merges them into one
func flatten(o *opt, a, b []string) func(w *alog, r *http.Request) string {
	return func(w *alog, r *http.Request) string {
		lf := new(logfmt)
		lf.withTime(o).withRequest(r).withResponse(w)

		buf := new(bytes.Buffer)
		for i, s := range a {
			switch s {
			case "":
				buf.WriteString(b[i])
			case "%h":
				buf.WriteString(lf.remoteHostname())
			case "%l":
				buf.WriteString("-")
			case "%u":
				buf.WriteString(lf.username())
			case "%t":
				buf.WriteString(lf.timeFormatted("[02/01/2006:03:04:05 -0700]"))
			case "%r":
				buf.WriteString(lf.requestLine())
			case "%s", "%>s":
				buf.WriteString(lf.status())
			case "%b":
				buf.WriteString(lf.bytesWritten())
			case "%D":
				buf.WriteString(lf.timeElapsed())
			default:
				if len(s) > 4 && s[:2] == "%{" && s[len(s)-2] == '}' {
					label := s[2 : len(s)-2]
					switch s[len(s)-1] {
					case 'i':
						buf.WriteString(r.Header.Get(label))
					case 't':
						buf.WriteString(convertTimeFormat(lf.ti, label))
					}
				}
			}
		}
		return buf.String()
	}
}

// FormatWith accepts a format using Apache formatting and returns a function accepting option functions
// which then returns a function that can handle standard HTTP middleware. This is to have better standard
// logging functions accessible from the library
func FormatWith(format string) func(...optFunc) func(http.Handler) http.Handler {
	return func(opts ...optFunc) func(http.Handler) http.Handler {
		return Format(format, opts...)
	}
}

// Format accepts a format using Apache formatting directives with option functions and returns a function that can handle standard HTTP middleware.
func Format(format string, opts ...optFunc) func(http.Handler) http.Handler {
	options := newOpt()
	for _, opt := range opts {
		opt(options)
	}

	var directives, betweens = make([]string, 0, 50), make([]string, 0, 50)
	var mBuf *bytes.Buffer
	aBuf, bBuf := new(bytes.Buffer), new(bytes.Buffer)
	mBuf = bBuf

	var isDirective, isEnclosure bool
	for i, r := range format {
		switch r {
		case '%':
			if isDirective {
				mBuf.WriteRune(r)
				continue
			}
			isDirective = true
			if i != 0 {
				directives = append(directives, aBuf.String())
				betweens = append(betweens, bBuf.String())
				aBuf.Reset()
				bBuf.Reset()
			}
			mBuf = aBuf
		case '{':
			isEnclosure = true
		case '}':
			isEnclosure = false
		case '>':
			// do the same thing
		default:
			if isDirective && !isEnclosure && !unicode.IsLetter(r) {
				isDirective = false
				isEnclosure = false
				if i != 0 {
					directives = append(directives, aBuf.String())
					betweens = append(betweens, bBuf.String())
					aBuf.Reset()
					bBuf.Reset()
				}
				mBuf = bBuf
			}
		}
		mBuf.WriteRune(r)
	}

	directives = append(directives, aBuf.String())
	betweens = append(betweens, bBuf.String())
	aBuf.Reset()
	bBuf.Reset()

	logFunc := flatten(options, directives, betweens)

	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			a := &alog{ResponseWriter: w}
			a.startTime()
			next.ServeHTTP(a, r)
			fmt.Fprintln(options.Out, logFunc(a, r))
		})
	}
}
<|endoftext|>


<commit_before>package repository

import (
	"bytes"
	"io/ioutil"
)

// ByteContentStorage wraps a ContentStorage and provides it with
// SetBytes and GetBytes methods
type ByteContentStorage struct {
	ContentStorage
}

// newByteContentStorage returns a new ByteContentStorage instance,
// wrapping the given ContentStorage.
func newByteContentStorage(s ContentStorage) *ByteContentStorage {
	return &ByteContentStorage{ContentStorage: s}
}

// GetBytes returns the data stored under the given id as bytes.
func (s *ByteContentStorage) GetBytes(id string) ([]byte, error) {
	r, err := s.ContentStorage.Get(id)
	if err != nil {
		return nil, err
	}
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	return data, nil
}

// SetBytes initially sets or updates the bytes for the given id.
func (s *ByteContentStorage) SetBytes(id string, data []byte) error {
	r := bytes.NewReader(data)
	return s.ContentStorage.Set(id, r)
}
<commit_msg>repository: fixed a file leak in the GetBytes function of the ByteContentStorage.<commit_after>package repository

import (
	"bytes"
	"io/ioutil"
)

// ByteContentStorage wraps a ContentStorage and provides it with
// SetBytes and GetBytes methods
type ByteContentStorage struct {
	ContentStorage
}

// newByteContentStorage returns a new ByteContentStorage instance,
// wrapping the given ContentStorage.
func newByteContentStorage(s ContentStorage) *ByteContentStorage {
	return &ByteContentStorage{ContentStorage: s}
}

// GetBytes returns the data stored under the given id as bytes.
func (s *ByteContentStorage) GetBytes(id string) ([]byte, error) {
	r, err := s.ContentStorage.Get(id)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	return data, nil
}

// SetBytes initially sets or updates the bytes for the given id.
func (s *ByteContentStorage) SetBytes(id string, data []byte) error {
	r := bytes.NewReader(data)
	return s.ContentStorage.Set(id, r)
}
<|endoftext|>


<commit_before>package bridge

import (
	"net"

	"github.com/docker/libnetwork"
)

const (
	NetworkType = "simplebridge"
	VethPrefix  = "veth"
)

type Configuration struct {
	BridgeName         string
	AddressIPv4        *net.IPNet
	FixedCIDR          *net.IPNet
	FixedCIDRv6        *net.IPNet
	EnableIPv6         bool
	EnableIPTables     bool
	EnableIPForwarding bool
}

func init() {
	libnetwork.RegisterNetworkType(NetworkType, Create, &Configuration{})
}

/*

func Create()

- NewBridgeInterface(*Configuration) (*BridgeInterface, error)
	. Issues LinkByName on config.BridgeName
- Create BridgeSetup instance with sequence of steps
- if !bridgeInterface.Exists()
	. Add DeviceCreation (error if non-default name)
	. Add AddressIPv4: set IPv4 (with automatic election if necessary)
- General case
	. Add option EnableIPv6 if no IPv6 on bridge (disable_ipv6=0 + set IPv6)
	. Verify configured addresses (with v4 and v6 updated in config)
	. Add FixedCIDR v4: register subnet on IP Allocator
	. Add FixedCIDR v6: register subnet on IP Allocator, route
- Add IPTables setup
- Add IPForward setup (depends on FixedCIDRv6)
- err := bridgeSetup.Apply()

*/

func Create(config *Configuration) (libnetwork.Network, error) {
	bridgeIntfc := NewInterface(config)
	bridgeSetup := NewBridgeSetup(bridgeIntfc)

	// If the bridge interface doesn't exist, we need to start the setup steps
	// by creating a new device and assigning it an IPv4 address.
	bridgeAlreadyExists := bridgeIntfc.Exists()
	if !bridgeAlreadyExists {
		bridgeSetup.QueueStep(SetupDevice)
		bridgeSetup.QueueStep(SetupBridgeIPv4)
	}

	// Conditionnally queue setup steps depending on configuration values.
	for _, step := range []struct {
		Condition bool
		Fn        SetupStep
	}{
		// Enable IPv6 on the bridge if required. We do this even for a
		// previously  existing bridge, as it may be here from a previous
		// installation where IPv6 wasn't supported yet and needs to be
		// assigned an IPv6 link-local address.
		{config.EnableIPv6, SetupBridgeIPv6},

		// We ensure that the bridge has the expectedIPv4 and IPv6 addresses in
		// the case of a previously existing device.
		{bridgeAlreadyExists, SetupVerifyConfiguredAddresses},

		// Setup the bridge to allocate containers IPv4 addresses in the
		// specified subnet.
		{config.FixedCIDR != nil, SetupFixedCIDRv4},

		// Setup the bridge to allocate containers global IPv6 addresses in the
		// specified subnet.
		{config.FixedCIDRv6 != nil, SetupFixedCIDRv6},

		// Setup IPTables.
		{config.EnableIPTables, SetupIPTables},

		// Setup IP forwarding.
		{config.EnableIPForwarding, SetupIPForwarding},
	} {
		if step.Condition {
			bridgeSetup.QueueStep(step.Fn)
		}
	}

	// Apply the prepared list of steps, and abort at the first error.
	bridgeSetup.QueueStep(SetupDeviceUp)
	if err := bridgeSetup.Apply(); err != nil {
		return nil, err
	}

	return &bridgeNetwork{*config}, nil
}

/*
func Create(config *Configuration) (libnetwork.Network, error) {
	var (
		addrv4  netlink.Addr
		addrsv6 []netlink.Addr
	)

	b := &bridgeNetwork{Config: *config}
	if b.Config.BridgeName == "" {
		b.Config.BridgeName = DefaultBridge
	}

	link, err := netlink.LinkByName(b.Config.BridgeName)
	if err != nil {
		// The bridge interface doesn't exist, but we only attempt to create it
		// if using the default name.
		if b.Config.BridgeName != DefaultBridge {
			return nil, err
		}

		// Create the bridge interface.
		if addrv4, addrsv6, err = createBridge(&b.Config); err != nil {
			return nil, err
		}
	} else {
		// The bridge interface exists: start by getting its configured
		// addresses and verify if it matches the requested configuration.
		addrv4, addrsv6, err = getInterfaceAddr(link)
		if err != nil {
			return nil, err
		}

		if b.Config.AddressIPv4 != "" {
			bridgeIP, _, err := net.ParseCIDR(b.Config.AddressIPv4)
			if err != nil {
				return nil, err
			}
			if !addrv4.IP.Equal(bridgeIP) {
				return nil, fmt.Errorf("Bridge IP %s does not match requested configuration %s", addrv4.IP, bridgeIP)
			}
		}

		// A bridge might exist but not have any IPv6 addr associated with it
		// yet (for example, an existing Docker installation that has only been
		// used with IPv4 and docker0 already is set up). In that case, we can
		// perform the bridge init for IPv6 here, else we will error out below
		// if --ipv6=true.
		if len(addrsv6) == 0 && config.EnableIPv6 {
			if err := setupIPv6Bridge(iface, config); err != nil {
				return nil, err
			}
		}
	}

	return b, nil
}
*/

/*
func createBridge(config *Configuration) (netlink.Addr, []netlink.Addr, error) {
	// Formats an error return with default values.
	fmtError := func(format string, params ...interface{}) (netlink.Addr, []netlink.Addr, error) {
		return netlink.Addr{}, nil, fmt.Errorf(format, params...)
	}

	// Elect a subnet for the bridge interface.
	bridgeIPNet, err := electBridgeNetwork(config)
	if err != nil {
		return fmtError("Failed to elect bridge network: %v", err)
	}
	log.Debugf("Creating bridge interface %q with network %s", config.BridgeName, bridgeIPNet)

	// We attempt to create the bridge, and ignore the returned error if it is
	// already existing.
	iface, err := createBridgeInterface(config.BridgeName)
	if err != nil && !os.IsExist(err) {
		return netlink.Addr{}, nil, err
	}

	// Configure bridge IPv4.
	if err := netlink.AddrAdd(iface, &netlink.Addr{bridgeIPNet, ""}); err != nil {
		return fmtError("Failed to add address %s to bridge: %v", bridgeIPNet, err)
	}

	// Configure bridge IPv6.
	if config.EnableIPv6 {
		if err := setupIPv6Bridge(iface, config); err != nil {
			return fmtError("Failed to setup bridge IPv6: %v", err)
		}
	}

	// Up the bridge interface.
	if err := netlink.LinkSetUp(iface); err != nil {
		return fmtError("Failed to up network bridge: %v", err)
	}

	if config.FixedCIDRv6 != "" {
		dest, network, err := net.ParseCIDR(config.FixedCIDRv6)
		if err != nil {
			return fmtError("Invalid bridge fixed CIDR IPv6 %q: %v", config.FixedCIDRv6, err)
		}

		// Set route to global IPv6 subnet
		log.Infof("Adding route to IPv6 network %q via device %q", dest, iface)
		if err := netlink.RouteAdd(&netlink.Route{Dst: network, LinkIndex: iface.Attrs().Index}); err != nil {
			return fmtError("Could not add route to IPv6 network %q via device %q", config.FixedCIDRv6, iface)
		}
	}

	return getInterfaceAddrByName(config.BridgeName)
}
*/

type bridgeNetwork struct {
	Config Configuration
}

func (b *bridgeNetwork) Type() string {
	return NetworkType
}

func (b *bridgeNetwork) Link(name string) ([]*libnetwork.Interface, error) {
	return nil, nil
}
<commit_msg>WIP - Remove commented code<commit_after>package bridge

import (
	"net"

	"github.com/docker/libnetwork"
)

const (
	NetworkType = "simplebridge"
	VethPrefix  = "veth"
)

type Configuration struct {
	BridgeName         string
	AddressIPv4        *net.IPNet
	FixedCIDR          *net.IPNet
	FixedCIDRv6        *net.IPNet
	EnableIPv6         bool
	EnableIPTables     bool
	EnableIPForwarding bool
}

func init() {
	libnetwork.RegisterNetworkType(NetworkType, Create, &Configuration{})
}

func Create(config *Configuration) (libnetwork.Network, error) {
	bridgeIntfc := NewInterface(config)
	bridgeSetup := NewBridgeSetup(bridgeIntfc)

	// If the bridge interface doesn't exist, we need to start the setup steps
	// by creating a new device and assigning it an IPv4 address.
	bridgeAlreadyExists := bridgeIntfc.Exists()
	if !bridgeAlreadyExists {
		bridgeSetup.QueueStep(SetupDevice)
		bridgeSetup.QueueStep(SetupBridgeIPv4)
	}

	// Conditionnally queue setup steps depending on configuration values.
	for _, step := range []struct {
		Condition bool
		Fn        SetupStep
	}{
		// Enable IPv6 on the bridge if required. We do this even for a
		// previously  existing bridge, as it may be here from a previous
		// installation where IPv6 wasn't supported yet and needs to be
		// assigned an IPv6 link-local address.
		{config.EnableIPv6, SetupBridgeIPv6},

		// We ensure that the bridge has the expectedIPv4 and IPv6 addresses in
		// the case of a previously existing device.
		{bridgeAlreadyExists, SetupVerifyConfiguredAddresses},

		// Setup the bridge to allocate containers IPv4 addresses in the
		// specified subnet.
		{config.FixedCIDR != nil, SetupFixedCIDRv4},

		// Setup the bridge to allocate containers global IPv6 addresses in the
		// specified subnet.
		{config.FixedCIDRv6 != nil, SetupFixedCIDRv6},

		// Setup IPTables.
		{config.EnableIPTables, SetupIPTables},

		// Setup IP forwarding.
		{config.EnableIPForwarding, SetupIPForwarding},
	} {
		if step.Condition {
			bridgeSetup.QueueStep(step.Fn)
		}
	}

	// Apply the prepared list of steps, and abort at the first error.
	bridgeSetup.QueueStep(SetupDeviceUp)
	if err := bridgeSetup.Apply(); err != nil {
		return nil, err
	}

	return &bridgeNetwork{*config}, nil
}

type bridgeNetwork struct {
	Config Configuration
}

func (b *bridgeNetwork) Type() string {
	return NetworkType
}

func (b *bridgeNetwork) Link(name string) ([]*libnetwork.Interface, error) {
	return nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elasticsearch

import (
	"code.google.com/p/log4go"
	"encoding/json"
	"github.com/cloudawan/kubernetes_management/utility/logger"
	elastigo "github.com/mattbaird/elastigo/lib"
	"strconv"
)

var log log4go.Logger = logger.GetLogger("utility")

type ElasticSearchClient struct {
	connection *elastigo.Conn
	host       []string
	port       int
}

func CreateElasticSearchClient(host []string, port int) *ElasticSearchClient {
	elasticSearchClient := &ElasticSearchClient{nil, host, port}
	elasticSearchClient.GetConnection()
	return elasticSearchClient
}

func (elasticSearchClient *ElasticSearchClient) GetConnection() *elastigo.Conn {
	if elasticSearchClient.connection != nil {
		return elasticSearchClient.connection
	} else {
		c := elastigo.NewConn()
		c.SetHosts(elasticSearchClient.host)
		c.SetPort(strconv.Itoa(elasticSearchClient.port))

		elasticSearchClient.connection = c
		return elasticSearchClient.connection
	}
}

func (elasticSearchClient *ElasticSearchClient) CloseConnection() {
	elasticSearchClient.connection.Close()
	elasticSearchClient.connection = nil
}

func (elasticSearchClient *ElasticSearchClient) Reconnect() {
	elasticSearchClient.CloseConnection()
	elasticSearchClient.GetConnection()
}

// Bulk Process
type BulkProcessor struct {
	bulkIndexer *elastigo.BulkIndexer
}

// WARNING: Due to goelstic's hard code channel size, if the buffered document amount reach 100, it will get stuck
func (bulkProcessor *BulkProcessor) BufferIndex(
	index string, _type string, id string, jsonMap map[string]interface{}) error {
	return bulkProcessor.bulkIndexer.Index(index, _type, id, "", nil, jsonMap, false)
}

func (bulkProcessor *BulkProcessor) FlushAndStopBulkProcessor() {
	bulkProcessor.bulkIndexer.Flush()
	bulkProcessor.bulkIndexer.Stop()
}

func (elasticSearchClient *ElasticSearchClient) CreateBulkProcessor(maxConnection int) *BulkProcessor {
	connection := elasticSearchClient.GetConnection()
	bulkIndexer := connection.NewBulkIndexer(maxConnection)
	bulkIndexer.Start()
	return &BulkProcessor{bulkIndexer}
}

// Get all types for index
func (elasticSearchClient *ElasticSearchClient) GetAllTypeForIndex(index string) ([]string, error) {
	connection := elasticSearchClient.GetConnection()
	request, err := connection.NewRequest("GET", "/"+index, "")
	if err != nil {
		log.Error(err)
		return nil, err
	} else {
		_, bodyBytes, err := request.Do(nil)
		if err != nil {
			log.Error(err)
			return nil, err
		} else {
			responseJsonMap := make(map[string]interface{})
			err := json.Unmarshal(bodyBytes, &responseJsonMap)
			if err != nil {
				log.Error(err)
				return nil, err
			} else {
				typeSlice := make([]string, 0)

				mappingMap, _ := responseJsonMap[index].(map[string]interface{})["mappings"].(map[string]interface{})
				for key, _ := range mappingMap {
					if key != "_default_" {
						typeSlice = append(typeSlice, key)
					}
				}

				return typeSlice, nil
			}
		}
	}
}
<commit_msg>Change to the change to library<commit_after>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elasticsearch

import (
	"code.google.com/p/log4go"
	"encoding/json"
	"github.com/cloudawan/kubernetes_management/utility/logger"
	elastigo "github.com/mattbaird/elastigo/lib"
	"strconv"
)

var log log4go.Logger = logger.GetLogger("utility")

type ElasticSearchClient struct {
	connection *elastigo.Conn
	host       []string
	port       int
}

func CreateElasticSearchClient(host []string, port int) *ElasticSearchClient {
	elasticSearchClient := &ElasticSearchClient{nil, host, port}
	elasticSearchClient.GetConnection()
	return elasticSearchClient
}

func (elasticSearchClient *ElasticSearchClient) GetConnection() *elastigo.Conn {
	if elasticSearchClient.connection != nil {
		return elasticSearchClient.connection
	} else {
		c := elastigo.NewConn()
		c.SetHosts(elasticSearchClient.host)
		c.SetPort(strconv.Itoa(elasticSearchClient.port))

		elasticSearchClient.connection = c
		return elasticSearchClient.connection
	}
}

func (elasticSearchClient *ElasticSearchClient) CloseConnection() {
	elasticSearchClient.connection.Close()
	elasticSearchClient.connection = nil
}

func (elasticSearchClient *ElasticSearchClient) Reconnect() {
	elasticSearchClient.CloseConnection()
	elasticSearchClient.GetConnection()
}

// Bulk Process
type BulkProcessor struct {
	bulkIndexer *elastigo.BulkIndexer
}

// WARNING: Due to goelstic's hard code channel size, if the buffered document amount reach 100, it will get stuck
func (bulkProcessor *BulkProcessor) BufferIndex(
	index string, _type string, id string, jsonMap map[string]interface{}) error {
	return bulkProcessor.bulkIndexer.Index(index, _type, id, "", "", nil, jsonMap)
}

func (bulkProcessor *BulkProcessor) FlushAndStopBulkProcessor() {
	bulkProcessor.bulkIndexer.Flush()
	bulkProcessor.bulkIndexer.Stop()
}

func (elasticSearchClient *ElasticSearchClient) CreateBulkProcessor(maxConnection int) *BulkProcessor {
	connection := elasticSearchClient.GetConnection()
	bulkIndexer := connection.NewBulkIndexer(maxConnection)
	bulkIndexer.Start()
	return &BulkProcessor{bulkIndexer}
}

// Get all types for index
func (elasticSearchClient *ElasticSearchClient) GetAllTypeForIndex(index string) ([]string, error) {
	connection := elasticSearchClient.GetConnection()
	request, err := connection.NewRequest("GET", "/"+index, "")
	if err != nil {
		log.Error(err)
		return nil, err
	} else {
		_, bodyBytes, err := request.Do(nil)
		if err != nil {
			log.Error(err)
			return nil, err
		} else {
			responseJsonMap := make(map[string]interface{})
			err := json.Unmarshal(bodyBytes, &responseJsonMap)
			if err != nil {
				log.Error(err)
				return nil, err
			} else {
				typeSlice := make([]string, 0)

				mappingMap, _ := responseJsonMap[index].(map[string]interface{})["mappings"].(map[string]interface{})
				for key, _ := range mappingMap {
					if key != "_default_" {
						typeSlice = append(typeSlice, key)
					}
				}

				return typeSlice, nil
			}
		}
	}
}
<|endoftext|>