<commit_before>package parg

import (
	"strings"
	"testing" //import go package for testing related functionality
)

// TestExtractFlags_NoArgs tests to ensure that when no arguments are provided,
// no flags or arguments are returned by extractFlags.
func TestExtractFlags_NoArgs(t *testing.T) {
	var noArgs []string
	flags, args := extractFlags(noArgs...)

	if len(flags) != 0 {
		t.Error("No flags should have been extracted")
	}

	if len(args) != 0 {
		t.Error("No arguments should have been extracted")
	}
}

// TestExtractFlags_OnlyFlags tests to ensure that if only flag arguments are
// provided, that the same number of flag arguments are returned & with no
// additional arguments by extractFlags.
func TestExtractFlags_OnlyFlags(t *testing.T) {
	onlyFlagArgs := []string{"-f", "--foobar"}
	flags, args := extractFlags(onlyFlagArgs...)

	if len(flags) != len(onlyFlagArgs) {
		t.Errorf(
			"%d number of flags expected, but only %d were extracted",
			len(onlyFlagArgs),
			len(flags),
		)
	}

	if len(args) != 0 {
		t.Error("No arguments should have been extracted")
	}
}

// TestExtractFlags_MutliShortFlags tests to ensure that multiple short-flags
// residing beside each other are properly recognized and extract individually,
// and no other arguments are returned.
func TestExtractFlags_MultiShortFlags(t *testing.T) {
	shortFlags := []string{"a", "b", "c"}
	shortFlagArgs := []string{"-" + strings.Join(shortFlags, "")}

	flags, args := extractFlags(shortFlagArgs...)

	if len(flags) != len(shortFlags) {
		t.Errorf(
			"%d number of flags expected, but only %d were extracted",
			len(shortFlags),
			len(flags),
		)
	}

	if len(args) != 0 {
		t.Error("No arguments should have been extracted")
	}
}

// TestExtractFlags_OnlyArgs tests to ensure that if only passive arguments are
// provided, that the same number of passive arguments are returned & with no
// flags extracted by extractFlags.
func TestExtractFlags_OnlyArgs(t *testing.T) {
	onlyArgs := []string{"arg1", "arg2", "arg3", "arg4"}
	flags, args := extractFlags(onlyArgs...)

	if len(args) != len(onlyArgs) {
		t.Errorf(
			"%d number of passive argumentss expected, but only %d were extracted",
			len(onlyArgs),
			len(args),
		)
	}

	if len(flags) != 0 {
		t.Error("No flags should have been extracted")
	}
}

// TestExtractFlags tests to ensure that the expected number of flags & passive
// arguments are extracted by extractFlags.
func TestExtractFlags(t *testing.T) {
	allArgs := []string{"-f", "foobar", "--fizzbuzz", "four", "five", "-irtusc"}
	numFlags := 8
	numArgs := 3

	flags, args := extractFlags(allArgs...)

	if len(flags) != numFlags {
		t.Errorf(
			"%d number of flags expected, but only %d were extracted",
			numFlags,
			len(flags),
		)
	}

	if len(args) != numArgs {
		t.Errorf(
			"%d number of passive argumentss expected, but only %d were extracted",
			numArgs,
			len(args),
		)
	}
}

// TestGetScreenWidth tests to ensure that a positive, non-zero integer value is returned
// to represent the width of the current screen.
func TestGetScreenWidth(t *testing.T) {
	// I am not really sure the best way to test this.
	// TODO: make a better test!

	width := getScreenWidth()
	if width <= 0 {
		t.Error("Retrieved screen width should be a positive, non-zero integer")
	}
}

// TestJoin tests to ensure that a variety of string slices can be joined in the
// correct, expected manner.
func TestJoin(t *testing.T) {
	testStrings := [][]string{
		[]string{"one", "two"},
		[]string{""},
		[]string{"three", "four", "five"},
	}

	expectedStrings := []string{
		"one two",
		"",
		"three four five",
	}

	for i, test := range testStrings {
		actual := join(" ", test...)
		expected := expectedStrings[i]

		if actual != expected {
			t.Errorf(
				"Expected: '%s' but received: '%s'",
				expected,
				actual,
			)
		}
	}

	actual := join("-*-", "abc", "def")
	expected := "abc-*-def"
	if actual != expected {
		t.Errorf(
			"Expected: '%s' but received: '%s'",
			expected,
			actual,
		)
	}
}

// TestSpacer tests to make sure the proper length strings are returned, as expected.
func TestSpacer(t *testing.T) {
	intTests := []int{-1000, -100, -10, -1, 0, 1, 10, 100, 1000}

	for _, test := range intTests {
		actual := spacer(test)
		expectedLen := test
		if expectedLen < 0 {
			expectedLen = 0
		}

		if len(actual) != expectedLen {
			if len(actual) != 0 {
				t.Errorf("Expected string of length: %d but received: '%s'", expectedLen, actual)
			}
		}
	}
}
<commit_msg>Finished utils testing<commit_after>package parg

import (
	"strings"
	"testing" //import go package for testing related functionality
)

// TestExtractFlags_NoArgs tests to ensure that when no arguments are provided,
// no flags or arguments are returned by extractFlags.
func TestExtractFlags_NoArgs(t *testing.T) {
	var noArgs []string
	flags, args := extractFlags(noArgs...)

	if len(flags) != 0 {
		t.Error("No flags should have been extracted")
	}

	if len(args) != 0 {
		t.Error("No arguments should have been extracted")
	}
}

// TestExtractFlags_OnlyFlags tests to ensure that if only flag arguments are
// provided, that the same number of flag arguments are returned & with no
// additional arguments by extractFlags.
func TestExtractFlags_OnlyFlags(t *testing.T) {
	onlyFlagArgs := []string{"-f", "--foobar"}
	flags, args := extractFlags(onlyFlagArgs...)

	if len(flags) != len(onlyFlagArgs) {
		t.Errorf(
			"%d number of flags expected, but only %d were extracted",
			len(onlyFlagArgs),
			len(flags),
		)
	}

	if len(args) != 0 {
		t.Error("No arguments should have been extracted")
	}
}

// TestExtractFlags_MutliShortFlags tests to ensure that multiple short-flags
// residing beside each other are properly recognized and extract individually,
// and no other arguments are returned.
func TestExtractFlags_MultiShortFlags(t *testing.T) {
	shortFlags := []string{"a", "b", "c"}
	shortFlagArgs := []string{"-" + strings.Join(shortFlags, "")}

	flags, args := extractFlags(shortFlagArgs...)

	if len(flags) != len(shortFlags) {
		t.Errorf(
			"%d number of flags expected, but only %d were extracted",
			len(shortFlags),
			len(flags),
		)
	}

	if len(args) != 0 {
		t.Error("No arguments should have been extracted")
	}
}

// TestExtractFlags_OnlyArgs tests to ensure that if only passive arguments are
// provided, that the same number of passive arguments are returned & with no
// flags extracted by extractFlags.
func TestExtractFlags_OnlyArgs(t *testing.T) {
	onlyArgs := []string{"arg1", "arg2", "arg3", "arg4"}
	flags, args := extractFlags(onlyArgs...)

	if len(args) != len(onlyArgs) {
		t.Errorf(
			"%d number of passive argumentss expected, but only %d were extracted",
			len(onlyArgs),
			len(args),
		)
	}

	if len(flags) != 0 {
		t.Error("No flags should have been extracted")
	}
}

// TestExtractFlags tests to ensure that the expected number of flags & passive
// arguments are extracted by extractFlags.
func TestExtractFlags(t *testing.T) {
	allArgs := []string{"-f", "foobar", "--fizzbuzz", "four", "five", "-irtusc"}
	numFlags := 8
	numArgs := 3

	flags, args := extractFlags(allArgs...)

	if len(flags) != numFlags {
		t.Errorf(
			"%d number of flags expected, but only %d were extracted",
			numFlags,
			len(flags),
		)
	}

	if len(args) != numArgs {
		t.Errorf(
			"%d number of passive argumentss expected, but only %d were extracted",
			numArgs,
			len(args),
		)
	}
}

// TestGetScreenWidth tests to ensure that a positive, non-zero integer value is returned
// to represent the width of the current screen.
func TestGetScreenWidth(t *testing.T) {
	// I am not really sure the best way to test this.
	// TODO: make a better test!

	width := getScreenWidth()
	if width <= 0 {
		t.Error("Retrieved screen width should be a positive, non-zero integer")
	}
}

// TestJoin tests to ensure that a variety of string slices can be joined in the
// correct, expected manner.
func TestJoin(t *testing.T) {
	testStrings := [][]string{
		[]string{"one", "two"},
		[]string{""},
		[]string{"three", "four", "five"},
	}

	expectedStrings := []string{
		"one two",
		"",
		"three four five",
	}

	for i, test := range testStrings {
		actual := join(" ", test...)
		expected := expectedStrings[i]

		if actual != expected {
			t.Errorf(
				"Expected: '%s' but received: '%s'",
				expected,
				actual,
			)
		}
	}

	actual := join("-*-", "abc", "def")
	expected := "abc-*-def"
	if actual != expected {
		t.Errorf(
			"Expected: '%s' but received: '%s'",
			expected,
			actual,
		)
	}
}

// TestSpacer tests to make sure the proper length strings are returned, as expected.
func TestSpacer(t *testing.T) {
	intTests := []int{-1000, -100, -10, -1, 0, 1, 10, 100, 1000}

	for _, test := range intTests {
		actual := spacer(test)
		expectedLen := test
		if expectedLen < 0 {
			expectedLen = 0
		}

		if len(actual) != expectedLen {
			if len(actual) != 0 {
				t.Errorf("Expected string of length: %d but received: '%s'", expectedLen, actual)
			}
		}
	}
}

// TestWordWrap tests to ensure strings will be broken into the appropriate
// word-length limited slice of strings.
func TestWordWrap(t *testing.T) {
	oneLine := "This text is below the limit."
	threeLines := "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum dolor justo, tempor quis"

	if len(wordWrap(oneLine, 35)) != 1 {
		t.Error("wordWrap did not return a slice of lenth 1")
	}

	if len(wordWrap(threeLines, 35)) != 3 {
		t.Error("wordWrap did not return a slice of length 3")
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"errors"
	"fmt"
	"reflect"
	"time"

	"github.com/keybase/client/go/logger"
	"github.com/keybase/kbfs/tlf"
	"golang.org/x/net/context"
)

// StateChecker verifies that the server-side state for KBFS is
// consistent.  Useful mostly for testing because it isn't scalable
// and loads all the state in memory.
type StateChecker struct {
	config Config
	log    logger.Logger
}

// NewStateChecker returns a new StateChecker instance.
func NewStateChecker(config Config) *StateChecker {
	return &StateChecker{config, config.MakeLogger("")}
}

// findAllFileBlocks adds all file blocks found under this block to
// the blocksFound map, if the given path represents an indirect
// block.
func (sc *StateChecker) findAllFileBlocks(ctx context.Context,
	lState *lockState, ops *folderBranchOps, kmd KeyMetadata,
	file path, blockSizes map[BlockPointer]uint32) error {
	fblock, err := ops.blocks.GetFileBlockForReading(ctx, lState, kmd,
		file.tailPointer(), file.Branch, file)
	if err != nil {
		return err
	}

	if !fblock.IsInd {
		return nil
	}

	parentPath := file.parentPath()
	for _, childPtr := range fblock.IPtrs {
		blockSizes[childPtr.BlockPointer] = childPtr.EncodedSize
		p := parentPath.ChildPath(file.tailName(), childPtr.BlockPointer)
		err := sc.findAllFileBlocks(ctx, lState, ops, kmd, p, blockSizes)
		if err != nil {
			return err
		}
	}
	return nil
}

// findAllBlocksInPath adds all blocks found within this directory to
// the blockSizes map, and then recursively checks all
// subdirectories.
func (sc *StateChecker) findAllBlocksInPath(ctx context.Context,
	lState *lockState, ops *folderBranchOps, kmd KeyMetadata,
	dir path, blockSizes map[BlockPointer]uint32) error {
	dblock, err := ops.blocks.GetDirBlockForReading(ctx, lState, kmd,
		dir.tailPointer(), dir.Branch, dir)
	if err != nil {
		return err
	}

	for name, de := range dblock.Children {
		if de.Type == Sym {
			continue
		}

		blockSizes[de.BlockPointer] = de.EncodedSize
		p := dir.ChildPath(name, de.BlockPointer)

		if de.Type == Dir {
			err := sc.findAllBlocksInPath(ctx, lState, ops, kmd, p, blockSizes)
			if err != nil {
				return err
			}
		} else {
			// If it's a file, check to see if it's indirect.
			err := sc.findAllFileBlocks(ctx, lState, ops, kmd, p, blockSizes)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (sc *StateChecker) getLastGCData(ctx context.Context,
	tlf tlf.ID) (time.Time, MetadataRevision) {
	config, ok := sc.config.(*ConfigLocal)
	if !ok {
		return time.Time{}, MetadataRevisionUninitialized
	}

	var latestTime time.Time
	var latestRev MetadataRevision
	for _, c := range *config.allKnownConfigsForTesting {
		ops := c.KBFSOps().(*KBFSOpsStandard).getOpsNoAdd(
			FolderBranch{tlf, MasterBranch})
		rt, rev := ops.fbm.getLastQRData()
		if rt.After(latestTime) && rev > latestRev {
			latestTime = rt
			latestRev = rev
		}
	}
	if latestTime == (time.Time{}) {
		return latestTime, latestRev
	}

	sc.log.CDebugf(ctx, "Last qr data for TLF %s: revTime=%s, rev=%d",
		tlf, latestTime, latestRev)
	return latestTime.Add(-sc.config.QuotaReclamationMinUnrefAge()), latestRev
}

// CheckMergedState verifies that the state for the given tlf is
// consistent.
func (sc *StateChecker) CheckMergedState(ctx context.Context, tlf tlf.ID) error {
	// Blow away MD cache so we don't have any lingering re-embedded
	// block changes (otherwise we won't be able to learn their sizes).
	sc.config.SetMDCache(NewMDCacheStandard(defaultMDCacheCapacity))

	// Fetch all the MD updates for this folder, and use the block
	// change lists to build up the set of currently referenced blocks.
	rmds, err := getMergedMDUpdates(ctx, sc.config, tlf,
		MetadataRevisionInitial)
	if err != nil {
		return err
	}
	if len(rmds) == 0 {
		sc.log.CDebugf(ctx, "No state to check for folder %s", tlf)
		return nil
	}

	lState := makeFBOLockState()

	// Re-embed block changes.
	kbfsOps, ok := sc.config.KBFSOps().(*KBFSOpsStandard)
	if !ok {
		return errors.New("Unexpected KBFSOps type")
	}

	fb := FolderBranch{tlf, MasterBranch}
	ops := kbfsOps.getOpsNoAdd(fb)
	lastGCRevisionTime, lastGCRev := sc.getLastGCData(ctx, tlf)

	// Build the expected block list.
	expectedLiveBlocks := make(map[BlockPointer]bool)
	expectedRef := uint64(0)
	archivedBlocks := make(map[BlockPointer]bool)
	actualLiveBlocks := make(map[BlockPointer]uint32)

	// See what the last GC op revision is.  All unref'd pointers from
	// that revision or earlier should be deleted from the block
	// server.
	gcRevision := MetadataRevisionUninitialized
	for _, rmd := range rmds {
		// Don't process copies.
		if rmd.IsWriterMetadataCopiedSet() {
			continue
		}

		for _, op := range rmd.data.Changes.Ops {
			GCOp, ok := op.(*GCOp)
			if !ok {
				continue
			}
			gcRevision = GCOp.LatestRev
		}
	}

	for _, rmd := range rmds {
		// Don't process copies.
		if rmd.IsWriterMetadataCopiedSet() {
			continue
		}
		// Any unembedded block changes also count towards the actual size
		if info := rmd.data.cachedChanges.Info; info.BlockPointer != zeroPtr {
			sc.log.CDebugf(ctx, "Unembedded block change: %v, %d",
				info.BlockPointer, info.EncodedSize)
			actualLiveBlocks[info.BlockPointer] = info.EncodedSize

			// Any child block change pointers?
			fblock, err := ops.blocks.GetFileBlockForReading(ctx, lState,
				rmd.ReadOnly(), info.BlockPointer, MasterBranch, path{})
			if err != nil {
				return err
			}
			for _, iptr := range fblock.IPtrs {
				sc.log.CDebugf(ctx, "Unembedded child block change: %v, %d",
					iptr.BlockPointer, iptr.EncodedSize)
				actualLiveBlocks[iptr.BlockPointer] = iptr.EncodedSize
			}
		}

		var hasGCOp bool
		for _, op := range rmd.data.Changes.Ops {
			_, isGCOp := op.(*GCOp)
			hasGCOp = hasGCOp || isGCOp

			opRefs := make(map[BlockPointer]bool)
			for _, ptr := range op.Refs() {
				if ptr != zeroPtr {
					expectedLiveBlocks[ptr] = true
					opRefs[ptr] = true
				}
			}
			if !isGCOp {
				for _, ptr := range op.Unrefs() {
					delete(expectedLiveBlocks, ptr)
					if ptr != zeroPtr {
						// If the revision has been garbage-collected,
						// or if the pointer has been referenced and
						// unreferenced within the same op (which
						// indicates a failed and retried sync), the
						// corresponding block should already be
						// cleaned up.
						if rmd.Revision() <= gcRevision || opRefs[ptr] {
							delete(archivedBlocks, ptr)
						} else {
							archivedBlocks[ptr] = true
						}
					}
				}
			}
			for _, update := range op.allUpdates() {
				delete(expectedLiveBlocks, update.Unref)
				if update.Unref != zeroPtr && update.Ref != update.Unref {
					if rmd.Revision() <= gcRevision {
						delete(archivedBlocks, update.Unref)
					} else {
						archivedBlocks[update.Unref] = true
					}
				}
				if update.Ref != zeroPtr {
					expectedLiveBlocks[update.Ref] = true
				}
			}
		}
		expectedRef += rmd.RefBytes()
		expectedRef -= rmd.UnrefBytes()

		if len(rmd.data.Changes.Ops) == 1 && hasGCOp {
			// Don't check GC status for GC revisions
			continue
		}

		// Make sure that if this revision should be covered by a GC
		// op, it is.  Note that this assumes that if QR is ever run,
		// it will be run completely and not left partially done due
		// to there being too many pointers to collect in one sweep.
		mtime := time.Unix(0, rmd.data.Dir.Mtime)
		if !lastGCRevisionTime.Before(mtime) && rmd.Revision() <= lastGCRev &&
			rmd.Revision() > gcRevision {
			return fmt.Errorf("Revision %d happened on or before the last "+
				"gc time %s rev %d, but was not included in the latest "+
				"gc op revision %d", rmd.Revision(), lastGCRevisionTime,
				lastGCRev, gcRevision)
		}
	}
	sc.log.CDebugf(ctx, "Folder %v has %d expected live blocks, total %d bytes",
		tlf, len(expectedLiveBlocks), expectedRef)

	currMD := rmds[len(rmds)-1]
	expectedUsage := currMD.DiskUsage()
	if expectedUsage != expectedRef {
		return fmt.Errorf("Expected ref bytes %d doesn't match latest disk "+
			"usage %d", expectedRef, expectedUsage)
	}

	// Then, using the current MD head, start at the root of the FS
	// and recursively walk the directory tree to find all the blocks
	// that are currently accessible.
	rootNode, _, _, err := ops.getRootNode(ctx)
	if err != nil {
		return err
	}
	rootPath := ops.nodeCache.PathFromNode(rootNode)
	if g, e := rootPath.tailPointer(), currMD.data.Dir.BlockPointer; g != e {
		return fmt.Errorf("Current MD root pointer %v doesn't match root "+
			"node pointer %v", e, g)
	}
	actualLiveBlocks[rootPath.tailPointer()] = currMD.data.Dir.EncodedSize
	if err := sc.findAllBlocksInPath(ctx, lState, ops, currMD.ReadOnly(),
		rootPath, actualLiveBlocks); err != nil {
		return err
	}
	sc.log.CDebugf(ctx, "Folder %v has %d actual live blocks",
		tlf, len(actualLiveBlocks))

	// Compare the two and see if there are any differences. Don't use
	// reflect.DeepEqual so we can print out exactly what's wrong.
	var extraBlocks []BlockPointer
	actualSize := uint64(0)
	for ptr, size := range actualLiveBlocks {
		actualSize += uint64(size)
		if !expectedLiveBlocks[ptr] {
			extraBlocks = append(extraBlocks, ptr)
		}
	}
	if len(extraBlocks) != 0 {
		sc.log.CWarningf(ctx, "%v: Extra live blocks found: %v",
			tlf, extraBlocks)
		return fmt.Errorf("Folder %v has inconsistent state", tlf)
	}
	var missingBlocks []BlockPointer
	for ptr := range expectedLiveBlocks {
		if _, ok := actualLiveBlocks[ptr]; !ok {
			missingBlocks = append(missingBlocks, ptr)
		}
	}
	if len(missingBlocks) != 0 {
		sc.log.CWarningf(ctx, "%v: Expected live blocks not found: %v",
			tlf, missingBlocks)
		return fmt.Errorf("Folder %v has inconsistent state", tlf)
	}

	if actualSize != expectedRef {
		return fmt.Errorf("Actual size %d doesn't match expected size %d",
			actualSize, expectedRef)
	}

	// Check that the set of referenced blocks matches exactly what
	// the block server knows about.
	bserverLocal, ok := sc.config.BlockServer().(blockServerLocal)
	if !ok {
		if jbs, jok := sc.config.BlockServer().(journalBlockServer); jok {
			bserverLocal, ok = jbs.BlockServer.(blockServerLocal)
			if !ok {
				sc.log.CDebugf(ctx, "Bad block server: %T", jbs.BlockServer)
			}
		}
	}
	if !ok {
		return errors.New("StateChecker only works against " +
			"BlockServerLocal")
	}
	bserverKnownBlocks, err := bserverLocal.getAllRefsForTest(ctx, tlf)
	if err != nil {
		return err
	}

	blockRefsByID := make(map[BlockID]blockRefMap)
	for ptr := range expectedLiveBlocks {
		if _, ok := blockRefsByID[ptr.ID]; !ok {
			blockRefsByID[ptr.ID] = make(blockRefMap)
		}
		blockRefsByID[ptr.ID].put(ptr.BlockContext, liveBlockRef, "")
	}
	for ptr := range archivedBlocks {
		if _, ok := blockRefsByID[ptr.ID]; !ok {
			blockRefsByID[ptr.ID] = make(blockRefMap)
		}
		blockRefsByID[ptr.ID].put(ptr.BlockContext, archivedBlockRef, "")
	}

	if g, e := bserverKnownBlocks, blockRefsByID; !reflect.DeepEqual(g, e) {
		for id, eRefs := range e {
			if gRefs := g[id]; !reflect.DeepEqual(gRefs, eRefs) {
				sc.log.CDebugf(ctx, "Refs for ID %v don't match.  "+
					"Got %v, expected %v", id, gRefs, eRefs)
			}
		}
		for id, gRefs := range g {
			if _, ok := e[id]; !ok {
				sc.log.CDebugf(ctx, "Did not find matching expected "+
					"ID for found block %v (with refs %v)", id, gRefs)
			}
		}

		return fmt.Errorf("Folder %v has inconsistent state", tlf)
	}

	// TODO: Check the archived and deleted blocks as well.
	return nil
}
<commit_msg>state_checker: use fileData instead of directly using IPtrs<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"errors"
	"fmt"
	"reflect"
	"time"

	"github.com/keybase/client/go/logger"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/kbfs/tlf"
	"golang.org/x/net/context"
)

// StateChecker verifies that the server-side state for KBFS is
// consistent.  Useful mostly for testing because it isn't scalable
// and loads all the state in memory.
type StateChecker struct {
	config Config
	log    logger.Logger
}

// NewStateChecker returns a new StateChecker instance.
func NewStateChecker(config Config) *StateChecker {
	return &StateChecker{config, config.MakeLogger("")}
}

func (sc *StateChecker) newFileData(lState *lockState,
	file path, kmd KeyMetadata, ops *folderBranchOps) *fileData {
	var uid keybase1.UID // reads don't need UID
	return newFileData(file, uid, sc.config.Crypto(),
		sc.config.BlockSplitter(), kmd,
		// We shouldn't ever be fetching dirty blocks during state
		// checking.
		func(ctx context.Context, kmd KeyMetadata, ptr BlockPointer,
			file path, rtype blockReqType) (*FileBlock, bool, error) {
			block, err := ops.blocks.GetFileBlockForReading(
				ctx, lState, kmd, ptr, file.Branch, file)
			if err != nil {
				return nil, false, err
			}
			return block, false, nil
		},
		func(ptr BlockPointer, block Block) error {
			return nil
		}, sc.log)
}

// findAllFileBlocks adds all file blocks found under this block to
// the blocksFound map, if the given path represents an indirect
// block.
func (sc *StateChecker) findAllFileBlocks(ctx context.Context,
	lState *lockState, ops *folderBranchOps, kmd KeyMetadata,
	file path, blockSizes map[BlockPointer]uint32) error {
	fd := sc.newFileData(lState, file, kmd, ops)
	infos, err := fd.getIndirectFileBlockInfos(ctx)
	if err != nil {
		return err
	}

	for _, info := range infos {
		blockSizes[info.BlockPointer] = info.EncodedSize
	}
	return nil
}

// findAllBlocksInPath adds all blocks found within this directory to
// the blockSizes map, and then recursively checks all
// subdirectories.
func (sc *StateChecker) findAllBlocksInPath(ctx context.Context,
	lState *lockState, ops *folderBranchOps, kmd KeyMetadata,
	dir path, blockSizes map[BlockPointer]uint32) error {
	dblock, err := ops.blocks.GetDirBlockForReading(ctx, lState, kmd,
		dir.tailPointer(), dir.Branch, dir)
	if err != nil {
		return err
	}

	for name, de := range dblock.Children {
		if de.Type == Sym {
			continue
		}

		blockSizes[de.BlockPointer] = de.EncodedSize
		p := dir.ChildPath(name, de.BlockPointer)

		if de.Type == Dir {
			err := sc.findAllBlocksInPath(ctx, lState, ops, kmd, p, blockSizes)
			if err != nil {
				return err
			}
		} else {
			// If it's a file, check to see if it's indirect.
			err := sc.findAllFileBlocks(ctx, lState, ops, kmd, p, blockSizes)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (sc *StateChecker) getLastGCData(ctx context.Context,
	tlf tlf.ID) (time.Time, MetadataRevision) {
	config, ok := sc.config.(*ConfigLocal)
	if !ok {
		return time.Time{}, MetadataRevisionUninitialized
	}

	var latestTime time.Time
	var latestRev MetadataRevision
	for _, c := range *config.allKnownConfigsForTesting {
		ops := c.KBFSOps().(*KBFSOpsStandard).getOpsNoAdd(
			FolderBranch{tlf, MasterBranch})
		rt, rev := ops.fbm.getLastQRData()
		if rt.After(latestTime) && rev > latestRev {
			latestTime = rt
			latestRev = rev
		}
	}
	if latestTime == (time.Time{}) {
		return latestTime, latestRev
	}

	sc.log.CDebugf(ctx, "Last qr data for TLF %s: revTime=%s, rev=%d",
		tlf, latestTime, latestRev)
	return latestTime.Add(-sc.config.QuotaReclamationMinUnrefAge()), latestRev
}

// CheckMergedState verifies that the state for the given tlf is
// consistent.
func (sc *StateChecker) CheckMergedState(ctx context.Context, tlf tlf.ID) error {
	// Blow away MD cache so we don't have any lingering re-embedded
	// block changes (otherwise we won't be able to learn their sizes).
	sc.config.SetMDCache(NewMDCacheStandard(defaultMDCacheCapacity))

	// Fetch all the MD updates for this folder, and use the block
	// change lists to build up the set of currently referenced blocks.
	rmds, err := getMergedMDUpdates(ctx, sc.config, tlf,
		MetadataRevisionInitial)
	if err != nil {
		return err
	}
	if len(rmds) == 0 {
		sc.log.CDebugf(ctx, "No state to check for folder %s", tlf)
		return nil
	}

	lState := makeFBOLockState()

	// Re-embed block changes.
	kbfsOps, ok := sc.config.KBFSOps().(*KBFSOpsStandard)
	if !ok {
		return errors.New("Unexpected KBFSOps type")
	}

	fb := FolderBranch{tlf, MasterBranch}
	ops := kbfsOps.getOpsNoAdd(fb)
	lastGCRevisionTime, lastGCRev := sc.getLastGCData(ctx, tlf)

	// Build the expected block list.
	expectedLiveBlocks := make(map[BlockPointer]bool)
	expectedRef := uint64(0)
	archivedBlocks := make(map[BlockPointer]bool)
	actualLiveBlocks := make(map[BlockPointer]uint32)

	// See what the last GC op revision is.  All unref'd pointers from
	// that revision or earlier should be deleted from the block
	// server.
	gcRevision := MetadataRevisionUninitialized
	for _, rmd := range rmds {
		// Don't process copies.
		if rmd.IsWriterMetadataCopiedSet() {
			continue
		}

		for _, op := range rmd.data.Changes.Ops {
			GCOp, ok := op.(*GCOp)
			if !ok {
				continue
			}
			gcRevision = GCOp.LatestRev
		}
	}

	for _, rmd := range rmds {
		// Don't process copies.
		if rmd.IsWriterMetadataCopiedSet() {
			continue
		}
		// Any unembedded block changes also count towards the actual size
		if info := rmd.data.cachedChanges.Info; info.BlockPointer != zeroPtr {
			sc.log.CDebugf(ctx, "Unembedded block change: %v, %d",
				info.BlockPointer, info.EncodedSize)
			actualLiveBlocks[info.BlockPointer] = info.EncodedSize

			// Any child block change pointers?
			file := path{FolderBranch{tlf, MasterBranch},
				[]pathNode{{
					info.BlockPointer,
					fmt.Sprintf("<MD with revision %d>", rmd.Revision())}}}
			err := sc.findAllFileBlocks(ctx, lState, ops, rmd.ReadOnly(),
				file, actualLiveBlocks)
			if err != nil {
				return err
			}
		}

		var hasGCOp bool
		for _, op := range rmd.data.Changes.Ops {
			_, isGCOp := op.(*GCOp)
			hasGCOp = hasGCOp || isGCOp

			opRefs := make(map[BlockPointer]bool)
			for _, ptr := range op.Refs() {
				if ptr != zeroPtr {
					expectedLiveBlocks[ptr] = true
					opRefs[ptr] = true
				}
			}
			if !isGCOp {
				for _, ptr := range op.Unrefs() {
					delete(expectedLiveBlocks, ptr)
					if ptr != zeroPtr {
						// If the revision has been garbage-collected,
						// or if the pointer has been referenced and
						// unreferenced within the same op (which
						// indicates a failed and retried sync), the
						// corresponding block should already be
						// cleaned up.
						if rmd.Revision() <= gcRevision || opRefs[ptr] {
							delete(archivedBlocks, ptr)
						} else {
							archivedBlocks[ptr] = true
						}
					}
				}
			}
			for _, update := range op.allUpdates() {
				delete(expectedLiveBlocks, update.Unref)
				if update.Unref != zeroPtr && update.Ref != update.Unref {
					if rmd.Revision() <= gcRevision {
						delete(archivedBlocks, update.Unref)
					} else {
						archivedBlocks[update.Unref] = true
					}
				}
				if update.Ref != zeroPtr {
					expectedLiveBlocks[update.Ref] = true
				}
			}
		}
		expectedRef += rmd.RefBytes()
		expectedRef -= rmd.UnrefBytes()

		if len(rmd.data.Changes.Ops) == 1 && hasGCOp {
			// Don't check GC status for GC revisions
			continue
		}

		// Make sure that if this revision should be covered by a GC
		// op, it is.  Note that this assumes that if QR is ever run,
		// it will be run completely and not left partially done due
		// to there being too many pointers to collect in one sweep.
		mtime := time.Unix(0, rmd.data.Dir.Mtime)
		if !lastGCRevisionTime.Before(mtime) && rmd.Revision() <= lastGCRev &&
			rmd.Revision() > gcRevision {
			return fmt.Errorf("Revision %d happened on or before the last "+
				"gc time %s rev %d, but was not included in the latest "+
				"gc op revision %d", rmd.Revision(), lastGCRevisionTime,
				lastGCRev, gcRevision)
		}
	}
	sc.log.CDebugf(ctx, "Folder %v has %d expected live blocks, total %d bytes",
		tlf, len(expectedLiveBlocks), expectedRef)

	currMD := rmds[len(rmds)-1]
	expectedUsage := currMD.DiskUsage()
	if expectedUsage != expectedRef {
		return fmt.Errorf("Expected ref bytes %d doesn't match latest disk "+
			"usage %d", expectedRef, expectedUsage)
	}

	// Then, using the current MD head, start at the root of the FS
	// and recursively walk the directory tree to find all the blocks
	// that are currently accessible.
	rootNode, _, _, err := ops.getRootNode(ctx)
	if err != nil {
		return err
	}
	rootPath := ops.nodeCache.PathFromNode(rootNode)
	if g, e := rootPath.tailPointer(), currMD.data.Dir.BlockPointer; g != e {
		return fmt.Errorf("Current MD root pointer %v doesn't match root "+
			"node pointer %v", e, g)
	}
	actualLiveBlocks[rootPath.tailPointer()] = currMD.data.Dir.EncodedSize
	if err := sc.findAllBlocksInPath(ctx, lState, ops, currMD.ReadOnly(),
		rootPath, actualLiveBlocks); err != nil {
		return err
	}
	sc.log.CDebugf(ctx, "Folder %v has %d actual live blocks",
		tlf, len(actualLiveBlocks))

	// Compare the two and see if there are any differences. Don't use
	// reflect.DeepEqual so we can print out exactly what's wrong.
	var extraBlocks []BlockPointer
	actualSize := uint64(0)
	for ptr, size := range actualLiveBlocks {
		actualSize += uint64(size)
		if !expectedLiveBlocks[ptr] {
			extraBlocks = append(extraBlocks, ptr)
		}
	}
	if len(extraBlocks) != 0 {
		sc.log.CWarningf(ctx, "%v: Extra live blocks found: %v",
			tlf, extraBlocks)
		return fmt.Errorf("Folder %v has inconsistent state", tlf)
	}
	var missingBlocks []BlockPointer
	for ptr := range expectedLiveBlocks {
		if _, ok := actualLiveBlocks[ptr]; !ok {
			missingBlocks = append(missingBlocks, ptr)
		}
	}
	if len(missingBlocks) != 0 {
		sc.log.CWarningf(ctx, "%v: Expected live blocks not found: %v",
			tlf, missingBlocks)
		return fmt.Errorf("Folder %v has inconsistent state", tlf)
	}

	if actualSize != expectedRef {
		return fmt.Errorf("Actual size %d doesn't match expected size %d",
			actualSize, expectedRef)
	}

	// Check that the set of referenced blocks matches exactly what
	// the block server knows about.
	bserverLocal, ok := sc.config.BlockServer().(blockServerLocal)
	if !ok {
		if jbs, jok := sc.config.BlockServer().(journalBlockServer); jok {
			bserverLocal, ok = jbs.BlockServer.(blockServerLocal)
			if !ok {
				sc.log.CDebugf(ctx, "Bad block server: %T", jbs.BlockServer)
			}
		}
	}
	if !ok {
		return errors.New("StateChecker only works against " +
			"BlockServerLocal")
	}
	bserverKnownBlocks, err := bserverLocal.getAllRefsForTest(ctx, tlf)
	if err != nil {
		return err
	}

	blockRefsByID := make(map[BlockID]blockRefMap)
	for ptr := range expectedLiveBlocks {
		if _, ok := blockRefsByID[ptr.ID]; !ok {
			blockRefsByID[ptr.ID] = make(blockRefMap)
		}
		blockRefsByID[ptr.ID].put(ptr.BlockContext, liveBlockRef, "")
	}
	for ptr := range archivedBlocks {
		if _, ok := blockRefsByID[ptr.ID]; !ok {
			blockRefsByID[ptr.ID] = make(blockRefMap)
		}
		blockRefsByID[ptr.ID].put(ptr.BlockContext, archivedBlockRef, "")
	}

	if g, e := bserverKnownBlocks, blockRefsByID; !reflect.DeepEqual(g, e) {
		for id, eRefs := range e {
			if gRefs := g[id]; !reflect.DeepEqual(gRefs, eRefs) {
				sc.log.CDebugf(ctx, "Refs for ID %v don't match.  "+
					"Got %v, expected %v", id, gRefs, eRefs)
			}
		}
		for id, gRefs := range g {
			if _, ok := e[id]; !ok {
				sc.log.CDebugf(ctx, "Did not find matching expected "+
					"ID for found block %v (with refs %v)", id, gRefs)
			}
		}

		return fmt.Errorf("Folder %v has inconsistent state", tlf)
	}

	// TODO: Check the archived and deleted blocks as well.
	return nil
}
<|endoftext|>


<commit_before>package uwsgi

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"testing"
	"time"
)

func writeKV(fd io.Writer, k, v string) {
	var b [2]byte
	binary.LittleEndian.PutUint16(b[:], uint16(len(k)))
	fd.Write(b[:])
	fd.Write([]byte(k))
	binary.LittleEndian.PutUint16(b[:], uint16(len(v)))
	fd.Write(b[:])
	fd.Write([]byte(v))
}

func TestBasic(t *testing.T) {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen error: %v", err)
	}
	addr, _ := l.Addr().(*net.TCPAddr)

	var lastReq *http.Request
	reqNum := 0
	handler := http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		reqNum++

		v := fmt.Sprintf("bar%d", reqNum)
		if req.FormValue("foo") == v {
			fmt.Fprintf(res, "req=%d", reqNum)
		}
		lastReq = req
	})

	server := &http.Server{Handler: handler}
	go server.Serve(&Listener{l})

	m := map[string]string{
		"HOST":              "localhost",
		"REQUEST_METHOD":    "POST",
		"REQUEST_URI":       "/foo",
		"CONTENT_LENGTH":    "8",
		"SERVER_PROTOCOL":   "HTTP/1.1",
		"HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded",
		"HTTP_USER_AGENT":   "go",
	}

	var b [2]byte
	var head [4]byte
	for n := 1; n <= 3; n++ {
		fd, _ := net.Dial("tcp", addr.String())
		s := 0
		for k, v := range m {
			s += (len([]byte(k)) + len([]byte(v)) + 4)
		}
		binary.LittleEndian.PutUint16(b[:], uint16(s))
		head[1] = b[0]
		head[2] = b[1]
		fd.Write(head[:])
		for k, v := range m {
			writeKV(fd, k, v)
		}
		fmt.Fprintf(fd, "foo=bar%d", n)
		time.Sleep(1e9)

		res, _ := http.ReadResponse(bufio.NewReader(fd), lastReq)
		got := res.Request.Method
		expected := "POST"
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}

		got = res.Request.URL.Path
		expected = "/foo"
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}
		body, _ := ioutil.ReadAll(res.Body)
		res.Body.Close()
		got = string(body)
		expected = fmt.Sprintf("req=%d", n)
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}
		fd.Close()
		fd = nil
	}

	l.Close()
}

func TestServer(t *testing.T) {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen error: %v", err)
	}
	addr, _ := l.Addr().(*net.TCPAddr)

	var lastReq *http.Request

	passenger := &Passenger{"tcp", "127.0.0.1"}
	handler := http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		passenger.ServeHTTP(res, req)
		panic("stop")
	})
	server := &http.Server{Handler: handler}
	go server.ListenAndServe()

	m := map[string]string{
		"REQUEST_METHOD":    "POST",
		"REQUEST_URI":       "/foo",
		"CONTENT_LENGTH":    "8",
		"SERVER_PROTOCOL":   "HTTP/1.1",
		"HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded",
		"HTTP_USER_AGENT":   "go",
	}
	var b [2]byte
	var head [4]byte
	for n := 1; n <= 3; n++ {
		fd, _ := net.Dial("tcp", addr.String())
		s := 0
		for k, v := range m {
			s += (len([]byte(k)) + len([]byte(v)) + 4)
		}
		binary.LittleEndian.PutUint16(b[:], uint16(s))
		head[1] = b[0]
		head[2] = b[1]
		fd.Write(head[:])
		for k, v := range m {
			writeKV(fd, k, v)
		}
		fmt.Fprintf(fd, "foo=bar%d", n)
		time.Sleep(1e9)

		res, _ := http.ReadResponse(bufio.NewReader(fd), lastReq)
		got := res.Request.Method
		expected := "POST"
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}

		got = res.Request.URL.Path
		expected = "/foo"
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}
		body, _ := ioutil.ReadAll(res.Body)
		res.Body.Close()
		got = string(body)
		expected = fmt.Sprintf("req=%d", n)
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}
		fd.Close()
		fd = nil
	}

	l.Close()
}
<commit_msg>remove TestServer<commit_after>package uwsgi

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"testing"
	"time"
)

func writeKV(fd io.Writer, k, v string) {
	var b [2]byte
	binary.LittleEndian.PutUint16(b[:], uint16(len(k)))
	fd.Write(b[:])
	fd.Write([]byte(k))
	binary.LittleEndian.PutUint16(b[:], uint16(len(v)))
	fd.Write(b[:])
	fd.Write([]byte(v))
}

func TestBasic(t *testing.T) {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen error: %v", err)
	}
	addr, _ := l.Addr().(*net.TCPAddr)

	var lastReq *http.Request
	reqNum := 0
	handler := http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		reqNum++

		v := fmt.Sprintf("bar%d", reqNum)
		if req.FormValue("foo") == v {
			fmt.Fprintf(res, "req=%d", reqNum)
		}
		lastReq = req
	})

	server := &http.Server{Handler: handler}
	go server.Serve(&Listener{l})

	m := map[string]string{
		"HOST":              "localhost",
		"REQUEST_METHOD":    "POST",
		"REQUEST_URI":       "/foo",
		"CONTENT_LENGTH":    "8",
		"SERVER_PROTOCOL":   "HTTP/1.1",
		"HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded",
		"HTTP_USER_AGENT":   "go",
	}

	var b [2]byte
	var head [4]byte
	for n := 1; n <= 3; n++ {
		fd, _ := net.Dial("tcp", addr.String())
		s := 0
		for k, v := range m {
			s += (len([]byte(k)) + len([]byte(v)) + 4)
		}
		binary.LittleEndian.PutUint16(b[:], uint16(s))
		head[1] = b[0]
		head[2] = b[1]
		fd.Write(head[:])
		for k, v := range m {
			writeKV(fd, k, v)
		}
		fmt.Fprintf(fd, "foo=bar%d", n)
		time.Sleep(1e9)

		res, _ := http.ReadResponse(bufio.NewReader(fd), lastReq)
		got := res.Request.Method
		expected := "POST"
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}

		got = res.Request.URL.Path
		expected = "/foo"
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}
		body, _ := ioutil.ReadAll(res.Body)
		res.Body.Close()
		got = string(body)
		expected = fmt.Sprintf("req=%d", n)
		if string(got) != expected {
			t.Errorf("Unexpected response for request #1; got %q; expected %q",
				string(got), expected)
		}
		fd.Close()
		fd = nil
	}

	l.Close()
}
<|endoftext|>


<commit_before><commit_msg>[daemon] use go conventients for error handling<commit_after><|endoftext|>


<commit_before><commit_msg>bump to version 2.7.1<commit_after><|endoftext|>


<commit_before><commit_msg>*: bump to v0.0.2<commit_after><|endoftext|>


<commit_before><commit_msg>vsphere: report clone and power-on errors to Sentry<commit_after><|endoftext|>


<commit_before><commit_msg>Remove the not used package in web/web.go<commit_after><|endoftext|>


<commit_before>package golog

import (
	"fmt"
	"path"
	"runtime"
	"strings"
	"time"
)

type LevelLogger interface {
	Log(level int, vals ...interface{})
	Logf(level int, f string, vals ...interface{})
	Logc(level int, closure func() string)
	FailNow()
	SetMinLogLevel(int)
}

type levelLoggerImpl struct {
	Logger
	// TODO comment this
	// Skip 0 refers to the function calling getLocation.
	getLocation func(skip int) *LogLocation
}

func NoLocation(skip int) *LogLocation { return nil }

func FullLocation(skip int) *LogLocation {
	pc, file, line, ok := runtime.Caller(skip + 1)
	if !ok {
		return nil
	} else {
		// TODO Make sure this is compiler agnostic.
		funcParts := strings.SplitN(runtime.FuncForPC(pc).Name(), ".", 2)
		return &LogLocation{
			Package:  funcParts[0],
			File:     path.Base(file),
			Function: funcParts[1],
			Line:     line,
		}
	}

	panic("Flow never reaches here, this mollifies the compiler")
}

func NewLevelLogger(l Logger, locFunc func(int) *LogLocation) LevelLogger {
	return &levelLoggerImpl{l, locFunc}
}

func (l *levelLoggerImpl) makeLogClosure(level int, msg func() string) func() *LogMessage {
	// Evaluate this early.
	ns := time.Nanoseconds()
	// TODO Be less brittle.
	// Skip over makeLogClosure, logCommon, and Log
	location := l.getLocation(3)

	return func() *LogMessage {
		return &LogMessage{
			Level:       level,
			Message:     msg(),
			Nanoseconds: ns,
			Location:    location,
		}
	}
}

func (l *levelLoggerImpl) logCommon(level int, closure func() string) {
	l.Logger.Log(level, l.makeLogClosure(level, closure))
}

func (l *levelLoggerImpl) Log(level int, msg ...interface{}) {
	l.logCommon(level, func() string { return fmt.Sprint(msg...) })
}

func (l *levelLoggerImpl) Logf(level int, f string, msg ...interface{}) {
	l.logCommon(level, func() string { return fmt.Sprintf(f, msg...) })
}

func (l *levelLoggerImpl) Logc(level int, closure func() string) {
	l.logCommon(level, closure)
}
<commit_msg>LocationLogger adds location<commit_after>package golog

import (
	"fmt"
	"path"
	"runtime"
	"strings"
	"time"
)

type LocationLogger interface {
	LogDepth(level int, closure func() string, depth int)
	FailNow()
	SetMinLogLevel(int)
}

type LevelLogger interface {
	Log(level int, vals ...interface{})
	Logf(level int, f string, vals ...interface{})
	Logc(level int, closure func() string)
	LocationLogger
}

type levelLoggerImpl struct {
	Logger
	// TODO comment this
	// Skip 0 refers to the function calling getLocation.
	getLocation func(skip int) *LogLocation
}

func NoLocation(skip int) *LogLocation { return nil }

func FullLocation(skip int) *LogLocation {
	pc, file, line, ok := runtime.Caller(skip + 1)
	if !ok {
		return nil
	} else {
		// TODO Make sure this is compiler agnostic.
		funcParts := strings.SplitN(runtime.FuncForPC(pc).Name(), ".", 2)
		return &LogLocation{
			Package:  funcParts[0],
			File:     path.Base(file),
			Function: funcParts[1],
			Line:     line,
		}
	}

	panic("Flow never reaches here, this mollifies the compiler")
}

func NewLevelLogger(l Logger, locFunc func(int) *LogLocation) LevelLogger {
	return &levelLoggerImpl{l, locFunc}
}

func (l *levelLoggerImpl) makeLogClosure(level int, msg func() string, skip int) func() *LogMessage {
	// Evaluate this early.
	ns := time.Nanoseconds()
	location := l.getLocation(skip + 1)

	return func() *LogMessage {
		return &LogMessage{
			Level:       level,
			Message:     msg(),
			Nanoseconds: ns,
			Location:    location,
		}
	}
}

func (l *levelLoggerImpl) LogDepth(level int, closure func() string, depth int) {
	l.Logger.Log(level, l.makeLogClosure(level, closure, depth + 1))
}

func (l *levelLoggerImpl) Log(level int, msg ...interface{}) {
	l.LogDepth(level, func() string { return fmt.Sprint(msg...) }, 1)
}

func (l *levelLoggerImpl) Logf(level int, f string, msg ...interface{}) {
	l.LogDepth(level, func() string { return fmt.Sprintf(f, msg...) }, 1)
}

func (l *levelLoggerImpl) Logc(level int, closure func() string) {
	l.LogDepth(level, closure, 1)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/hanwen/termite/termite"
	"log"
	"os"
	"path/filepath"
	"rpc"
	"strings"
)

const _SHELL = "/bin/sh"

func TryRunDirect(cmd string) {
	if cmd == ":" {
		os.Exit(0)
	}

	parsed := termite.ParseCommand(cmd)
	if len(parsed) == 0 {
		return
	}

	if parsed[0] == "echo" {
		fmt.Println(strings.Join(parsed[1:], " "))
		os.Exit(0)
	}
	if parsed[0] == "true" {
		os.Exit(0)
	}
	if parsed[0] == "false" {
		os.Exit(1)
	}

	// TODO mkdir, rm, expr, others?
}

func Refresh() {
	socket := termite.FindSocket()
	conn := termite.OpenSocketConnection(socket, termite.RPC_CHANNEL)

	client := rpc.NewClient(conn)

	req := 1
	rep := 1
	err := client.Call("LocalMaster.RefreshAttributeCache", &req, &rep)
	client.Close()
	if err != nil {
		log.Fatal("LocalMaster.RefreshAttributeCache: ", err)
	}
	conn.Close()
}

func cleanEnv(input []string) []string {
	env := []string{}
	for _, v := range input {
		comps := strings.SplitN(v, "=", 2)
		if comps[1] == "termite-make" {
			// TODO - more generic.
			v = fmt.Sprintf("%s=%s", comps[0], "make")
		} else if comps[0] == "MAKE_SHELL" {
			continue
		}
		env = append(env, v)
	}
	return env
}

func TryRunLocally(command string, topdir string) (exit *os.Waitmsg, rule termite.LocalRule) {
	decider := termite.NewLocalDecider(topdir)
	if !(len(os.Args) == 3 && os.Args[0] == _SHELL && os.Args[1] == "-c") {
		return
	}

	rule = decider.ShouldRunLocally(command)
	if rule.Local {
		env := os.Environ()
		if !rule.Recurse {
			env = cleanEnv(env)
		}

		proc, err := os.StartProcess(_SHELL, os.Args, &os.ProcAttr{
			Env:   env,
			Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
		})
		if err != nil {
			log.Fatalf("os.StartProcess() for %s: %v", command, err)
		}
		msg, err := proc.Wait(0)
		if err != nil {
			log.Fatalf("proc.Wait() for %s: %v", command, err)
		}
		return msg, rule
	}

	return
}

func main() {
	command := flag.String("c", "", "command to run.")
	refresh := flag.Bool("refresh", false, "refresh master file cache.")
	debug := flag.Bool("dbg", false, "set on debugging in request.")
	flag.Parse()

	if *refresh {
		Refresh()
	}

	if *command == "" {
		return
	}
	os.Args[0] = _SHELL
	TryRunDirect(*command)

	socket := termite.FindSocket()
	if socket == "" {
		log.Fatal("Could not find .termite-socket")
	}
	topDir, _ := filepath.Split(socket)

	localWaitMsg, localRule := TryRunLocally(*command, topDir)
	if localWaitMsg != nil && !localRule.SkipRefresh {
		Refresh()
	}

	wd, err := os.Getwd()
	if err != nil {
		log.Fatal("Getwd", err)
	}

	conn := termite.OpenSocketConnection(socket, termite.RPC_CHANNEL)

	// TODO - could skip the shell if we can deduce it is a
	// no-frills command invocation.
	req := termite.WorkRequest{
		Binary:     _SHELL,
		Argv:       []string{"/bin/sh", "-c", *command},
		Env:        cleanEnv(os.Environ()),
		Dir:        wd,
		RanLocally: localWaitMsg != nil,
	}
	req.Debug = localRule.Debug || os.Getenv("TERMITE_DEBUG") != "" || *debug
	client := rpc.NewClient(conn)

	rep := termite.WorkReply{}
	err = client.Call("LocalMaster.Run", &req, &rep)
	client.Close()
	if err != nil {
		log.Fatal("LocalMaster.Run: ", err)
	}

	os.Stdout.Write([]byte(rep.Stdout))
	os.Stderr.Write([]byte(rep.Stderr))

	// TODO -something with signals.
	if localWaitMsg == nil {
		localWaitMsg = &rep.Exit
	}
	conn.Close()
	os.Exit(localWaitMsg.ExitStatus())
}
<commit_msg>Print failed command.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/hanwen/termite/termite"
	"log"
	"os"
	"path/filepath"
	"rpc"
	"strings"
)

const _SHELL = "/bin/sh"

func TryRunDirect(cmd string) {
	if cmd == ":" {
		os.Exit(0)
	}

	parsed := termite.ParseCommand(cmd)
	if len(parsed) == 0 {
		return
	}

	if parsed[0] == "echo" {
		fmt.Println(strings.Join(parsed[1:], " "))
		os.Exit(0)
	}
	if parsed[0] == "true" {
		os.Exit(0)
	}
	if parsed[0] == "false" {
		os.Exit(1)
	}

	// TODO mkdir, rm, expr, others?
}

func Refresh() {
	socket := termite.FindSocket()
	conn := termite.OpenSocketConnection(socket, termite.RPC_CHANNEL)

	client := rpc.NewClient(conn)

	req := 1
	rep := 1
	err := client.Call("LocalMaster.RefreshAttributeCache", &req, &rep)
	client.Close()
	if err != nil {
		log.Fatal("LocalMaster.RefreshAttributeCache: ", err)
	}
	conn.Close()
}

func cleanEnv(input []string) []string {
	env := []string{}
	for _, v := range input {
		comps := strings.SplitN(v, "=", 2)
		if comps[1] == "termite-make" {
			// TODO - more generic.
			v = fmt.Sprintf("%s=%s", comps[0], "make")
		} else if comps[0] == "MAKE_SHELL" {
			continue
		}
		env = append(env, v)
	}
	return env
}

func TryRunLocally(command string, topdir string) (exit *os.Waitmsg, rule termite.LocalRule) {
	decider := termite.NewLocalDecider(topdir)
	if !(len(os.Args) == 3 && os.Args[0] == _SHELL && os.Args[1] == "-c") {
		return
	}

	rule = decider.ShouldRunLocally(command)
	if rule.Local {
		env := os.Environ()
		if !rule.Recurse {
			env = cleanEnv(env)
		}

		proc, err := os.StartProcess(_SHELL, os.Args, &os.ProcAttr{
			Env:   env,
			Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
		})
		if err != nil {
			log.Fatalf("os.StartProcess() for %s: %v", command, err)
		}
		msg, err := proc.Wait(0)
		if err != nil {
			log.Fatalf("proc.Wait() for %s: %v", command, err)
		}
		return msg, rule
	}

	return
}

func main() {
	command := flag.String("c", "", "command to run.")
	refresh := flag.Bool("refresh", false, "refresh master file cache.")
	debug := flag.Bool("dbg", false, "set on debugging in request.")
	flag.Parse()

	if *refresh {
		Refresh()
	}

	if *command == "" {
		return
	}
	os.Args[0] = _SHELL
	TryRunDirect(*command)

	socket := termite.FindSocket()
	if socket == "" {
		log.Fatal("Could not find .termite-socket")
	}
	topDir, _ := filepath.Split(socket)

	localWaitMsg, localRule := TryRunLocally(*command, topDir)
	if localWaitMsg != nil && !localRule.SkipRefresh {
		Refresh()
	}

	wd, err := os.Getwd()
	if err != nil {
		log.Fatal("Getwd", err)
	}

	conn := termite.OpenSocketConnection(socket, termite.RPC_CHANNEL)

	// TODO - could skip the shell if we can deduce it is a
	// no-frills command invocation.
	req := termite.WorkRequest{
		Binary:     _SHELL,
		Argv:       []string{"/bin/sh", "-c", *command},
		Env:        cleanEnv(os.Environ()),
		Dir:        wd,
		RanLocally: localWaitMsg != nil,
	}
	req.Debug = localRule.Debug || os.Getenv("TERMITE_DEBUG") != "" || *debug
	client := rpc.NewClient(conn)

	rep := termite.WorkReply{}
	err = client.Call("LocalMaster.Run", &req, &rep)
	client.Close()
	if err != nil {
		log.Fatal("LocalMaster.Run: ", err)
	}

	os.Stdout.Write([]byte(rep.Stdout))
	os.Stderr.Write([]byte(rep.Stderr))

	// TODO -something with signals.
	if localWaitMsg == nil {
		localWaitMsg = &rep.Exit
	}
	if localWaitMsg.ExitStatus() != 0 {
		log.Printf("Failed: %q", *command)
	}
	conn.Close()
	os.Exit(localWaitMsg.ExitStatus())
}
<|endoftext|>


<commit_before>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package glfw

import (
	"fmt"
	"unsafe"

	"golang.org/x/sys/windows"
)

const (
	processQueryLimitedInformation = 0x1000
)

var (
	kernel32 = windows.NewLazySystemDLL("kernel32.dll")
	user32   = windows.NewLazySystemDLL("user32.dll")

	getCurrentProcessIdProc      = kernel32.NewProc("GetCurrentProcessId")
	getConsoleWindowProc         = kernel32.NewProc("GetConsoleWindow")
	getWindowThreadProcessIdProc = user32.NewProc("GetWindowThreadProcessId")
	showWindowAsyncProc          = user32.NewProc("ShowWindowAsync")
)

func getCurrentProcessId() (uint32, error) {
	r, _, e := getCurrentProcessIdProc.Call()
	if e != nil && e.(windows.Errno) != 0 {
		return 0, fmt.Errorf("ui: GetCurrentProcessId failed: %d", e)
	}
	return uint32(r), nil
}

func getWindowThreadProcessId(hwnd uintptr) (uint32, error) {
	pid := uint32(0)
	r, _, e := getWindowThreadProcessIdProc.Call(hwnd, uintptr(unsafe.Pointer(&pid)))
	if r == 0 {
		return 0, fmt.Errorf("ui: GetWindowThreadProcessId failed: %d", e)
	}
	return pid, nil
}

func getConsoleWindow() (uintptr, error) {
	r, _, e := getConsoleWindowProc.Call()
	if e != nil && e.(windows.Errno) != 0 {
		return 0, fmt.Errorf("ui: GetConsoleWindow failed: %d", e)
	}
	return r, nil
}

func showWindowAsync(hwnd uintptr, show int) error {
	if _, _, e := showWindowAsyncProc.Call(hwnd, uintptr(show)); e != nil && e.(windows.Errno) != 0 {
		return fmt.Errorf("ui: ShowWindowAsync failed: %d", e)
	}
	return nil
}

// hideConsoleWindowOnWindows will hide the console window that is showing when
// compiling on Windows without specifying the '-ldflags "-Hwindowsgui"' flag.
func hideConsoleWindowOnWindows() {
	pid, err := getCurrentProcessId()
	if err != nil {
		// Ignore errors because:
		// 1. It is not critical if the console can't be hid.
		// 2. There is nothing to do when errors happen.
		return
	}
	w, err := getConsoleWindow()
	if err != nil {
		// Ignore errors
		return
	}
	// Get the process ID of the console's creator.
	cpid, err := getWindowThreadProcessId(w)
	if err != nil {
		// Ignore errors
		return
	}
	if pid == cpid {
		// The current process created its own console. Hide this.
		showWindowAsync(w, windows.SW_HIDE)
	}
}
<commit_msg>internal/uidriver/glfw: use FreeConsole instead of hiding (#1961)<commit_after>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package glfw

import (
	"fmt"
	"unsafe"

	"golang.org/x/sys/windows"
)

const (
	processQueryLimitedInformation = 0x1000
)

var (
	kernel32 = windows.NewLazySystemDLL("kernel32.dll")
	user32   = windows.NewLazySystemDLL("user32.dll")

	getCurrentProcessIdProc      = kernel32.NewProc("GetCurrentProcessId")
	getConsoleWindowProc         = kernel32.NewProc("GetConsoleWindow")
	freeConsoleWindowProc        = kernel32.NewProc("FreeConsole")
	getWindowThreadProcessIdProc = user32.NewProc("GetWindowThreadProcessId")
)

func getCurrentProcessId() (uint32, error) {
	r, _, e := getCurrentProcessIdProc.Call()
	if e != nil && e.(windows.Errno) != 0 {
		return 0, fmt.Errorf("ui: GetCurrentProcessId failed: %d", e)
	}
	return uint32(r), nil
}

func getWindowThreadProcessId(hwnd uintptr) (uint32, error) {
	pid := uint32(0)
	r, _, e := getWindowThreadProcessIdProc.Call(hwnd, uintptr(unsafe.Pointer(&pid)))
	if r == 0 {
		return 0, fmt.Errorf("ui: GetWindowThreadProcessId failed: %d", e)
	}
	return pid, nil
}

func getConsoleWindow() (uintptr, error) {
	r, _, e := getConsoleWindowProc.Call()
	if e != nil && e.(windows.Errno) != 0 {
		return 0, fmt.Errorf("ui: GetConsoleWindow failed: %d", e)
	}
	return r, nil
}

func freeConsole() error {
	_, _, e := freeConsoleWindowProc.Call()
	if e != nil && e.(windows.Errno) != 0 {
		return fmt.Errorf("ui: FreeConsole failed: %d", e)
	}
	return nil
}

// hideConsoleWindowOnWindows will hide the console window that is showing when
// compiling on Windows without specifying the '-ldflags "-Hwindowsgui"' flag.
func hideConsoleWindowOnWindows() {
	pid, err := getCurrentProcessId()
	if err != nil {
		// Ignore errors because:
		// 1. It is not critical if the console can't be hid.
		// 2. There is nothing to do when errors happen.
		return
	}
	w, err := getConsoleWindow()
	if err != nil {
		// Ignore errors
		return
	}
	// Get the process ID of the console's creator.
	cpid, err := getWindowThreadProcessId(w)
	if err != nil {
		// Ignore errors
		return
	}
	if pid == cpid {
		// The current process created its own console. Hide this.
		// Ignore error
		freeConsole()
	}
}
<|endoftext|>


<commit_before>package cachestore

import "sync"

// Store interface of cache store
type Store interface {
	// Load load value with given key
	// Return value and whether load successfully
	Load(key string) (value interface{}, ok bool)
	// Store store value with given key
	Store(key string, value interface{})
	// Delete delete value from store with given key
	Delete(key string)
	// Flush flush store
	Flush()
	// LoadInterface Load load value with given key
	// Return loaded value or nil if load fail
	LoadInterface(key string) interface{}
}

// MapStore store which stores value in map.
// You should confirm safe for concurrent by yourself.
type MapStore map[string]interface{}

// Load load value with given key
// Return value and whether load successfully
func (m *MapStore) Load(key string) (value interface{}, ok bool) {
	v, ok := (*m)[key]
	return v, ok
}

// Store sotre value with given key
func (m *MapStore) Store(key string, value interface{}) {
	(*m)[key] = value
}

// LoadInterface Load load value with given key
// Return loaded value or nil if load fail
func (m *MapStore) LoadInterface(key string) interface{} {
	return (*m)[key]
}

// Delete delete value from store with given key
func (m *MapStore) Delete(key string) {
	delete(*m, key)
}

// Flush flush store
func (m *MapStore) Flush() {
	s := MapStore(map[string]interface{}{})
	*m = s
}

// NewMapStore create new map store
func NewMapStore() *MapStore {
	s := MapStore(map[string]interface{}{})
	return &s
}

// SyncMapStore store which stores value in sync.map.
type SyncMapStore struct {
	Map *sync.Map
}

// Load load value with given key
// Return value and whether load successfully
func (m *SyncMapStore) Load(key string) (value interface{}, ok bool) {
	return m.Map.Load(key)
}

// Store sotre value with given key
func (m *SyncMapStore) Store(key string, value interface{}) {
	m.Map.Store(key, value)
}

// Delete delete value from store with given key
func (m *SyncMapStore) Delete(key string) {
	m.Map.Delete(key)
}

// Flush flush store
func (m *SyncMapStore) Flush() {
	m.Map = &sync.Map{}
}

// LoadInterface Load load value with given key
// Return loaded value or nil if load fail
func (m *SyncMapStore) LoadInterface(key string) interface{} {
	v, _ := m.Load(key)
	return v
}

// NewSyncMapStore create new sync.map store
func NewSyncMapStore() *SyncMapStore {
	return &SyncMapStore{
		Map: &sync.Map{},
	}
}
<commit_msg>update<commit_after>package cachestore

import (
	"sync"
	"sync/atomic"
	"unsafe"
)

// Store interface of cache store
type Store interface {
	// Load load value with given key
	// Return value and whether load successfully
	Load(key string) (value interface{}, ok bool)
	// Store store value with given key
	Store(key string, value interface{})
	// Delete delete value from store with given key
	Delete(key string)
	// Flush flush store
	Flush()
	// LoadInterface Load load value with given key
	// Return loaded value or nil if load fail
	LoadInterface(key string) interface{}
}

// MapStore store which stores value in map.
// You should confirm safe for concurrent by yourself.
type MapStore map[string]interface{}

// Load load value with given key
// Return value and whether load successfully
func (m *MapStore) Load(key string) (value interface{}, ok bool) {
	v, ok := (*m)[key]
	return v, ok
}

// Store sotre value with given key
func (m *MapStore) Store(key string, value interface{}) {
	(*m)[key] = value
}

// LoadInterface Load load value with given key
// Return loaded value or nil if load fail
func (m *MapStore) LoadInterface(key string) interface{} {
	return (*m)[key]
}

// Delete delete value from store with given key
func (m *MapStore) Delete(key string) {
	delete(*m, key)
}

// Flush flush store
func (m *MapStore) Flush() {
	s := MapStore(map[string]interface{}{})
	*m = s
}

// NewMapStore create new map store
func NewMapStore() *MapStore {
	s := MapStore(map[string]interface{}{})
	return &s
}

// SyncMapStore store which stores value in sync.map.
type SyncMapStore struct {
	syncmap unsafe.Pointer
}

func (m *SyncMapStore) Map() *sync.Map {
	return (*sync.Map)(atomic.LoadPointer(&m.syncmap))
}

func (m *SyncMapStore) SetMap(smap *sync.Map) {
	atomic.StorePointer(&m.syncmap, unsafe.Pointer(smap))
}

// Load load value with given key
// Return value and whether load successfully
func (m *SyncMapStore) Load(key string) (value interface{}, ok bool) {
	return m.Map().Load(key)
}

// Store sotre value with given key
func (m *SyncMapStore) Store(key string, value interface{}) {
	m.Map().Store(key, value)
}

// Delete delete value from store with given key
func (m *SyncMapStore) Delete(key string) {
	m.Map().Delete(key)
}

// Flush flush store
func (m *SyncMapStore) Flush() {
	m.SetMap(&sync.Map{})
}

// LoadInterface Load load value with given key
// Return loaded value or nil if load fail
func (m *SyncMapStore) LoadInterface(key string) interface{} {
	v, _ := m.Load(key)
	return v
}

// NewSyncMapStore create new sync.map store
func NewSyncMapStore() *SyncMapStore {
	m := &SyncMapStore{}
	m.SetMap(&sync.Map{})
	return m
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"os"
	"time"
)

type Randomizable struct {
	Key			string
	Value		float64
	Min 		float64
	Max 		float64
	Variance 	float64
}

type Randomized struct {
	Key			string		`json:"key"`
	Value		float64		`json:"value"`
}

func handler(writer http.ResponseWriter, req *http.Request) {
	var randomizables []Randomizable
	var results []Randomized

	if req.Body == nil {
		http.Error(writer, "Please send a request body", 400)
		return
	}

	err := json.NewDecoder(req.Body).Decode(&randomizables)
	if err != nil {
		http.Error(writer, err.Error(), 400)
		return
	}

	for _, randomizable := range randomizables {
		results = append(results, randomize(randomizable))
	}

	resultJson, err := json.Marshal(results)
	if err != nil {
		http.Error(writer, err.Error(), 500)
		return
	}

	fmt.Fprintf(writer, string(resultJson))
}

func randomize(randomizable Randomizable) Randomized {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	result := r.NormFloat64() * randomizable.Variance + randomizable.Value

	if (result > randomizable.Max) {
		result = randomizable.Max
	}

	if (result < randomizable.Min) {
		result = randomizable.Min
	}

	return Randomized { randomizable.Key, result }
}

func main() {
	http.HandleFunc("/", handler)

	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("PORT environment variable was not set")
	}

	log.Println("Attempting to listen on port: ", port)
	err := http.ListenAndServe(":" + port, nil)

	if err != nil {
		log.Fatal("Could not listen: ", err)
	}
}
<commit_msg>More code style tweaks<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"os"
	"time"
)

type Randomizable struct {
	Key		string
	Value		float64
	Min 		float64
	Max 		float64
	Variance 	float64
}

type Randomized struct {
	Key		string	`json:"key"`
	Value		float64	`json:"value"`
}

func handler(writer http.ResponseWriter, req *http.Request) {
	var randomizables []Randomizable
	var results []Randomized

	if req.Body == nil {
		http.Error(writer, "Please send a request body", 400)
		return
	}

	err := json.NewDecoder(req.Body).Decode(&randomizables)
	if err != nil {
		http.Error(writer, err.Error(), 400)
		return
	}

	for _, randomizable := range randomizables {
		results = append(results, randomize(randomizable))
	}

	resultJson, err := json.Marshal(results)
	if err != nil {
		http.Error(writer, err.Error(), 500)
		return
	}

	fmt.Fprintf(writer, string(resultJson))
}

func randomize(randomizable Randomizable) Randomized {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	result := r.NormFloat64() * randomizable.Variance + randomizable.Value

	if result > randomizable.Max {
		result = randomizable.Max
	}

	if result < randomizable.Min {
		result = randomizable.Min
	}

	return Randomized { randomizable.Key, result }
}

func main() {
	http.HandleFunc("/", handler)

	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("PORT environment variable was not set")
	}

	log.Println("Attempting to listen on port: ", port)
	err := http.ListenAndServe(":" + port, nil)

	if err != nil {
		log.Fatal("Could not listen: ", err)
	}
}
<|endoftext|>


<commit_before>package qbit

import (
	"github.com/advantageous/go-qbit/logging"
	"errors"
)

type BasicSendQueue struct {
	channel    chan []interface{}
	owner      Queue
	batchSize  int
	logger     logging.Logger
	index      int
	queueLocal []interface{}
}

func NewSendQueue(channel chan []interface{}, owner Queue, batchSize  int, logger logging.Logger) SendQueue{

	if logger == nil {
		logger = logging.GetSimpleLogger("QBIT_SIMPLE_QUEUE", owner.Name() + "-sender")
	}

	queueLocal := make([]interface{}, batchSize)

	return &BasicSendQueue{
		channel: channel,
		owner: owner,
		batchSize: batchSize,
		logger: logger,
		queueLocal: queueLocal,
	}
}

func (bsq *BasicSendQueue) Send(item interface{}) error {

	err := bsq.flushIfOverBatch()
	if err!=nil {
		return err
	}
	bsq.queueLocal[bsq.index] = item
	bsq.index++
	return err
}

func (bsq *BasicSendQueue) flushIfOverBatch() error  {
	if ( bsq.index < bsq.batchSize ) {
		return nil
	} else {
		return  bsq.sendLocalQueue()
	}
}

func (bsq *BasicSendQueue) sendLocalQueue() error {
	var err error
	if bsq.index > 0 {
		slice := make([]interface{}, bsq.index)
		copy(slice, bsq.queueLocal)

		select {
		case bsq.channel <- slice:
			bsq.index = 0
			for i := 0; i < len(bsq.queueLocal); i++ {
				bsq.queueLocal[i] = nil
			}
		default:
			err = errors.New("Unable to send")
		}
	}
	return err
}

func (bsq *BasicSendQueue) FlushSends() error {
	return bsq.sendLocalQueue()
}

func (bsq *BasicSendQueue) Size() int {
	return len(bsq.channel)
}

func (bsq *BasicSendQueue) Name() string {
	return bsq.owner.Name()
}
<commit_msg>might make this optional.<commit_after>package qbit

import (
	"github.com/advantageous/go-qbit/logging"
	"errors"
)

type BasicSendQueue struct {
	channel    chan []interface{}
	owner      Queue
	batchSize  int
	logger     logging.Logger
	index      int
	queueLocal []interface{}
}

func NewSendQueue(channel chan []interface{}, owner Queue, batchSize  int, logger logging.Logger) SendQueue{

	if logger == nil {
		logger = logging.GetSimpleLogger("QBIT_SIMPLE_QUEUE", owner.Name() + "-sender")
	}

	queueLocal := make([]interface{}, batchSize)

	return &BasicSendQueue{
		channel: channel,
		owner: owner,
		batchSize: batchSize,
		logger: logger,
		queueLocal: queueLocal,
	}
}

func (bsq *BasicSendQueue) Send(item interface{}) error {

	err := bsq.flushIfOverBatch()
	if err!=nil {
		return err
	}
	bsq.queueLocal[bsq.index] = item
	bsq.index++
	return err
}

func (bsq *BasicSendQueue) flushIfOverBatch() error  {
	if ( bsq.index < bsq.batchSize ) {
		return nil
	} else {
		return  bsq.sendLocalQueue()
	}
}

func (bsq *BasicSendQueue) sendLocalQueue() error {
	var err error
	if bsq.index > 0 {
		slice := make([]interface{}, bsq.index)
		copy(slice, bsq.queueLocal)

		select {
		case bsq.channel <- slice:
			bsq.index = 0
			//for i := 0; i < len(bsq.queueLocal); i++ {
			//	bsq.queueLocal[i] = nil
			//}
		default:
			err = errors.New("Unable to send")
		}
	}
	return err
}

func (bsq *BasicSendQueue) FlushSends() error {
	return bsq.sendLocalQueue()
}

func (bsq *BasicSendQueue) Size() int {
	return len(bsq.channel)
}

func (bsq *BasicSendQueue) Name() string {
	return bsq.owner.Name()
}
<|endoftext|>


<commit_before>package dev

import (
	"log"

	"google.golang.org/grpc"
)

type managerOptions struct {
	grpcDialOpts []grpc.DialOption
	logger       *log.Logger
	noConnect    bool
	trace        bool
	selfAddr     string
	selfID       uint32
}

// ManagerOption provides a way to set different options on a new Manager.
type ManagerOption func(*managerOptions)

// WithGrpcDialOptions returns a ManagerOption which sets any gRPC dial options
// the Manager should use when initially connecting to each node in its
// pool.
func WithGrpcDialOptions(opts ...grpc.DialOption) ManagerOption {
	return func(o *managerOptions) {
		o.grpcDialOpts = opts
	}
}

// WithLogger returns a ManagerOption which sets an optional error logger for
// the Manager.
func WithLogger(logger *log.Logger) ManagerOption {
	return func(o *managerOptions) {
		o.logger = logger
	}
}

// WithNoConnect returns a ManagerOption which instructs the Manager not to
// connect to any of its nodes. Mainly used for testing purposes.
func WithNoConnect() ManagerOption {
	return func(o *managerOptions) {
		o.noConnect = true
	}
}

// WithSelfAddr returns a ManagerOption which instructs the Manager not to connect
// to the node with network address addr. The address must be present in the
// list of node addresses provided to the Manager.
func WithSelfAddr(addr string) ManagerOption {
	return func(o *managerOptions) {
		o.selfAddr = addr
	}
}

// WithSelfID returns a ManagerOption which instructs the Manager not to
// connect to the node with the given id. The node must be present in the list
// of node addresses provided to the Manager.
func WithSelfID(id uint32) ManagerOption {
	return func(o *managerOptions) {
		o.selfID = id
	}
}

// WithTracing controls whether to trace qourum calls for this Manager instance
// using the golang.org/x/net/trace package.
func WithTracing() ManagerOption {
	return func(o *managerOptions) {
		o.trace = true
	}
}
<commit_msg>dev: add note about tracing<commit_after>package dev

import (
	"log"

	"google.golang.org/grpc"
)

type managerOptions struct {
	grpcDialOpts []grpc.DialOption
	logger       *log.Logger
	noConnect    bool
	trace        bool
	selfAddr     string
	selfID       uint32
}

// ManagerOption provides a way to set different options on a new Manager.
type ManagerOption func(*managerOptions)

// WithGrpcDialOptions returns a ManagerOption which sets any gRPC dial options
// the Manager should use when initially connecting to each node in its
// pool.
func WithGrpcDialOptions(opts ...grpc.DialOption) ManagerOption {
	return func(o *managerOptions) {
		o.grpcDialOpts = opts
	}
}

// WithLogger returns a ManagerOption which sets an optional error logger for
// the Manager.
func WithLogger(logger *log.Logger) ManagerOption {
	return func(o *managerOptions) {
		o.logger = logger
	}
}

// WithNoConnect returns a ManagerOption which instructs the Manager not to
// connect to any of its nodes. Mainly used for testing purposes.
func WithNoConnect() ManagerOption {
	return func(o *managerOptions) {
		o.noConnect = true
	}
}

// WithSelfAddr returns a ManagerOption which instructs the Manager not to connect
// to the node with network address addr. The address must be present in the
// list of node addresses provided to the Manager.
func WithSelfAddr(addr string) ManagerOption {
	return func(o *managerOptions) {
		o.selfAddr = addr
	}
}

// WithSelfID returns a ManagerOption which instructs the Manager not to
// connect to the node with the given id. The node must be present in the list
// of node addresses provided to the Manager.
func WithSelfID(id uint32) ManagerOption {
	return func(o *managerOptions) {
		o.selfID = id
	}
}

// WithTracing controls whether to trace qourum calls for this Manager instance
// using the golang.org/x/net/trace package. Tracing is currently only supported
// for regular quorum calls.
func WithTracing() ManagerOption {
	return func(o *managerOptions) {
		o.trace = true
	}
}
<|endoftext|>


<commit_before>// Application that captures webpage archives on a CT worker and uploads it to
// Google Storage.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/skia-dev/glog"

	"strings"
	"time"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	workerNum     = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	pagesetType   = flag.String("pageset_type", util.PAGESET_TYPE_MOBILE_10k, "The type of pagesets to create from the Alexa CSV list. Eg: 10k, Mobile10k, All.")
	chromiumBuild = flag.String("chromium_build", "", "The chromium build to use for this capture_archives run.")
)

func main() {
	common.Init()
	defer util.TimeTrack(time.Now(), "Capturing Archives")
	defer glog.Flush()

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES))
	defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)

	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}

	// Reset the local chromium checkout.
	if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
		return
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Delete and remake the local webpage archives directory.
	pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType)
	skutil.RemoveAll(pathToArchives)
	skutil.MkdirAll(pathToArchives, 0700)

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the specified chromium build if it does not exist locally.
	if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
		glog.Error(err)
		return
	}

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
	chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
	recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR)
	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs
	// Loop through all pagesets.
	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}
	// TODO(rmistry): Remove this hack once the 1M webpage archives have been captured.
	glog.Infof("The length of fileInfos is: %s", len(fileInfos))
	fileInfos = fileInfos[16000:17500]
	glog.Infof("The fileInfos are: %s", fileInfos)
	for _, fileInfo := range fileInfos {
		pagesetBaseName := filepath.Base(fileInfo.Name())
		if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
			// Ignore timestamp files and .pyc files.
			continue
		}

		// Convert the filename into a format consumable by the record_wpr binary.
		pagesetArchiveName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
		pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())

		glog.Infof("===== Processing %s =====", pagesetPath)
		args := []string{
			"--extra-browser-args=--disable-setuid-sandbox",
			"--browser=exact",
			"--browser-executable=" + chromiumBinary,
			fmt.Sprintf("%s_page_set", pagesetArchiveName),
			"--page-set-base-dir=" + pathToPagesets,
		}
		env := []string{
			fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets),
			"DISPLAY=:0",
		}
		skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
	}

	// Write timestamp to the webpage archives dir.
	skutil.LogErr(util.CreateTimestampFile(pathToArchives))

	// Upload webpage archives dir to Google Storage.
	if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
}
<commit_msg>Capture CT archives from 17500-18499<commit_after>// Application that captures webpage archives on a CT worker and uploads it to
// Google Storage.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/skia-dev/glog"

	"strings"
	"time"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	workerNum     = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	pagesetType   = flag.String("pageset_type", util.PAGESET_TYPE_MOBILE_10k, "The type of pagesets to create from the Alexa CSV list. Eg: 10k, Mobile10k, All.")
	chromiumBuild = flag.String("chromium_build", "", "The chromium build to use for this capture_archives run.")
)

func main() {
	common.Init()
	defer util.TimeTrack(time.Now(), "Capturing Archives")
	defer glog.Flush()

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES))
	defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)

	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}

	// Reset the local chromium checkout.
	if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
		return
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Delete and remake the local webpage archives directory.
	pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType)
	skutil.RemoveAll(pathToArchives)
	skutil.MkdirAll(pathToArchives, 0700)

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the specified chromium build if it does not exist locally.
	if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
		glog.Error(err)
		return
	}

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
	chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
	recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR)
	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs
	// Loop through all pagesets.
	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}
	// TODO(rmistry): Remove this hack once the 1M webpage archives have been captured.
	glog.Infof("The length of fileInfos is: %s", len(fileInfos))
	fileInfos = fileInfos[17500:18500]
	glog.Infof("The fileInfos are: %s", fileInfos)
	for _, fileInfo := range fileInfos {
		pagesetBaseName := filepath.Base(fileInfo.Name())
		if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
			// Ignore timestamp files and .pyc files.
			continue
		}

		// Convert the filename into a format consumable by the record_wpr binary.
		pagesetArchiveName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
		pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())

		glog.Infof("===== Processing %s =====", pagesetPath)
		args := []string{
			"--extra-browser-args=--disable-setuid-sandbox",
			"--browser=exact",
			"--browser-executable=" + chromiumBinary,
			fmt.Sprintf("%s_page_set", pagesetArchiveName),
			"--page-set-base-dir=" + pathToPagesets,
		}
		env := []string{
			fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets),
			"DISPLAY=:0",
		}
		skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
	}

	// Write timestamp to the webpage archives dir.
	skutil.LogErr(util.CreateTimestampFile(pathToArchives))

	// Upload webpage archives dir to Google Storage.
	if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
}
<|endoftext|>


<commit_before>package pathfinding

func Dijkstra() {
	panic("Not yet implemented")
}
<commit_msg>Pretty lame implementation (from wikipedia) to get started.<commit_after>package pathfinding

func minDist(dist map[*Node] int) *Node {
	var result_node *Node
	min := int(^uint(0) >> 1)
	for node, _dist := range dist {
		if min >= _dist {
			result_node = node
			min = _dist
		}
	}
	return result_node
}

func removeFromQ(Q []*Node, n *Node) []*Node {
	var result []*Node
	for _, node := range Q {
		if node != n {
			result = append(result, node)
		}
	}
	return result
}

func dist_between(n1 *Node, n2 *Node) int {
	return 0
}

func Dijkstra(graph *Graph) []*Node {
	MAX_INT := int(^uint(0) >> 1)
	var path []*Node
	var Q []*Node
	dist := make(map[*Node] int, len(graph.nodes))

	for _, node := range graph.nodes {
		dist[node] = MAX_INT
	}
	dist[graph.start] = 0
	copy(graph.nodes, Q)
	for len(Q) != 0 {
		u := minDist(dist)
		if dist[u] == MAX_INT {
			break
		}
		Q = removeFromQ(Q, u)
		for _, v := range graph.adjacentNodes(u) {
			alt := dist[u] + dist_between(u, v)
			if alt < dist[u] {
				dist[v] = alt
				v.parent = u
				//Reorder v in Q
			}
		}
	}
	return path
}
<|endoftext|>


<commit_before>package lrcompress

import (
	"bufio"
	"encoding/binary"
	"errors"
	"io"
)

// ring buffer (yep, size baked in)
const CompHistBits = 22           // log2 bytes of history for compression
const rMask = 1<<CompHistBits - 1 // &rMask turns offset into ring pos

// compression hashtable
const hBits = 18           // log2 hashtable size
const hMask = 1<<hBits - 1 // c.hTbl[h>>hShift&hMask] is current hashtable entry
const hShift = 32 - hBits
const fMask = 1<<fBits - 1             // hit hashtable if fBits are 1111...
const fBits = CompHistBits - hBits + 1 // 1/2 fill the table

// output format choices
const window = 64          // bytes that must overlap to match
const maxLiteral = 1 << 16 // we'll write this size literal
const maxMatch = 1 << 18   // max match we output (we'll read larger)

type compRing [1 << CompHistBits]byte
type compHtbl [1 << hBits]int64

// Compressor is a Writer into which you can dump content.
type Compressor struct {
	pos        int64     // count of bytes ever written
	ring       compRing  // the bytes
	h          uint32    // current rolling hash
	matchPos   int64     // current match start or 0
	matchLen   int64     // current match length or 0
	cursor     int64     // "expected" match start
	w          io.Writer // compressed output
	literalLen int64     // current literal length or 0
	encodeBuf  [16]byte  // for varints
	hTbl       compHtbl  // hashtable holding offsets into source file
}

// Make a compressor with 1<<CompHistBits of memory, writing output to w.
func NewCompressor(w io.Writer) *Compressor {
	return &Compressor{w: w, pos: 1, cursor: 1}
}

func (c *Compressor) putInt(i int64) (err error) {
	n := binary.PutVarint(c.encodeBuf[:], i)
	_, err = c.w.Write(c.encodeBuf[:n])
	return
}

func (c *Compressor) putMatch(matchPos, matchLen int64) (err error) {
	err = c.putInt(matchLen)
	if err != nil {
		return
	}
	err = c.putInt(matchPos - c.cursor)
	c.cursor = matchPos + matchLen
	return
}

func (c *Compressor) putLiteral(pos, literalLen int64) (err error) {
	if literalLen == 0 {
		return
	}
	err = c.putInt(-literalLen)
	if err != nil {
		return
	}
	if literalLen > pos&rMask {
		_, err = c.w.Write(c.ring[(pos-literalLen)&rMask:])
		if err != nil {
			return
		}
		_, err = c.w.Write(c.ring[:pos&rMask])
	} else {
		_, err = c.w.Write(c.ring[(pos-literalLen)&rMask : pos&rMask])
	}
	if err != nil {
		return
	}
	c.cursor += literalLen
	return
}

// found a potential match; see if it checks out and use it if so
func (c *Compressor) tryMatch(ring *compRing, pos, literalLen, match int64) (matchLen_ int64, err error) {
	matchPos, matchLen := match, int64(1) // 1 because cur. byte matched
	min := pos - rMask + maxLiteral
	if min < 0 {
		min = 0
	}
	// extend backwards
	for literalLen > 0 &&
		matchPos-1 > min &&
		ring[(pos-matchLen)&rMask] == ring[(matchPos-1)&rMask] {
		literalLen--
		matchPos--
		matchLen++
	}
	if matchLen >= window { // long enough match, flush literal and use it
		// this literal ends before pos-matchLen+1, not pos
		if err = c.putLiteral(pos-matchLen+1, literalLen); err != nil {
			return
		}
		return matchLen, error(nil)
	} else { // short match, ignore
		return 0, error(nil)
	}
}

func (c *Compressor) Write(p []byte) (n int, err error) {
	h, ring, hTbl, pos, matchPos, matchLen, literalLen := c.h, &c.ring, &c.hTbl, c.pos, c.matchPos, c.matchLen, c.literalLen
	for _, b := range p {
		// can use any 32-bit const with least sig. bits=10b and some higher
		// bits set; even *=6 eventually mixes lower bits into the top ones
		h *= ((0x703a03ac|1)*2)&(1<<32-1) | 1<<31
		h ^= uint32(b)
		// if we're in a match, extend or end it
		if matchLen > 0 {
			// try to extend it
			if ring[(matchPos+matchLen)&rMask] == b &&
				matchLen < maxMatch {
				matchLen++
			} else {
				// can't extend it, flush out what we have
				if err = c.putMatch(matchPos, matchLen); err != nil {
					return
				}
				matchPos, matchLen = 0, 0
			}
		} else if literalLen > window && h&fMask == fMask {
			match := hTbl[h>>hShift&hMask]
			// check if it's in usable range and cur. byte matches, then tryMatch
			if match > 0 && b == ring[match&rMask] && match > pos-rMask+maxLiteral {
				matchLen, err = c.tryMatch(ring, pos, literalLen, match)
				if matchLen > 0 {
					literalLen = 0
					matchPos = match - matchLen + 1
				} else if err != nil {
					return
				}
			}
		}
		// (still) not in a match, so just extend the literal
		if matchLen == 0 {
			if literalLen == maxLiteral {
				if err = c.putLiteral(pos, literalLen); err != nil {
					return
				}
				literalLen = 0
			}
			literalLen++
		}

		// update hashtable and ring
		ring[pos&rMask] = b
		if h&fMask == fMask {
			hTbl[h>>hShift&hMask] = pos
		}
		pos++
	}
	c.h, c.pos, c.matchPos, c.matchLen, c.literalLen = h, pos, matchPos, matchLen, literalLen
	return len(p), nil
}

func (c *Compressor) Flush() (err error) {
	if c.matchLen > 0 {
		err = c.putMatch(c.matchPos, c.matchLen)
		c.matchPos, c.matchLen = 0, 0
	} else {
		err = c.putLiteral(c.pos, c.literalLen)
		c.literalLen = 0
	}
	return
}

// Flush if needed, then write an end marker to the diff output.
func (c *Compressor) Close() (err error) {
	if err = c.Flush(); err != nil {
		return
	}
	return c.putInt(0)
}

// ring is a ring-buffer of bytes with Copy and Write operations. Writes and
// copies are teed to an io.Writer provided on initialization.
type ring struct {
	pos  int64     // count of bytes ever written
	mask int64     // &mask turns pos into a ring offset
	w    io.Writer // output of writes/copies goes here as well as ring
	ring []byte    // the bytes
}

func newRing(sizeBits uint, w io.Writer) ring {
	return ring{
		pos:  0,
		mask: 1<<sizeBits - 1,
		w:    w,
		ring: make([]byte, 1<<sizeBits),
	}
}

func (r *ring) Write(p []byte) (n int, err error) {
	n, err = r.w.Write(p)
	if err != nil {
		return
	}
	for len(p) > 0 && err == nil {
		l := len(p)
		pos := int(r.pos & r.mask)
		if pos+l > len(r.ring) {
			l = len(r.ring) - pos
		}
		copy(r.ring[pos:pos+l], p[:l])
		p = p[l:]
		r.pos += int64(l)
	}
	return n, nil
}

// ring.Copy copies old content to the current position. If the copy source
// overlaps the destination, Copy will produce repeats.
func (r *ring) Copy(start int64, n int) (err error) {
	N := n
	for N > 0 && err == nil {
		n = N
		q := int(start & r.mask)
		// lower piece size (n) if needed
		if start == r.pos { // unsupported but don't hang forever
			return errors.New("zero offset for copy unsupported")
		} else if start+int64(n) > r.pos { // src overlaps dest
			n = int(r.pos - start)
		}
		if q+n > len(r.ring) { // source wraps around
			n = len(r.ring) - q
		}
		// do the copy and any write
		start += int64(n)
		if _, err := r.Write(r.ring[q : q+n]); err != nil {
			return err
		}
		N -= n
	}
	return
}

// Decompress input from rd to w in one shot. Does not handle framing format.
func Decompress(historyBits uint, rd io.Reader, w io.Writer) error {
	br := bufio.NewReader(rd)
	r := newRing(historyBits, w)
	cursor := int64(0)
	for {
		instr, err := binary.ReadVarint(br)
		if err != nil {
			return err
		}
		if instr > 0 { // copy!
			l := instr
			cursorMove, err := binary.ReadVarint(br)
			if err != nil {
				return err
			}
			cursor += cursorMove
			if err = r.Copy(cursor, int(l)); err != nil {
				return err
			}
			cursor += l
		}
		if instr == 0 { // end of stream!
			return io.EOF
		}
		if instr < 0 { // literal!
			l := -instr
			cursor += l
			if _, err := io.CopyN(&r, br, l); err != nil {
				return err
			}
		}
	}
}

func NewDecompressor(historyBits uint, rd io.Reader) io.Reader {
	pr, pw := io.Pipe()
	go func() {
		pw.CloseWithError(Decompress(historyBits, rd, pw))
	}()
	return pr
}
<commit_msg>fix regression in Decompress speed<commit_after>package lrcompress

import (
	"bufio"
	"encoding/binary"
	"errors"
	"io"
)

// ring buffer (yep, size baked in)
const CompHistBits = 22           // log2 bytes of history for compression
const rMask = 1<<CompHistBits - 1 // &rMask turns offset into ring pos

// compression hashtable
const hBits = 18           // log2 hashtable size
const hMask = 1<<hBits - 1 // c.hTbl[h>>hShift&hMask] is current hashtable entry
const hShift = 32 - hBits
const fMask = 1<<fBits - 1             // hit hashtable if fBits are 1111...
const fBits = CompHistBits - hBits + 1 // 1/2 fill the table

// output format choices
const window = 64          // bytes that must overlap to match
const maxLiteral = 1 << 16 // we'll write this size literal
const maxMatch = 1 << 18   // max match we output (we'll read larger)

type compRing [1 << CompHistBits]byte
type compHtbl [1 << hBits]int64

// Compressor is a Writer into which you can dump content.
type Compressor struct {
	pos        int64     // count of bytes ever written
	ring       compRing  // the bytes
	h          uint32    // current rolling hash
	matchPos   int64     // current match start or 0
	matchLen   int64     // current match length or 0
	cursor     int64     // "expected" match start
	w          io.Writer // compressed output
	literalLen int64     // current literal length or 0
	encodeBuf  [16]byte  // for varints
	hTbl       compHtbl  // hashtable holding offsets into source file
}

// Make a compressor with 1<<CompHistBits of memory, writing output to w.
func NewCompressor(w io.Writer) *Compressor {
	return &Compressor{w: w, pos: 1, cursor: 1}
}

func (c *Compressor) putInt(i int64) (err error) {
	n := binary.PutVarint(c.encodeBuf[:], i)
	_, err = c.w.Write(c.encodeBuf[:n])
	return
}

func (c *Compressor) putMatch(matchPos, matchLen int64) (err error) {
	err = c.putInt(matchLen)
	if err != nil {
		return
	}
	err = c.putInt(matchPos - c.cursor)
	c.cursor = matchPos + matchLen
	return
}

func (c *Compressor) putLiteral(pos, literalLen int64) (err error) {
	if literalLen == 0 {
		return
	}
	err = c.putInt(-literalLen)
	if err != nil {
		return
	}
	if literalLen > pos&rMask {
		_, err = c.w.Write(c.ring[(pos-literalLen)&rMask:])
		if err != nil {
			return
		}
		_, err = c.w.Write(c.ring[:pos&rMask])
	} else {
		_, err = c.w.Write(c.ring[(pos-literalLen)&rMask : pos&rMask])
	}
	if err != nil {
		return
	}
	c.cursor += literalLen
	return
}

// found a potential match; see if it checks out and use it if so
func (c *Compressor) tryMatch(ring *compRing, pos, literalLen, match int64) (matchLen_ int64, err error) {
	matchPos, matchLen := match, int64(1) // 1 because cur. byte matched
	min := pos - rMask + maxLiteral
	if min < 0 {
		min = 0
	}
	// extend backwards
	for literalLen > 0 &&
		matchPos-1 > min &&
		ring[(pos-matchLen)&rMask] == ring[(matchPos-1)&rMask] {
		literalLen--
		matchPos--
		matchLen++
	}
	if matchLen >= window { // long enough match, flush literal and use it
		// this literal ends before pos-matchLen+1, not pos
		if err = c.putLiteral(pos-matchLen+1, literalLen); err != nil {
			return
		}
		return matchLen, error(nil)
	} else { // short match, ignore
		return 0, error(nil)
	}
}

func (c *Compressor) Write(p []byte) (n int, err error) {
	h, ring, hTbl, pos, matchPos, matchLen, literalLen := c.h, &c.ring, &c.hTbl, c.pos, c.matchPos, c.matchLen, c.literalLen
	for _, b := range p {
		// can use any 32-bit const with least sig. bits=10b and some higher
		// bits set; even *=6 eventually mixes lower bits into the top ones
		h *= ((0x703a03ac|1)*2)&(1<<32-1) | 1<<31
		h ^= uint32(b)
		// if we're in a match, extend or end it
		if matchLen > 0 {
			// try to extend it
			if ring[(matchPos+matchLen)&rMask] == b &&
				matchLen < maxMatch {
				matchLen++
			} else {
				// can't extend it, flush out what we have
				if err = c.putMatch(matchPos, matchLen); err != nil {
					return
				}
				matchPos, matchLen = 0, 0
			}
		} else if literalLen > window && h&fMask == fMask {
			match := hTbl[h>>hShift&hMask]
			// check if it's in usable range and cur. byte matches, then tryMatch
			if match > 0 && b == ring[match&rMask] && match > pos-rMask+maxLiteral {
				matchLen, err = c.tryMatch(ring, pos, literalLen, match)
				if matchLen > 0 {
					literalLen = 0
					matchPos = match - matchLen + 1
				} else if err != nil {
					return
				}
			}
		}
		// (still) not in a match, so just extend the literal
		if matchLen == 0 {
			if literalLen == maxLiteral {
				if err = c.putLiteral(pos, literalLen); err != nil {
					return
				}
				literalLen = 0
			}
			literalLen++
		}

		// update hashtable and ring
		ring[pos&rMask] = b
		if h&fMask == fMask {
			hTbl[h>>hShift&hMask] = pos
		}
		pos++
	}
	c.h, c.pos, c.matchPos, c.matchLen, c.literalLen = h, pos, matchPos, matchLen, literalLen
	return len(p), nil
}

func (c *Compressor) Flush() (err error) {
	if c.matchLen > 0 {
		err = c.putMatch(c.matchPos, c.matchLen)
		c.matchPos, c.matchLen = 0, 0
	} else {
		err = c.putLiteral(c.pos, c.literalLen)
		c.literalLen = 0
	}
	return
}

// Flush if needed, then write an end marker to the diff output.
func (c *Compressor) Close() (err error) {
	if err = c.Flush(); err != nil {
		return
	}
	return c.putInt(0)
}

// ring is a ring-buffer of bytes with Copy and Write operations. Writes and
// copies are teed to an io.Writer provided on initialization.
type ring struct {
	pos  int64     // count of bytes ever written
	mask int64     // &mask turns pos into a ring offset
	w    io.Writer // output of writes/copies goes here as well as ring
	ring []byte    // the bytes
}

func newRing(sizeBits uint, w io.Writer) ring {
	return ring{
		pos:  0,
		mask: 1<<sizeBits - 1,
		w:    w,
		ring: make([]byte, 1<<sizeBits),
	}
}

func (r *ring) Write(p []byte) (n int, err error) {
	n, err = r.w.Write(p)
	if err != nil {
		return
	}
	for len(p) > 0 && err == nil {
		l := len(p)
		pos := int(r.pos & r.mask)
		if pos+l > len(r.ring) {
			l = len(r.ring) - pos
		}
		copy(r.ring[pos:pos+l], p[:l])
		p = p[l:]
		r.pos += int64(l)
	}
	return n, nil
}

// ring.Copy copies old content to the current position. If the copy source
// overlaps the destination, Copy will produce repeats.
func (r *ring) Copy(start int64, n int) (err error) {
	N := n
	for N > 0 && err == nil {
		n = N
		q := int(start & r.mask)
		// lower piece size (n) if needed
		if start == r.pos { // unsupported but don't hang forever
			return errors.New("zero offset for copy unsupported")
		} else if start+int64(n) > r.pos { // src overlaps dest
			n = int(r.pos - start)
		}
		if q+n > len(r.ring) { // source wraps around
			n = len(r.ring) - q
		}
		// do the copy and any write
		start += int64(n)
		if _, err := r.Write(r.ring[q : q+n]); err != nil {
			return err
		}
		N -= n
	}
	return
}

// Decompress input from rd to w in one shot. Does not handle framing format.
func Decompress(historyBits uint, rd io.Reader, w io.Writer) error {
	br := bufio.NewReader(rd)
	r := newRing(historyBits, w)
	cursor := int64(0)
	var literalBuf [maxLiteral]byte
	for {
		instr, err := binary.ReadVarint(br)
		if err != nil {
			return err
		}
		if instr > 0 { // copy!
			l := instr
			cursorMove, err := binary.ReadVarint(br)
			if err != nil {
				return err
			}
			cursor += cursorMove
			if err = r.Copy(cursor, int(l)); err != nil {
				return err
			}
			cursor += l
		}
		if instr == 0 { // end of stream!
			return io.EOF
		}
		if instr < 0 { // literal!
			l := -instr
			cursor += l
			for l > 0 {
				chunk := int(l)
				if chunk > maxLiteral {
					chunk = maxLiteral
				}
				n, err := br.Read(literalBuf[:chunk])
				if err == io.EOF && n < chunk {
					return io.ErrUnexpectedEOF
				} else if err != nil {
					return err
				}
				r.Write(literalBuf[:n])
				l -= int64(n)
			}
		}
	}
}

func NewDecompressor(historyBits uint, rd io.Reader) io.Reader {
	pr, pw := io.Pipe()
	go func() {
		pw.CloseWithError(Decompress(historyBits, rd, pw))
	}()
	return pr
}
<|endoftext|>