<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cvtesting reduces boilerplate in tests.
package cvtesting

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"os"
	"testing"
	"time"

	nativeDatastore "cloud.google.com/go/datastore"
	"google.golang.org/api/option"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/clock/testclock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"
	"go.chromium.org/luci/gae/filter/txndefer"
	"go.chromium.org/luci/gae/impl/cloud"
	"go.chromium.org/luci/gae/impl/memory"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/gae/service/info"
	serverauth "go.chromium.org/luci/server/auth"
	"go.chromium.org/luci/server/tq"
	"go.chromium.org/luci/server/tq/tqtesting"

	"go.chromium.org/luci/cv/internal/config"
	gf "go.chromium.org/luci/cv/internal/gerrit/gerritfake"

	. "github.com/smartystreets/goconvey/convey"
)

// TODO(tandrii): add fake config generation facilities.

// Test encapsulates typical setup for CV test.
//
// Typical use:
//   ct := cvtesting.Test{}
//   ctx, cancel := ct.SetUp()
//   defer cancel()
type Test struct {
	// Cfg manipulates CV config.
	Cfg config.TestController
	// GFake is a Gerrit fake. Defaults to an empty one.
	GFake *gf.Fake
	// TQ allows to run TQ tasks.
	TQ *tqtesting.Scheduler
	// Clock allows to move time forward.
	// By default, the time is moved automatically is something waits on it.
	Clock testclock.TestClock

	// MaxDuration limits how long a test can run as a fail safe.
	//
	// Defaults to 10s to most likely finish in pre/post submit tests,
	// with limited CPU resources.
	// Set to ~10ms when debugging a hung test.
	MaxDuration time.Duration

	// cleanups are executed in reverse order in cleanup().
	cleanups []func()
}

func (t *Test) SetUp() (ctx context.Context, deferme func()) {
	// Set defaults.
	if t.MaxDuration == time.Duration(0) {
		if os.Getenv("DATASTORE_PROJECT") == "" {
			t.MaxDuration = 10 * time.Second
		} else {
			t.MaxDuration = 60 * time.Second
		}
	}
	if t.GFake == nil {
		t.GFake = &gf.Fake{}
	}

	ctx = context.Background()
	if testing.Verbose() {
		ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
	}

	topCtx, cancel := context.WithTimeout(ctx, t.MaxDuration)
	t.cleanups = append(t.cleanups, func() {
		// Fail the test if the topCtx has timed out.
		So(topCtx.Err(), ShouldBeNil)
		cancel()
	})

	// Use a date-time that is easy to eyeball in logs.
	utc := time.Date(2020, time.February, 2, 10, 30, 00, 0, time.UTC)
	// But set it up in a clock as a local time to expose incorrect assumptions of UTC.
	now := time.Date(2020, time.February, 2, 13, 30, 00, 0, time.FixedZone("Fake local", 3*60*60))
	So(now.Equal(utc), ShouldBeTrue)
	ctx, t.Clock = testclock.UseTime(topCtx, now)
	t.Clock.SetTimerCallback(func(dur time.Duration, _ clock.Timer) {
		// Move fake time forward whenever someone's waiting for it.
		t.Clock.Add(dur)
	})

	ctx = t.installDS(ctx)
	ctx = txndefer.FilterRDS(ctx)

	ctx = t.GFake.Install(ctx)
	ctx, t.TQ = tq.TestingContext(ctx, nil)
	return ctx, t.cleanup
}

func (t *Test) cleanup() {
	for i := len(t.cleanups) - 1; i >= 0; i-- {
		t.cleanups[i]()
	}
}

func (t *Test) installDS(ctx context.Context) context.Context {
	if ctx, ok := t.installDSReal(ctx); ok {
		return ctx
	}
	if ctx, ok := t.installDSEmulator(ctx); ok {
		return ctx
	}
	ctx = memory.Use(ctx)
	// CV runs against Firestore backend, which is consistent.
	datastore.GetTestable(ctx).Consistent(true)
	datastore.GetTestable(ctx).AutoIndex(true)
	return ctx
}

// installDSProd configures CV tests to run with actual DS.
//
// If DATASTORE_PROJECT ENV var isn't set, returns false.
//
// To use, first
//
//    $ luci-auth context -- bash
//    $ export DATASTORE_PROJECT=my-cloud-project-with-datastore
//
// and then run go tests the usual way, e.g.:
//
//    $ go test ./...
func (t *Test) installDSReal(ctx context.Context) (context.Context, bool) {
	project := os.Getenv("DATASTORE_PROJECT")
	if project == "" {
		return ctx, false
	}
	if project == "luci-change-verifier" {
		panic("Don't use production CV project. Using -dev is OK.")
	}

	at := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{
		Scopes: serverauth.CloudOAuthScopes,
	})
	ts, err := at.TokenSource()
	if err != nil {
		err = errors.Annotate(err, "failed to initialize the token source (are you in `$ luci-auth context`?)").Err()
		So(err, ShouldBeNil)
	}

	logging.Debugf(ctx, "Using DS of project %q", project)
	client, err := nativeDatastore.NewClient(ctx, project, option.WithTokenSource(ts))
	So(err, ShouldBeNil)
	return t.installDSshared(ctx, client), true
}

// installDSEmulator configures CV tests to run with DS emulator.
//
// If DATASTORE_EMULATOR_HOST ENV var isn't set, returns false.
//
// To use, run
//
//     $ gcloud beta emulators datastore start --consistency=1.0
//
// and export DATASTORE_EMULATOR_HOST as printed by above command.
//
// NOTE: as of Feb 2021, emulator runs in legacy Datastore mode,
// not Firestore.
func (t *Test) installDSEmulator(ctx context.Context) (context.Context, bool) {
	emulatorHost := os.Getenv("DATASTORE_EMULATOR_HOST")
	if emulatorHost == "" {
		return ctx, false
	}

	logging.Debugf(ctx, "Using DS emulator at %q", emulatorHost)
	client, err := nativeDatastore.NewClient(ctx, "luci-gae-test")
	So(err, ShouldBeNil)
	return t.installDSshared(ctx, client), true
}

func (t *Test) installDSshared(ctx context.Context, client *nativeDatastore.Client) context.Context {
	t.cleanups = append(t.cleanups, func() {
		if err := client.Close(); err != nil {
			logging.Errorf(ctx, "failed to close DS client: %s", err)
		}
	})
	ctx = (&cloud.Config{ProjectID: "luci-ds-emulator-test", DS: client}).Use(ctx, nil)

	// Enter a namespace for this tests.
	randNamespace := make([]byte, 8)
	if _, err := rand.Read(randNamespace); err != nil {
		panic(err)
	}
	ns := fmt.Sprintf("testing-%s-%s", time.Now().Format("2006-01-02"), hex.EncodeToString(randNamespace))
	ctx = info.MustNamespace(ctx, ns)

	// Failure to clear is hard before the test,
	// ignored after the test.
	So(clearDS(ctx), ShouldBeNil)
	t.cleanups = append(t.cleanups, func() {
		if err := clearDS(ctx); err != nil {
			logging.Errorf(ctx, "failed to clean DS namespace %s: %s", ns, err)
		}
	})
	return ctx
}

func clearDS(ctx context.Context) error {
	// Execute a kindless query to clear entire namespace.
	q := datastore.NewQuery("").KeysOnly(true)
	var allKeys []*datastore.Key
	if err := datastore.GetAll(ctx, q, &allKeys); err != nil {
		return errors.Annotate(err, "failed to get entities").Err()
	}
	if err := datastore.Delete(ctx, allKeys); err != nil {
		return errors.Annotate(err, "failed to delete %d entities", len(allKeys)).Err()
	}
	return nil
}
<commit_msg>[cv][testing] cleanup old testing namespaces in emulator/real DS.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cvtesting reduces boilerplate in tests.
package cvtesting

import (
	"context"
	cryptorand "crypto/rand"
	"encoding/hex"
	"fmt"
	"math/rand"
	"os"
	"regexp"
	"testing"
	"time"

	nativeDatastore "cloud.google.com/go/datastore"
	"google.golang.org/api/option"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/clock/testclock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"
	"go.chromium.org/luci/gae/filter/txndefer"
	"go.chromium.org/luci/gae/impl/cloud"
	"go.chromium.org/luci/gae/impl/memory"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/gae/service/info"
	serverauth "go.chromium.org/luci/server/auth"
	"go.chromium.org/luci/server/tq"
	"go.chromium.org/luci/server/tq/tqtesting"

	"go.chromium.org/luci/cv/internal/config"
	gf "go.chromium.org/luci/cv/internal/gerrit/gerritfake"

	. "github.com/smartystreets/goconvey/convey"
)

// TODO(tandrii): add fake config generation facilities.

// Test encapsulates typical setup for CV test.
//
// Typical use:
//   ct := cvtesting.Test{}
//   ctx, cancel := ct.SetUp()
//   defer cancel()
type Test struct {
	// Cfg manipulates CV config.
	Cfg config.TestController
	// GFake is a Gerrit fake. Defaults to an empty one.
	GFake *gf.Fake
	// TQ allows to run TQ tasks.
	TQ *tqtesting.Scheduler
	// Clock allows to move time forward.
	// By default, the time is moved automatically is something waits on it.
	Clock testclock.TestClock

	// MaxDuration limits how long a test can run as a fail safe.
	//
	// Defaults to 10s to most likely finish in pre/post submit tests,
	// with limited CPU resources.
	// Set to ~10ms when debugging a hung test.
	MaxDuration time.Duration

	// cleanups are executed in reverse order in cleanup().
	cleanups []func()
}

func (t *Test) SetUp() (ctx context.Context, deferme func()) {
	// Set defaults.
	if t.MaxDuration == time.Duration(0) {
		if os.Getenv("DATASTORE_PROJECT") == "" {
			t.MaxDuration = 10 * time.Second
		} else {
			t.MaxDuration = 60 * time.Second
		}
	}
	if t.GFake == nil {
		t.GFake = &gf.Fake{}
	}

	ctx = context.Background()
	if testing.Verbose() {
		ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
	}

	topCtx, cancel := context.WithTimeout(ctx, t.MaxDuration)
	t.cleanups = append(t.cleanups, func() {
		// Fail the test if the topCtx has timed out.
		So(topCtx.Err(), ShouldBeNil)
		cancel()
	})

	// Use a date-time that is easy to eyeball in logs.
	utc := time.Date(2020, time.February, 2, 10, 30, 00, 0, time.UTC)
	// But set it up in a clock as a local time to expose incorrect assumptions of UTC.
	now := time.Date(2020, time.February, 2, 13, 30, 00, 0, time.FixedZone("Fake local", 3*60*60))
	So(now.Equal(utc), ShouldBeTrue)
	ctx, t.Clock = testclock.UseTime(topCtx, now)
	t.Clock.SetTimerCallback(func(dur time.Duration, _ clock.Timer) {
		// Move fake time forward whenever someone's waiting for it.
		t.Clock.Add(dur)
	})

	ctx = t.installDS(ctx)
	ctx = txndefer.FilterRDS(ctx)

	ctx = t.GFake.Install(ctx)
	ctx, t.TQ = tq.TestingContext(ctx, nil)
	return ctx, t.cleanup
}

func (t *Test) cleanup() {
	for i := len(t.cleanups) - 1; i >= 0; i-- {
		t.cleanups[i]()
	}
}

func (t *Test) installDS(ctx context.Context) context.Context {
	if ctx, ok := t.installDSReal(ctx); ok {
		return ctx
	}
	if ctx, ok := t.installDSEmulator(ctx); ok {
		return ctx
	}
	ctx = memory.Use(ctx)
	// CV runs against Firestore backend, which is consistent.
	datastore.GetTestable(ctx).Consistent(true)
	datastore.GetTestable(ctx).AutoIndex(true)
	return ctx
}

// installDSProd configures CV tests to run with actual DS.
//
// If DATASTORE_PROJECT ENV var isn't set, returns false.
//
// To use, first
//
//    $ luci-auth context -- bash
//    $ export DATASTORE_PROJECT=my-cloud-project-with-datastore
//
// and then run go tests the usual way, e.g.:
//
//    $ go test ./...
func (t *Test) installDSReal(ctx context.Context) (context.Context, bool) {
	project := os.Getenv("DATASTORE_PROJECT")
	if project == "" {
		return ctx, false
	}
	if project == "luci-change-verifier" {
		panic("Don't use production CV project. Using -dev is OK.")
	}

	at := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{
		Scopes: serverauth.CloudOAuthScopes,
	})
	ts, err := at.TokenSource()
	if err != nil {
		err = errors.Annotate(err, "failed to initialize the token source (are you in `$ luci-auth context`?)").Err()
		So(err, ShouldBeNil)
	}

	logging.Debugf(ctx, "Using DS of project %q", project)
	client, err := nativeDatastore.NewClient(ctx, project, option.WithTokenSource(ts))
	So(err, ShouldBeNil)
	return t.installDSshared(ctx, project, client), true
}

// installDSEmulator configures CV tests to run with DS emulator.
//
// If DATASTORE_EMULATOR_HOST ENV var isn't set, returns false.
//
// To use, run
//
//     $ gcloud beta emulators datastore start --consistency=1.0
//
// and export DATASTORE_EMULATOR_HOST as printed by above command.
//
// NOTE: as of Feb 2021, emulator runs in legacy Datastore mode,
// not Firestore.
func (t *Test) installDSEmulator(ctx context.Context) (context.Context, bool) {
	emulatorHost := os.Getenv("DATASTORE_EMULATOR_HOST")
	if emulatorHost == "" {
		return ctx, false
	}

	logging.Debugf(ctx, "Using DS emulator at %q", emulatorHost)
	client, err := nativeDatastore.NewClient(ctx, "luci-gae-emulator-test")
	So(err, ShouldBeNil)
	return t.installDSshared(ctx, "luci-gae-emulator-test", client), true
}

func (t *Test) installDSshared(ctx context.Context, cloudProject string, client *nativeDatastore.Client) context.Context {
	t.cleanups = append(t.cleanups, func() {
		if err := client.Close(); err != nil {
			logging.Errorf(ctx, "failed to close DS client: %s", err)
		}
	})
	ctx = (&cloud.Config{ProjectID: cloudProject, DS: client}).Use(ctx, nil)
	maybeCleanupOldDSNamespaces(ctx)

	// Enter a namespace for this tests.
	ns := genDSNamespaceName(time.Now())
	logging.Debugf(ctx, "Using %q DS namespace", ns)
	ctx = info.MustNamespace(ctx, ns)
	// Failure to clear is hard before the test,
	// ignored after the test.
	So(clearDS(ctx), ShouldBeNil)
	t.cleanups = append(t.cleanups, func() {
		if err := clearDS(ctx); err != nil {
			logging.Errorf(ctx, "failed to clean DS namespace %s: %s", ns, err)
		}
	})
	return ctx
}

func genDSNamespaceName(t time.Time) string {
	rnd := make([]byte, 8)
	if _, err := cryptorand.Read(rnd); err != nil {
		panic(err)
	}
	return fmt.Sprintf("testing-%s-%s", time.Now().Format("2006-01-02"), hex.EncodeToString(rnd))
}

var dsNamespaceRegexp = regexp.MustCompile(`^testing-(\d{4}-\d\d-\d\d)-[0-9a-f]+$`)

func isOldTestDSNamespace(ns string, now time.Time) bool {
	m := dsNamespaceRegexp.FindSubmatch([]byte(ns))
	if len(m) == 0 {
		return false
	}
	// Anything up ~2 days old should be kept to avoid accidentally removing
	// currently under test namespace in presence of timezones and out of sync
	// clocks.
	const maxAge = 2 * 24 * time.Hour
	t, err := time.Parse("2006-01-02", string(m[1]))
	if err != nil {
		panic(err)
	}
	return now.Sub(t) > maxAge
}

func clearDS(ctx context.Context) error {
	// Execute a kindless query to clear entire namespace.
	q := datastore.NewQuery("").KeysOnly(true)
	var allKeys []*datastore.Key
	if err := datastore.GetAll(ctx, q, &allKeys); err != nil {
		return errors.Annotate(err, "failed to get entities").Err()
	}
	if err := datastore.Delete(ctx, allKeys); err != nil {
		return errors.Annotate(err, "failed to delete %d entities", len(allKeys)).Err()
	}
	return nil
}

func maybeCleanupOldDSNamespaces(ctx context.Context) {
	if rand.Intn(1024) < 1020 { // ~99% of cases.
		return
	}
	q := datastore.NewQuery("__namespace__").KeysOnly(true)
	var allKeys []*datastore.Key
	if err := datastore.GetAll(ctx, q, &allKeys); err != nil {
		logging.Warningf(ctx, "failed to query all namespaces: %s", err)
		return
	}
	now := time.Now()
	var toDelete []string
	for _, k := range allKeys {
		ns := k.StringID()
		if isOldTestDSNamespace(ns, now) {
			toDelete = append(toDelete, ns)
		}
	}
	logging.Debugf(ctx, "cleaning up %d old namespaces", len(toDelete))
	for _, ns := range toDelete {
		logging.Debugf(ctx, "cleaning up %s", ns)
		if err := clearDS(info.MustNamespace(ctx, ns)); err != nil {
			logging.Errorf(ctx, "failed to clean old DS namespace %s: %s", ns, err)
		}
	}
}
<|endoftext|>


<commit_before>package miner

import (
	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// ProcessConsensusDigest will update the miner's most recent block.
func (m *Miner) ProcessConsensusChange(cc modules.ConsensusChange) {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Adjust the height of the miner. The miner height is initialized to zero,
	// but the genesis block is actually height zero. For the genesis block
	// only, the height will be left at zero.
	//
	// Checking the height here eliminates the need to initialize the miner to
	// an underflowed types.BlockHeight, which was deemed the worse of the two
	// evils.
	if m.persist.Height != 0 || cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID() != m.cs.GenesisBlock().ID() {
		m.persist.Height -= types.BlockHeight(len(cc.RevertedBlocks))
		m.persist.Height += types.BlockHeight(len(cc.AppliedBlocks))
	}

	// Update the unsolved block.
	var exists1, exists2 bool
	m.persist.UnsolvedBlock.ParentID = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID()
	m.persist.Target, exists1 = m.cs.ChildTarget(m.persist.UnsolvedBlock.ParentID)
	m.persist.UnsolvedBlock.Timestamp, exists2 = m.cs.MinimumValidChildTimestamp(m.persist.UnsolvedBlock.ParentID)
	if build.DEBUG && !exists1 {
		panic("could not get child target")
	}
	if build.DEBUG && !exists2 {
		panic("could not get child earliest timestamp")
	}

	// There is a new parent block, the source block should be updated to keep
	// the stale rate as low as possible.
	m.newSourceBlock()
	m.persist.RecentChange = cc.ID

	// Save the new consensus information.
	err := m.save()
	if err != nil {
		m.log.Println("ERROR:", err)
	}
}

// ReceiveUpdatedUnconfirmedTransactions will replace the current unconfirmed
// set of transactions with the input transactions.
func (m *Miner) ReceiveUpdatedUnconfirmedTransactions(unconfirmedTransactions []types.Transaction, _ modules.ConsensusChange) {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Edge case - if there are no transactions, set the block's transactions
	// to nil and return.
	if len(unconfirmedTransactions) == 0 {
		m.persist.UnsolvedBlock.Transactions = nil
		return
	}

	// Add transactions to the block until the block size limit is reached.
	// Transactions are assumed to be in a sensible order.
	var i int
	remainingSize := int(types.BlockSizeLimit - 5e3)
	for i = range unconfirmedTransactions {
		remainingSize -= len(encoding.Marshal(unconfirmedTransactions[i]))
		if remainingSize < 0 {
			break
		}
	}
	m.persist.UnsolvedBlock.Transactions = unconfirmedTransactions[:i+1]
}
<commit_msg>improved logging during ProcessConsensusChange<commit_after>package miner

import (
	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// ProcessConsensusDigest will update the miner's most recent block.
func (m *Miner) ProcessConsensusChange(cc modules.ConsensusChange) {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Adjust the height of the miner. The miner height is initialized to zero,
	// but the genesis block is actually height zero. For the genesis block
	// only, the height will be left at zero.
	//
	// Checking the height here eliminates the need to initialize the miner to
	// an underflowed types.BlockHeight, which was deemed the worse of the two
	// evils.
	if m.persist.Height != 0 || cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID() != m.cs.GenesisBlock().ID() {
		m.persist.Height -= types.BlockHeight(len(cc.RevertedBlocks))
		m.persist.Height += types.BlockHeight(len(cc.AppliedBlocks))
	}

	// Update the unsolved block.
	var exists1, exists2 bool
	m.persist.UnsolvedBlock.ParentID = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID()
	m.persist.Target, exists1 = m.cs.ChildTarget(m.persist.UnsolvedBlock.ParentID)
	m.persist.UnsolvedBlock.Timestamp, exists2 = m.cs.MinimumValidChildTimestamp(m.persist.UnsolvedBlock.ParentID)
	if build.DEBUG && !exists1 {
		panic("could not get child target")
	}
	if build.DEBUG && !exists2 {
		panic("could not get child earliest timestamp")
	}

	// There is a new parent block, the source block should be updated to keep
	// the stale rate as low as possible.
	m.newSourceBlock()
	m.persist.RecentChange = cc.ID

	// Save the new consensus information.
	err := m.save()
	if err != nil {
		m.log.Println("ERROR: could not save during ProcessConsensusChange:", err)
	}
}

// ReceiveUpdatedUnconfirmedTransactions will replace the current unconfirmed
// set of transactions with the input transactions.
func (m *Miner) ReceiveUpdatedUnconfirmedTransactions(unconfirmedTransactions []types.Transaction, _ modules.ConsensusChange) {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Edge case - if there are no transactions, set the block's transactions
	// to nil and return.
	if len(unconfirmedTransactions) == 0 {
		m.persist.UnsolvedBlock.Transactions = nil
		return
	}

	// Add transactions to the block until the block size limit is reached.
	// Transactions are assumed to be in a sensible order.
	var i int
	remainingSize := int(types.BlockSizeLimit - 5e3)
	for i = range unconfirmedTransactions {
		remainingSize -= len(encoding.Marshal(unconfirmedTransactions[i]))
		if remainingSize < 0 {
			break
		}
	}
	m.persist.UnsolvedBlock.Transactions = unconfirmedTransactions[:i+1]
}
<|endoftext|>


<commit_before>//go:build (linux && !appengine) || darwin || freebsd || netbsd || openbsd
// +build linux,!appengine darwin freebsd netbsd openbsd

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"bytes"
	"fmt"
	"os"
	"sync"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

func access(name string) error {
	if err := unix.Access(name, unix.R_OK|unix.W_OK); err != nil {
		return &os.PathError{Op: "lstat", Path: name, Err: err}
	}
	return nil
}

// Forked from Golang but chooses fast path upon os.Mkdir()
// error to avoid os.Lstat() call.
//
// osMkdirAll creates a directory named path,
// along with any necessary parents, and returns nil,
// or else returns an error.
// The permission bits perm (before umask) are used for all
// directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
func osMkdirAll(dirPath string, perm os.FileMode) error {
	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
	err := Access(dirPath)
	if err == nil {
		return nil
	}
	if !osIsNotExist(err) {
		return &os.PathError{Op: "mkdir", Path: dirPath, Err: err}
	}

	// Slow path: make sure parent exists and then call Mkdir for path.
	i := len(dirPath)
	for i > 0 && os.IsPathSeparator(dirPath[i-1]) { // Skip trailing path separator.
		i--
	}

	j := i
	for j > 0 && !os.IsPathSeparator(dirPath[j-1]) { // Scan backward over element.
		j--
	}

	if j > 1 {
		// Create parent.
		if err = osMkdirAll(dirPath[:j-1], perm); err != nil {
			return err
		}
	}

	// Parent now exists; invoke Mkdir and use its result.
	if err = Mkdir(dirPath, perm); err != nil {
		if osIsExist(err) {
			return nil
		}
		return err
	}

	return nil
}

// The buffer must be at least a block long.
// refer https://github.com/golang/go/issues/24015
const blockSize = 8 << 10 // 8192

// By default atleast 128 entries in single getdents call (1MiB buffer)
var (
	direntPool = sync.Pool{
		New: func() interface{} {
			buf := make([]byte, blockSize*128)
			return &buf
		},
	}

	direntNamePool = sync.Pool{
		New: func() interface{} {
			buf := make([]byte, blockSize)
			return &buf
		},
	}
)

// unexpectedFileMode is a sentinel (and bogus) os.FileMode
// value used to represent a syscall.DT_UNKNOWN Dirent.Type.
const unexpectedFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice

func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err error) {
	// golang.org/issue/15653
	dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[0]))
	if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v {
		return consumed, nil, typ, fmt.Errorf("buf size of %d smaller than dirent header size %d", len(buf), v)
	}
	if len(buf) < int(dirent.Reclen) {
		return consumed, nil, typ, fmt.Errorf("buf size %d < record length %d", len(buf), dirent.Reclen)
	}
	consumed = int(dirent.Reclen)
	if direntInode(dirent) == 0 { // File absent in directory.
		return
	}
	switch dirent.Type {
	case syscall.DT_REG:
		typ = 0
	case syscall.DT_DIR:
		typ = os.ModeDir
	case syscall.DT_LNK:
		typ = os.ModeSymlink
	default:
		// Skip all other file types. Revisit if/when this code needs
		// to handle such files, MinIO is only interested in
		// files and directories.
		typ = unexpectedFileMode
	}

	nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
	nameLen, err := direntNamlen(dirent)
	if err != nil {
		return consumed, nil, typ, err
	}

	return consumed, nameBuf[:nameLen], typ, nil
}

// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
// the directory itself, if the dirPath doesn't exist this function doesn't return
// an error.
func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) error {
	f, err := Open(dirPath)
	if err != nil {
		if osErrToFileErr(err) == errFileNotFound {
			return nil
		}
		return osErrToFileErr(err)
	}
	defer f.Close()

	bufp := direntPool.Get().(*[]byte)
	defer direntPool.Put(bufp)
	buf := *bufp

	boff := 0 // starting read position in buf
	nbuf := 0 // end valid data in buf

	for {
		if boff >= nbuf {
			boff = 0
			stop := globalOSMetrics.time(osMetricReadDirent)
			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
			stop()
			if err != nil {
				if isSysErrNotDir(err) {
					return nil
				}
				err = osErrToFileErr(err)
				if err == errFileNotFound {
					return nil
				}
				return err
			}
			if nbuf <= 0 {
				break // EOF
			}
		}
		consumed, name, typ, err := parseDirEnt(buf[boff:nbuf])
		if err != nil {
			return err
		}
		boff += consumed
		if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) {
			continue
		}

		// Fallback for filesystems (like old XFS) that don't
		// support Dirent.Type and have DT_UNKNOWN (0) there
		// instead.
		if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink {
			fi, err := Stat(pathJoin(dirPath, string(name)))
			if err != nil {
				// It got deleted in the meantime, not found
				// or returns too many symlinks ignore this
				// file/directory.
				if osIsNotExist(err) || isSysErrPathNotFound(err) ||
					isSysErrTooManySymlinks(err) {
					continue
				}
				return err
			}

			// Ignore symlinked directories.
			if typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
				continue
			}

			typ = fi.Mode() & os.ModeType
		}
		if err = fn(string(name), typ); err == errDoneForNow {
			// fn() requested to return by caller.
			return nil
		}
	}

	return err
}

// Return count entries at the directory dirPath and all entries
// if count is set to -1
func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
	f, err := OpenFile(dirPath, readMode, 0)
	if err != nil {
		return nil, osErrToFileErr(err)
	}
	defer f.Close()

	bufp := direntPool.Get().(*[]byte)
	defer direntPool.Put(bufp)
	buf := *bufp

	nameTmp := direntNamePool.Get().(*[]byte)
	defer direntNamePool.Put(nameTmp)
	tmp := *nameTmp

	boff := 0 // starting read position in buf
	nbuf := 0 // end valid data in buf

	count := opts.count

	for count != 0 {
		if boff >= nbuf {
			boff = 0
			stop := globalOSMetrics.time(osMetricReadDirent)
			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
			stop()
			if err != nil {
				if isSysErrNotDir(err) {
					return nil, errFileNotFound
				}
				return nil, osErrToFileErr(err)
			}
			if nbuf <= 0 {
				break
			}
		}
		consumed, name, typ, err := parseDirEnt(buf[boff:nbuf])
		if err != nil {
			return nil, err
		}
		boff += consumed
		if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) {
			continue
		}

		// Fallback for filesystems (like old XFS) that don't
		// support Dirent.Type and have DT_UNKNOWN (0) there
		// instead.
		if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink {
			fi, err := Stat(pathJoin(dirPath, string(name)))
			if err != nil {
				// It got deleted in the meantime, not found
				// or returns too many symlinks ignore this
				// file/directory.
				if osIsNotExist(err) || isSysErrPathNotFound(err) ||
					isSysErrTooManySymlinks(err) {
					continue
				}
				return nil, err
			}

			// Ignore symlinked directories.
			if !opts.followDirSymlink && typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
				continue
			}

			typ = fi.Mode() & os.ModeType
		}

		var nameStr string
		if typ.IsRegular() {
			nameStr = string(name)
		} else if typ.IsDir() {
			// Use temp buffer to append a slash to avoid string concat.
			tmp = tmp[:len(name)+1]
			copy(tmp, name)
			tmp[len(tmp)-1] = '/' // SlashSeparator
			nameStr = string(tmp)
		}

		count--
		entries = append(entries, nameStr)
	}

	return
}

func globalSync() {
	defer globalOSMetrics.time(osMetricSync)
	syscall.Sync()
}
<commit_msg>fix: readdir fallback on root of the drive (#15457)<commit_after>//go:build (linux && !appengine) || darwin || freebsd || netbsd || openbsd
// +build linux,!appengine darwin freebsd netbsd openbsd

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"bytes"
	"fmt"
	"os"
	"sync"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

func access(name string) error {
	if err := unix.Access(name, unix.R_OK|unix.W_OK); err != nil {
		return &os.PathError{Op: "lstat", Path: name, Err: err}
	}
	return nil
}

// Forked from Golang but chooses fast path upon os.Mkdir()
// error to avoid os.Lstat() call.
//
// osMkdirAll creates a directory named path,
// along with any necessary parents, and returns nil,
// or else returns an error.
// The permission bits perm (before umask) are used for all
// directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
func osMkdirAll(dirPath string, perm os.FileMode) error {
	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
	err := Access(dirPath)
	if err == nil {
		return nil
	}
	if !osIsNotExist(err) {
		return &os.PathError{Op: "mkdir", Path: dirPath, Err: err}
	}

	// Slow path: make sure parent exists and then call Mkdir for path.
	i := len(dirPath)
	for i > 0 && os.IsPathSeparator(dirPath[i-1]) { // Skip trailing path separator.
		i--
	}

	j := i
	for j > 0 && !os.IsPathSeparator(dirPath[j-1]) { // Scan backward over element.
		j--
	}

	if j > 1 {
		// Create parent.
		if err = osMkdirAll(dirPath[:j-1], perm); err != nil {
			return err
		}
	}

	// Parent now exists; invoke Mkdir and use its result.
	if err = Mkdir(dirPath, perm); err != nil {
		if osIsExist(err) {
			return nil
		}
		return err
	}

	return nil
}

// The buffer must be at least a block long.
// refer https://github.com/golang/go/issues/24015
const blockSize = 8 << 10 // 8192

// By default atleast 128 entries in single getdents call (1MiB buffer)
var (
	direntPool = sync.Pool{
		New: func() interface{} {
			buf := make([]byte, blockSize*128)
			return &buf
		},
	}

	direntNamePool = sync.Pool{
		New: func() interface{} {
			buf := make([]byte, blockSize)
			return &buf
		},
	}
)

// unexpectedFileMode is a sentinel (and bogus) os.FileMode
// value used to represent a syscall.DT_UNKNOWN Dirent.Type.
const unexpectedFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice

func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err error) {
	// golang.org/issue/15653
	dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[0]))
	if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v {
		return consumed, nil, typ, fmt.Errorf("buf size of %d smaller than dirent header size %d", len(buf), v)
	}
	if len(buf) < int(dirent.Reclen) {
		return consumed, nil, typ, fmt.Errorf("buf size %d < record length %d", len(buf), dirent.Reclen)
	}
	consumed = int(dirent.Reclen)
	if direntInode(dirent) == 0 { // File absent in directory.
		return
	}
	switch dirent.Type {
	case syscall.DT_REG:
		typ = 0
	case syscall.DT_DIR:
		typ = os.ModeDir
	case syscall.DT_LNK:
		typ = os.ModeSymlink
	default:
		// Skip all other file types. Revisit if/when this code needs
		// to handle such files, MinIO is only interested in
		// files and directories.
		typ = unexpectedFileMode
	}

	nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
	nameLen, err := direntNamlen(dirent)
	if err != nil {
		return consumed, nil, typ, err
	}

	return consumed, nameBuf[:nameLen], typ, nil
}

// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
// the directory itself, if the dirPath doesn't exist this function doesn't return
// an error.
func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) error {
	f, err := Open(dirPath)
	if err != nil {
		if osErrToFileErr(err) == errFileNotFound {
			return nil
		}
		return osErrToFileErr(err)
	}
	defer f.Close()

	bufp := direntPool.Get().(*[]byte)
	defer direntPool.Put(bufp)
	buf := *bufp

	boff := 0 // starting read position in buf
	nbuf := 0 // end valid data in buf

	for {
		if boff >= nbuf {
			boff = 0
			stop := globalOSMetrics.time(osMetricReadDirent)
			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
			stop()
			if err != nil {
				if isSysErrNotDir(err) {
					return nil
				}
				err = osErrToFileErr(err)
				if err == errFileNotFound {
					return nil
				}
				return err
			}
			if nbuf <= 0 {
				break // EOF
			}
		}
		consumed, name, typ, err := parseDirEnt(buf[boff:nbuf])
		if err != nil {
			return err
		}
		boff += consumed
		if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) {
			continue
		}

		// Fallback for filesystems (like old XFS) that don't
		// support Dirent.Type and have DT_UNKNOWN (0) there
		// instead.
		if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink {
			fi, err := Stat(pathJoin(dirPath, string(name)))
			if err != nil {
				// It got deleted in the meantime, not found
				// or returns too many symlinks ignore this
				// file/directory.
				if osIsNotExist(err) || isSysErrPathNotFound(err) ||
					isSysErrTooManySymlinks(err) {
					continue
				}
				return err
			}

			// Ignore symlinked directories.
			if typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
				continue
			}

			typ = fi.Mode() & os.ModeType
		}
		if err = fn(string(name), typ); err == errDoneForNow {
			// fn() requested to return by caller.
			return nil
		}
	}

	return err
}

// Return count entries at the directory dirPath and all entries
// if count is set to -1
func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
	f, err := OpenFile(dirPath, readMode, 0o666)
	if err != nil {
		if osIsPermission(err) {
			f, err = Open(dirPath)
			if err != nil {
				return nil, osErrToFileErr(err)
			}
		} else {
			return nil, osErrToFileErr(err)
		}
	}
	defer f.Close()

	bufp := direntPool.Get().(*[]byte)
	defer direntPool.Put(bufp)
	buf := *bufp

	nameTmp := direntNamePool.Get().(*[]byte)
	defer direntNamePool.Put(nameTmp)
	tmp := *nameTmp

	boff := 0 // starting read position in buf
	nbuf := 0 // end valid data in buf

	count := opts.count

	for count != 0 {
		if boff >= nbuf {
			boff = 0
			stop := globalOSMetrics.time(osMetricReadDirent)
			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
			stop()
			if err != nil {
				if isSysErrNotDir(err) {
					return nil, errFileNotFound
				}
				return nil, osErrToFileErr(err)
			}
			if nbuf <= 0 {
				break
			}
		}
		consumed, name, typ, err := parseDirEnt(buf[boff:nbuf])
		if err != nil {
			return nil, err
		}
		boff += consumed
		if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) {
			continue
		}

		// Fallback for filesystems (like old XFS) that don't
		// support Dirent.Type and have DT_UNKNOWN (0) there
		// instead.
		if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink {
			fi, err := Stat(pathJoin(dirPath, string(name)))
			if err != nil {
				// It got deleted in the meantime, not found
				// or returns too many symlinks ignore this
				// file/directory.
				if osIsNotExist(err) || isSysErrPathNotFound(err) ||
					isSysErrTooManySymlinks(err) {
					continue
				}
				return nil, err
			}

			// Ignore symlinked directories.
			if !opts.followDirSymlink && typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
				continue
			}

			typ = fi.Mode() & os.ModeType
		}

		var nameStr string
		if typ.IsRegular() {
			nameStr = string(name)
		} else if typ.IsDir() {
			// Use temp buffer to append a slash to avoid string concat.
			tmp = tmp[:len(name)+1]
			copy(tmp, name)
			tmp[len(tmp)-1] = '/' // SlashSeparator
			nameStr = string(tmp)
		}

		count--
		entries = append(entries, nameStr)
	}

	return
}

func globalSync() {
	defer globalOSMetrics.time(osMetricSync)
	syscall.Sync()
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"errors"
	"strings"
	"time"

	"github.com/nytlabs/gojee"
)

func Burp(j []byte, i Iterator, inherited Point) ([]Point, string, error) {
	points, _, jsonFragment, err := process(j, i, inherited, true)
	return points, jsonFragment, err
}

func Chew(j []byte, i Iterator, inherited Point) ([]Point, error) {
	points, _, _, err := process(j, i, inherited, false)
	return points, err
}

func process(j []byte, i Iterator, inherited Point, test bool) ([]Point, bool, string, error) {
	var results []Point

	selected, err := queryBytes(j, i.Selector)
	if err != nil {
		return results, false, "", err
	}

	var elements []interface{}

	err = json.Unmarshal(selected, &elements)
	if err != nil {
		return results, false, "", err
	}

	for _, element := range elements {
		point := inherited.Copy()
		elem, err := json.MarshalIndent(element, "", "  ")

		if err != nil {
			return results, false, "", err
		}

		if i.Time != "" {
			out, err := query(elem, i.Time)
			if err != nil {
				return results, false, "", err
			}
			if f, ok := out.(float64); ok {
				point.Timestamp = time.Unix(int64(f)/1000, 0)
			} else {
				return results, false, " ", errors.New("Time could not be read")
			}
		}

		for key, selector := range i.Values {
			out, err := query(elem, selector)
			if err != nil {
				return results, false, "", err
			}
			point.Values[key] = out
		}

		for key, selector := range i.Tags {
			out, err := queryBytes(elem, selector)
			if err != nil {
				return results, false, "", err
			}
			trimmed := strings.Trim(string(out), "\"")
			point.Tags[key] = trimmed
		}

		if len(i.Iterators) > 0 {
			for _, iterator := range i.Iterators {
				processed, stop, jsonFragment, err := process(elem, iterator, point, test)
				if err != nil {
					return results, false, "", err
				}
				results = append(results, processed...)
				if stop {
					return results, stop, jsonFragment, nil
				}
			}
		} else {
			results = append(results, point)
			if test {
				return results, true, string(elem), nil
			}
		}
	}

	return results, false, "", nil
}

func queryBytes(j []byte, q string) ([]byte, error) {
	result, err := query(j, q)
	if err != nil {
		return []byte{}, err
	}
	return json.Marshal(result)
}

func query(j []byte, q string) (interface{}, error) {
	var umsg jee.BMsg
	l, err := jee.Lexer(q)
	if err != nil {
		return nil, err
	}

	tree, err := jee.Parser(l)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(j, &umsg)
	if err != nil {
		return nil, err
	}

	result, err := jee.Eval(tree, umsg)
	if err != nil {
		return nil, err
	}
	return result, nil
}
<commit_msg>also print json if no iterator is defined<commit_after>package cmd

import (
	"encoding/json"
	"errors"
	"strings"
	"time"

	"github.com/nytlabs/gojee"
)

func Burp(j []byte, i Iterator, inherited Point) ([]Point, string, error) {
	var points []Point
	if i.Selector == "" {
		return points, string(j), errors.New("No selector definded")
	}
	points, _, jsonFragment, err := process(j, i, inherited, true)
	return points, jsonFragment, err
}

func Chew(j []byte, i Iterator, inherited Point) ([]Point, error) {
	var points []Point
	if i.Selector == "" {
		return points, errors.New("No selector definded")
	}
	points, _, _, err := process(j, i, inherited, false)
	return points, err
}

func process(j []byte, i Iterator, inherited Point, test bool) ([]Point, bool, string, error) {
	var results []Point

	selected, err := queryBytes(j, i.Selector)
	if err != nil {
		return results, false, "", err
	}

	var elements []interface{}

	err = json.Unmarshal(selected, &elements)
	if err != nil {
		return results, false, "", err
	}

	for _, element := range elements {
		point := inherited.Copy()
		elem, err := json.MarshalIndent(element, "", "  ")

		if err != nil {
			return results, false, "", err
		}

		if i.Time != "" {
			out, err := query(elem, i.Time)
			if err != nil {
				return results, false, "", err
			}
			if f, ok := out.(float64); ok {
				point.Timestamp = time.Unix(int64(f)/1000, 0)
			} else {
				return results, false, " ", errors.New("Time could not be read")
			}
		}

		for key, selector := range i.Values {
			out, err := query(elem, selector)
			if err != nil {
				return results, false, "", err
			}
			point.Values[key] = out
		}

		for key, selector := range i.Tags {
			out, err := queryBytes(elem, selector)
			if err != nil {
				return results, false, "", err
			}
			trimmed := strings.Trim(string(out), "\"")
			point.Tags[key] = trimmed
		}

		if len(i.Iterators) > 0 {
			for _, iterator := range i.Iterators {
				processed, stop, jsonFragment, err := process(elem, iterator, point, test)
				if err != nil {
					return results, false, "", err
				}
				results = append(results, processed...)
				if stop {
					return results, stop, jsonFragment, nil
				}
			}
		} else {
			results = append(results, point)
			if test {
				return results, true, string(elem), nil
			}
		}
	}

	return results, false, "", nil
}

func queryBytes(j []byte, q string) ([]byte, error) {
	result, err := query(j, q)
	if err != nil {
		return []byte{}, err
	}
	return json.Marshal(result)
}

func query(j []byte, q string) (interface{}, error) {
	var umsg jee.BMsg
	l, err := jee.Lexer(q)
	if err != nil {
		return nil, err
	}

	tree, err := jee.Parser(l)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(j, &umsg)
	if err != nil {
		return nil, err
	}

	result, err := jee.Eval(tree, umsg)
	if err != nil {
		return nil, err
	}
	return result, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/sh0rez/tanka/pkg/config/v1alpha1"
	"github.com/spf13/cobra"
)

// initCmd creates a new application
func initCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "init",
		Short: "Create the directory structure",
	}
	force := cmd.Flags().BoolP("force", "f", false, "ignore the working directory not being empty")
	provider := cmd.Flags().StringP("provider", "p", "", fmt.Sprintf("one of %s", listProviders()))
	cmd.Run = func(cmd *cobra.Command, args []string) {
		files, err := ioutil.ReadDir(".")
		if err != nil {
			log.Fatalln("Error listing files:", err)
		}
		if len(files) > 0 && !*force {
			log.Fatalln("Error: directory not empty. Use `-f` to force")
		}

		if err := writeNewFile("jsonnetfile.json", "{}"); err != nil {
			log.Fatalln("Error creating `jsonnetfile.json`:", err)
		}

		if err := os.Mkdir("vendor", os.ModePerm); err != nil {
			log.Fatalln("Error creating `vendor/` folder:", err)
		}

		if err := os.Mkdir("lib", os.ModePerm); err != nil {
			log.Fatalln("Error creating `vendor/` folder:", err)
		}

		if err := os.MkdirAll("environments/default", os.ModePerm); err != nil {
			log.Fatalln("Error creating environments folder")
		}

		if err := writeNewFile("environments/default/main.jsonnet", "{}"); err != nil {
			log.Fatalln("Error creating `main.jsonnet`:", err)
		}

		cfg := v1alpha1.Config{
			APIVersion: "tanka.dev/v1alpha1",
			Kind:       "Environment",
		}
		if *provider != "" {
			cfg.Spec = map[string]interface{}{}
			cfg.Spec["provider"] = map[string]interface{}{
				*provider: map[string]interface{}{},
			}
		}

		spec, err := json.Marshal(&cfg)
		if err != nil {
			log.Fatalln("Error creating spec.json:", err)
		}

		if err := writeNewFile("environments/default/spec.json", string(spec)); err != nil {
			log.Fatalln("Error creating `environments/default/spec.json`:", err)
		}

	}
	return cmd
}

// writeNewFile writes the content to a file if it does not exist
func writeNewFile(name, content string) error {
	if _, err := os.Stat(name); os.IsNotExist(err) {
		return ioutil.WriteFile(name, []byte(content), 0644)
	}
	return nil
}
<commit_msg>feat(cmd): init indent json<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/sh0rez/tanka/pkg/config/v1alpha1"
	"github.com/spf13/cobra"
)

// initCmd creates a new application
func initCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "init",
		Short: "Create the directory structure",
	}
	force := cmd.Flags().BoolP("force", "f", false, "ignore the working directory not being empty")
	provider := cmd.Flags().StringP("provider", "p", "", fmt.Sprintf("one of %s", listProviders()))
	cmd.Run = func(cmd *cobra.Command, args []string) {
		files, err := ioutil.ReadDir(".")
		if err != nil {
			log.Fatalln("Error listing files:", err)
		}
		if len(files) > 0 && !*force {
			log.Fatalln("Error: directory not empty. Use `-f` to force")
		}

		if err := writeNewFile("jsonnetfile.json", "{}"); err != nil {
			log.Fatalln("Error creating `jsonnetfile.json`:", err)
		}

		if err := os.Mkdir("vendor", os.ModePerm); err != nil {
			log.Fatalln("Error creating `vendor/` folder:", err)
		}

		if err := os.Mkdir("lib", os.ModePerm); err != nil {
			log.Fatalln("Error creating `vendor/` folder:", err)
		}

		if err := os.MkdirAll("environments/default", os.ModePerm); err != nil {
			log.Fatalln("Error creating environments folder")
		}

		if err := writeNewFile("environments/default/main.jsonnet", "{}"); err != nil {
			log.Fatalln("Error creating `main.jsonnet`:", err)
		}

		cfg := v1alpha1.Config{
			APIVersion: "tanka.dev/v1alpha1",
			Kind:       "Environment",
		}
		if *provider != "" {
			cfg.Spec = map[string]interface{}{}
			cfg.Spec["provider"] = map[string]interface{}{
				*provider: map[string]interface{}{},
			}
		}

		spec, err := json.MarshalIndent(&cfg, "", "  ")
		if err != nil {
			log.Fatalln("Error creating spec.json:", err)
		}

		if err := writeNewFile("environments/default/spec.json", string(spec)); err != nil {
			log.Fatalln("Error creating `environments/default/spec.json`:", err)
		}

	}
	return cmd
}

// writeNewFile writes the content to a file if it does not exist
func writeNewFile(name, content string) error {
	if _, err := os.Stat(name); os.IsNotExist(err) {
		return ioutil.WriteFile(name, []byte(content), 0644)
	}
	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/bpicode/fritzctl/config"
	"github.com/spf13/cobra"
)

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "print version of this application",
	Long:  "Print the version of this application as one-line semantic version string.",
	RunE:  printVersion,
}

func init() {
	RootCmd.AddCommand(versionCmd)
}

func printVersion(cmd *cobra.Command, args []string) error {
	fmt.Println(config.Version)
	return nil
}
<commit_msg>Issue #38: capitalize<commit_after>package cmd

import (
	"fmt"

	"github.com/bpicode/fritzctl/config"
	"github.com/spf13/cobra"
)

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print version of this application",
	Long:  "Print the version of this application as one-line semantic version string.",
	RunE:  printVersion,
}

func init() {
	RootCmd.AddCommand(versionCmd)
}

func printVersion(cmd *cobra.Command, args []string) error {
	fmt.Println(config.Version)
	return nil
}
<|endoftext|>


<commit_before>package main

const Version string = "0.3.0"
<commit_msg>Bump version 0.3.1<commit_after>package main

const Version string = "0.3.1"
<|endoftext|>


<commit_before>package zenodb

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/cloudfoundry/gosigar"
	"github.com/dustin/go-humanize"
	"github.com/getlantern/goexpr/geo"
	"github.com/getlantern/goexpr/isp"
	geredis "github.com/getlantern/goexpr/redis"
	"github.com/getlantern/golog"
	"github.com/getlantern/vtime"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/planner"
	"github.com/getlantern/zenodb/sql"
	"github.com/rickar/props"
	"gopkg.in/redis.v5"
)

var (
	log = golog.LoggerFor("zenodb")

	systemRAM float64
)

func init() {
	mem := sigar.Mem{}
	err := mem.Get()
	if err != nil {
		panic(fmt.Sprintf("Unable to get system memory: %v", err))
	}
	systemRAM = float64(mem.Total)
}

// DBOpts provides options for configuring the database.
type DBOpts struct {
	// Dir points at the directory that contains the data files.
	Dir string
	// SchemaFile points at a YAML schema file that configures the tables and
	// views in the database.
	SchemaFile string
	// AliasesFile points at a file that contains expression aliases in the form
	// name=template(%v, %v), with one alias per line.
	AliasesFile string
	// EnableGeo enables geolocation functions
	EnableGeo bool
	// ISPProvider configures a provider of ISP lookups. Specify this to allow the
	// use of ISP functions.
	ISPProvider isp.Provider
	// RedisClient provides a connection to redis which enables the use of Redis
	// expressions like HGET.
	RedisClient *redis.Client
	// RedisCacheSize controls the size of redis hash caches
	RedisCacheSize int
	// VirtualTime, if true, tells zenodb to use a virtual clock that advances
	// based on the timestamps of Points received via inserts.
	VirtualTime bool
	// WALSyncInterval governs how frequently to sync the WAL to disk. 0 means
	// it syncs after every write (which is not great for performance).
	WALSyncInterval time.Duration
	// MaxWALSize limits how much WAL data to keep (in bytes)
	MaxWALSize int
	// WALCompressionSize specifies the size beyond which to compress WAL segments
	WALCompressionSize int
	// MaxMemoryRatio caps the maximum memory of this process. When the system
	// comes under memory pressure, it will start flushing table memstores.
	MaxMemoryRatio float64
	// Passthrough flags this node as a passthrough (won't store data in tables,
	// just WAL). Passthrough nodes will also outsource queries to specific
	// partition handlers. Requires that NumPartitions be specified.
	Passthrough bool
	// NumPartitions identifies how many partitions to split data from
	// passthrough nodes.
	NumPartitions int
	// Partition identies the partition owned by this follower
	Partition int
	// MaxFollowAge limits how far back to go when follower pulls data from
	// leader
	MaxFollowAge time.Duration
	// Follow is a function that allows a follower to request following a stream
	// from a passthrough node.
	Follow                     func(f func() *common.Follow, cb func(data []byte, newOffset wal.Offset) error)
	RegisterRemoteQueryHandler func(partition int, query planner.QueryClusterFN)
}

// DB is a zenodb database.
type DB struct {
	opts                 *DBOpts
	clock                vtime.Clock
	tables               map[string]*table
	orderedTables        []*table
	streams              map[string]*wal.WAL
	newStreamSubscriber  map[string]chan *tableWithOffset
	tablesMutex          sync.RWMutex
	isSorting            bool
	nextTableToSort      int
	memory               uint64
	flushMutex           sync.Mutex
	followerJoined       chan *follower
	processFollowersOnce sync.Once
	remoteQueryHandlers  map[int]chan planner.QueryClusterFN
	closed               bool
}

// NewDB creates a database using the given options.
func NewDB(opts *DBOpts) (*DB, error) {
	var err error
	db := &DB{
		opts:                opts,
		clock:               vtime.RealClock,
		tables:              make(map[string]*table),
		streams:             make(map[string]*wal.WAL),
		newStreamSubscriber: make(map[string]chan *tableWithOffset),
		followerJoined:      make(chan *follower, opts.NumPartitions),
		remoteQueryHandlers: make(map[int]chan planner.QueryClusterFN),
	}
	if opts.VirtualTime {
		db.clock = vtime.NewVirtualClock(time.Time{})
	}
	if opts.MaxWALSize <= 0 {
		opts.MaxWALSize = 10 * 1024768 // 10 MB
	}
	if opts.WALCompressionSize <= 0 {
		opts.WALCompressionSize = opts.MaxWALSize / 10
	}

	// Create db dir
	err = os.MkdirAll(opts.Dir, 0755)
	if err != nil && !os.IsExist(err) {
		return nil, fmt.Errorf("Unable to create db dir at %v: %v", opts.Dir, err)
	}

	if opts.EnableGeo {
		log.Debug("Enabling geolocation functions")
		err = geo.Init(filepath.Join(opts.Dir, "geoip.dat"))
		if err != nil {
			return nil, fmt.Errorf("Unable to initialize geo: %v", err)
		}
	}

	if opts.ISPProvider != nil {
		log.Debugf("Setting ISP provider to %v", opts.ISPProvider)
		isp.SetProvider(opts.ISPProvider)
	}

	if opts.AliasesFile != "" {
		registerAliases(opts.AliasesFile)
	}

	if opts.RedisClient != nil && opts.RedisCacheSize > 0 {
		log.Debug("Enabling redis expressions")
		geredis.Configure(opts.RedisClient, opts.RedisCacheSize)
	}

	if opts.SchemaFile != "" {
		err = db.pollForSchema(opts.SchemaFile)
		if err != nil {
			return nil, fmt.Errorf("Unable to apply schema: %v", err)
		}
	}
	log.Debugf("Dir: %v    SchemaFile: %v", opts.Dir, opts.SchemaFile)

	if db.opts.RegisterRemoteQueryHandler != nil {
		go db.opts.RegisterRemoteQueryHandler(db.opts.Partition, db.queryForRemote)
	}

	if db.opts.MaxMemoryRatio > 0 {
		log.Debugf("Limiting maximum memstore memory to %v", humanize.Bytes(db.maxMemoryBytes()))
	}
	go db.trackMemStats()

	return db, err
}

func (db *DB) Close() {
	log.Debug("Closing")
	db.tablesMutex.Lock()
	for name, stream := range db.streams {
		log.Debugf("Closing stream %v", name)
		stream.Close()
		delete(db.streams, name)
	}
	db.tablesMutex.Unlock()
}

func registerAliases(aliasesFile string) {
	log.Debugf("Registering aliases from file at %v", aliasesFile)

	file, err := os.Open(aliasesFile)
	if err != nil {
		log.Errorf("Unable to open aliases file at %v: %v", aliasesFile, err)
		return
	}
	defer file.Close()

	p, err := props.Read(file)
	if err != nil {
		log.Errorf("Unable to read aliases file at %v: %v", aliasesFile, err)
		return
	}

	for _, alias := range p.Names() {
		template := strings.TrimSpace(p.Get(alias))
		alias = strings.TrimSpace(alias)
		sql.RegisterAlias(alias, template)
		log.Debugf("Registered alias %v = %v", alias, template)
	}
}

// TableStats returns the TableStats for the named table.
func (db *DB) TableStats(table string) TableStats {
	t := db.getTable(table)
	if t == nil {
		return TableStats{}
	}
	t.statsMutex.RLock()
	defer t.statsMutex.RUnlock()
	return t.stats
}

// AllTableStats returns all TableStats for all tables, keyed to the table
// names.
func (db *DB) AllTableStats() map[string]TableStats {
	m := make(map[string]TableStats)
	tables := make(map[string]*table, 0)
	db.tablesMutex.RLock()
	for name, t := range db.tables {
		tables[name] = t
	}
	db.tablesMutex.RUnlock()
	for name, t := range tables {
		t.statsMutex.RLock()
		m[name] = t.stats
		t.statsMutex.RUnlock()
	}
	return m
}

// PrintTableStats prints the stats for the named table to a string.
func (db *DB) PrintTableStats(table string) string {
	stats := db.TableStats(table)
	now := db.clock.Now()
	return fmt.Sprintf("%v (%v)\tFiltered: %v    Queued: %v    Inserted: %v    Dropped: %v    Expired: %v",
		table,
		now.In(time.UTC),
		humanize.Comma(stats.FilteredPoints),
		humanize.Comma(stats.QueuedPoints),
		humanize.Comma(stats.InsertedPoints),
		humanize.Comma(stats.DroppedPoints),
		humanize.Comma(stats.ExpiredValues))
}

func (db *DB) getTable(table string) *table {
	db.tablesMutex.RLock()
	t := db.tables[strings.ToLower(table)]
	db.tablesMutex.RUnlock()
	return t
}

func (db *DB) now(table string) time.Time {
	return db.clock.Now()
}

func (db *DB) getFields(table string) (core.Fields, error) {
	t := db.getTable(table)
	if t == nil {
		return nil, fmt.Errorf("Table '%v' not found", table)
	}
	return t.Fields, nil
}

func (db *DB) getFieldsOptional(table string) (core.Fields, error) {
	t := db.getTable(table)
	if t == nil {
		return nil, nil
	}
	return t.Fields, nil
}

func (db *DB) capWALAge(wal *wal.WAL) {
	for {
		time.Sleep(1 * time.Minute)
		err := wal.TruncateToSize(int64(db.opts.MaxWALSize))
		if err != nil {
			log.Errorf("Error truncating WAL: %v", err)
		}
		err = wal.CompressBeforeSize(int64(db.opts.WALCompressionSize))
		if err != nil {
			log.Errorf("Error compressing WAL: %v", err)
		}
	}
}

func (db *DB) trackMemStats() {
	for {
		db.updateMemStats()
		time.Sleep(2 * time.Second)
	}
}

func (db *DB) updateMemStats() {
	memstats := &runtime.MemStats{}
	runtime.ReadMemStats(memstats)
	atomic.StoreUint64(&db.memory, memstats.HeapInuse)
}

func (db *DB) capMemStoreSize() {
	if db.opts.MaxMemoryRatio <= 0 {
		return
	}

	db.tablesMutex.RLock()
	sizes := make(byCurrentSize, 0, len(db.tables))
	for _, table := range db.tables {
		if !table.Virtual {
			sizes = append(sizes, &memStoreSize{table, table.memStoreSize()})
		}
	}
	db.tablesMutex.RUnlock()

	db.flushMutex.Lock()
	actual := atomic.LoadUint64(&db.memory)
	allowed := db.maxMemoryBytes()
	if actual > allowed {
		// Force flushing on the table with the largest memstore
		sort.Sort(sizes)
		log.Debugf("Memory usage of %v exceeds allowed %v, forcing flush on %v", humanize.Bytes(actual), humanize.Bytes(allowed), sizes[0].t.Name)
		sizes[0].t.forceFlush()
		db.updateMemStats()
		log.Debugf("Done forcing flush on %v", sizes[0].t.Name)
	}
	db.flushMutex.Unlock()
}

func (db *DB) maxMemoryBytes() uint64 {
	return uint64(systemRAM * db.opts.MaxMemoryRatio)
}

type memStoreSize struct {
	t    *table
	size int
}

type byCurrentSize []*memStoreSize

func (a byCurrentSize) Len() int           { return len(a) }
func (a byCurrentSize) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byCurrentSize) Less(i, j int) bool { return a[i].size > a[j].size }
<commit_msg>Using allocated size instead of inuse to determine memory usage<commit_after>package zenodb

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/cloudfoundry/gosigar"
	"github.com/dustin/go-humanize"
	"github.com/getlantern/goexpr/geo"
	"github.com/getlantern/goexpr/isp"
	geredis "github.com/getlantern/goexpr/redis"
	"github.com/getlantern/golog"
	"github.com/getlantern/vtime"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/planner"
	"github.com/getlantern/zenodb/sql"
	"github.com/rickar/props"
	"gopkg.in/redis.v5"
)

var (
	log = golog.LoggerFor("zenodb")

	systemRAM float64
)

func init() {
	mem := sigar.Mem{}
	err := mem.Get()
	if err != nil {
		panic(fmt.Sprintf("Unable to get system memory: %v", err))
	}
	systemRAM = float64(mem.Total)
}

// DBOpts provides options for configuring the database.
type DBOpts struct {
	// Dir points at the directory that contains the data files.
	Dir string
	// SchemaFile points at a YAML schema file that configures the tables and
	// views in the database.
	SchemaFile string
	// AliasesFile points at a file that contains expression aliases in the form
	// name=template(%v, %v), with one alias per line.
	AliasesFile string
	// EnableGeo enables geolocation functions
	EnableGeo bool
	// ISPProvider configures a provider of ISP lookups. Specify this to allow the
	// use of ISP functions.
	ISPProvider isp.Provider
	// RedisClient provides a connection to redis which enables the use of Redis
	// expressions like HGET.
	RedisClient *redis.Client
	// RedisCacheSize controls the size of redis hash caches
	RedisCacheSize int
	// VirtualTime, if true, tells zenodb to use a virtual clock that advances
	// based on the timestamps of Points received via inserts.
	VirtualTime bool
	// WALSyncInterval governs how frequently to sync the WAL to disk. 0 means
	// it syncs after every write (which is not great for performance).
	WALSyncInterval time.Duration
	// MaxWALSize limits how much WAL data to keep (in bytes)
	MaxWALSize int
	// WALCompressionSize specifies the size beyond which to compress WAL segments
	WALCompressionSize int
	// MaxMemoryRatio caps the maximum memory of this process. When the system
	// comes under memory pressure, it will start flushing table memstores.
	MaxMemoryRatio float64
	// Passthrough flags this node as a passthrough (won't store data in tables,
	// just WAL). Passthrough nodes will also outsource queries to specific
	// partition handlers. Requires that NumPartitions be specified.
	Passthrough bool
	// NumPartitions identifies how many partitions to split data from
	// passthrough nodes.
	NumPartitions int
	// Partition identies the partition owned by this follower
	Partition int
	// MaxFollowAge limits how far back to go when follower pulls data from
	// leader
	MaxFollowAge time.Duration
	// Follow is a function that allows a follower to request following a stream
	// from a passthrough node.
	Follow                     func(f func() *common.Follow, cb func(data []byte, newOffset wal.Offset) error)
	RegisterRemoteQueryHandler func(partition int, query planner.QueryClusterFN)
}

// DB is a zenodb database.
type DB struct {
	opts                 *DBOpts
	clock                vtime.Clock
	tables               map[string]*table
	orderedTables        []*table
	streams              map[string]*wal.WAL
	newStreamSubscriber  map[string]chan *tableWithOffset
	tablesMutex          sync.RWMutex
	isSorting            bool
	nextTableToSort      int
	memory               uint64
	flushMutex           sync.Mutex
	followerJoined       chan *follower
	processFollowersOnce sync.Once
	remoteQueryHandlers  map[int]chan planner.QueryClusterFN
	closed               bool
}

// NewDB creates a database using the given options.
func NewDB(opts *DBOpts) (*DB, error) {
	var err error
	db := &DB{
		opts:                opts,
		clock:               vtime.RealClock,
		tables:              make(map[string]*table),
		streams:             make(map[string]*wal.WAL),
		newStreamSubscriber: make(map[string]chan *tableWithOffset),
		followerJoined:      make(chan *follower, opts.NumPartitions),
		remoteQueryHandlers: make(map[int]chan planner.QueryClusterFN),
	}
	if opts.VirtualTime {
		db.clock = vtime.NewVirtualClock(time.Time{})
	}
	if opts.MaxWALSize <= 0 {
		opts.MaxWALSize = 10 * 1024768 // 10 MB
	}
	if opts.WALCompressionSize <= 0 {
		opts.WALCompressionSize = opts.MaxWALSize / 10
	}

	// Create db dir
	err = os.MkdirAll(opts.Dir, 0755)
	if err != nil && !os.IsExist(err) {
		return nil, fmt.Errorf("Unable to create db dir at %v: %v", opts.Dir, err)
	}

	if opts.EnableGeo {
		log.Debug("Enabling geolocation functions")
		err = geo.Init(filepath.Join(opts.Dir, "geoip.dat"))
		if err != nil {
			return nil, fmt.Errorf("Unable to initialize geo: %v", err)
		}
	}

	if opts.ISPProvider != nil {
		log.Debugf("Setting ISP provider to %v", opts.ISPProvider)
		isp.SetProvider(opts.ISPProvider)
	}

	if opts.AliasesFile != "" {
		registerAliases(opts.AliasesFile)
	}

	if opts.RedisClient != nil && opts.RedisCacheSize > 0 {
		log.Debug("Enabling redis expressions")
		geredis.Configure(opts.RedisClient, opts.RedisCacheSize)
	}

	if opts.SchemaFile != "" {
		err = db.pollForSchema(opts.SchemaFile)
		if err != nil {
			return nil, fmt.Errorf("Unable to apply schema: %v", err)
		}
	}
	log.Debugf("Dir: %v    SchemaFile: %v", opts.Dir, opts.SchemaFile)

	if db.opts.RegisterRemoteQueryHandler != nil {
		go db.opts.RegisterRemoteQueryHandler(db.opts.Partition, db.queryForRemote)
	}

	if db.opts.MaxMemoryRatio > 0 {
		log.Debugf("Limiting maximum memstore memory to %v", humanize.Bytes(db.maxMemoryBytes()))
	}
	go db.trackMemStats()

	return db, err
}

func (db *DB) Close() {
	log.Debug("Closing")
	db.tablesMutex.Lock()
	for name, stream := range db.streams {
		log.Debugf("Closing stream %v", name)
		stream.Close()
		delete(db.streams, name)
	}
	db.tablesMutex.Unlock()
}

func registerAliases(aliasesFile string) {
	log.Debugf("Registering aliases from file at %v", aliasesFile)

	file, err := os.Open(aliasesFile)
	if err != nil {
		log.Errorf("Unable to open aliases file at %v: %v", aliasesFile, err)
		return
	}
	defer file.Close()

	p, err := props.Read(file)
	if err != nil {
		log.Errorf("Unable to read aliases file at %v: %v", aliasesFile, err)
		return
	}

	for _, alias := range p.Names() {
		template := strings.TrimSpace(p.Get(alias))
		alias = strings.TrimSpace(alias)
		sql.RegisterAlias(alias, template)
		log.Debugf("Registered alias %v = %v", alias, template)
	}
}

// TableStats returns the TableStats for the named table.
func (db *DB) TableStats(table string) TableStats {
	t := db.getTable(table)
	if t == nil {
		return TableStats{}
	}
	t.statsMutex.RLock()
	defer t.statsMutex.RUnlock()
	return t.stats
}

// AllTableStats returns all TableStats for all tables, keyed to the table
// names.
func (db *DB) AllTableStats() map[string]TableStats {
	m := make(map[string]TableStats)
	tables := make(map[string]*table, 0)
	db.tablesMutex.RLock()
	for name, t := range db.tables {
		tables[name] = t
	}
	db.tablesMutex.RUnlock()
	for name, t := range tables {
		t.statsMutex.RLock()
		m[name] = t.stats
		t.statsMutex.RUnlock()
	}
	return m
}

// PrintTableStats prints the stats for the named table to a string.
func (db *DB) PrintTableStats(table string) string {
	stats := db.TableStats(table)
	now := db.clock.Now()
	return fmt.Sprintf("%v (%v)\tFiltered: %v    Queued: %v    Inserted: %v    Dropped: %v    Expired: %v",
		table,
		now.In(time.UTC),
		humanize.Comma(stats.FilteredPoints),
		humanize.Comma(stats.QueuedPoints),
		humanize.Comma(stats.InsertedPoints),
		humanize.Comma(stats.DroppedPoints),
		humanize.Comma(stats.ExpiredValues))
}

func (db *DB) getTable(table string) *table {
	db.tablesMutex.RLock()
	t := db.tables[strings.ToLower(table)]
	db.tablesMutex.RUnlock()
	return t
}

func (db *DB) now(table string) time.Time {
	return db.clock.Now()
}

func (db *DB) getFields(table string) (core.Fields, error) {
	t := db.getTable(table)
	if t == nil {
		return nil, fmt.Errorf("Table '%v' not found", table)
	}
	return t.Fields, nil
}

func (db *DB) getFieldsOptional(table string) (core.Fields, error) {
	t := db.getTable(table)
	if t == nil {
		return nil, nil
	}
	return t.Fields, nil
}

func (db *DB) capWALAge(wal *wal.WAL) {
	for {
		time.Sleep(1 * time.Minute)
		err := wal.TruncateToSize(int64(db.opts.MaxWALSize))
		if err != nil {
			log.Errorf("Error truncating WAL: %v", err)
		}
		err = wal.CompressBeforeSize(int64(db.opts.WALCompressionSize))
		if err != nil {
			log.Errorf("Error compressing WAL: %v", err)
		}
	}
}

func (db *DB) trackMemStats() {
	for {
		db.updateMemStats()
		time.Sleep(2 * time.Second)
	}
}

func (db *DB) updateMemStats() {
	memstats := &runtime.MemStats{}
	runtime.ReadMemStats(memstats)
	atomic.StoreUint64(&db.memory, memstats.Alloc)
}

func (db *DB) capMemStoreSize() {
	if db.opts.MaxMemoryRatio <= 0 {
		return
	}

	db.tablesMutex.RLock()
	sizes := make(byCurrentSize, 0, len(db.tables))
	for _, table := range db.tables {
		if !table.Virtual {
			sizes = append(sizes, &memStoreSize{table, table.memStoreSize()})
		}
	}
	db.tablesMutex.RUnlock()

	db.flushMutex.Lock()
	actual := atomic.LoadUint64(&db.memory)
	allowed := db.maxMemoryBytes()
	if actual > allowed {
		// Force flushing on the table with the largest memstore
		sort.Sort(sizes)
		log.Debugf("Memory usage of %v exceeds allowed %v, forcing flush on %v", humanize.Bytes(actual), humanize.Bytes(allowed), sizes[0].t.Name)
		sizes[0].t.forceFlush()
		db.updateMemStats()
		log.Debugf("Done forcing flush on %v", sizes[0].t.Name)
	}
	db.flushMutex.Unlock()
}

func (db *DB) maxMemoryBytes() uint64 {
	return uint64(systemRAM * db.opts.MaxMemoryRatio)
}

type memStoreSize struct {
	t    *table
	size int
}

type byCurrentSize []*memStoreSize

func (a byCurrentSize) Len() int           { return len(a) }
func (a byCurrentSize) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byCurrentSize) Less(i, j int) bool { return a[i].size > a[j].size }
<|endoftext|>


<commit_before>// package server provides a server that can wrap a node and serve an http api
// for interacting with the node.
package server

import (
	"net"
	"net/http"
	"strings"

	"github.com/NebulousLabs/Sia/node"
	"github.com/NebulousLabs/Sia/node/api"

	"github.com/NebulousLabs/errors"
)

// A Server is a collection of siad modules that can be communicated with over
// an http api.
type Server struct {
	api               *api.API
	apiServer         *http.Server
	done              chan struct{}
	listener          net.Listener
	node              *node.Node
	requiredUserAgent string
	serveErr          error
}

// serve listens for and handles API calls. It is a blocking function.
func (srv *Server) serve() error {
	// The server will run until an error is encountered or the listener is
	// closed, via either the Close method or by signal handling.  Closing the
	// listener will result in the benign error handled below.
	err := srv.apiServer.Serve(srv.listener)
	if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") {
		return err
	}
	return nil
}

// Close closes the Server's listener, causing the HTTP server to shut down.
func (srv *Server) Close() error {
	// Stop accepting API requests.
	err := srv.listener.Close()
	// Wait for serve() to return and capture its error.
	<-srv.done
	err = errors.Compose(err, srv.serveErr)
	// Shutdown modules.
	err = errors.Compose(err, srv.node.Close())
	return errors.AddContext(err, "error while closing server")
}

// New creates a new API server from the provided modules. The API will
// require authentication using HTTP basic auth if the supplied password is not
// the empty string. Usernames are ignored for authentication. This type of
// authentication sends passwords in plaintext and should therefore only be
// used if the APIaddr is localhost.
func New(APIaddr string, requiredUserAgent string, requiredPassword string, nodeParams node.NodeParams) (*Server, error) {
	// Create the server listener.
	listener, err := net.Listen("tcp", APIaddr)
	if err != nil {
		return nil, err
	}

	// Create the Sia node for the server.
	node, err := node.New(nodeParams)
	if err != nil {
		return nil, errors.AddContext(err, "server is unable to create the Sia node")
	}

	// Create the api for the server.
	api := api.New(requiredUserAgent, requiredPassword, node.ConsensusSet, node.Explorer, node.Gateway, node.Host, node.Miner, node.Renter, node.TransactionPool, node.Wallet)
	srv := &Server{
		api: api,
		apiServer: &http.Server{
			Handler: api,
		},
		done:              make(chan struct{}),
		listener:          listener,
		requiredUserAgent: requiredUserAgent,
	}

	// Spin up a goroutine that serves the API and closes srv.done when
	// finished.
	go func() {
		srv.serveErr = srv.serve()
		close(srv.done)
	}()

	return srv, nil
}
<commit_msg>server.go: fix golint warning<commit_after>// Package server provides a server that can wrap a node and serve an http api
// for interacting with the node.
package server

import (
	"net"
	"net/http"
	"strings"

	"github.com/NebulousLabs/Sia/node"
	"github.com/NebulousLabs/Sia/node/api"

	"github.com/NebulousLabs/errors"
)

// A Server is a collection of siad modules that can be communicated with over
// an http api.
type Server struct {
	api               *api.API
	apiServer         *http.Server
	done              chan struct{}
	listener          net.Listener
	node              *node.Node
	requiredUserAgent string
	serveErr          error
}

// serve listens for and handles API calls. It is a blocking function.
func (srv *Server) serve() error {
	// The server will run until an error is encountered or the listener is
	// closed, via either the Close method or by signal handling.  Closing the
	// listener will result in the benign error handled below.
	err := srv.apiServer.Serve(srv.listener)
	if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") {
		return err
	}
	return nil
}

// Close closes the Server's listener, causing the HTTP server to shut down.
func (srv *Server) Close() error {
	// Stop accepting API requests.
	err := srv.listener.Close()
	// Wait for serve() to return and capture its error.
	<-srv.done
	err = errors.Compose(err, srv.serveErr)
	// Shutdown modules.
	err = errors.Compose(err, srv.node.Close())
	return errors.AddContext(err, "error while closing server")
}

// New creates a new API server from the provided modules. The API will
// require authentication using HTTP basic auth if the supplied password is not
// the empty string. Usernames are ignored for authentication. This type of
// authentication sends passwords in plaintext and should therefore only be
// used if the APIaddr is localhost.
func New(APIaddr string, requiredUserAgent string, requiredPassword string, nodeParams node.NodeParams) (*Server, error) {
	// Create the server listener.
	listener, err := net.Listen("tcp", APIaddr)
	if err != nil {
		return nil, err
	}

	// Create the Sia node for the server.
	node, err := node.New(nodeParams)
	if err != nil {
		return nil, errors.AddContext(err, "server is unable to create the Sia node")
	}

	// Create the api for the server.
	api := api.New(requiredUserAgent, requiredPassword, node.ConsensusSet, node.Explorer, node.Gateway, node.Host, node.Miner, node.Renter, node.TransactionPool, node.Wallet)
	srv := &Server{
		api: api,
		apiServer: &http.Server{
			Handler: api,
		},
		done:              make(chan struct{}),
		listener:          listener,
		requiredUserAgent: requiredUserAgent,
	}

	// Spin up a goroutine that serves the API and closes srv.done when
	// finished.
	go func() {
		srv.serveErr = srv.serve()
		close(srv.done)
	}()

	return srv, nil
}
<|endoftext|>


<commit_before>package cobra

import (
	"bytes"
	"fmt"
	"strings"
	"testing"
)

var _ = fmt.Println

var tp, te, tt, t1 []string
var flagb1, flagb2, flagb3, flagbr bool
var flags1, flags2, flags3 string
var flagi1, flagi2, flagi3, flagir int
var globalFlag1 bool
var flagEcho, rootcalled bool

var cmdPrint = &Command{
	Use:   "print [string to print]",
	Short: "Print anything to the screen",
	Long:  `an utterly useless command for testing.`,
	Run: func(cmd *Command, args []string) {
		tp = args
	},
}

var cmdEcho = &Command{
	Use:   "echo [string to echo]",
	Short: "Echo anything to the screen",
	Long:  `an utterly useless command for testing.`,
	Run: func(cmd *Command, args []string) {
		te = args
	},
}

var cmdTimes = &Command{
	Use:   "times [# times] [string to echo]",
	Short: "Echo anything to the screen more times",
	Long:  `an slightly useless command for testing.`,
	Run:   timesRunner,
}

var cmdRootNoRun = &Command{
	Use:   "cobra-test",
	Short: "The root can run it's own function",
	Long:  "The root description for help",
}

var cmdRootWithRun = &Command{
	Use:   "cobra-test",
	Short: "The root can run it's own function",
	Long:  "The root description for help",
	Run: func(cmd *Command, args []string) {
		rootcalled = true
	},
}

func timesRunner(cmd *Command, args []string) {
	tt = args
}

func flagInit() {
	cmdEcho.ResetFlags()
	cmdPrint.ResetFlags()
	cmdTimes.ResetFlags()
	cmdRootNoRun.ResetFlags()
	cmdRootWithRun.ResetFlags()
	cmdEcho.Flags().IntVarP(&flagi1, "intone", "i", 123, "help message for flag intone")
	cmdTimes.Flags().IntVarP(&flagi2, "inttwo", "j", 234, "help message for flag inttwo")
	cmdPrint.Flags().IntVarP(&flagi3, "intthree", "i", 345, "help message for flag intthree")
	cmdEcho.PersistentFlags().StringVarP(&flags1, "strone", "s", "one", "help message for flag strone")
	cmdTimes.PersistentFlags().StringVarP(&flags2, "strtwo", "t", "two", "help message for flag strtwo")
	cmdPrint.PersistentFlags().StringVarP(&flags3, "strthree", "s", "three", "help message for flag strthree")
	cmdEcho.Flags().BoolVarP(&flagb1, "boolone", "b", true, "help message for flag boolone")
	cmdTimes.Flags().BoolVarP(&flagb2, "booltwo", "c", false, "help message for flag booltwo")
	cmdPrint.Flags().BoolVarP(&flagb3, "boolthree", "b", true, "help message for flag boolthree")
}

func commandInit() {
	cmdEcho.ResetCommands()
	cmdPrint.ResetCommands()
	cmdTimes.ResetCommands()
}

func initialize() *Command {
	tt, tp, te = nil, nil, nil
	var c = cmdRootNoRun
	flagInit()
	commandInit()
	return c
}

func initializeWithRootCmd() *Command {
	cmdRootWithRun.ResetCommands()
	tt, tp, te, rootcalled = nil, nil, nil, false
	flagInit()
	cmdRootWithRun.Flags().BoolVarP(&flagbr, "boolroot", "b", false, "help message for flag boolroot")
	cmdRootWithRun.Flags().IntVarP(&flagir, "introot", "i", 321, "help message for flag intthree")
	commandInit()
	return cmdRootWithRun
}

func TestSingleCommand(t *testing.T) {
	c := initialize()
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("print one two", " "))
	c.Execute()

	if te != nil || tt != nil {
		t.Error("Wrong command called")
	}
	if tp == nil {
		t.Error("Wrong command called")
	}
	if strings.Join(tp, " ") != "one two" {
		t.Error("Command didn't parse correctly")
	}
}

func TestChildCommand(t *testing.T) {
	c := initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo times one two", " "))
	c.Execute()

	if te != nil || tp != nil {
		t.Error("Wrong command called")
	}
	if tt == nil {
		t.Error("Wrong command called")
	}
	if strings.Join(tt, " ") != "one two" {
		t.Error("Command didn't parse correctly")
	}
}

func TestFlagLong(t *testing.T) {
	c := initialize()
	c.AddCommand(cmdPrint, cmdEcho, cmdTimes)
	c.SetArgs(strings.Split("echo --intone=13 something here", " "))
	c.Execute()

	if strings.Join(te, " ") != "something here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", te)
	}
	if flagi1 != 13 {
		t.Errorf("int flag didn't get correct value, had %d", flagi1)
	}
	if flagi2 != 234 {
		t.Errorf("default flag value changed, 234 expected, %d given", flagi2)
	}
}

func TestFlagShort(t *testing.T) {
	c := initialize()
	c.AddCommand(cmdPrint, cmdEcho, cmdTimes)
	c.SetArgs(strings.Split("echo -i13 something here", " "))
	c.Execute()

	if strings.Join(te, " ") != "something here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", te)
	}
	if flagi1 != 13 {
		t.Errorf("int flag didn't get correct value, had %d", flagi1)
	}
	if flagi2 != 234 {
		t.Errorf("default flag value changed, 234 expected, %d given", flagi2)
	}

	c = initialize()
	c.AddCommand(cmdPrint, cmdEcho, cmdTimes)
	c.SetArgs(strings.Split("echo -i 13 something here", " "))
	c.Execute()

	if strings.Join(te, " ") != "something here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", te)
	}
	if flagi1 != 13 {
		t.Errorf("int flag didn't get correct value, had %d", flagi1)
	}
	if flagi2 != 234 {
		t.Errorf("default flag value changed, 234 expected, %d given", flagi2)
	}

	// Testing same shortcode, different command
	c = initialize()
	c.AddCommand(cmdPrint, cmdEcho, cmdTimes)
	c.SetArgs(strings.Split("print -i99 one two", " "))
	c.Execute()

	if strings.Join(tp, " ") != "one two" {
		t.Errorf("flags didn't leave proper args remaining..%s given", tp)
	}
	if flagi3 != 99 {
		t.Errorf("int flag didn't get correct value, had %d", flagi3)
	}
	if flagi1 != 123 {
		t.Errorf("default flag value changed on different command with same shortname, 234 expected, %d given", flagi2)
	}
}

func TestChildCommandFlags(t *testing.T) {
	c := initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo times -j 99 one two", " "))
	c.Execute()

	if strings.Join(tt, " ") != "one two" {
		t.Errorf("flags didn't leave proper args remaining..%s given", tt)
	}

	buf := new(bytes.Buffer)
	// Testing with flag that shouldn't be persistent
	c = initialize()
	c.SetOutput(buf)
	// define children
	c.AddCommand(cmdPrint, cmdEcho)
	// define grandchild
	cmdEcho.AddCommand(cmdTimes)
	c.SetArgs(strings.Split("echo times -j 99 -i77 one two", " "))
	e := c.Execute()

	if e == nil {
		t.Errorf("invalid flag should generate error")
	}

	if !strings.Contains(buf.String(), "unknown shorthand") {
		t.Errorf("Wrong error message displayed, \n %s", buf.String())
	}

	if flagi2 != 99 {
		t.Errorf("flag value should be 99, %d given", flagi2)
	}

	if flagi1 != 123 {
		t.Errorf("unset flag should have default value, expecting 123, given %d", flagi1)
	}

	// Testing with flag only existing on child
	buf.Reset()
	c = initialize()
	c.SetOutput(buf)
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo -j 99 -i77 one two", " "))
	err := c.Execute()

	if err == nil {
		t.Errorf("invalid flag should generate error")
	}

	if !strings.Contains(buf.String(), "intone=123") {
		t.Errorf("Wrong error message displayed, \n %s", buf.String())
	}
}

func TestTrailingCommandFlags(t *testing.T) {
	buf := new(bytes.Buffer)
	c := initialize()
	c.SetOutput(buf)
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo two -x", " "))
	e3 := c.Execute()

	if e3 == nil {
		t.Errorf("invalid flag should generate error")
	}
}

func TestPersistentFlags(t *testing.T) {
	c := initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo -s something more here", " "))
	c.Execute()

	// persistentFlag should act like normal flag on it's own command
	if strings.Join(te, " ") != "more here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", te)
	}

	// persistentFlag should act like normal flag on it's own command
	if flags1 != "something" {
		t.Errorf("string flag didn't get correct value, had %v", flags1)
	}

	c = initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo times -s again -c test here", " "))
	c.Execute()

	if strings.Join(tt, " ") != "test here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", tt)
	}

	if flags1 != "again" {
		t.Errorf("string flag didn't get correct value, had %v", flags1)
	}

	if flagb2 != true {
		t.Errorf("local flag not parsed correctly. Expected false, had %v", flagb2)
	}
}

func TestHelpCommand(t *testing.T) {
	buf := new(bytes.Buffer)
	c := initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("help echo", " "))
	c.SetOutput(buf)
	c.Execute()

	if !strings.Contains(buf.String(), cmdEcho.Long) {
		t.Errorf("Wrong error message displayed, \n %s", buf.String())
	}

	buf.Reset()
	c = initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("help echo times", " "))
	c.SetOutput(buf)
	c.Execute()

	if !strings.Contains(buf.String(), cmdTimes.Long) {
		t.Errorf("Wrong error message displayed, \n %s", buf.String())
	}
}

func TestRunnableRootCommand(t *testing.T) {
	c := initializeWithRootCmd()
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs([]string(nil))
	c.Execute()

	if rootcalled != true {
		t.Errorf("Root Function was not called")
	}
}

func TestRootFlags(t *testing.T) {
	c := initializeWithRootCmd()
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("-i 17 -b", " "))
	c.Execute()

	if flagbr != true {
		t.Errorf("flag value should be true, %v given", flagbr)
	}

	if flagir != 17 {
		t.Errorf("flag value should be 17, %d given", flagir)
	}

}
<commit_msg>Reset root command lists in testing<commit_after>package cobra

import (
	"bytes"
	"fmt"
	"strings"
	"testing"
)

var _ = fmt.Println

var tp, te, tt, t1 []string
var flagb1, flagb2, flagb3, flagbr bool
var flags1, flags2, flags3 string
var flagi1, flagi2, flagi3, flagir int
var globalFlag1 bool
var flagEcho, rootcalled bool

var cmdPrint = &Command{
	Use:   "print [string to print]",
	Short: "Print anything to the screen",
	Long:  `an utterly useless command for testing.`,
	Run: func(cmd *Command, args []string) {
		tp = args
	},
}

var cmdEcho = &Command{
	Use:   "echo [string to echo]",
	Short: "Echo anything to the screen",
	Long:  `an utterly useless command for testing.`,
	Run: func(cmd *Command, args []string) {
		te = args
	},
}

var cmdTimes = &Command{
	Use:   "times [# times] [string to echo]",
	Short: "Echo anything to the screen more times",
	Long:  `an slightly useless command for testing.`,
	Run:   timesRunner,
}

var cmdRootNoRun = &Command{
	Use:   "cobra-test",
	Short: "The root can run it's own function",
	Long:  "The root description for help",
}

var cmdRootWithRun = &Command{
	Use:   "cobra-test",
	Short: "The root can run it's own function",
	Long:  "The root description for help",
	Run: func(cmd *Command, args []string) {
		rootcalled = true
	},
}

func timesRunner(cmd *Command, args []string) {
	tt = args
}

func flagInit() {
	cmdEcho.ResetFlags()
	cmdPrint.ResetFlags()
	cmdTimes.ResetFlags()
	cmdRootNoRun.ResetFlags()
	cmdRootWithRun.ResetFlags()
	cmdEcho.Flags().IntVarP(&flagi1, "intone", "i", 123, "help message for flag intone")
	cmdTimes.Flags().IntVarP(&flagi2, "inttwo", "j", 234, "help message for flag inttwo")
	cmdPrint.Flags().IntVarP(&flagi3, "intthree", "i", 345, "help message for flag intthree")
	cmdEcho.PersistentFlags().StringVarP(&flags1, "strone", "s", "one", "help message for flag strone")
	cmdTimes.PersistentFlags().StringVarP(&flags2, "strtwo", "t", "two", "help message for flag strtwo")
	cmdPrint.PersistentFlags().StringVarP(&flags3, "strthree", "s", "three", "help message for flag strthree")
	cmdEcho.Flags().BoolVarP(&flagb1, "boolone", "b", true, "help message for flag boolone")
	cmdTimes.Flags().BoolVarP(&flagb2, "booltwo", "c", false, "help message for flag booltwo")
	cmdPrint.Flags().BoolVarP(&flagb3, "boolthree", "b", true, "help message for flag boolthree")
}

func commandInit() {
	cmdEcho.ResetCommands()
	cmdPrint.ResetCommands()
	cmdTimes.ResetCommands()
	cmdRootNoRun.ResetCommands()
	cmdRootWithRun.ResetCommands()
}

func initialize() *Command {
	tt, tp, te = nil, nil, nil
	var c = cmdRootNoRun
	flagInit()
	commandInit()
	return c
}

func initializeWithRootCmd() *Command {
	cmdRootWithRun.ResetCommands()
	tt, tp, te, rootcalled = nil, nil, nil, false
	flagInit()
	cmdRootWithRun.Flags().BoolVarP(&flagbr, "boolroot", "b", false, "help message for flag boolroot")
	cmdRootWithRun.Flags().IntVarP(&flagir, "introot", "i", 321, "help message for flag intthree")
	commandInit()
	return cmdRootWithRun
}

func TestSingleCommand(t *testing.T) {
	c := initialize()
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("print one two", " "))
	c.Execute()

	if te != nil || tt != nil {
		t.Error("Wrong command called")
	}
	if tp == nil {
		t.Error("Wrong command called")
	}
	if strings.Join(tp, " ") != "one two" {
		t.Error("Command didn't parse correctly")
	}
}

func TestChildCommand(t *testing.T) {
	c := initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo times one two", " "))
	c.Execute()

	if te != nil || tp != nil {
		t.Error("Wrong command called")
	}
	if tt == nil {
		t.Error("Wrong command called")
	}
	if strings.Join(tt, " ") != "one two" {
		t.Error("Command didn't parse correctly")
	}
}

func TestFlagLong(t *testing.T) {
	c := initialize()
	c.AddCommand(cmdPrint, cmdEcho, cmdTimes)
	c.SetArgs(strings.Split("echo --intone=13 something here", " "))
	c.Execute()

	if strings.Join(te, " ") != "something here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", te)
	}
	if flagi1 != 13 {
		t.Errorf("int flag didn't get correct value, had %d", flagi1)
	}
	if flagi2 != 234 {
		t.Errorf("default flag value changed, 234 expected, %d given", flagi2)
	}
}

func TestFlagShort(t *testing.T) {
	c := initialize()
	c.AddCommand(cmdPrint, cmdEcho, cmdTimes)
	c.SetArgs(strings.Split("echo -i13 something here", " "))
	c.Execute()

	if strings.Join(te, " ") != "something here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", te)
	}
	if flagi1 != 13 {
		t.Errorf("int flag didn't get correct value, had %d", flagi1)
	}
	if flagi2 != 234 {
		t.Errorf("default flag value changed, 234 expected, %d given", flagi2)
	}

	c = initialize()
	c.AddCommand(cmdPrint, cmdEcho, cmdTimes)
	c.SetArgs(strings.Split("echo -i 13 something here", " "))
	c.Execute()

	if strings.Join(te, " ") != "something here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", te)
	}
	if flagi1 != 13 {
		t.Errorf("int flag didn't get correct value, had %d", flagi1)
	}
	if flagi2 != 234 {
		t.Errorf("default flag value changed, 234 expected, %d given", flagi2)
	}

	// Testing same shortcode, different command
	c = initialize()
	c.AddCommand(cmdPrint, cmdEcho, cmdTimes)
	c.SetArgs(strings.Split("print -i99 one two", " "))
	c.Execute()

	if strings.Join(tp, " ") != "one two" {
		t.Errorf("flags didn't leave proper args remaining..%s given", tp)
	}
	if flagi3 != 99 {
		t.Errorf("int flag didn't get correct value, had %d", flagi3)
	}
	if flagi1 != 123 {
		t.Errorf("default flag value changed on different command with same shortname, 234 expected, %d given", flagi2)
	}
}

func TestChildCommandFlags(t *testing.T) {
	c := initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo times -j 99 one two", " "))
	c.Execute()

	if strings.Join(tt, " ") != "one two" {
		t.Errorf("flags didn't leave proper args remaining..%s given", tt)
	}

	buf := new(bytes.Buffer)
	// Testing with flag that shouldn't be persistent
	c = initialize()
	c.SetOutput(buf)
	// define children
	c.AddCommand(cmdPrint, cmdEcho)
	// define grandchild
	cmdEcho.AddCommand(cmdTimes)
	c.SetArgs(strings.Split("echo times -j 99 -i77 one two", " "))
	e := c.Execute()

	if e == nil {
		t.Errorf("invalid flag should generate error")
	}

	if !strings.Contains(buf.String(), "unknown shorthand") {
		t.Errorf("Wrong error message displayed, \n %s", buf.String())
	}

	if flagi2 != 99 {
		t.Errorf("flag value should be 99, %d given", flagi2)
	}

	if flagi1 != 123 {
		t.Errorf("unset flag should have default value, expecting 123, given %d", flagi1)
	}

	// Testing with flag only existing on child
	buf.Reset()
	c = initialize()
	c.SetOutput(buf)
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo -j 99 -i77 one two", " "))
	err := c.Execute()

	if err == nil {
		t.Errorf("invalid flag should generate error")
	}

	if !strings.Contains(buf.String(), "intone=123") {
		t.Errorf("Wrong error message displayed, \n %s", buf.String())
	}
}

func TestTrailingCommandFlags(t *testing.T) {
	buf := new(bytes.Buffer)
	c := initialize()
	c.SetOutput(buf)
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo two -x", " "))
	e3 := c.Execute()

	if e3 == nil {
		t.Errorf("invalid flag should generate error")
	}
}

func TestPersistentFlags(t *testing.T) {
	c := initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo -s something more here", " "))
	c.Execute()

	// persistentFlag should act like normal flag on it's own command
	if strings.Join(te, " ") != "more here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", te)
	}

	// persistentFlag should act like normal flag on it's own command
	if flags1 != "something" {
		t.Errorf("string flag didn't get correct value, had %v", flags1)
	}

	c = initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("echo times -s again -c test here", " "))
	c.Execute()

	if strings.Join(tt, " ") != "test here" {
		t.Errorf("flags didn't leave proper args remaining..%s given", tt)
	}

	if flags1 != "again" {
		t.Errorf("string flag didn't get correct value, had %v", flags1)
	}

	if flagb2 != true {
		t.Errorf("local flag not parsed correctly. Expected false, had %v", flagb2)
	}
}

func TestHelpCommand(t *testing.T) {
	buf := new(bytes.Buffer)
	c := initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("help echo", " "))
	c.SetOutput(buf)
	c.Execute()

	if !strings.Contains(buf.String(), cmdEcho.Long) {
		t.Errorf("Wrong error message displayed, \n %s", buf.String())
	}

	buf.Reset()
	c = initialize()
	cmdEcho.AddCommand(cmdTimes)
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("help echo times", " "))
	c.SetOutput(buf)
	c.Execute()

	if !strings.Contains(buf.String(), cmdTimes.Long) {
		t.Errorf("Wrong error message displayed, \n %s", buf.String())
	}
}

func TestRunnableRootCommand(t *testing.T) {
	c := initializeWithRootCmd()
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs([]string(nil))
	c.Execute()

	if rootcalled != true {
		t.Errorf("Root Function was not called")
	}
}

func TestRootFlags(t *testing.T) {
	c := initializeWithRootCmd()
	c.AddCommand(cmdPrint, cmdEcho)
	c.SetArgs(strings.Split("-i 17 -b", " "))
	c.Execute()

	if flagbr != true {
		t.Errorf("flag value should be true, %v given", flagbr)
	}

	if flagir != 17 {
		t.Errorf("flag value should be 17, %d given", flagir)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/sbinet/go-commander"
	"github.com/sbinet/go-flag"
)

func hwaf_make_cmd_version() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_version,
		UsageLine: "version",
		Short:     "print version and exit",
		Long: `
print version and exit.

ex:
 $ hwaf version
 hwaf-20121212
`,
		Flag: *flag.NewFlagSet("hwaf-version", flag.ExitOnError),
	}
	return cmd
}

func hwaf_run_cmd_version(cmd *commander.Command, args []string) {
	fmt.Printf("hwaf-20121218\n")
}

// EOF
<commit_msg>update version<commit_after>package main

import (
	"fmt"

	"github.com/sbinet/go-commander"
	"github.com/sbinet/go-flag"
)

func hwaf_make_cmd_version() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_version,
		UsageLine: "version",
		Short:     "print version and exit",
		Long: `
print version and exit.

ex:
 $ hwaf version
 hwaf-20121212
`,
		Flag: *flag.NewFlagSet("hwaf-version", flag.ExitOnError),
	}
	return cmd
}

func hwaf_run_cmd_version(cmd *commander.Command, args []string) {
	fmt.Printf("hwaf-20121219\n")
}

// EOF
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"net"

	"code.google.com/p/goprotobuf/proto"

	"github.com/supershabam/griak/pb/riak"
)

const RiakAddr = "104.131.63.89:8087"

func main() {
	log.Print("HI")
	req := &riak.RpbGetBucketReq{
		Bucket: []byte("stats.droplet.1234.cpu"),
		Type:   []byte("metricgroup"),
	}
	conn, err := net.Dial("tcp", RiakAddr)
	if err != nil {
		log.Fatal(err)
	}
	data, err := proto.Marshal(req)
	if err != nil {
		log.Fatal(err)
	}

	// Build message with header: <length:32> <msg_code:8> <pbmsg>
	// i := int32(len(pbmsg) + 1)
	// msgbuf := []byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i), code}
	// msgbuf = append(msgbuf, pbmsg...)

	var code byte
	code = 19
	i := int32(len(data) + 1)
	msgbuf := []byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i), code}
	msgbuf = append(msgbuf, data...)

	n, err := conn.Write(msgbuf)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("wrote %d bytes", n)
	code, data, err = ReadRiak(conn)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("code: %d", code)
	log.Printf("data: %s", data)
}

func ReadRiak(r io.Reader) (code byte, data []byte, err error) {
	lenbuf := make([]byte, 4)
	codebuf := make([]byte, 1)
	_, err = io.ReadFull(r, lenbuf)
	if err != nil {
		return
	}
	_, err = io.ReadFull(r, codebuf)
	if err != nil {
		return
	}
	code = codebuf[0]
	length := int(lenbuf[0])<<24 +
		int(lenbuf[1])<<16 +
		int(lenbuf[2])<<8 +
		int(lenbuf[3]) - 1
	log.Printf("reading %d", length)
	data = make([]byte, length)
	_, err = io.ReadFull(r, data)
	return
}
<commit_msg>adds WriteRiak function<commit_after>package main

import (
	"io"
	"log"
	"net"

	"code.google.com/p/goprotobuf/proto"

	"github.com/supershabam/griak/pb/riak"
)

const RiakAddr = "104.131.63.89:8087"

func main() {
	log.Print("HI")
	req := &riak.RpbGetBucketReq{
		Bucket: []byte("stats.droplet.1234.cpu"),
		Type:   []byte("metricgroup"),
	}
	conn, err := net.Dial("tcp", RiakAddr)
	if err != nil {
		log.Fatal(err)
	}
	data, err := proto.Marshal(req)
	if err != nil {
		log.Fatal(err)
	}

	err = WriteRiak(conn, 19, data)
	if err != nil {
		log.Fatal(err)
	}
	code, data, err := ReadRiak(conn)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("code: %d", code)
	log.Printf("data: %s", data)
}

func WriteRiak(w io.Writer, code byte, data []byte) error {
	// Build message with header: <length:32> <msg_code:8> <pbmsg>
	var length int32 = int32(len(data) + 1)
	lenbuf := []byte{
		byte(length >> 24),
		byte(length >> 16),
		byte(length >> 8),
		byte(length),
	}
	buf := []byte{}
	buf = append(buf, lenbuf...)
	buf = append(buf, code)
	buf = append(buf, data...)
	_, err := w.Write(buf)
	return err
}

func ReadRiak(r io.Reader) (code byte, data []byte, err error) {
	lenbuf := make([]byte, 4)
	codebuf := make([]byte, 1)
	_, err = io.ReadFull(r, lenbuf)
	if err != nil {
		return
	}
	_, err = io.ReadFull(r, codebuf)
	if err != nil {
		return
	}
	code = codebuf[0]
	length := int(lenbuf[0])<<24 +
		int(lenbuf[1])<<16 +
		int(lenbuf[2])<<8 +
		int(lenbuf[3]) - 1
	log.Printf("reading %d", length)
	data = make([]byte, length)
	_, err = io.ReadFull(r, data)
	return
}
<|endoftext|>