<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"path/filepath"
	"time"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/runutil"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/util"
	"v.io/x/devtools/internal/xunit"
)

const (
	defaultProjectTestTimeout = 5 * time.Minute
)

// runProjectTest is a helper for running project tests.
func runProjectTest(ctx *tool.Context, testName, projectName, target string, env map[string]string, profiles []string) (_ *test.Result, e error) {
	// Initialize the test.
	cleanup, err := initTest(ctx, testName, profiles)
	if err != nil {
		return nil, err
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Navigate to project directory.
	root, err := util.V23Root()
	if err != nil {
		return nil, err
	}
	testDir := filepath.Join(root, "release", "projects", projectName)
	if err := ctx.Run().Chdir(testDir); err != nil {
		return nil, err
	}

	// Clean.
	if err := ctx.Run().Command("make", "clean"); err != nil {
		return nil, err
	}

	// Set environment from the env argument map.
	opts := ctx.Run().Opts()
	for k, v := range env {
		opts.Env[k] = v
	}

	// Run the tests.
	if err := ctx.Run().TimedCommandWithOpts(defaultProjectTestTimeout, opts, "make", target); err != nil {
		if err == runutil.CommandTimedOutErr {
			return &test.Result{
				Status:       test.TimedOut,
				TimeoutValue: defaultProjectTestTimeout,
			}, nil
		} else {
			return nil, internalTestError{err, "Make " + target}
		}
	}

	return &test.Result{Status: test.Passed}, nil
}

// vanadiumBrowserTest runs the tests for the Vanadium browser.
func vanadiumBrowserTest(ctx *tool.Context, testName string, _ ...Opt) (*test.Result, error) {
	env := map[string]string{
		"XUNIT_OUTPUT_FILE": xunit.ReportPath(testName),
	}
	return runProjectTest(ctx, testName, "browser", "test", env, []string{"web"})
}

// vanadiumChatShellTest runs the tests for the chat shell client.
func vanadiumChatShellTest(ctx *tool.Context, testName string, _ ...Opt) (*test.Result, error) {
	return runProjectTest(ctx, testName, "chat", "test-shell", nil, nil)
}

// vanadiumChatWebTest runs the tests for the chat web client.
func vanadiumChatWebTest(ctx *tool.Context, testName string, _ ...Opt) (*test.Result, error) {
	return runProjectTest(ctx, testName, "chat", "test-web", nil, []string{"web"})
}

// vanadiumPipe2BrowserTest runs the tests for pipe2browser.
func vanadiumPipe2BrowserTest(ctx *tool.Context, testName string, _ ...Opt) (*test.Result, error) {
	return runProjectTest(ctx, testName, "pipe2browser", "test", nil, []string{"web"})
}
<commit_msg>TBR v23/internal/test: Increasing timeout for projects since shutdown() has become slower. We can revert if/when https://github.com/vanadium/issues/issues/374 is resolved.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"path/filepath"
	"time"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/runutil"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/util"
	"v.io/x/devtools/internal/xunit"
)

const (
	defaultProjectTestTimeout = 10 * time.Minute
)

// runProjectTest is a helper for running project tests.
func runProjectTest(ctx *tool.Context, testName, projectName, target string, env map[string]string, profiles []string) (_ *test.Result, e error) {
	// Initialize the test.
	cleanup, err := initTest(ctx, testName, profiles)
	if err != nil {
		return nil, err
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Navigate to project directory.
	root, err := util.V23Root()
	if err != nil {
		return nil, err
	}
	testDir := filepath.Join(root, "release", "projects", projectName)
	if err := ctx.Run().Chdir(testDir); err != nil {
		return nil, err
	}

	// Clean.
	if err := ctx.Run().Command("make", "clean"); err != nil {
		return nil, err
	}

	// Set environment from the env argument map.
	opts := ctx.Run().Opts()
	for k, v := range env {
		opts.Env[k] = v
	}

	// Run the tests.
	if err := ctx.Run().TimedCommandWithOpts(defaultProjectTestTimeout, opts, "make", target); err != nil {
		if err == runutil.CommandTimedOutErr {
			return &test.Result{
				Status:       test.TimedOut,
				TimeoutValue: defaultProjectTestTimeout,
			}, nil
		} else {
			return nil, internalTestError{err, "Make " + target}
		}
	}

	return &test.Result{Status: test.Passed}, nil
}

// vanadiumBrowserTest runs the tests for the Vanadium browser.
func vanadiumBrowserTest(ctx *tool.Context, testName string, _ ...Opt) (*test.Result, error) {
	env := map[string]string{
		"XUNIT_OUTPUT_FILE": xunit.ReportPath(testName),
	}
	return runProjectTest(ctx, testName, "browser", "test", env, []string{"web"})
}

// vanadiumChatShellTest runs the tests for the chat shell client.
func vanadiumChatShellTest(ctx *tool.Context, testName string, _ ...Opt) (*test.Result, error) {
	return runProjectTest(ctx, testName, "chat", "test-shell", nil, nil)
}

// vanadiumChatWebTest runs the tests for the chat web client.
func vanadiumChatWebTest(ctx *tool.Context, testName string, _ ...Opt) (*test.Result, error) {
	return runProjectTest(ctx, testName, "chat", "test-web", nil, []string{"web"})
}

// vanadiumPipe2BrowserTest runs the tests for pipe2browser.
func vanadiumPipe2BrowserTest(ctx *tool.Context, testName string, _ ...Opt) (*test.Result, error) {
	return runProjectTest(ctx, testName, "pipe2browser", "test", nil, []string{"web"})
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
)

// Export returns a lair project by id
func Export(w http.ResponseWriter, req *http.Request) {
}
<commit_msg>Remove unused file<commit_after><|endoftext|>


<commit_before>package errors

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
)

type _errorh struct {
	ErrStatus  string `json:"status"`
	ErrMessage string `json:"message"`
	ErrCode    int    `json:"code"`
	*stack     `json:"-"`
}

func (e _errorh) Error() string { return e.ErrMessage }

func (e _errorh) Status() string { return e.ErrStatus }

func (e _errorh) Code() int { return e.ErrCode }

func (e _errorh) Message() string { return e.ErrMessage }

func (e _errorh) JSONError() error {
	b, _ := json.Marshal(e)
	return errors.New(string(b))
}

func (e _errorh) Format(s fmt.State, verb rune) {
	switch verb {
	case 'v':
		if s.Flag('+') {
			io.WriteString(s, e.ErrMessage)
			fmt.Fprintf(s, "%+v", e.StackTrace())
			return
		}
		fallthrough
	case 's':
		io.WriteString(s, e.ErrMessage)
	}
}

// New returns an error with the supplied message.
func Newh(code int, message string) error {
	return _errorh{
		http.StatusText(code),
		message,
		code,
		callers(),
	}
}

// Errorf formats according to a format specifier and returns the string
// as a value that satisfies error.
func Errorhf(code int, format string, args ...interface{}) error {
	return _errorh{
		http.StatusText(code),
		fmt.Sprintf(format, args...),
		code,
		callers(),
	}
}

func ParseJSON(err string) _errorh {
	e := _errorh{}
	errr := json.Unmarshal([]byte(err), &e)
	if errr != nil {
		e.ErrStatus = http.StatusText(500)
		e.ErrMessage = errr.Error()
		e.ErrCode = 500
		e.stack = callers()
	}

	e.stack = callers()
	return e
}

func BadRequest(message string) error {
	return &_errorh{
		http.StatusText(400),
		message,
		400,
		callers(),
	}
}

func Unauthorized(message string) error {
	return &_errorh{
		http.StatusText(401),
		message,
		401,
		callers(),
	}
}

func Forbidden(message string) error {
	return &_errorh{
		http.StatusText(403),
		message,
		403,
		callers(),
	}
}

func NotFound(message string) error {
	return &_errorh{
		http.StatusText(404),
		message,
		404,
		callers(),
	}
}

func InternalServerError(message string) error {
	return &_errorh{
		http.StatusText(500),
		message,
		500,
		callers(),
	}
}
<commit_msg>update<commit_after>package errors

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
)

type _errorh struct {
	ErrMessage string `json:"message"`
	ErrCode    int    `json:"code"`
	*stack     `json:"-"`
}

func (e _errorh) Error() string { return e.ErrMessage }

func (e _errorh) Code() int { return e.ErrCode }

func (e _errorh) Message() string { return e.ErrMessage }

func (e _errorh) JSONError() error {
	b, _ := json.Marshal(e)
	return errors.New(string(b))
}

func (e _errorh) Format(s fmt.State, verb rune) {
	switch verb {
	case 'v':
		if s.Flag('+') {
			io.WriteString(s, e.ErrMessage)
			fmt.Fprintf(s, "%+v", e.StackTrace())
			return
		}
		fallthrough
	case 's':
		io.WriteString(s, e.ErrMessage)
	}
}

// New returns an error with the supplied message.
func Newh(code int, message string) error {
	return _errorh{
		message,
		code,
		callers(),
	}
}

// Errorf formats according to a format specifier and returns the string
// as a value that satisfies error.
func Errorhf(code int, format string, args ...interface{}) error {
	return _errorh{
		fmt.Sprintf(format, args...),
		code,
		callers(),
	}
}

func ParseJSON(err string) _errorh {
	e := _errorh{}
	errr := json.Unmarshal([]byte(err), &e)
	if errr != nil {
		e.ErrMessage = errr.Error()
		e.ErrCode = 500
		e.stack = callers()
	}

	e.stack = callers()
	return e
}

func BadRequest(message string) error {
	return &_errorh{
		message,
		400,
		callers(),
	}
}

func Unauthorized(message string) error {
	return &_errorh{
		message,
		401,
		callers(),
	}
}

func Forbidden(message string) error {
	return &_errorh{
		message,
		403,
		callers(),
	}
}

func NotFound(message string) error {
	return &_errorh{
		message,
		404,
		callers(),
	}
}

func InternalServerError(message string) error {
	return &_errorh{
		message,
		500,
		callers(),
	}
}
<|endoftext|>


<commit_before>package attackplugin

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/cloudfoundry/cli/plugin"
	"github.com/xchapter7x/cf-app-attack/vegetaclihelper"
	"github.com/xchapter7x/lo"
)

var VegetaRunner = vegetaclihelper.VegetaCliExecute

func (c *AppAttack) Run(cliConnection plugin.CliConnection, args []string) {
	switch args[0] {
	case CmdBench:
		appname := args[1]
		vegetaArgs := args[2:]

		if appModel, err := cliConnection.GetApp(appname); err == nil {
			appHost := fmt.Sprintf("%s.%s", appModel.Routes[0].Host, appModel.Routes[0].Domain.Name)
			VegetaRunner(vegetaArgs, appHost)
		} else {
			lo.G.Error("error on app query: ", err.Error())
			panic(err)
		}

	default:
		fmt.Println("Invalid command:", args[0])
	}
}

func (c *AppAttack) GetVersionType() plugin.VersionType {
	versionArray := strings.Split(strings.TrimPrefix(c.Version, "v"), ".")
	major, _ := strconv.Atoi(versionArray[0])
	minor, _ := strconv.Atoi(versionArray[1])
	build, _ := strconv.Atoi(versionArray[2])
	return plugin.VersionType{
		Major: major,
		Minor: minor,
		Build: build,
	}
}

func (c *AppAttack) GetMetadata() plugin.PluginMetadata {
	return plugin.PluginMetadata{
		Name:    PluginName,
		Version: c.GetVersionType(),
		Commands: []plugin.Command{
			plugin.Command{
				Name:     CmdBench,
				HelpText: "Run a performance/load test function for an app in cf",
				UsageDetails: plugin.Usage{
					Usage: fmt.Sprintf("cf %s mycoolappname [vegeta globals] [attack | report | dump] [vegeta args]", CmdBench),
				},
			},
		},
	}
}
<commit_msg>adding debug stuff<commit_after>package attackplugin

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/cloudfoundry/cli/plugin"
	"github.com/xchapter7x/cf-app-attack/vegetaclihelper"
	"github.com/xchapter7x/lo"
)

var VegetaRunner = vegetaclihelper.VegetaCliExecute

func (c *AppAttack) Run(cliConnection plugin.CliConnection, args []string) {
	switch args[0] {
	case CmdBench:
		appname := args[1]
		vegetaArgs := args[2:]

		if appModel, err := cliConnection.GetApp(appname); err == nil {
			lo.G.Debug("appmodel: ", appModel)
			appHost := fmt.Sprintf("%s.%s", appModel.Routes[0].Host, appModel.Routes[0].Domain.Name)
			VegetaRunner(vegetaArgs, appHost)
		} else {
			lo.G.Error("error on app query: ", err.Error())
			panic(err)
		}

	default:
		fmt.Println("Invalid command:", args[0])
	}
}

func (c *AppAttack) GetVersionType() plugin.VersionType {
	versionArray := strings.Split(strings.TrimPrefix(c.Version, "v"), ".")
	major, _ := strconv.Atoi(versionArray[0])
	minor, _ := strconv.Atoi(versionArray[1])
	build, _ := strconv.Atoi(versionArray[2])
	return plugin.VersionType{
		Major: major,
		Minor: minor,
		Build: build,
	}
}

func (c *AppAttack) GetMetadata() plugin.PluginMetadata {
	return plugin.PluginMetadata{
		Name:    PluginName,
		Version: c.GetVersionType(),
		Commands: []plugin.Command{
			plugin.Command{
				Name:     CmdBench,
				HelpText: "Run a performance/load test function for an app in cf",
				UsageDetails: plugin.Usage{
					Usage: fmt.Sprintf("cf %s mycoolappname [vegeta globals] [attack | report | dump] [vegeta args]", CmdBench),
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"runtime/pprof"

	"github.com/keybase/client/go/client"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/logger"
	keybase1 "github.com/keybase/client/protocol/go"
	"golang.org/x/net/context"
)

func getMDServerAddr() string {
	// XXX TODO: the source of this will likely change soon
	return os.Getenv(EnvMDServerAddr)
}

func useLocalMDServer() bool {
	return len(getMDServerAddr()) == 0
}

func useLocalKeyServer() bool {
	// currently the remote MD server also acts as the key server.
	return useLocalMDServer()
}

func makeMDServer(config Config, serverRootDir *string) (
	MDServer, error) {
	if serverRootDir == nil {
		// local in-memory MD server
		return NewMDServerMemory(config)
	}

	if useLocalMDServer() {
		// local persistent MD server
		handlePath := filepath.Join(*serverRootDir, "kbfs_handles")
		mdPath := filepath.Join(*serverRootDir, "kbfs_md")
		revPath := filepath.Join(*serverRootDir, "kbfs_revisions")
		return NewMDServerLocal(
			config, handlePath, mdPath, revPath)
	}

	// remote MD server. this can't fail. reconnection attempts
	// will be automatic.
	mdServer := NewMDServerRemote(context.TODO(), config, getMDServerAddr())
	return mdServer, nil
}

func makeKeyServer(config Config, serverRootDir *string) (
	KeyServer, error) {
	if serverRootDir == nil {
		// local in-memory key server
		return NewKeyServerMemory(config)
	}

	if useLocalKeyServer() {
		// local persistent key server
		keyPath := filepath.Join(*serverRootDir, "kbfs_key")
		return NewKeyServerLocal(config, keyPath)
	}

	// currently the remote MD server also acts as the key server.
	keyServer := config.MDServer().(*MDServerRemote)
	return keyServer, nil
}

func makeBlockServer(config Config, serverRootDir *string) (BlockServer, error) {
	bServerAddr := os.Getenv(EnvBServerAddr)
	if len(bServerAddr) == 0 {
		if serverRootDir == nil {
			return NewBlockServerMemory(config)
		}

		blockPath := filepath.Join(*serverRootDir, "kbfs_block")
		return NewBlockServerLocal(config, blockPath)
	}

	fmt.Printf("Using remote bserver %s\n", bServerAddr)
	return NewBlockServerRemote(context.TODO(), config, bServerAddr), nil
}

func makeKBPKIClient(config Config, serverRootDir *string, localUser string) (KBPKI, error) {
	if localUser == "" {
		libkb.G.ConfigureSocketInfo()
		return NewKBPKIClient(libkb.G)
	}

	users := []string{"strib", "max", "chris", "fred"}
	userIndex := -1
	for i := range users {
		if localUser == users[i] {
			userIndex = i
			break
		}
	}
	if userIndex < 0 {
		return nil, fmt.Errorf("user %s not in list %v", localUser, users)
	}

	localUsers := MakeLocalUsers(users)

	// TODO: Auto-generate these, too?
	localUsers[0].Asserts = []string{"github:strib"}
	localUsers[1].Asserts = []string{"twitter:maxtaco"}
	localUsers[2].Asserts = []string{"twitter:malgorithms"}
	localUsers[3].Asserts = []string{"twitter:fakalin"}

	var localUID keybase1.UID
	if userIndex >= 0 {
		localUID = localUsers[userIndex].UID
	}

	if serverRootDir == nil {
		return NewKBPKIMemory(localUID, localUsers), nil
	}

	favPath := filepath.Join(*serverRootDir, "kbfs_favs")
	return NewKBPKILocal(localUID, localUsers, favPath, config.Codec())
}

// Init initializes a config and returns it. If localUser is
// non-empty, libkbfs does not communicate to any remote servers and
// instead uses fake implementations of various servers.
//
// If serverRootDir is nil, an in-memory server is used. If it is
// non-nil and points to the empty string, the current working
// directory is used. Otherwise, the pointed-to string is treated as a
// path.
//
// onInterruptFn is called whenever an interrupt signal is received
// (e.g., if the user hits Ctrl-C).
//
// Init should be called at the beginning of main. Shutdown (see
// below) should then be called at the end of main (usually via
// defer).
func Init(localUser string, serverRootDir *string, cpuProfilePath,
	memProfilePath string, onInterruptFn func(), debug bool) (Config, error) {
	if cpuProfilePath != "" {
		// Let the GC/OS clean up the file handle.
		f, err := os.Create(cpuProfilePath)
		if err != nil {
			return nil, err
		}
		pprof.StartCPUProfile(f)
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, os.Interrupt)
	go func() {
		_ = <-sigchan

		Shutdown(memProfilePath)

		if onInterruptFn != nil {
			onInterruptFn()
		}

		os.Exit(1)
	}()

	config := NewConfigLocal()

	// Set logging
	config.SetLoggerMaker(func(module string) logger.Logger {
		lg := logger.New(fmt.Sprintf("kbfs(%s)", module))
		if debug {
			// Turn on debugging.  TODO: allow a proper log file and
			// style to be specified.
			lg.Configure("", true, "")
		}
		return lg
	})

	libkb.G.Init()
	libkb.G.ConfigureConfig()
	libkb.G.ConfigureLogging()
	libkb.G.ConfigureCaches()
	libkb.G.ConfigureMerkleClient()

	mdServer, err := makeMDServer(config, serverRootDir)
	if err != nil {
		return nil, fmt.Errorf("problem creating MD server: %v", err)
	}
	config.SetMDServer(mdServer)

	keyServer, err := makeKeyServer(config, serverRootDir)
	if err != nil {
		return nil, fmt.Errorf("problem creating key server: %v", err)
	}
	config.SetKeyServer(keyServer)

	client.InitUI()
	libkb.G.UI.Configure()

	k, err := makeKBPKIClient(config, serverRootDir, localUser)
	if err != nil {
		return nil, fmt.Errorf("problem creating KBPKI client: %s", err)
	}
	config.SetKBPKI(k)

	if localUser == "" {
		c, err := NewCryptoClient(config.Codec(), libkb.G)
		if err != nil {
			return nil, fmt.Errorf("Could not get Crypto: %v", err)
		}
		config.SetCrypto(c)
	} else {
		signingKey := MakeLocalUserSigningKeyOrBust(localUser)
		cryptPrivateKey := MakeLocalUserCryptPrivateKeyOrBust(localUser)
		config.SetCrypto(NewCryptoLocal(config.Codec(), signingKey,
			cryptPrivateKey))
	}

	bserv, err := makeBlockServer(config, serverRootDir)
	if err != nil {
		return nil, fmt.Errorf("cannot open block database: %v", err)
	}
	config.SetBlockServer(bserv)

	return config, nil
}

// Shutdown does any necessary shutdown tasks for libkbfs. Shutdown
// should be called at the end of main.
func Shutdown(memProfilePath string) error {
	pprof.StopCPUProfile()

	if memProfilePath != "" {
		// Let the GC/OS clean up the file handle.
		f, err := os.Create(memProfilePath)
		if err != nil {
			return err
		}

		pprof.WriteHeapProfile(f)
	}

	return nil
}
<commit_msg>init: allow for bare "kbfs" logger module<commit_after>package libkbfs

import (
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"runtime/pprof"

	"github.com/keybase/client/go/client"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/logger"
	keybase1 "github.com/keybase/client/protocol/go"
	"golang.org/x/net/context"
)

func getMDServerAddr() string {
	// XXX TODO: the source of this will likely change soon
	return os.Getenv(EnvMDServerAddr)
}

func useLocalMDServer() bool {
	return len(getMDServerAddr()) == 0
}

func useLocalKeyServer() bool {
	// currently the remote MD server also acts as the key server.
	return useLocalMDServer()
}

func makeMDServer(config Config, serverRootDir *string) (
	MDServer, error) {
	if serverRootDir == nil {
		// local in-memory MD server
		return NewMDServerMemory(config)
	}

	if useLocalMDServer() {
		// local persistent MD server
		handlePath := filepath.Join(*serverRootDir, "kbfs_handles")
		mdPath := filepath.Join(*serverRootDir, "kbfs_md")
		revPath := filepath.Join(*serverRootDir, "kbfs_revisions")
		return NewMDServerLocal(
			config, handlePath, mdPath, revPath)
	}

	// remote MD server. this can't fail. reconnection attempts
	// will be automatic.
	mdServer := NewMDServerRemote(context.TODO(), config, getMDServerAddr())
	return mdServer, nil
}

func makeKeyServer(config Config, serverRootDir *string) (
	KeyServer, error) {
	if serverRootDir == nil {
		// local in-memory key server
		return NewKeyServerMemory(config)
	}

	if useLocalKeyServer() {
		// local persistent key server
		keyPath := filepath.Join(*serverRootDir, "kbfs_key")
		return NewKeyServerLocal(config, keyPath)
	}

	// currently the remote MD server also acts as the key server.
	keyServer := config.MDServer().(*MDServerRemote)
	return keyServer, nil
}

func makeBlockServer(config Config, serverRootDir *string) (BlockServer, error) {
	bServerAddr := os.Getenv(EnvBServerAddr)
	if len(bServerAddr) == 0 {
		if serverRootDir == nil {
			return NewBlockServerMemory(config)
		}

		blockPath := filepath.Join(*serverRootDir, "kbfs_block")
		return NewBlockServerLocal(config, blockPath)
	}

	fmt.Printf("Using remote bserver %s\n", bServerAddr)
	return NewBlockServerRemote(context.TODO(), config, bServerAddr), nil
}

func makeKBPKIClient(config Config, serverRootDir *string, localUser string) (KBPKI, error) {
	if localUser == "" {
		libkb.G.ConfigureSocketInfo()
		return NewKBPKIClient(libkb.G)
	}

	users := []string{"strib", "max", "chris", "fred"}
	userIndex := -1
	for i := range users {
		if localUser == users[i] {
			userIndex = i
			break
		}
	}
	if userIndex < 0 {
		return nil, fmt.Errorf("user %s not in list %v", localUser, users)
	}

	localUsers := MakeLocalUsers(users)

	// TODO: Auto-generate these, too?
	localUsers[0].Asserts = []string{"github:strib"}
	localUsers[1].Asserts = []string{"twitter:maxtaco"}
	localUsers[2].Asserts = []string{"twitter:malgorithms"}
	localUsers[3].Asserts = []string{"twitter:fakalin"}

	var localUID keybase1.UID
	if userIndex >= 0 {
		localUID = localUsers[userIndex].UID
	}

	if serverRootDir == nil {
		return NewKBPKIMemory(localUID, localUsers), nil
	}

	favPath := filepath.Join(*serverRootDir, "kbfs_favs")
	return NewKBPKILocal(localUID, localUsers, favPath, config.Codec())
}

// Init initializes a config and returns it. If localUser is
// non-empty, libkbfs does not communicate to any remote servers and
// instead uses fake implementations of various servers.
//
// If serverRootDir is nil, an in-memory server is used. If it is
// non-nil and points to the empty string, the current working
// directory is used. Otherwise, the pointed-to string is treated as a
// path.
//
// onInterruptFn is called whenever an interrupt signal is received
// (e.g., if the user hits Ctrl-C).
//
// Init should be called at the beginning of main. Shutdown (see
// below) should then be called at the end of main (usually via
// defer).
func Init(localUser string, serverRootDir *string, cpuProfilePath,
	memProfilePath string, onInterruptFn func(), debug bool) (Config, error) {
	if cpuProfilePath != "" {
		// Let the GC/OS clean up the file handle.
		f, err := os.Create(cpuProfilePath)
		if err != nil {
			return nil, err
		}
		pprof.StartCPUProfile(f)
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, os.Interrupt)
	go func() {
		_ = <-sigchan

		Shutdown(memProfilePath)

		if onInterruptFn != nil {
			onInterruptFn()
		}

		os.Exit(1)
	}()

	config := NewConfigLocal()

	// Set logging
	config.SetLoggerMaker(func(module string) logger.Logger {
		mname := "kbfs"
		if module != "" {
			mname += fmt.Sprintf("(%s)", module)
		}
		lg := logger.New(mname)
		if debug {
			// Turn on debugging.  TODO: allow a proper log file and
			// style to be specified.
			lg.Configure("", true, "")
		}
		return lg
	})

	libkb.G.Init()
	libkb.G.ConfigureConfig()
	libkb.G.ConfigureLogging()
	libkb.G.ConfigureCaches()
	libkb.G.ConfigureMerkleClient()

	mdServer, err := makeMDServer(config, serverRootDir)
	if err != nil {
		return nil, fmt.Errorf("problem creating MD server: %v", err)
	}
	config.SetMDServer(mdServer)

	keyServer, err := makeKeyServer(config, serverRootDir)
	if err != nil {
		return nil, fmt.Errorf("problem creating key server: %v", err)
	}
	config.SetKeyServer(keyServer)

	client.InitUI()
	libkb.G.UI.Configure()

	k, err := makeKBPKIClient(config, serverRootDir, localUser)
	if err != nil {
		return nil, fmt.Errorf("problem creating KBPKI client: %s", err)
	}
	config.SetKBPKI(k)

	if localUser == "" {
		c, err := NewCryptoClient(config.Codec(), libkb.G)
		if err != nil {
			return nil, fmt.Errorf("Could not get Crypto: %v", err)
		}
		config.SetCrypto(c)
	} else {
		signingKey := MakeLocalUserSigningKeyOrBust(localUser)
		cryptPrivateKey := MakeLocalUserCryptPrivateKeyOrBust(localUser)
		config.SetCrypto(NewCryptoLocal(config.Codec(), signingKey,
			cryptPrivateKey))
	}

	bserv, err := makeBlockServer(config, serverRootDir)
	if err != nil {
		return nil, fmt.Errorf("cannot open block database: %v", err)
	}
	config.SetBlockServer(bserv)

	return config, nil
}

// Shutdown does any necessary shutdown tasks for libkbfs. Shutdown
// should be called at the end of main.
func Shutdown(memProfilePath string) error {
	pprof.StopCPUProfile()

	if memProfilePath != "" {
		// Let the GC/OS clean up the file handle.
		f, err := os.Create(memProfilePath)
		if err != nil {
			return err
		}

		pprof.WriteHeapProfile(f)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import "github.com/kidoman/embd" // ew this shouldn't be here TODO
import "encoding/json"
import "io/ioutil"

type State struct {
	timelines    TimelineRunners
	timelineFile string
	dinos        DinoStates
	rawDinos     Dinos
}

func (s *State) LoadConfig(filename string) error {
	// we should have a specific Config type, really
	str, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	var dinos Dinos
	err = json.Unmarshal([]byte(str), &dinos)
	if err != nil {
		return err
	}

	s.rawDinos = dinos

	for _, d := range dinos {
		ds := DinoState{d, nil}
		s.dinos = append(s.dinos, ds)
	}

	return nil
}

func (s *State) RestoreTimelines(filename string) error {
	s.timelineFile = filename

	str, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	var tls []Timeline
	err = json.Unmarshal([]byte(str), &tls)
	if err != nil {
		return err
	}

	state.timelines.ReplaceAllTimelines(tls)
	return nil
}

func (s *State) SaveTimelines() error {
	tls := s.timelines.GetAllTimelines()
	json, _ := json.Marshal(tls)
	return ioutil.WriteFile(s.timelineFile, json, 0664)
}

func (s *State) Init() {
	for i := range s.dinos {
		gpio := s.gpioBackendFor(s.dinos[i].Dino)
		SetupGPIOForDino(s.dinos[i].Dino, gpio)
		s.dinos[i].gpio = gpio
	}
}

func (s *State) gpioBackendFor(d Dino) GPIO {
	var gpio GPIO

	if d.Backend == "mqtt" {
		gpio = GPIO(&MQTT_GPIO{})
	} else {
		host, _, _ := embd.DetectHost()
		if host == embd.HostNull {
			println("No real GPIO, using fake_gpio")
			gpio = GPIO(FakeGPIO{})
		} else {
			println("Using embd GPIO")
			gpio = GPIO(&LocalGPIO{})
		}
	}

	gpio = &MQTT_GPIO{}
	gpio.Init()

	return gpio
}

var state State

func getCurrentDinoList() Dinos {
	return state.rawDinos
}

func getCurrentTimelines() []Timeline {
	return state.timelines.GetAllTimelines()
}

func replaceAllTimelines(ts []Timeline) []Timeline {
	state.timelines.ReplaceAllTimelines(ts)
	return state.timelines.GetAllTimelines()
}

func addOrReplaceTimeline(ts Timeline) []Timeline {
	state.timelines.AddOrReplaceTimeline(ts)
	state.SaveTimelines()
	return state.timelines.GetAllTimelines()
}

func deleteAllTimelines() []Timeline {
	state.timelines.DeleteAllTimelines()
	return state.timelines.GetAllTimelines()
}

func trigger(dino, sensor string) {
	state.timelines.Trigger(dino, sensor)
}
<commit_msg>Don't overwrite our carefully hand-crafted artisanal new GPIO instance with an mqtt one unconditionally.  Whoops.  Getting tired.<commit_after>package main

import "github.com/kidoman/embd" // ew this shouldn't be here TODO
import "encoding/json"
import "io/ioutil"

type State struct {
	timelines    TimelineRunners
	timelineFile string
	dinos        DinoStates
	rawDinos     Dinos
}

func (s *State) LoadConfig(filename string) error {
	// we should have a specific Config type, really
	str, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	var dinos Dinos
	err = json.Unmarshal([]byte(str), &dinos)
	if err != nil {
		return err
	}

	s.rawDinos = dinos

	for _, d := range dinos {
		ds := DinoState{d, nil}
		s.dinos = append(s.dinos, ds)
	}

	return nil
}

func (s *State) RestoreTimelines(filename string) error {
	s.timelineFile = filename

	str, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	var tls []Timeline
	err = json.Unmarshal([]byte(str), &tls)
	if err != nil {
		return err
	}

	state.timelines.ReplaceAllTimelines(tls)
	return nil
}

func (s *State) SaveTimelines() error {
	tls := s.timelines.GetAllTimelines()
	json, _ := json.Marshal(tls)
	return ioutil.WriteFile(s.timelineFile, json, 0664)
}

func (s *State) Init() {
	for i := range s.dinos {
		gpio := s.gpioBackendFor(s.dinos[i].Dino)
		SetupGPIOForDino(s.dinos[i].Dino, gpio)
		s.dinos[i].gpio = gpio
	}
}

func (s *State) gpioBackendFor(d Dino) GPIO {
	var gpio GPIO

	if d.Backend == "mqtt" {
		gpio = GPIO(&MQTT_GPIO{})
	} else {
		host, _, _ := embd.DetectHost()
		if host == embd.HostNull {
			println("No real GPIO, using fake_gpio")
			gpio = GPIO(FakeGPIO{})
		} else {
			println("Using embd GPIO")
			gpio = GPIO(&LocalGPIO{})
		}
	}

	gpio.Init()

	return gpio
}

var state State

func getCurrentDinoList() Dinos {
	return state.rawDinos
}

func getCurrentTimelines() []Timeline {
	return state.timelines.GetAllTimelines()
}

func replaceAllTimelines(ts []Timeline) []Timeline {
	state.timelines.ReplaceAllTimelines(ts)
	return state.timelines.GetAllTimelines()
}

func addOrReplaceTimeline(ts Timeline) []Timeline {
	state.timelines.AddOrReplaceTimeline(ts)
	state.SaveTimelines()
	return state.timelines.GetAllTimelines()
}

func deleteAllTimelines() []Timeline {
	state.timelines.DeleteAllTimelines()
	return state.timelines.GetAllTimelines()
}

func trigger(dino, sensor string) {
	state.timelines.Trigger(dino, sensor)
}
<|endoftext|>


<commit_before>package node

import (
	"reflect"
	"sync"
)

type NodeStats struct {
	lock          sync.Mutex
	messages      chan string
	messageCounts map[string]int64
}

func NewNodeStats() *NodeStats {
	stats := &NodeStats{
		messages: make(chan string, 1024),
		messageCounts: make(map[string]int64, 0),
	}
	go stats.CountEventsRoutine()
	return stats
}

func (stats *NodeStats) CountEventsRoutine() {
	for msg := range stats.messages {
		stats.lock.Lock()
		stats.messageCounts[msg]++
		stats.lock.Unlock()
	}
}

// LogMessage puts a message on the queue for consumption by CountEventsRoutine.
// We are just using reflection here to log different message types.
func (stats *NodeStats) LogMessage(msg *Message) {
	typeName := reflect.TypeOf(*msg).Elem().Name()
	stats.messages <- typeName
}

// LogMessage puts a message on the queue for consumption by CountEventsRoutine.
// We are counting the occurences of each unique string.
func (stats *NodeStats) LogEvent(msg string) {
	stats.messages <- msg
}

// Register the getUsage method on the node's internal agent.
func (node *node) RegisterGetUsage() {
	options := make(map[string]interface{}, 0)
	endpoint := string(node.agent.pdid + "/getUsage")
	node.agent.Register(endpoint, node.stats.GetUsage, options)
}

// GetUsage is meant to be registered as an RPC function.  Returns a map of
// strings to counters.
func (stats *NodeStats) GetUsage(args []interface{}, kwargs map[string]interface{}, details map[string]interface{}) (result *CallResult) {
	counts := make(map[string]int64, 0)

	stats.lock.Lock()
	for k, v := range stats.messageCounts {
		counts[k] = v
	}
	stats.lock.Unlock()

	// Set up the return value: a one element slice containing the map of
	// strings to counts.
	result = &CallResult {
		Args: []interface{}{counts},
	}
	return
}
<commit_msg>Added a start time for counter interpretation.<commit_after>package node

import (
	"reflect"
	"sync"
	"time"
)

type NodeStats struct {
	startTime     int64
	lock          sync.Mutex
	messages      chan string
	messageCounts map[string]int64
}

func NewNodeStats() *NodeStats {
	stats := &NodeStats{
		startTime: int64(time.Now().Unix()),
		messages: make(chan string, 1024),
		messageCounts: make(map[string]int64, 0),
	}
	go stats.CountEventsRoutine()
	return stats
}

func (stats *NodeStats) CountEventsRoutine() {
	for msg := range stats.messages {
		stats.lock.Lock()
		stats.messageCounts[msg]++
		stats.lock.Unlock()
	}
}

// LogMessage puts a message on the queue for consumption by CountEventsRoutine.
// We are just using reflection here to log different message types.
func (stats *NodeStats) LogMessage(msg *Message) {
	typeName := reflect.TypeOf(*msg).Elem().Name()
	stats.messages <- typeName
}

// LogMessage puts a message on the queue for consumption by CountEventsRoutine.
// We are counting the occurences of each unique string.
func (stats *NodeStats) LogEvent(msg string) {
	stats.messages <- msg
}

// Register the getUsage method on the node's internal agent.
func (node *node) RegisterGetUsage() {
	options := make(map[string]interface{}, 0)
	endpoint := string(node.agent.pdid + "/getUsage")
	node.agent.Register(endpoint, node.stats.GetUsage, options)
}

// GetUsage is meant to be registered as an RPC function.  Returns a map of
// strings to counters.
func (stats *NodeStats) GetUsage(args []interface{}, kwargs map[string]interface{}, details map[string]interface{}) (result *CallResult) {
	counts := make(map[string]int64, 0)

	stats.lock.Lock()
	for k, v := range stats.messageCounts {
		counts[k] = v
	}

	// The start time tells the receiver when our counters were initialized.
	counts["start"] = stats.startTime
	stats.lock.Unlock()

	// Set up the return value: a one element slice containing the map of
	// strings to counts.
	result = &CallResult {
		Args: []interface{}{counts},
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"time"

	"gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
)

type stats struct {
	Total   int
	Last30d int
	Last7d  int
	Last24h int
	Lines   int
	Time    time.Time
}

func (s stats) FormattedTime() string {
	return s.Time.In(time.UTC).Format(time.RFC850)
}

type tokeiLanguageReport struct {
	Blanks   int `json:"blanks"`
	Code     int `json:"code"`
	Comments int `json:"comments"`
	Stats    []struct {
		Blanks   int    `json:"blanks"`
		Code     int    `json:"code"`
		Comments int    `json:"comments"`
		Lines    int    `json:"lines"`
		Name     string `json:"name"`
	} `json:"stats"`
	Lines int `json:"lines"`
}

type TokeiReport struct {
	CSS        tokeiLanguageReport `json:"Css"`
	Dockerfile tokeiLanguageReport `json:"Dockerfile"`
	Go         tokeiLanguageReport `json:"Go"`
	Hex        tokeiLanguageReport `json:"Hex"`
	HTML       tokeiLanguageReport `json:"Html"`
	Makefile   tokeiLanguageReport `json:"Makefile"`
	Markdown   tokeiLanguageReport `json:"Markdown"`
	Sh         tokeiLanguageReport `json:"Sh"`
	Text       tokeiLanguageReport `json:"Text"`
	Toml       tokeiLanguageReport `json:"Toml"`
	Yaml       tokeiLanguageReport `json:"Yaml"`
}

func getTokeiReport(path string) (*TokeiReport, error) {
	cmd := exec.Command("tokei", "-e", "vendor/", path, "-o", "json")
	buf := new(bytes.Buffer)
	cmd.Stdout = buf
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, err
	}
	decoder := json.NewDecoder(buf)
	r := new(TokeiReport)
	if err := decoder.Decode(r); err != nil {
		return nil, err
	}
	return r, nil
}

func getStats(fetch bool) (*stats, error) {
	n := time.Now()
	last30Days := n.AddDate(0, 0, -30)
	last7Days := n.AddDate(0, 0, -7)
	last24h := n.AddDate(0, 0, -1)
	var total, last30DaysTotal, last7DaysTotal, last24hTotal, lines int
	for _, name := range []string{
		"stun", "turn", "sdp", "web", "stund", "tech-status", "ice", "rtc", "gortcd",
		"ansible-role-nginx", "ansible-go", "api", "docs", "dtls", "neo",
	} {
		p := "/tmp/gortc-analyze/" + name
		r, err := git.PlainClone(p, false, &git.CloneOptions{
			URL: "https://github.com/gortc/" + name,
		})
		if err == git.ErrRepositoryAlreadyExists {
			r, err = git.PlainOpen(p)
			if err != nil {
				return nil, err
			}
			w, err := r.Worktree()
			if err != nil {
				return nil, err
			}
			if fetch {
				err = w.Pull(&git.PullOptions{
					Force:      true,
					RemoteName: "origin",
				})
				log.Println("pull", name, err)
				if err == git.NoErrAlreadyUpToDate {
					err = nil
				}
			}
		}
		if err != nil {
			return nil, err
		}
		rep, err := getTokeiReport(p)
		if err != nil {
			return nil, err
		}
		if name == "dtls" {
			// It's not fair to count vendored code.
			// TODO: count delta.
			rep.Go.Lines = 0
		}
		lines += rep.Go.Lines + rep.Yaml.Lines + rep.Dockerfile.Lines
		ref, err := r.Head()
		if err != nil {
			return nil, err
		}
		fmt.Println(name, "head", ref)
		b, err := r.Log(&git.LogOptions{
			From: ref.Hash(),
		})
		var commits int
		countAuthors := map[string]struct{}{
			"ar@cydev.ru":                          {},
			"ernado@ya.ru":                         {},
			"mail@backkem.me":                      {},
			"ar@gortc.io":                          {},
			"songjiayang@users.noreply.github.com": {},
			"a.razumov@corp.mail.ru":               {},
		}
		if err = b.ForEach(func(commit *object.Commit) error {
			if _, ok := countAuthors[commit.Author.Email]; !ok {
				return nil
			}
			commits++
			if commit.Author.When.After(last30Days) {
				last30DaysTotal++
			}
			if commit.Author.When.After(last7Days) {
				last7DaysTotal++
			}
			if commit.Author.When.After(last24h) {
				last24hTotal++
			}
			return nil
		}); err != nil {
			return nil, err
		}
		total += commits
	}
	return &stats{
		Time:    time.Now(),
		Total:   total,
		Last30d: last30DaysTotal,
		Last7d:  last7DaysTotal,
		Last24h: last24hTotal,
		Lines:   lines,
	}, nil
}
<commit_msg>stats: add turnc to list<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"time"

	"gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
)

type stats struct {
	Total   int
	Last30d int
	Last7d  int
	Last24h int
	Lines   int
	Time    time.Time
}

func (s stats) FormattedTime() string {
	return s.Time.In(time.UTC).Format(time.RFC850)
}

type tokeiLanguageReport struct {
	Blanks   int `json:"blanks"`
	Code     int `json:"code"`
	Comments int `json:"comments"`
	Stats    []struct {
		Blanks   int    `json:"blanks"`
		Code     int    `json:"code"`
		Comments int    `json:"comments"`
		Lines    int    `json:"lines"`
		Name     string `json:"name"`
	} `json:"stats"`
	Lines int `json:"lines"`
}

type TokeiReport struct {
	CSS        tokeiLanguageReport `json:"Css"`
	Dockerfile tokeiLanguageReport `json:"Dockerfile"`
	Go         tokeiLanguageReport `json:"Go"`
	Hex        tokeiLanguageReport `json:"Hex"`
	HTML       tokeiLanguageReport `json:"Html"`
	Makefile   tokeiLanguageReport `json:"Makefile"`
	Markdown   tokeiLanguageReport `json:"Markdown"`
	Sh         tokeiLanguageReport `json:"Sh"`
	Text       tokeiLanguageReport `json:"Text"`
	Toml       tokeiLanguageReport `json:"Toml"`
	Yaml       tokeiLanguageReport `json:"Yaml"`
}

func getTokeiReport(path string) (*TokeiReport, error) {
	cmd := exec.Command("tokei", "-e", "vendor/", path, "-o", "json")
	buf := new(bytes.Buffer)
	cmd.Stdout = buf
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, err
	}
	decoder := json.NewDecoder(buf)
	r := new(TokeiReport)
	if err := decoder.Decode(r); err != nil {
		return nil, err
	}
	return r, nil
}

func getStats(fetch bool) (*stats, error) {
	n := time.Now()
	last30Days := n.AddDate(0, 0, -30)
	last7Days := n.AddDate(0, 0, -7)
	last24h := n.AddDate(0, 0, -1)
	var total, last30DaysTotal, last7DaysTotal, last24hTotal, lines int
	for _, name := range []string{
		"stun", "turn", "sdp", "web", "stund", "tech-status", "ice", "rtc", "gortcd",
		"ansible-role-nginx", "ansible-go", "api", "docs", "dtls", "neo", "turnc",
	} {
		p := "/tmp/gortc-analyze/" + name
		r, err := git.PlainClone(p, false, &git.CloneOptions{
			URL: "https://github.com/gortc/" + name,
		})
		if err == git.ErrRepositoryAlreadyExists {
			r, err = git.PlainOpen(p)
			if err != nil {
				return nil, err
			}
			w, err := r.Worktree()
			if err != nil {
				return nil, err
			}
			if fetch {
				err = w.Pull(&git.PullOptions{
					Force:      true,
					RemoteName: "origin",
				})
				log.Println("pull", name, err)
				if err == git.NoErrAlreadyUpToDate {
					err = nil
				}
			}
		}
		if err != nil {
			return nil, err
		}
		rep, err := getTokeiReport(p)
		if err != nil {
			return nil, err
		}
		if name == "dtls" {
			// It's not fair to count vendored code.
			// TODO: count delta.
			rep.Go.Lines = 0
		}
		lines += rep.Go.Lines + rep.Yaml.Lines + rep.Dockerfile.Lines
		ref, err := r.Head()
		if err != nil {
			return nil, err
		}
		fmt.Println(name, "head", ref)
		b, err := r.Log(&git.LogOptions{
			From: ref.Hash(),
		})
		var commits int
		countAuthors := map[string]struct{}{
			"ar@cydev.ru":                          {},
			"ernado@ya.ru":                         {},
			"mail@backkem.me":                      {},
			"ar@gortc.io":                          {},
			"songjiayang@users.noreply.github.com": {},
			"a.razumov@corp.mail.ru":               {},
		}
		if err = b.ForEach(func(commit *object.Commit) error {
			if _, ok := countAuthors[commit.Author.Email]; !ok {
				return nil
			}
			commits++
			if commit.Author.When.After(last30Days) {
				last30DaysTotal++
			}
			if commit.Author.When.After(last7Days) {
				last7DaysTotal++
			}
			if commit.Author.When.After(last24h) {
				last24hTotal++
			}
			return nil
		}); err != nil {
			return nil, err
		}
		total += commits
	}
	return &stats{
		Time:    time.Now(),
		Total:   total,
		Last30d: last30DaysTotal,
		Last7d:  last7DaysTotal,
		Last24h: last24hTotal,
		Lines:   lines,
	}, nil
}
<|endoftext|>


<commit_before>package oauthmw

import (
	"encoding/gob"
	"time"

	"github.com/knq/sessionmw/jsonstore"

	"golang.org/x/oauth2"
)

// Store is the object used by oauthmw in the session.
type Store struct {
	// Provider name of token.
	Provider string `json:"provider"`

	// Token is redeemed oauth2 token.
	Token *oauth2.Token `json:"token,omitempty"`

	// States are the passed states sent to oauth2 providers.
	States map[string]StoreState `json:"states"`
}

// StoreState is storage for a passed oauth2 in a session.
type StoreState struct {
	// Provider name of state.
	Provider string `json:"provider"`

	// Expiration is when the state expires.
	Expiration time.Time `json:"expiration"`

	// Redeemed indicates whether or not the state has been previously redeemed.
	Redeemed bool `json:"redeemed"`
}

func init() {
	// register oauthmw stores for use by gob.encode/decode (ie various binary sessionmw.Store types)
	gob.RegisterName("knq.oauthmw.Store", Store{})
	gob.RegisterName("knq.oauthmw.StoreState", StoreState{})

	// register oauthmw for use with jsonstore
	jsonstore.RegisterSchema("knq.oauthmw.Store", Store{})
	jsonstore.RegisterSchema("knq.oauthmw.StoreState", StoreState{})
}
<commit_msg>Temporarily commenting out non-public jsonstore hook<commit_after>package oauthmw

import (
	"encoding/gob"
	"time"

	//"github.com/knq/sessionmw/jsonstore"

	"golang.org/x/oauth2"
)

// Store is the object used by oauthmw in the session.
type Store struct {
	// Provider name of token.
	Provider string `json:"provider"`

	// Token is redeemed oauth2 token.
	Token *oauth2.Token `json:"token,omitempty"`

	// States are the passed states sent to oauth2 providers.
	States map[string]StoreState `json:"states"`
}

// StoreState is storage for a passed oauth2 in a session.
type StoreState struct {
	// Provider name of state.
	Provider string `json:"provider"`

	// Expiration is when the state expires.
	Expiration time.Time `json:"expiration"`

	// Redeemed indicates whether or not the state has been previously redeemed.
	Redeemed bool `json:"redeemed"`
}

func init() {
	// register oauthmw stores for use by gob.encode/decode (ie various binary sessionmw.Store types)
	gob.RegisterName("knq.oauthmw.Store", Store{})
	gob.RegisterName("knq.oauthmw.StoreState", StoreState{})

	// register oauthmw for use with jsonstore
	//jsonstore.RegisterSchema("knq.oauthmw.Store", Store{})
	//jsonstore.RegisterSchema("knq.oauthmw.StoreState", StoreState{})
}
<|endoftext|>


<commit_before>package godb

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"sync"
)

type store struct {
	idx *btree
	buf *bytes.Buffer
	sync.RWMutex
}

func OpenStore(path string) (*store, error) {
	idx := &btree{ngin: new(engine)}
	if err := idx.open(path); err != nil {
		return nil, err
	}
	st := &store{
		idx: idx,
		buf: bytes.NewBuffer(make([]byte, maxKey, maxKey)),
	}
	st.buf.Reset()
	return st, nil
}

func (s *store) Add(key, val interface{}) error {
	s.Lock()
	defer s.Unlock()
	k, err := s.genKey(key)
	if err != nil {
		return fmt.Errorf("store[add]: error while generating key -> %q", err)
	}
	v, err := json.Marshal(val)
	if err != nil {
		return fmt.Errorf("store[add]: error while attempting to marshal -> %q", err)
	}
	if err := verify(k, v); err != nil {
		return fmt.Errorf("store[add]: error while doing bounds check -> %q", err)
	}
	if err := s.idx.add(k, v); err != nil {
		return fmt.Errorf("store[add]: error while adding to index -> %q", err)
	}
	return nil
}

func (s *store) Set(key, val interface{}) error {
	s.Lock()
	defer s.Unlock()
	k, err := s.genKey(key)
	if err != nil {
		return fmt.Errorf("store[set]: error while generating key -> %q", err)
	}
	v, err := json.Marshal(val)
	if err != nil {
		return fmt.Errorf("store[set]: error while attempting to marshal -> %q", err)
	}
	if err := verify(k, v); err != nil {
		return fmt.Errorf("store[set]: error while doing bounds check -> %q", err)
	}
	if err := s.idx.set(k, v); err != nil {
		return fmt.Errorf("store[set]: error while setting in index -> %q", err)
	}
	return nil
}

func (s *store) Get(key, ptr interface{}) error {
	s.RLock()
	defer s.RUnlock()
	k, err := s.genKey(key)
	if err != nil {
		return fmt.Errorf("store[get]: error while generating key -> %q", err)
	}
	v, err := s.idx.get(k)
	if err != nil {
		return fmt.Errorf("store[get]: error while getting value from index -> %q", err)
	}
	if err := json.Unmarshal(v, ptr); err != nil {
		return fmt.Errorf("store[get]: error while attempting to un-marshal -> %q", err)
	}
	return nil
}

func (s *store) Del(key interface{}) error {
	s.Lock()
	defer s.Unlock()
	k, err := s.genKey(key)
	if err != nil {
		return fmt.Errorf("store[del]: error while generating key -> %q", err)
	}
	if err := s.idx.del(k); err != nil {
		return fmt.Errorf("store[del]: error while deleting value from index -> %q", err)
	}
	return nil
}

func (s *store) Qry(model json.Unmarshaler) <-chan json.Unmarshaler {
	s.RLock()
	defer s.RUnlock()
	m := make(chan json.Unmarshaler)
	go func() {
		for val := range s.idx.next() {
			if val == nil {
				break
			}
			err := model.UnmarshalJSON(val)
			if err == nil {
				m <- model
			}
		}
		close(m)
	}()
	return m
}

func (s *store) Count() int {
	s.RLock()
	defer s.RUnlock()
	return s.idx.count
}

func (s *store) Close() error {
	s.Lock()
	defer s.Unlock()
	return s.idx.close()
}

// do verify by doing bounds check
func verify(key, val []byte) error {
	// key bounds check
	if len(key) > maxKey {
		return fmt.Errorf("store[verify]: key exceeds maximum key length of %d bytes, by %d bytes\n", maxKey, len(key)-maxKey)
	}
	// val bounds check
	if len(val) > maxVal {
		return fmt.Errorf("store[verify]: val exceeds maximum val length of %d bytes, by %d bytes\n", maxVal, len(val)-maxVal)
	}
	// passed bounds check, no errors so return nil
	return nil
}

func (s *store) genKey(k interface{}) ([]byte, error) {
	switch k.(type) {
	case string:
		k = []byte(k.(string))
	case int:
		k = int64(k.(int))
	case uint:
		k = uint64(k.(uint))
	}
	if err := binary.Write(s.buf, binary.BigEndian, k); err != nil {
		s.buf.Reset()
		return nil, err
	}
	key := make([]byte, maxKey, maxKey)
	if s.buf.Len() > maxKey {
		s.buf.Truncate(maxKey)
	}
	copy(key[maxKey-s.buf.Len():], s.buf.Bytes())
	s.buf.Reset()
	return key, nil
}
<commit_msg>putting msgpack encoding in place<commit_after>package godb

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"sync"

	"github.com/cagnosolutions/msgpack"
)

type store struct {
	idx *btree
	buf *bytes.Buffer
	sync.RWMutex
}

func OpenStore(path string) (*store, error) {
	idx := &btree{ngin: new(engine)}
	if err := idx.open(path); err != nil {
		return nil, err
	}
	st := &store{
		idx: idx,
		buf: bytes.NewBuffer(make([]byte, maxKey, maxKey)),
	}
	st.buf.Reset()
	return st, nil
}

func (s *store) Add(key, val interface{}) error {
	s.Lock()
	defer s.Unlock()
	k, err := s.genKey(key)
	if err != nil {
		return fmt.Errorf("store[add]: error while generating key -> %q", err)
	}
	//v, err := json.Marshal(val)
	v, err := msgpack.Marshal(val)
	if err != nil {
		return fmt.Errorf("store[add]: error while attempting to marshal -> %q", err)
	}
	if err := verify(k, v); err != nil {
		return fmt.Errorf("store[add]: error while doing bounds check -> %q", err)
	}
	if err := s.idx.add(k, v); err != nil {
		return fmt.Errorf("store[add]: error while adding to index -> %q", err)
	}
	return nil
}

func (s *store) Set(key, val interface{}) error {
	s.Lock()
	defer s.Unlock()
	k, err := s.genKey(key)
	if err != nil {
		return fmt.Errorf("store[set]: error while generating key -> %q", err)
	}
	//v, err := json.Marshal(val)
	v, err := msgpack.Marshal(val)
	if err != nil {
		return fmt.Errorf("store[set]: error while attempting to marshal -> %q", err)
	}
	if err := verify(k, v); err != nil {
		return fmt.Errorf("store[set]: error while doing bounds check -> %q", err)
	}
	if err := s.idx.set(k, v); err != nil {
		return fmt.Errorf("store[set]: error while setting in index -> %q", err)
	}
	return nil
}

func (s *store) Get(key, ptr interface{}) error {
	s.RLock()
	defer s.RUnlock()
	k, err := s.genKey(key)
	if err != nil {
		return fmt.Errorf("store[get]: error while generating key -> %q", err)
	}
	v, err := s.idx.get(k)
	if err != nil {
		return fmt.Errorf("store[get]: error while getting value from index -> %q", err)
	}
	//if err := json.Unmarshal(v, ptr); err != nil {
	if err := msgpack.Unmarshal(v, ptr); err != nil {
		return fmt.Errorf("store[get]: error while attempting to un-marshal -> %q", err)
	}
	return nil
}

func (s *store) Del(key interface{}) error {
	s.Lock()
	defer s.Unlock()
	k, err := s.genKey(key)
	if err != nil {
		return fmt.Errorf("store[del]: error while generating key -> %q", err)
	}
	if err := s.idx.del(k); err != nil {
		return fmt.Errorf("store[del]: error while deleting value from index -> %q", err)
	}
	return nil
}

func (s *store) Qry(model json.Unmarshaler) <-chan json.Unmarshaler {
	s.RLock()
	defer s.RUnlock()
	m := make(chan json.Unmarshaler)
	go func() {
		for val := range s.idx.next() {
			if val == nil {
				break
			}
			err := model.UnmarshalJSON(val)
			if err == nil {
				m <- model
			}
		}
		close(m)
	}()
	return m
}

func (s *store) Count() int {
	s.RLock()
	defer s.RUnlock()
	return s.idx.count
}

func (s *store) Close() error {
	s.Lock()
	defer s.Unlock()
	return s.idx.close()
}

// do verify by doing bounds check
func verify(key, val []byte) error {
	// key bounds check
	if len(key) > maxKey {
		return fmt.Errorf("store[verify]: key exceeds maximum key length of %d bytes, by %d bytes\n", maxKey, len(key)-maxKey)
	}
	// val bounds check
	if len(val) > maxVal {
		return fmt.Errorf("store[verify]: val exceeds maximum val length of %d bytes, by %d bytes\n", maxVal, len(val)-maxVal)
	}
	// passed bounds check, no errors so return nil
	return nil
}

func (s *store) genKeyMsgpack(k interface{}) ([]byte, error) {
	b, err := msgpack.Marshal(k)
	if err != nil {
		return nil, err
	}
	if len(b) > maxKey {
		b = b[:maxKey] // truncate to len of maxKey
		return b, nil
	}
	b = append(make([]byte, maxKey-len(b)), b...)
	return b, nil
}

func (s *store) genKey(k interface{}) ([]byte, error) {
	switch k.(type) {
	case string:
		k = []byte(k.(string))
	case int:
		k = int64(k.(int))
	case uint:
		k = uint64(k.(uint))
	}
	if err := binary.Write(s.buf, binary.BigEndian, k); err != nil {
		s.buf.Reset()
		return nil, err
	}
	key := make([]byte, maxKey, maxKey)
	if s.buf.Len() > maxKey {
		s.buf.Truncate(maxKey)
	}
	copy(key[maxKey-s.buf.Len():], s.buf.Bytes())
	s.buf.Reset()
	return key, nil
}
<|endoftext|>


<commit_before>package jet

import (
	"errors"
	"fmt"
	"math"
)

const (
	TableName  string = "migrations"
	ColumnName string = "version"
)

var (
	EOM error = errors.New("EOM")
)

type Suite struct {
	Migrations []*Migration
	Stmts      *Stmts
}

type Stmts struct {
	CreateTableSQL   string
	SelectVersionSQL string
	InsertVersionSQL string
	UpdateVersionSQL string
}

func (s *Suite) Add(m *Migration) {
	if m == nil {
		panic("nil migration")
	}
	if s.Stmts == nil {
		s.Stmts = &Stmts{
			CreateTableSQL:   fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%s" ( "%s" INTEGER )`, TableName, ColumnName),
			SelectVersionSQL: fmt.Sprintf(`SELECT "%s" FROM "%s" LIMIT 1`, ColumnName, TableName),
			InsertVersionSQL: fmt.Sprintf(`INSERT INTO "%s" ( "%s" ) VALUES ( $1 )`, TableName, ColumnName),
			UpdateVersionSQL: fmt.Sprintf(`UPDATE "%s" SET "%s" = $1 WHERE "%s" = $2`, TableName, ColumnName, ColumnName),
		}
	}
	if s.Migrations == nil {
		s.Migrations = []*Migration{m}
	} else {
		s.Migrations = append(s.Migrations, m)
	}
}

func (s *Suite) AddSQL(up, down string) {
	s.Add(&Migration{
		Up:   up,
		Down: down,
	})
}

func (s *Suite) Step(db Db) (error, int64) {
	return s.Run(db, true, 1)
}

func (s *Suite) Rollback(db Db) (error, int64) {
	return s.Run(db, false, 1)
}

func (s *Suite) Migrate(db Db) (error, int64) {
	return s.Run(db, true, math.MaxInt32)
}

func (s *Suite) Reset(db Db) (error, int64) {
	return s.Run(db, false, math.MaxInt32)
}

func (s *Suite) Run(db Db, up bool, maxSteps int) (error, int64) {
	if l := len(s.Migrations); l == 0 {
		return errors.New("cannot run suite, no migrations set"), -1
	}
	err := db.Query(s.Stmts.CreateTableSQL).Run()
	if err != nil {
		return err, -1
	}
	var row struct {
		Version int64
	}
	row.Version = -1
	err = db.Query(s.Stmts.SelectVersionSQL).Rows(&row, 1)
	if err != nil {
		return err, -1
	}
	step := 0
	current := row.Version
	for _, m := range s.buildList(up, row.Version) {
		if step++; maxSteps > 0 && step > maxSteps {
			break
		}
		txn, err := db.Begin()
		if err != nil {
			return err, -1
		}
		next := m.Id
		if up {
			txn.Query(m.Up).Run()
		} else {
			next--
			txn.Query(m.Down).Run()
		}
		if current == -1 {
			txn.Query(s.Stmts.InsertVersionSQL, next).Run()
		} else {
			txn.Query(s.Stmts.UpdateVersionSQL, next, current).Run()
		}
		if err := txn.Commit(); err != nil {
			return err, -1
		}
		current = next
	}
	return nil, current
}

func (s *Suite) buildList(up bool, version int64) []*Migration {
	a := []*Migration{}
	for i, m := range s.Migrations {
		m.Id = int64(i + 1)
		if up {
			if m.Id > version {
				a = append(a, m)
			}
		} else {
			if m.Id <= version {
				a = append(a, m)
			}
		}
	}
	if !up {
		reverse(a)
	}
	return a
}

func reverse(a []*Migration) []*Migration {
	for i, j := 0, len(a)-1; i < j; i, j = i+1, j-1 {
		a[i], a[j] = a[j], a[i]
	}
	return a
}
<commit_msg>synchronize Run<commit_after>package jet

import (
	"errors"
	"fmt"
	"math"
	"sync"
)

const (
	TableName  string = "migrations"
	ColumnName string = "version"
)

var (
	EOM error = errors.New("EOM")
)

type Suite struct {
	Migrations []*Migration
	Stmts      *Stmts
	mtx        sync.Mutex
}

type Stmts struct {
	CreateTableSQL   string
	SelectVersionSQL string
	InsertVersionSQL string
	UpdateVersionSQL string
}

func (s *Suite) Add(m *Migration) {
	if m == nil {
		panic("nil migration")
	}
	if s.Stmts == nil {
		s.Stmts = &Stmts{
			CreateTableSQL:   fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%s" ( "%s" INTEGER )`, TableName, ColumnName),
			SelectVersionSQL: fmt.Sprintf(`SELECT "%s" FROM "%s" LIMIT 1`, ColumnName, TableName),
			InsertVersionSQL: fmt.Sprintf(`INSERT INTO "%s" ( "%s" ) VALUES ( $1 )`, TableName, ColumnName),
			UpdateVersionSQL: fmt.Sprintf(`UPDATE "%s" SET "%s" = $1 WHERE "%s" = $2`, TableName, ColumnName, ColumnName),
		}
	}
	if s.Migrations == nil {
		s.Migrations = []*Migration{m}
	} else {
		s.Migrations = append(s.Migrations, m)
	}
}

func (s *Suite) AddSQL(up, down string) {
	s.Add(&Migration{
		Up:   up,
		Down: down,
	})
}

func (s *Suite) Step(db Db) (error, int64) {
	return s.Run(db, true, 1)
}

func (s *Suite) Rollback(db Db) (error, int64) {
	return s.Run(db, false, 1)
}

func (s *Suite) Migrate(db Db) (error, int64) {
	return s.Run(db, true, math.MaxInt32)
}

func (s *Suite) Reset(db Db) (error, int64) {
	return s.Run(db, false, math.MaxInt32)
}

func (s *Suite) Run(db Db, up bool, maxSteps int) (error, int64) {
	s.mtx.Lock()
	defer s.mtx.Unlock()
	if l := len(s.Migrations); l == 0 {
		return errors.New("cannot run suite, no migrations set"), -1
	}
	err := db.Query(s.Stmts.CreateTableSQL).Run()
	if err != nil {
		return err, -1
	}
	var row struct {
		Version int64
	}
	row.Version = -1
	err = db.Query(s.Stmts.SelectVersionSQL).Rows(&row, 1)
	if err != nil {
		return err, -1
	}
	step := 0
	current := row.Version
	for _, m := range s.buildList(up, row.Version) {
		if step++; maxSteps > 0 && step > maxSteps {
			break
		}
		txn, err := db.Begin()
		if err != nil {
			return err, -1
		}
		next := m.Id
		if up {
			txn.Query(m.Up).Run()
		} else {
			next--
			txn.Query(m.Down).Run()
		}
		if current == -1 {
			txn.Query(s.Stmts.InsertVersionSQL, next).Run()
		} else {
			txn.Query(s.Stmts.UpdateVersionSQL, next, current).Run()
		}
		if err := txn.Commit(); err != nil {
			return err, -1
		}
		current = next
	}
	return nil, current
}

func (s *Suite) buildList(up bool, version int64) []*Migration {
	a := []*Migration{}
	for i, m := range s.Migrations {
		m.Id = int64(i + 1)
		if up {
			if m.Id > version {
				a = append(a, m)
			}
		} else {
			if m.Id <= version {
				a = append(a, m)
			}
		}
	}
	if !up {
		reverse(a)
	}
	return a
}

func reverse(a []*Migration) []*Migration {
	for i, j := 0, len(a)-1; i < j; i, j = i+1, j-1 {
		a[i], a[j] = a[j], a[i]
	}
	return a
}
<|endoftext|>


<commit_before>package cmagic

import (
	"strings"
	"fmt"
)

type filter struct {
	t  T
	rs []Relation
}

func (f *filter) generateWhere() (string, []interface{}) {
	strs := []string{}
	vals := []interface{}{}
	for _, r := range f.rs {
		s, v := r.cql()
		strs = append(strs, s)
		vals = append(vals, v...)
	}
	return "WHERE " + strings.Join(strs, " AND "), vals
}

func (f filter) Replace(i interface{}) error {
	return nil
}

// UPDATE keyspace.Movies SET col1 = val1, col2 = val2
func updateStatement(cfName string, pkName string, fieldNames []string) string {
	cols := []string{}
	for _, v := range fieldNames {
		cols = append(cols, v+" = ?")
	}
	return fmt.Sprintf("UPDATE %v SET "+strings.Join(cols, ", "), cfName)
}

func (f filter) Update(m map[string]interface{}) error {
	fields, values := keyValues(m)
	for k, v := range m {
		fields = append(fields, k)
		values = append(values, v)
	}
	str, wvals := f.generateWhere()
	stmt := updateStatement(f.t.keySpace.name, f.t.info.name, fields)
	sess := f.t.keySpace.session
	return sess.Query(stmt +" "+ str, append(values, wvals...)...).Exec()
}

func (f filter) Delete() error {
	// return c.keySpace.session.Query(g.DeleteById(c.keySpace.name, c.info.primaryKey), id).Exec()
	return nil
}

func (f filter) Query() Query {
	return &query{
		f: f,
	}
}
<commit_msg>More debug crap<commit_after>package cmagic

import (
	"strings"
	"fmt"
)

type filter struct {
	t  T
	rs []Relation
}

func (f *filter) generateWhere() (string, []interface{}) {
	strs := []string{}
	vals := []interface{}{}
	for _, r := range f.rs {
		s, v := r.cql()
		strs = append(strs, s)
		vals = append(vals, v...)
	}
	return "WHERE " + strings.Join(strs, " AND "), vals
}

func (f filter) Replace(i interface{}) error {
	return nil
}

// UPDATE keyspace.Movies SET col1 = val1, col2 = val2
func updateStatement(cfName string, pkName string, fieldNames []string) string {
	cols := []string{}
	for _, v := range fieldNames {
		cols = append(cols, v+" = ?")
	}
	return fmt.Sprintf("UPDATE %v SET "+strings.Join(cols, ", "), cfName)
}

func (f filter) Update(m map[string]interface{}) error {
	fields, values := keyValues(m)
	for k, v := range m {
		fields = append(fields, k)
		values = append(values, v)
	}
	str, wvals := f.generateWhere()
	stmt := updateStatement(f.t.keySpace.name, f.t.info.name, fields)
	sess := f.t.keySpace.session
	if f.t.keySpace.debugMode {
		fmt.Println(stmt, wvals)
	}
	return sess.Query(stmt +" "+ str, append(values, wvals...)...).Exec()
}

func (f filter) Delete() error {
	// return c.keySpace.session.Query(g.DeleteById(c.keySpace.name, c.info.primaryKey), id).Exec()
	return nil
}

func (f filter) Query() Query {
	return &query{
		f: f,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/samuel/go-zookeeper/zk"
)

const (
	zkPath         = "/aurora/scheduler"
	zkLeaderPrefix = "singleton_candidate_"
)

type finder interface {
	leaderURL() (string, error)
}

func newFinder(url string) (f finder, err error) {
	if strings.HasPrefix(url, "http://") {
		f = &httpFinder{url: url}
	}

	if strings.HasPrefix(url, "zk://") {
		f = newZkFinder(url)
	}

	if f == nil {
		err = errors.New("finder: bad address")
	}

	return f, err
}

type httpFinder struct {
	url string
}

func (f *httpFinder) leaderURL() (string, error) {
	// This will redirect us to the elected Aurora master
	schedulerURL := fmt.Sprintf("%s/scheduler", f.url)
	rr, err := http.NewRequest("GET", schedulerURL, nil)
	if err != nil {
		return "", err
	}

	rresp, err := httpClient.Transport.RoundTrip(rr)
	if err != nil {
		return "", err
	}
	defer rresp.Body.Close()

	masterLoc := rresp.Header.Get("Location")
	if masterLoc == "" {
		glog.V(6).Info("missing Location header in request")
		masterLoc = schedulerURL
	}

	return strings.TrimRight(masterLoc, "/scheduler"), nil
}

func hostsFromURL(urls string) (hosts []string, err error) {
	for _, s := range strings.Split(urls, ",") {
		u, err := url.Parse(s)
		if err != nil {
			return hosts, err
		}

		hosts = append(hosts, u.Host)
	}

	return hosts, err
}

type zkFinder struct {
	conn *zk.Conn

	sync.RWMutex
	leaderIP string
}

func newZkFinder(url string) *zkFinder {
	zkSrvs, err := hostsFromURL(url)
	if err != nil {
		panic(err)
	}

	conn, events, err := zk.Connect(zkSrvs, 20*time.Second)
	if err != nil {
		panic(err)
	}

	go func() {
		for ev := range events {
			glog.V(6).Infof("conn: %s server: %s", ev.State, ev.Server)
		}
	}()

	f := zkFinder{conn: conn}
	go f.watch()

	return &f
}

func (f *zkFinder) leaderzNode() (string, error) {
	children, stat, err := f.conn.Children(zkPath)
	if stat == nil {
		err = errors.New("zkFinder: children returned nil stat")
	}
	if err != nil {
		return "", err
	}

	var leaderSeq int
	var leader string
	for _, child := range children {
		path := strings.Split(child, zkLeaderPrefix)
		if len(path) > 1 {
			seq, err := strconv.Atoi(path[1])
			if err != nil {
				return "", err
			}

			if leader == "" {
				leader = child
			}

			if seq <= leaderSeq {
				leaderSeq = seq
				leader = child
			}
		}
	}

	if leader == "" {
		return leader, errors.New("zkFinder: zNode not found")
	}

	return fmt.Sprintf("%s/%s", zkPath, leader), nil
}

func (f *zkFinder) leaderURL() (string, error) {
	f.RLock()
	defer f.RUnlock()

	if f.leaderIP == "" {
		return "", errors.New("zkFinder: no leader found via ZooKeeper")
	}

	return fmt.Sprintf("http://%s:8081", f.leaderIP), nil
}

func (f *zkFinder) watch() {
	for _ = range time.NewTicker(1 * time.Second).C {
		zNode, err := f.leaderzNode()
		if err != nil {
			glog.Warning(err)
			continue
		}

		glog.V(6).Info("leader zNode at: ", zNode)

		data, stat, events, err := f.conn.GetW(zNode)
		if stat == nil {
			err = errors.New("get returned nil stat")
		}
		if err != nil {
			glog.Warning(err)
			continue
		}

		f.Lock()
		f.leaderIP = string(data)
		f.Unlock()

		for ev := range events {
			switch {
			case ev.Err != nil:
				err = fmt.Errorf("watcher error %+v", ev.Err)
			case ev.Type == zk.EventNodeDeleted:
				err = errors.New("leader zNode deleted")
			}

			if err != nil {
				glog.Warning(err)
				break
			}
		}
	}
}
<commit_msg>Match both http and https<commit_after>package main

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/samuel/go-zookeeper/zk"
)

const (
	zkPath         = "/aurora/scheduler"
	zkLeaderPrefix = "singleton_candidate_"
)

type finder interface {
	leaderURL() (string, error)
}

func newFinder(url string) (f finder, err error) {
	if strings.HasPrefix(url, "http") {
		f = &httpFinder{url: url}
	}

	if strings.HasPrefix(url, "zk://") {
		f = newZkFinder(url)
	}

	if f == nil {
		err = errors.New("finder: bad address")
	}

	return f, err
}

type httpFinder struct {
	url string
}

func (f *httpFinder) leaderURL() (string, error) {
	// This will redirect us to the elected Aurora master
	schedulerURL := fmt.Sprintf("%s/scheduler", f.url)
	rr, err := http.NewRequest("GET", schedulerURL, nil)
	if err != nil {
		return "", err
	}

	rresp, err := httpClient.Transport.RoundTrip(rr)
	if err != nil {
		return "", err
	}
	defer rresp.Body.Close()

	masterLoc := rresp.Header.Get("Location")
	if masterLoc == "" {
		glog.V(6).Info("missing Location header in request")
		masterLoc = schedulerURL
	}

	return strings.TrimRight(masterLoc, "/scheduler"), nil
}

func hostsFromURL(urls string) (hosts []string, err error) {
	for _, s := range strings.Split(urls, ",") {
		u, err := url.Parse(s)
		if err != nil {
			return hosts, err
		}

		hosts = append(hosts, u.Host)
	}

	return hosts, err
}

type zkFinder struct {
	conn *zk.Conn

	sync.RWMutex
	leaderIP string
}

func newZkFinder(url string) *zkFinder {
	zkSrvs, err := hostsFromURL(url)
	if err != nil {
		panic(err)
	}

	conn, events, err := zk.Connect(zkSrvs, 20*time.Second)
	if err != nil {
		panic(err)
	}

	go func() {
		for ev := range events {
			glog.V(6).Infof("conn: %s server: %s", ev.State, ev.Server)
		}
	}()

	f := zkFinder{conn: conn}
	go f.watch()

	return &f
}

func (f *zkFinder) leaderzNode() (string, error) {
	children, stat, err := f.conn.Children(zkPath)
	if stat == nil {
		err = errors.New("zkFinder: children returned nil stat")
	}
	if err != nil {
		return "", err
	}

	var leaderSeq int
	var leader string
	for _, child := range children {
		path := strings.Split(child, zkLeaderPrefix)
		if len(path) > 1 {
			seq, err := strconv.Atoi(path[1])
			if err != nil {
				return "", err
			}

			if leader == "" {
				leader = child
			}

			if seq <= leaderSeq {
				leaderSeq = seq
				leader = child
			}
		}
	}

	if leader == "" {
		return leader, errors.New("zkFinder: zNode not found")
	}

	return fmt.Sprintf("%s/%s", zkPath, leader), nil
}

func (f *zkFinder) leaderURL() (string, error) {
	f.RLock()
	defer f.RUnlock()

	if f.leaderIP == "" {
		return "", errors.New("zkFinder: no leader found via ZooKeeper")
	}

	return fmt.Sprintf("http://%s:8081", f.leaderIP), nil
}

func (f *zkFinder) watch() {
	for _ = range time.NewTicker(1 * time.Second).C {
		zNode, err := f.leaderzNode()
		if err != nil {
			glog.Warning(err)
			continue
		}

		glog.V(6).Info("leader zNode at: ", zNode)

		data, stat, events, err := f.conn.GetW(zNode)
		if stat == nil {
			err = errors.New("get returned nil stat")
		}
		if err != nil {
			glog.Warning(err)
			continue
		}

		f.Lock()
		f.leaderIP = string(data)
		f.Unlock()

		for ev := range events {
			switch {
			case ev.Err != nil:
				err = fmt.Errorf("watcher error %+v", ev.Err)
			case ev.Type == zk.EventNodeDeleted:
				err = errors.New("leader zNode deleted")
			}

			if err != nil {
				glog.Warning(err)
				break
			}
		}
	}
}
<|endoftext|>


<commit_before>package plugins

// PluginsConfiguration encapsulates all configuration data
//
// swagger:model
type Configuration struct {
	LastID  int     `json:"lastid"`
	Plugins []Plugin `json:"plugins"`
}

func NewConfiguration() *Configuration {
	var configuration = new(Configuration)

	configuration.LastID = 0
	configuration.Plugins = []Plugin{}

	return configuration
}

// Plugin represents a plugin configuration
//
// swagger:model
type Plugin struct {
	ID          int    `json:"id"`
	Name        string        `json:"title"`
	Description string    `json:"description"`
	Frontend    []Frontend `json:"frontend"`
	Backend     []Backend `json:"backend"`
}

func NewPlugin() (*Plugin) {
	var p = new(Plugin)

	return p
}

type Frontend struct {
	Key   string
	Value string
}

func NewFrontend() *Frontend {
	var p = new(Frontend)

	return p
}

type Backend struct {
	Key   string
	Value string
}

func NewBackend() *Backend {
	var p = new(Backend)

	return p
}
<commit_msg>[Add] full plugin structure<commit_after>package plugins

// PluginsConfiguration encapsulates all configuration data
//
// swagger:model
type Configuration struct {
	LastID  int     `json:"lastid"`
	Plugins []Plugin `json:"plugins"`
}

func NewConfiguration() *Configuration {
	var configuration = new(Configuration)

	configuration.LastID = 0
	configuration.Plugins = []Plugin{}

	return configuration
}

// Plugin represents a plugin configuration
//
// swagger:model
type Plugin struct {
	ID          int         `json:"id"`
	Name        string      `json:"name"`
	Description string      `json:"description"`
	Frontend    Frontend  `json:"frontend"`
	Backend     Backend   `json:"backend"`
}

func NewPlugin() (*Plugin) {
	var p = new(Plugin)

	p.Frontend = *NewFrontend()
	p.Backend = *NewBackend()

	return p
}

type Frontend struct {
	EltName   string `json:"eltName"`
	Cols      int `json:"cols"`
	Rows      int `json:"rows"`
	Fixedsize int `json:"fixedsize"`
	Props     []Props `json:"props"`
}

func NewFrontend() *Frontend {
	var f = new(Frontend)

	f.Props = []Props{}

	return f
}

type Props struct {
	Name        string `json:"name"`
	Type        string `json:"type"`
	Description string `json:"description"`
	Value       string `json:"value"`
}

type Backend struct {
	Ports []int                  `json:"ports"`
	Props map[string]interface{} `json:"props"`
}

func NewBackend() *Backend {
	var b = new(Backend)

	b.Ports = []int{}
	b.Props = make(map[string]interface{})

	return b
}
<|endoftext|>


<commit_before>package zenodb

import (
	"fmt"
	"math"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/bytemap"
	"github.com/getlantern/errors"
	"github.com/getlantern/goexpr"
	"github.com/getlantern/golog"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/sql"
)

// TableStats presents statistics for a given table (currently only since the
// last time the database process was started).
type TableStats struct {
	FilteredPoints int64
	QueuedPoints   int64
	InsertedPoints int64
	DroppedPoints  int64
	ExpiredValues  int64
}

// TableOpts configures a table.
type TableOpts struct {
	// Name is the name of the table.
	Name string
	// View indicates if this table is a view on top of an existing table.
	View bool
	// MinFlushLatency sets a lower bound on how frequently the memstore is
	// flushed to disk.
	MinFlushLatency time.Duration
	// MaxFlushLatency sets an upper bound on how long to wait before flushing the
	// memstore to disk.
	MaxFlushLatency time.Duration
	// RetentionPeriod limits how long data is kept in the table (based on the
	// timestamp of the data itself).
	RetentionPeriod time.Duration
	// PartitionBy can be used in clustered deployments to decide which
	// dimensions to use in partitioning data. If unspecified, all dimensions are
	// used for partitioning.
	PartitionBy []string
	// SQL is the SELECT query that determines the fields, filtering and input
	// source for this table.
	SQL string
	// Virtual, if true, means that the table's data isn't actually stored or
	// queryable. Virtual tables are useful for defining a base set of fields
	// from which other tables can select.
	Virtual      bool
	dependencyOf []*TableOpts
}

type table struct {
	*TableOpts
	sql.Query
	db                  *DB
	rowStore            *rowStore
	log                 golog.Logger
	whereMutex          sync.RWMutex
	stats               TableStats
	statsMutex          sync.RWMutex
	wal                 *wal.Reader
	readOffset          wal.Offset
	highWaterMarkDisk   int64
	highWaterMarkMemory int64
	highWaterMarkMx     sync.RWMutex
}

// CreateTable creates a table based on the given opts.
func (db *DB) CreateTable(opts *TableOpts) error {
	q, err := sql.Parse(opts.SQL, nil)
	if err != nil {
		return err
	}
	return db.doCreateTable(opts, q)
}

// CreateView creates a view based on the given opts.
func (db *DB) CreateView(opts *TableOpts) error {
	table, err := sql.TableFor(opts.SQL)
	if err != nil {
		return err
	}

	// Get existing fields from existing table
	t := db.getTable(table)
	if t == nil {
		return fmt.Errorf("Table '%v' not found", table)
	}
	q, err := sql.Parse(opts.SQL, db.getFieldsOptional)
	if err != nil {
		return err
	}

	// Point view at same stream as table
	// TODO: populate view with existing data from table
	q.From = t.From

	if q.GroupBy == nil {
		q.GroupBy = t.GroupBy
		q.GroupByAll = t.GroupByAll
	}

	if q.Resolution == 0 {
		q.Resolution = t.Resolution
	}

	if len(opts.PartitionBy) == 0 {
		opts.PartitionBy = t.PartitionBy
	}

	// Combine where clauses
	if t.Where != nil {
		if q.Where == nil {
			q.Where = t.Where
		} else {
			combined, err := goexpr.Binary("AND", q.Where, t.Where)
			if err != nil {
				return err
			}
			q.Where = combined
		}
	}

	return db.doCreateTable(opts, q)
}

func (db *DB) doCreateTable(opts *TableOpts, q *sql.Query) error {
	if !opts.Virtual {
		if opts.RetentionPeriod <= 0 {
			return errors.New("Please specify a positive RetentionPeriod")
		}
		if opts.MinFlushLatency <= 0 {
			log.Debug("MinFlushLatency disabled")
		}
		if opts.MaxFlushLatency <= 0 {
			opts.MaxFlushLatency = time.Duration(math.MaxInt64)
			log.Debug("MaxFlushLatency disabled")
		}
	}
	opts.Name = strings.ToLower(opts.Name)

	// prepend a magic _points field
	newFields := make([]core.Field, 0, len(q.Fields)+1)
	newFields = append(newFields, sql.PointsField)
	for _, field := range q.Fields {
		// Don't add _points twice
		if field.Name != "_points" {
			newFields = append(newFields, field)
		}
	}
	q.Fields = newFields

	t := &table{
		TableOpts: opts,
		Query:     *q,
		db:        db,
		log:       golog.LoggerFor("zenodb." + opts.Name),
	}

	t.applyWhere(q.Where)

	var rsErr error
	var walOffset wal.Offset
	if !t.Virtual {
		t.rowStore, walOffset, rsErr = t.openRowStore(&rowStoreOptions{
			dir:             filepath.Join(db.opts.Dir, t.Name),
			minFlushLatency: t.MinFlushLatency,
			maxFlushLatency: t.MaxFlushLatency,
		})
		if rsErr != nil {
			return rsErr
		}
		t.log.Debugf("Starting at WAL offset %v", walOffset)
	}

	db.tablesMutex.Lock()
	defer db.tablesMutex.Unlock()

	if db.tables[t.Name] != nil {
		return fmt.Errorf("Table %v already exists", t.Name)
	}
	db.tables[t.Name] = t
	db.orderedTables = append(db.orderedTables, t)

	if !t.Virtual {
		partitionKeys := t.PartitionBy
		partitionKeysString := partitionKeysToString(partitionKeys)
		if t.db.distinctPartitionKeys[partitionKeysString] == nil {
			distinctPartitionKeys := make(map[string][]string, len(t.db.distinctPartitionKeys))
			for key, value := range db.distinctPartitionKeys {
				distinctPartitionKeys[key] = value
			}
			distinctPartitionKeys[partitionKeysString] = partitionKeys
			db.distinctPartitionKeys = distinctPartitionKeys
		}

		if t.db.opts.Follow != nil {
			t.startFollowing(walOffset)
			return nil
		}
		return t.startWALProcessing(walOffset)
	}

	return nil
}

func (t *table) startFollowing(walOffset wal.Offset) {
	newSubscriber := t.db.newStreamSubscriber[t.From]
	if newSubscriber == nil {
		newSubscriber = make(chan *tableWithOffset, 100)
		go t.db.followLeader(t.From, newSubscriber)
		t.db.newStreamSubscriber[t.From] = newSubscriber
	}
	newSubscriber <- &tableWithOffset{t, walOffset}
}

func (t *table) startWALProcessing(walOffset wal.Offset) error {
	var walErr error
	w := t.db.streams[t.From]
	if w == nil {
		walDir := filepath.Join(t.db.opts.Dir, "_wal", t.From)
		dirErr := os.MkdirAll(walDir, 0755)
		if dirErr != nil && !os.IsExist(dirErr) {
			return dirErr
		}
		w, walErr = wal.Open(walDir, t.db.opts.WALSyncInterval)
		if walErr != nil {
			return walErr
		}
		go t.db.capWALAge(w)
		t.db.streams[t.From] = w
	}

	if t.db.opts.Passthrough {
		log.Debugf("Passthrough will not insert data to table %v", t.Name)
		return nil
	}

	log.Debugf("%v will read inserts from %v at offset %v", t.Name, t.From, walOffset)
	t.wal, walErr = w.NewReader(t.Name, walOffset)
	if walErr != nil {
		return fmt.Errorf("Unable to obtain WAL reader: %v", walErr)
	}

	go t.processWALInserts()
	if !t.db.opts.Passthrough {
		go t.logHighWaterMark()
	}
	return nil
}

func (t *table) applyWhere(where goexpr.Expr) {
	t.whereMutex.Lock()
	t.Where = where
	t.whereMutex.Unlock()
}

func (t *table) getWhere() goexpr.Expr {
	t.whereMutex.RLock()
	where := t.Where
	t.whereMutex.RUnlock()
	return where
}

func (t *table) truncateBefore() time.Time {
	return t.db.clock.Now().Add(-1 * t.RetentionPeriod)
}

func (t *table) iterate(fields []string, includeMemStore bool, onValue func(bytemap.ByteMap, []encoding.Sequence)) error {
	return t.rowStore.iterate(fields, includeMemStore, onValue)
}

// shouldSort determines whether or not a flush should be sorted. The flush will
// sort if the table is the next table in line to be sorted, and no other sort
// is currently happening. If shouldSort returns true, the flushing process
// must call stopSorting when finished so that other tables have a chance to
// sort.
func (t *table) shouldSort() bool {
	if t.db.opts.MaxMemoryRatio <= 0 {
		return false
	}

	t.db.tablesMutex.RLock()
	if t.db.nextTableToSort >= len(t.db.orderedTables) {
		t.db.nextTableToSort = 0
	}
	nextTableToSort := t.db.orderedTables[t.db.nextTableToSort]
	result := t.Name == nextTableToSort.Name && !t.db.isSorting
	t.db.tablesMutex.RUnlock()
	return result
}

func (t *table) stopSorting() {
	t.db.tablesMutex.RLock()
	t.db.isSorting = false
	t.db.nextTableToSort++
	t.db.tablesMutex.RUnlock()
}

func (t *table) memStoreSize() int {
	return t.rowStore.memStoreSize()
}

func (t *table) forceFlush() {
	t.rowStore.forceFlush()
}

func (t *table) logHighWaterMark() {
	for {
		time.Sleep(15 * time.Second)
		t.highWaterMarkMx.RLock()
		disk := t.highWaterMarkDisk
		memory := t.highWaterMarkMemory
		t.highWaterMarkMx.RUnlock()
		t.log.Debugf("High Water Mark    disk: %v    memory: %v", encoding.TimeFromInt(disk).In(time.UTC), encoding.TimeFromInt(memory).In(time.UTC))
	}
}

func (t *table) updateHighWaterMarkDisk(ts int64) {
	t.highWaterMarkMx.Lock()
	if ts > t.highWaterMarkDisk {
		t.highWaterMarkDisk = ts
	}
	t.highWaterMarkMx.Unlock()
}

func (t *table) updateHighWaterMarkMemory(ts int64) {
	t.highWaterMarkMx.Lock()
	if ts > t.highWaterMarkMemory {
		t.highWaterMarkMemory = ts
	}
	t.highWaterMarkMx.Unlock()
}
<commit_msg>Fixed logging of high water marks<commit_after>package zenodb

import (
	"fmt"
	"math"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/bytemap"
	"github.com/getlantern/errors"
	"github.com/getlantern/goexpr"
	"github.com/getlantern/golog"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/sql"
)

// TableStats presents statistics for a given table (currently only since the
// last time the database process was started).
type TableStats struct {
	FilteredPoints int64
	QueuedPoints   int64
	InsertedPoints int64
	DroppedPoints  int64
	ExpiredValues  int64
}

// TableOpts configures a table.
type TableOpts struct {
	// Name is the name of the table.
	Name string
	// View indicates if this table is a view on top of an existing table.
	View bool
	// MinFlushLatency sets a lower bound on how frequently the memstore is
	// flushed to disk.
	MinFlushLatency time.Duration
	// MaxFlushLatency sets an upper bound on how long to wait before flushing the
	// memstore to disk.
	MaxFlushLatency time.Duration
	// RetentionPeriod limits how long data is kept in the table (based on the
	// timestamp of the data itself).
	RetentionPeriod time.Duration
	// PartitionBy can be used in clustered deployments to decide which
	// dimensions to use in partitioning data. If unspecified, all dimensions are
	// used for partitioning.
	PartitionBy []string
	// SQL is the SELECT query that determines the fields, filtering and input
	// source for this table.
	SQL string
	// Virtual, if true, means that the table's data isn't actually stored or
	// queryable. Virtual tables are useful for defining a base set of fields
	// from which other tables can select.
	Virtual      bool
	dependencyOf []*TableOpts
}

type table struct {
	*TableOpts
	sql.Query
	db                  *DB
	rowStore            *rowStore
	log                 golog.Logger
	whereMutex          sync.RWMutex
	stats               TableStats
	statsMutex          sync.RWMutex
	wal                 *wal.Reader
	readOffset          wal.Offset
	highWaterMarkDisk   int64
	highWaterMarkMemory int64
	highWaterMarkMx     sync.RWMutex
}

// CreateTable creates a table based on the given opts.
func (db *DB) CreateTable(opts *TableOpts) error {
	q, err := sql.Parse(opts.SQL, nil)
	if err != nil {
		return err
	}
	return db.doCreateTable(opts, q)
}

// CreateView creates a view based on the given opts.
func (db *DB) CreateView(opts *TableOpts) error {
	table, err := sql.TableFor(opts.SQL)
	if err != nil {
		return err
	}

	// Get existing fields from existing table
	t := db.getTable(table)
	if t == nil {
		return fmt.Errorf("Table '%v' not found", table)
	}
	q, err := sql.Parse(opts.SQL, db.getFieldsOptional)
	if err != nil {
		return err
	}

	// Point view at same stream as table
	// TODO: populate view with existing data from table
	q.From = t.From

	if q.GroupBy == nil {
		q.GroupBy = t.GroupBy
		q.GroupByAll = t.GroupByAll
	}

	if q.Resolution == 0 {
		q.Resolution = t.Resolution
	}

	if len(opts.PartitionBy) == 0 {
		opts.PartitionBy = t.PartitionBy
	}

	// Combine where clauses
	if t.Where != nil {
		if q.Where == nil {
			q.Where = t.Where
		} else {
			combined, err := goexpr.Binary("AND", q.Where, t.Where)
			if err != nil {
				return err
			}
			q.Where = combined
		}
	}

	return db.doCreateTable(opts, q)
}

func (db *DB) doCreateTable(opts *TableOpts, q *sql.Query) error {
	if !opts.Virtual {
		if opts.RetentionPeriod <= 0 {
			return errors.New("Please specify a positive RetentionPeriod")
		}
		if opts.MinFlushLatency <= 0 {
			log.Debug("MinFlushLatency disabled")
		}
		if opts.MaxFlushLatency <= 0 {
			opts.MaxFlushLatency = time.Duration(math.MaxInt64)
			log.Debug("MaxFlushLatency disabled")
		}
	}
	opts.Name = strings.ToLower(opts.Name)

	// prepend a magic _points field
	newFields := make([]core.Field, 0, len(q.Fields)+1)
	newFields = append(newFields, sql.PointsField)
	for _, field := range q.Fields {
		// Don't add _points twice
		if field.Name != "_points" {
			newFields = append(newFields, field)
		}
	}
	q.Fields = newFields

	t := &table{
		TableOpts: opts,
		Query:     *q,
		db:        db,
		log:       golog.LoggerFor("zenodb." + opts.Name),
	}

	t.applyWhere(q.Where)

	var rsErr error
	var walOffset wal.Offset
	if !t.Virtual {
		t.rowStore, walOffset, rsErr = t.openRowStore(&rowStoreOptions{
			dir:             filepath.Join(db.opts.Dir, t.Name),
			minFlushLatency: t.MinFlushLatency,
			maxFlushLatency: t.MaxFlushLatency,
		})
		if rsErr != nil {
			return rsErr
		}
		t.log.Debugf("Starting at WAL offset %v", walOffset)
	}

	db.tablesMutex.Lock()
	defer db.tablesMutex.Unlock()

	if db.tables[t.Name] != nil {
		return fmt.Errorf("Table %v already exists", t.Name)
	}
	db.tables[t.Name] = t
	db.orderedTables = append(db.orderedTables, t)

	if !t.Virtual {
		partitionKeys := t.PartitionBy
		partitionKeysString := partitionKeysToString(partitionKeys)
		if t.db.distinctPartitionKeys[partitionKeysString] == nil {
			distinctPartitionKeys := make(map[string][]string, len(t.db.distinctPartitionKeys))
			for key, value := range db.distinctPartitionKeys {
				distinctPartitionKeys[key] = value
			}
			distinctPartitionKeys[partitionKeysString] = partitionKeys
			db.distinctPartitionKeys = distinctPartitionKeys
		}

		if !t.db.opts.Passthrough {
			go t.logHighWaterMark()
		}

		if t.db.opts.Follow != nil {
			t.startFollowing(walOffset)
			return nil
		}
		return t.startWALProcessing(walOffset)
	}

	return nil
}

func (t *table) startFollowing(walOffset wal.Offset) {
	newSubscriber := t.db.newStreamSubscriber[t.From]
	if newSubscriber == nil {
		newSubscriber = make(chan *tableWithOffset, 100)
		go t.db.followLeader(t.From, newSubscriber)
		t.db.newStreamSubscriber[t.From] = newSubscriber
	}
	newSubscriber <- &tableWithOffset{t, walOffset}
}

func (t *table) startWALProcessing(walOffset wal.Offset) error {
	var walErr error
	w := t.db.streams[t.From]
	if w == nil {
		walDir := filepath.Join(t.db.opts.Dir, "_wal", t.From)
		dirErr := os.MkdirAll(walDir, 0755)
		if dirErr != nil && !os.IsExist(dirErr) {
			return dirErr
		}
		w, walErr = wal.Open(walDir, t.db.opts.WALSyncInterval)
		if walErr != nil {
			return walErr
		}
		go t.db.capWALAge(w)
		t.db.streams[t.From] = w
	}

	if t.db.opts.Passthrough {
		log.Debugf("Passthrough will not insert data to table %v", t.Name)
		return nil
	}

	log.Debugf("%v will read inserts from %v at offset %v", t.Name, t.From, walOffset)
	t.wal, walErr = w.NewReader(t.Name, walOffset)
	if walErr != nil {
		return fmt.Errorf("Unable to obtain WAL reader: %v", walErr)
	}

	go t.processWALInserts()
	return nil
}

func (t *table) applyWhere(where goexpr.Expr) {
	t.whereMutex.Lock()
	t.Where = where
	t.whereMutex.Unlock()
}

func (t *table) getWhere() goexpr.Expr {
	t.whereMutex.RLock()
	where := t.Where
	t.whereMutex.RUnlock()
	return where
}

func (t *table) truncateBefore() time.Time {
	return t.db.clock.Now().Add(-1 * t.RetentionPeriod)
}

func (t *table) iterate(fields []string, includeMemStore bool, onValue func(bytemap.ByteMap, []encoding.Sequence)) error {
	return t.rowStore.iterate(fields, includeMemStore, onValue)
}

// shouldSort determines whether or not a flush should be sorted. The flush will
// sort if the table is the next table in line to be sorted, and no other sort
// is currently happening. If shouldSort returns true, the flushing process
// must call stopSorting when finished so that other tables have a chance to
// sort.
func (t *table) shouldSort() bool {
	if t.db.opts.MaxMemoryRatio <= 0 {
		return false
	}

	t.db.tablesMutex.RLock()
	if t.db.nextTableToSort >= len(t.db.orderedTables) {
		t.db.nextTableToSort = 0
	}
	nextTableToSort := t.db.orderedTables[t.db.nextTableToSort]
	result := t.Name == nextTableToSort.Name && !t.db.isSorting
	t.db.tablesMutex.RUnlock()
	return result
}

func (t *table) stopSorting() {
	t.db.tablesMutex.RLock()
	t.db.isSorting = false
	t.db.nextTableToSort++
	t.db.tablesMutex.RUnlock()
}

func (t *table) memStoreSize() int {
	return t.rowStore.memStoreSize()
}

func (t *table) forceFlush() {
	t.rowStore.forceFlush()
}

func (t *table) logHighWaterMark() {
	for {
		time.Sleep(15 * time.Second)
		t.highWaterMarkMx.RLock()
		disk := t.highWaterMarkDisk
		memory := t.highWaterMarkMemory
		t.highWaterMarkMx.RUnlock()
		t.log.Debugf("High Water Mark    disk: %v    memory: %v", encoding.TimeFromInt(disk).In(time.UTC), encoding.TimeFromInt(memory).In(time.UTC))
	}
}

func (t *table) updateHighWaterMarkDisk(ts int64) {
	t.highWaterMarkMx.Lock()
	if ts > t.highWaterMarkDisk {
		t.highWaterMarkDisk = ts
	}
	t.highWaterMarkMx.Unlock()
}

func (t *table) updateHighWaterMarkMemory(ts int64) {
	t.highWaterMarkMx.Lock()
	if ts > t.highWaterMarkMemory {
		t.highWaterMarkMemory = ts
	}
	t.highWaterMarkMx.Unlock()
}
<|endoftext|>


<commit_before>package schema

import (
	"database/sql"
	"fmt"
	"strings"

	"github.com/altipla-consulting/database"
	"github.com/altipla-consulting/schema/column"
	"github.com/juju/errors"
	"golang.org/x/net/context"
)

// Table stores information about a table
type Table struct {
	name    string
	columns []column.Column
}

// NewTable initializes the info about a new table.
func NewTable(name string) *Table {
	return &Table{name: name}
}

// Create sends the commands to create a new table in the DB.
func (t *Table) Create(ctx context.Context) error {
	conn := database.FromContext(ctx)

	lines := make([]string, len(t.columns))
	for i, col := range t.columns {
		lines[i] = col.CreateSQL()
	}

	content := strings.Join(lines, ",")
	statement := fmt.Sprintf("CREATE TABLE `%s` (%s) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci", t.name, content)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// Drop sends the commands to drop the table in the DB.
func (t *Table) Drop(ctx context.Context) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("DROP TABLE `%s`", t.name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// Exists checks if the table is already present in the DB.
func (t *Table) Exists(ctx context.Context) (bool, error) {
	conn := database.FromContext(ctx)

	var name string
	err := conn.DB.QueryRow(fmt.Sprintf("SHOW TABLES LIKE '%s';", t.name)).Scan(&name)
	switch {
	case err == sql.ErrNoRows:
		return false, nil

	case err != nil:
		return false, errors.Trace(err)

	default:
		return true, nil
	}
}

// CreateIfNotExists creates the table if it isn't already present.
func (t *Table) CreateIfNotExists(ctx context.Context) error {
	exists, err := t.Exists(ctx)
	switch {
	case err != nil:
		return errors.Trace(err)

	case !exists:
		if err := t.Create(ctx); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

// AddColumn add a new column to the list that will be added when the table is created.
func (t *Table) AddColumn(column column.Column) {
	t.columns = append(t.columns, column)
}

// RenameColumn changes the name of a column. It needs the current type of the column
// and is not recommended to change the type manually with that string (though it's possible).
func (t *Table) RenameColumn(ctx context.Context, oldName, name, columnType string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` CHANGE `%s` `%s` %s", t.name, oldName, name, columnType)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// CreateColumn directly creates a new column in an already present table.
func (t *Table) CreateColumn(ctx context.Context, col column.Column) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` ADD %s", t.name, col.CreateSQL())
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// DropColumn removes a column from the table.
func (t *Table) DropColumn(ctx context.Context, name string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` DROP COLUMN %s", t.name, name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// DropPrimaryKey removes the primary key from a table (not the column, only the index).
func (t *Table) DropPrimaryKey(ctx context.Context) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` DROP PRIMARY KEY", t.name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// AssignPrimaryKey sets the new primary key of the table (it should have been dropped
// before, or not exist in the first place)
func (t *Table) AssignPrimaryKey(ctx context.Context, name string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` ADD PRIMARY KEY (`%s`)", t.name, name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// AddUnique adds a new unique index to a column.
func (t *Table) AddUnique(ctx context.Context, column string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` ADD UNIQUE `%s`", t.name, column)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// DropUnique removes the unique index of a column.
func (t *Table) DropUnique(ctx context.Context, column string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` DROP INDEX `%s`", t.name, column)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// Rename changes the name of the table to a new one.
func (t *Table) Rename(ctx context.Context, name string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("RENAME TABLE `%s` TO `%s`", t.name, name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}
<commit_msg>Fix syntax of the creation of a new unique index.<commit_after>package schema

import (
	"database/sql"
	"fmt"
	"strings"

	"github.com/altipla-consulting/database"
	"github.com/altipla-consulting/schema/column"
	"github.com/juju/errors"
	"golang.org/x/net/context"
)

// Table stores information about a table
type Table struct {
	name    string
	columns []column.Column
}

// NewTable initializes the info about a new table.
func NewTable(name string) *Table {
	return &Table{name: name}
}

// Create sends the commands to create a new table in the DB.
func (t *Table) Create(ctx context.Context) error {
	conn := database.FromContext(ctx)

	lines := make([]string, len(t.columns))
	for i, col := range t.columns {
		lines[i] = col.CreateSQL()
	}

	content := strings.Join(lines, ",")
	statement := fmt.Sprintf("CREATE TABLE `%s` (%s) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci", t.name, content)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// Drop sends the commands to drop the table in the DB.
func (t *Table) Drop(ctx context.Context) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("DROP TABLE `%s`", t.name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// Exists checks if the table is already present in the DB.
func (t *Table) Exists(ctx context.Context) (bool, error) {
	conn := database.FromContext(ctx)

	var name string
	err := conn.DB.QueryRow(fmt.Sprintf("SHOW TABLES LIKE '%s';", t.name)).Scan(&name)
	switch {
	case err == sql.ErrNoRows:
		return false, nil

	case err != nil:
		return false, errors.Trace(err)

	default:
		return true, nil
	}
}

// CreateIfNotExists creates the table if it isn't already present.
func (t *Table) CreateIfNotExists(ctx context.Context) error {
	exists, err := t.Exists(ctx)
	switch {
	case err != nil:
		return errors.Trace(err)

	case !exists:
		if err := t.Create(ctx); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

// AddColumn add a new column to the list that will be added when the table is created.
func (t *Table) AddColumn(column column.Column) {
	t.columns = append(t.columns, column)
}

// RenameColumn changes the name of a column. It needs the current type of the column
// and is not recommended to change the type manually with that string (though it's possible).
func (t *Table) RenameColumn(ctx context.Context, oldName, name, columnType string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` CHANGE `%s` `%s` %s", t.name, oldName, name, columnType)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// CreateColumn directly creates a new column in an already present table.
func (t *Table) CreateColumn(ctx context.Context, col column.Column) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` ADD %s", t.name, col.CreateSQL())
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// DropColumn removes a column from the table.
func (t *Table) DropColumn(ctx context.Context, name string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` DROP COLUMN %s", t.name, name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// DropPrimaryKey removes the primary key from a table (not the column, only the index).
func (t *Table) DropPrimaryKey(ctx context.Context) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` DROP PRIMARY KEY", t.name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// AssignPrimaryKey sets the new primary key of the table (it should have been dropped
// before, or not exist in the first place)
func (t *Table) AssignPrimaryKey(ctx context.Context, name string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` ADD PRIMARY KEY (`%s`)", t.name, name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// AddUnique adds a new unique index to a column.
func (t *Table) AddUnique(ctx context.Context, column string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` ADD UNIQUE INDEX (`%s`)", t.name, column)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// DropUnique removes the unique index of a column.
func (t *Table) DropUnique(ctx context.Context, column string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("ALTER TABLE `%s` DROP INDEX `%s`", t.name, column)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// Rename changes the name of the table to a new one.
func (t *Table) Rename(ctx context.Context, name string) error {
	conn := database.FromContext(ctx)

	statement := fmt.Sprintf("RENAME TABLE `%s` TO `%s`", t.name, name)
	if _, err := conn.DB.Exec(statement); err != nil {
		return errors.Trace(err)
	}

	return nil
}
<|endoftext|>