<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"fmt"

	"golang.org/x/net/context"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	rpc "github.com/keybase/go-framed-msgpack-rpc"
)

type CmdPGPGen struct {
	libkb.Contextified
	arg engine.PGPKeyImportEngineArg
}

var SmallKey = 1024

func (v *CmdPGPGen) ParseArgv(ctx *cli.Context) (err error) {
	nargs := len(ctx.Args())
	if nargs != 0 {
		err = fmt.Errorf("pgp gen takes 0 args")
	} else {
		g := libkb.PGPGenArg{}
		g.PGPUids = ctx.StringSlice("pgp-uid")
		v.arg.DoExport = !ctx.Bool("no-export")
		v.arg.AllowMulti = ctx.Bool("multi")
		if ctx.Bool("debug") {
			g.PrimaryBits = SmallKey
			g.SubkeyBits = SmallKey
		}
		v.arg.Gen = &g
	}
	return err
}

// Why use CreatePGPIDs rather than MakeAllIds?
func (v *CmdPGPGen) Run() (err error) {
	protocols := []rpc.Protocol{
		NewSecretUIProtocol(G),
	}
	cli, err := GetPGPClient()
	if err != nil {
		return err
	}
	if err = RegisterProtocols(protocols); err != nil {
		return err
	}

	// Prompt for user IDs if none given on command line
	if len(v.arg.Gen.PGPUids) == 0 {
		if err = v.propmptPGPIDs(); err != nil {
			return err
		}
	} else if err = v.arg.Gen.CreatePGPIDs(); err != nil {
		return err
	}
	v.arg.PushSecret, err = v.G().UI.GetTerminalUI().PromptYesNo(PromptDescriptorPGPGenPushSecret, "Push an encrypted copy of your new secret key to the Keybase.io server?", libkb.PromptDefaultYes)
	if err != nil {
		return err
	}

	err = cli.PGPKeyGen(context.TODO(), v.arg.Export())
	err = AddPGPMultiInstructions(err)
	return err
}

var CheckRealName = libkb.Checker{
	F: func(s string) bool {
		nameID, err := libkb.ParseIdentity(s)
		if err != nil {
			return false
		}
		return len(nameID.Username) > 0 && len(nameID.Comment) == 0 && len(nameID.Email) == 0
	},
	Hint: "for example: \"Ned Snowben\"",
}

var CheckOptionalEmail = libkb.Checker{
	F: func(s string) bool {
		if len(s) == 0 {
			return true
		}
		return libkb.CheckEmail.F(s)
	},
	Hint: libkb.CheckEmail.Hint,
}

func (v *CmdPGPGen) propmptPGPIDs() (err error) {
	id := libkb.Identity{}
	prompt := "Enter your real name, which will be publicly visible in your new key"
	id.Username, err = PromptWithChecker(PromptDescriptorPGPGenEnterID, v.G().UI.GetTerminalUI(), prompt, false, CheckRealName)
	if err != nil {
		return
	}
	// Email required for primary ID
	prompt = "Enter a public email address for your key"
	id.Email, err = PromptWithChecker(PromptDescriptorPGPGenEnterID, v.G().UI.GetTerminalUI(), prompt, false, libkb.CheckEmail)
	if err != nil {
		return
	}
	v.arg.Gen.Ids = append(v.arg.Gen.Ids, id)

	emailsSeen := make(map[string]struct{})

	emailsSeen[id.Email] = struct{}{}

	idAdditional := libkb.Identity{
		Username: id.Username,
	}
	prompt = "Enter another email address (or <enter> when done)"
	for {
		idAdditional.Email, err = PromptWithChecker(PromptDescriptorPGPGenEnterID, v.G().UI.GetTerminalUI(), prompt, false, CheckOptionalEmail)
		if err != nil || len(idAdditional.Email) == 0 {
			break
		}

		// Make sure it hasn't been added already
		if _, ok := emailsSeen[idAdditional.Email]; ok {
			v.G().Log.Warning("Email already applied to this key")
			continue
		}

		emailsSeen[idAdditional.Email] = struct{}{}
		v.arg.Gen.Ids = append(v.arg.Gen.Ids, idAdditional)
	}

	return
}

func AddPGPMultiInstructions(err error) error {
	if err == nil {
		return nil
	}
	if kee, ok := err.(libkb.KeyExistsError); ok {
		return fmt.Errorf("You already have a PGP key registered (%s)\n"+
			"Specify the `--multi` flag to override this check",
			kee.Key.ToQuads())
	}
	// Not the right type. Return it as is.
	return err
}

func NewCmdPGPGen(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "gen",
		Usage: "Generate a new PGP key and write to local secret keychain",
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "d, debug",
				Usage: "Generate small keys for debugging.",
			},
			cli.StringSliceFlag{
				Name:  "pgp-uid",
				Usage: "Specify custom PGP uid(s).",
				Value: &cli.StringSlice{},
			},
			cli.BoolFlag{
				Name:  "multi",
				Usage: "Allow multiple PGP keys.",
			},
			cli.BoolFlag{
				Name:  "no-export",
				Usage: "Disable exporting of new keys to GPG keychain.",
			},
		},
		Description: `"keybase pgp gen" generates a new PGP key for this account.
   In all cases, it signs the public key with an exising device key,
   and pushes the signature to the server. Thus, the user will have a
   publicly-visible "PGP device" after running this operation.

   The secret half of the PGP key is written by default to the user's
   local Keybase keychain and encrypted with the "local key security"
   (LKS) protocol. (For more information, try 'keybase help keyring').
   Also, by default, the public half of the new PGP key
   is exported to the local GnuPG keyring, if one is found.  (For now,
   you must export the secret half to gpg manually with a command like
   'keybase pgp export -s | gpg --import'.)

   On subsequent secret key accesses --- say for PGP decryption or
   for signing --- access to the local GnuGP keyring is not required.
   Rather, keybase will access the secret PGP key in its own local keychain.

   By default, the secret half of the PGP key is never exported off
   of the local system, but users have a choice via terminal prompt
   to select storage of their encrypted secret PGP key on the Keybase
   servers.`,
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdPGPGen{Contextified: libkb.NewContextified(g)}, "gen", c)
		},
	}
}

func (v *CmdPGPGen) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		KbKeyring: true,
		API:       true,
	}
}
<commit_msg>Revert "Arjunravinarayan names" (#3840)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"fmt"

	"golang.org/x/net/context"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	rpc "github.com/keybase/go-framed-msgpack-rpc"
)

type CmdPGPGen struct {
	libkb.Contextified
	arg engine.PGPKeyImportEngineArg
}

var SmallKey = 1024

func (v *CmdPGPGen) ParseArgv(ctx *cli.Context) (err error) {
	nargs := len(ctx.Args())
	if nargs != 0 {
		err = fmt.Errorf("pgp gen takes 0 args")
	} else {
		g := libkb.PGPGenArg{}
		g.PGPUids = ctx.StringSlice("pgp-uid")
		v.arg.DoExport = !ctx.Bool("no-export")
		v.arg.AllowMulti = ctx.Bool("multi")
		if ctx.Bool("debug") {
			g.PrimaryBits = SmallKey
			g.SubkeyBits = SmallKey
		}
		v.arg.Gen = &g
	}
	return err
}

// Why use CreatePGPIDs rather than MakeAllIds?
func (v *CmdPGPGen) Run() (err error) {
	protocols := []rpc.Protocol{
		NewSecretUIProtocol(G),
	}
	cli, err := GetPGPClient()
	if err != nil {
		return err
	}
	if err = RegisterProtocols(protocols); err != nil {
		return err
	}

	// Prompt for user IDs if none given on command line
	if len(v.arg.Gen.PGPUids) == 0 {
		if err = v.propmptPGPIDs(); err != nil {
			return err
		}
	} else if err = v.arg.Gen.CreatePGPIDs(); err != nil {
		return err
	}
	v.arg.PushSecret, err = v.G().UI.GetTerminalUI().PromptYesNo(PromptDescriptorPGPGenPushSecret, "Push an encrypted copy of your new secret key to the Keybase.io server?", libkb.PromptDefaultYes)
	if err != nil {
		return err
	}

	err = cli.PGPKeyGen(context.TODO(), v.arg.Export())
	err = AddPGPMultiInstructions(err)
	return err
}

var CheckRealName = libkb.Checker{
	F: func(s string) bool {
		nameID, err := libkb.ParseIdentity(s)
		if err != nil {
			return false
		}
		return len(nameID.Username) > 0 && len(nameID.Comment) == 0 && len(nameID.Email) == 0
	},
	Hint: "for example: \"Ned Snowben\"",
}

var CheckOptionalEmail = libkb.Checker{
	F: func(s string) bool {
		if len(s) == 0 {
			return true
		}
		return libkb.CheckEmail.F(s)
	},
	Hint: libkb.CheckEmail.Hint,
}

func (v *CmdPGPGen) propmptPGPIDs() (err error) {
	id := libkb.Identity{}
	prompt := "Enter your real name, which will be publicly visible in your new key"
	id.Username, err = PromptWithChecker(PromptDescriptorPGPGenEnterID, v.G().UI.GetTerminalUI(), prompt, false, CheckRealName)
	if err != nil {
		return
	}
	// Email required for primary ID
	prompt = "Enter a public email address for your key"
	id.Email, err = PromptWithChecker(PromptDescriptorPGPGenEnterID, v.G().UI.GetTerminalUI(), prompt, false, libkb.CheckEmail)
	if err != nil {
		return
	}
	v.arg.Gen.Ids = append(v.arg.Gen.Ids, id)

	emailsSeen := make(map[string]struct{})

	emailsSeen[id.Email] = struct{}{}

	idAdditional := libkb.Identity{}
	prompt = "Enter another email address (or <enter> when done)"
	for {
		idAdditional.Email, err = PromptWithChecker(PromptDescriptorPGPGenEnterID, v.G().UI.GetTerminalUI(), prompt, false, CheckOptionalEmail)
		if err != nil || len(idAdditional.Email) == 0 {
			break
		}

		// Make sure it hasn't been added already
		if _, ok := emailsSeen[idAdditional.Email]; ok {
			v.G().Log.Warning("Email already applied to this key")
			continue
		}

		emailsSeen[idAdditional.Email] = struct{}{}
		v.arg.Gen.Ids = append(v.arg.Gen.Ids, idAdditional)
	}

	return
}

func AddPGPMultiInstructions(err error) error {
	if err == nil {
		return nil
	}
	if kee, ok := err.(libkb.KeyExistsError); ok {
		return fmt.Errorf("You already have a PGP key registered (%s)\n"+
			"Specify the `--multi` flag to override this check",
			kee.Key.ToQuads())
	}
	// Not the right type. Return it as is.
	return err
}

func NewCmdPGPGen(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "gen",
		Usage: "Generate a new PGP key and write to local secret keychain",
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "d, debug",
				Usage: "Generate small keys for debugging.",
			},
			cli.StringSliceFlag{
				Name:  "pgp-uid",
				Usage: "Specify custom PGP uid(s).",
				Value: &cli.StringSlice{},
			},
			cli.BoolFlag{
				Name:  "multi",
				Usage: "Allow multiple PGP keys.",
			},
			cli.BoolFlag{
				Name:  "no-export",
				Usage: "Disable exporting of new keys to GPG keychain.",
			},
		},
		Description: `"keybase pgp gen" generates a new PGP key for this account.
   In all cases, it signs the public key with an exising device key,
   and pushes the signature to the server. Thus, the user will have a
   publicly-visible "PGP device" after running this operation.

   The secret half of the PGP key is written by default to the user's
   local Keybase keychain and encrypted with the "local key security"
   (LKS) protocol. (For more information, try 'keybase help keyring').
   Also, by default, the public half of the new PGP key
   is exported to the local GnuPG keyring, if one is found.  (For now,
   you must export the secret half to gpg manually with a command like
   'keybase pgp export -s | gpg --import'.)

   On subsequent secret key accesses --- say for PGP decryption or
   for signing --- access to the local GnuGP keyring is not required.
   Rather, keybase will access the secret PGP key in its own local keychain.

   By default, the secret half of the PGP key is never exported off
   of the local system, but users have a choice via terminal prompt
   to select storage of their encrypted secret PGP key on the Keybase
   servers.`,
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdPGPGen{Contextified: libkb.NewContextified(g)}, "gen", c)
		},
	}
}

func (v *CmdPGPGen) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		KbKeyring: true,
		API:       true,
	}
}
<|endoftext|>


<commit_before>package gitstore

import (
	"context"
	"time"

	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/git/repograph"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/vcsinfo"
)

// GetRepoGraph returns *repograph.Graph backed by the given GitStore.
func GetRepoGraph(ctx context.Context, gs GitStore) (*repograph.Graph, error) {
	ri, err := NewGitStoreRepoImpl(ctx, gs)
	if err != nil {
		return nil, err
	}
	return repograph.NewWithRepoImpl(ctx, ri)
}

// gitStoreRepoImpl is an implementation of the repograph.RepoImpl interface
// which uses a GitStore to interact with a git repo.
type gitStoreRepoImpl struct {
	*repograph.MemCacheRepoImpl
	gs         GitStore
	lastUpdate time.Time
}

// NewGitStoreRepoImpl returns a repograph.RepoImpl instance which uses the
// given GitStore.
func NewGitStoreRepoImpl(ctx context.Context, gs GitStore) (repograph.RepoImpl, error) {
	rv := &gitStoreRepoImpl{
		MemCacheRepoImpl: repograph.NewMemCacheRepoImpl(nil, nil),
		gs:               gs,
	}
	if err := rv.Update(ctx); err != nil {
		return nil, err
	}
	return rv, nil
}

// See documentation for repograph.RepoImpl interface.
func (g *gitStoreRepoImpl) Update(ctx context.Context) error {
	branchPtrs, err := g.gs.GetBranches(ctx)
	if err != nil {
		return skerr.Wrapf(err, "Failed to read branches from GitStore")
	}
	branches := make([]*git.Branch, 0, len(branchPtrs))
	for name, ptr := range branchPtrs {
		if name != ALL_BRANCHES {
			branches = append(branches, &git.Branch{
				Name: name,
				Head: ptr.Head,
			})
		}
	}

	from := g.lastUpdate.Add(-10 * time.Minute)
	now := time.Now()
	to := now.Add(time.Second)
	indexCommits, err := g.gs.RangeByTime(ctx, from, to, ALL_BRANCHES)
	if err != nil {
		return skerr.Wrapf(err, "Failed to read IndexCommits from GitStore")
	}
	hashes := make([]string, 0, len(indexCommits))
	for _, c := range indexCommits {
		hashes = append(hashes, c.Hash)
	}
	commits, err := g.gs.Get(ctx, hashes)
	if err != nil {
		return skerr.Wrapf(err, "Failed to read LongCommits from GitStore")
	}
	commitsMap := make(map[string]*vcsinfo.LongCommit, len(commits))
	for _, c := range commits {
		commitsMap[c.Hash] = c
	}

	g.lastUpdate = now
	g.BranchList = branches
	g.Commits = commitsMap
	return nil
}

// See documentation for repograph.RepoImpl interface.
func (g *gitStoreRepoImpl) Details(ctx context.Context, hash string) (*vcsinfo.LongCommit, error) {
	d, err := g.MemCacheRepoImpl.Details(ctx, hash)
	if err == nil {
		return d, nil
	}
	// Update() should have pre-fetched all of the commits for us, so we
	// shouldn't have hit this code. Log a warning and fall back to
	// retrieving the commit from GitStore.
	sklog.Warningf("Commit %q not found in cache; performing explicit lookup.", hash)
	got, err := g.gs.Get(ctx, []string{hash})
	if err != nil {
		return nil, skerr.Wrapf(err, "Failed to read commit %s from GitStore", hash)
	}
	for _, c := range got {
		if c == nil {
			return nil, skerr.Fmt("Commit %s not present in GitStore.", hash)
		}
		g.Commits[c.Hash] = c
	}
	return got[0], nil
}
<commit_msg>[gitstore] Check for nil commits during RepoImpl.Update()<commit_after>package gitstore

import (
	"context"
	"time"

	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/git/repograph"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/vcsinfo"
)

// GetRepoGraph returns *repograph.Graph backed by the given GitStore.
func GetRepoGraph(ctx context.Context, gs GitStore) (*repograph.Graph, error) {
	ri, err := NewGitStoreRepoImpl(ctx, gs)
	if err != nil {
		return nil, err
	}
	return repograph.NewWithRepoImpl(ctx, ri)
}

// gitStoreRepoImpl is an implementation of the repograph.RepoImpl interface
// which uses a GitStore to interact with a git repo.
type gitStoreRepoImpl struct {
	*repograph.MemCacheRepoImpl
	gs         GitStore
	lastUpdate time.Time
}

// NewGitStoreRepoImpl returns a repograph.RepoImpl instance which uses the
// given GitStore.
func NewGitStoreRepoImpl(ctx context.Context, gs GitStore) (repograph.RepoImpl, error) {
	rv := &gitStoreRepoImpl{
		MemCacheRepoImpl: repograph.NewMemCacheRepoImpl(nil, nil),
		gs:               gs,
	}
	if err := rv.Update(ctx); err != nil {
		return nil, err
	}
	return rv, nil
}

// See documentation for repograph.RepoImpl interface.
func (g *gitStoreRepoImpl) Update(ctx context.Context) error {
	branchPtrs, err := g.gs.GetBranches(ctx)
	if err != nil {
		return skerr.Wrapf(err, "Failed to read branches from GitStore")
	}
	branches := make([]*git.Branch, 0, len(branchPtrs))
	for name, ptr := range branchPtrs {
		if name != ALL_BRANCHES {
			branches = append(branches, &git.Branch{
				Name: name,
				Head: ptr.Head,
			})
		}
	}

	from := g.lastUpdate.Add(-10 * time.Minute)
	now := time.Now()
	to := now.Add(time.Second)
	indexCommits, err := g.gs.RangeByTime(ctx, from, to, ALL_BRANCHES)
	if err != nil {
		return skerr.Wrapf(err, "Failed to read IndexCommits from GitStore")
	}
	hashes := make([]string, 0, len(indexCommits))
	for _, c := range indexCommits {
		hashes = append(hashes, c.Hash)
	}
	commits, err := g.gs.Get(ctx, hashes)
	if err != nil {
		return skerr.Wrapf(err, "Failed to read LongCommits from GitStore")
	}
	commitsMap := make(map[string]*vcsinfo.LongCommit, len(commits))
	for idx, c := range commits {
		if c == nil {
			return skerr.Fmt("Found IndexCommit but no LongCommit for %s; this may be due to an eventually-consistent DB implementation being slightly out of date, or it could be because GitSync is failing.", hashes[idx])
		}
		commitsMap[c.Hash] = c
	}

	g.lastUpdate = now
	g.BranchList = branches
	g.Commits = commitsMap
	return nil
}

// See documentation for repograph.RepoImpl interface.
func (g *gitStoreRepoImpl) Details(ctx context.Context, hash string) (*vcsinfo.LongCommit, error) {
	d, err := g.MemCacheRepoImpl.Details(ctx, hash)
	if err == nil {
		return d, nil
	}
	// Update() should have pre-fetched all of the commits for us, so we
	// shouldn't have hit this code. Log a warning and fall back to
	// retrieving the commit from GitStore.
	sklog.Warningf("Commit %q not found in cache; performing explicit lookup.", hash)
	got, err := g.gs.Get(ctx, []string{hash})
	if err != nil {
		return nil, skerr.Wrapf(err, "Failed to read commit %s from GitStore", hash)
	}
	for _, c := range got {
		if c == nil {
			return nil, skerr.Fmt("Commit %s not present in GitStore.", hash)
		}
		g.Commits[c.Hash] = c
	}
	return got[0], nil
}
<|endoftext|>


<commit_before>package detector

import (
	"encoding/json"

	"github.com/lovoo/goka"
	"github.com/lovoo/goka/examples/3-messaging"
	"github.com/lovoo/goka/examples/3-messaging/blocker"
)

const (
	minMessages = 1000
	maxRate     = 0.90
)

var (
	group goka.Group = "detector"
)

type Counters struct {
	Sent     int
	Received int
}

type CountersCodec struct{}

func (c *CountersCodec) Encode(value interface{}) ([]byte, error) {
	return json.Marshal(value)
}

func (c *CountersCodec) Decode(data []byte) (interface{}, error) {
	var m Counters
	return &m, json.Unmarshal(data, &m)
}

func getValue(ctx goka.Context) *Counters {
	if v := ctx.Value(); v != nil {
		return v.(*Counters)
	}
	return &Counters{}
}

func loopToSender(ctx goka.Context, msg interface{}) {
	m := msg.(*messaging.Message)
	ctx.Loopback(m.From, m)
}

func detectSpammer(ctx goka.Context, c *Counters) {
	var (
		total = float64(c.Sent + c.Received)
		rate  = float64(c.Sent) / total
	)
	if total >= minMessages && rate >= maxRate {
		ctx.Emit(blocker.Stream, ctx.Key(), new(blocker.BlockEvent))
	}
}

func Run(brokers []string) {
	g := goka.DefineGroup(group,
		goka.Input(messaging.SentStream, new(messaging.MessageCodec), func(ctx goka.Context, msg interface{}) {
			c := getValue(ctx)
			c.Received++
			ctx.SetValue(c)
			loopToSender(ctx, msg)
		}),
		goka.Loop(new(messaging.MessageCodec), func(ctx goka.Context, msg interface{}) {
			c := getValue(ctx)
			c.Sent++
			ctx.SetValue(c)
			detectSpammer(ctx, c)
		}),
		goka.Output(blocker.Stream, new(blocker.BlockEventCodec)),
		goka.Persist(new(CountersCodec)),
	)
	if p, err := goka.NewProcessor(brokers, g); err != nil {
		panic(err)
	} else if err = p.Start(); err != nil {
		panic(err)
	}
}
<commit_msg>fixing detector<commit_after>package detector

import (
	"encoding/json"

	"github.com/lovoo/goka"
	"github.com/lovoo/goka/examples/3-messaging"
	"github.com/lovoo/goka/examples/3-messaging/blocker"
)

const (
	minMessages = 200
	maxRate     = 0.5
)

var (
	group goka.Group = "detector"
)

type Counters struct {
	Sent     int
	Received int
}

type CountersCodec struct{}

func (c *CountersCodec) Encode(value interface{}) ([]byte, error) {
	return json.Marshal(value)
}

func (c *CountersCodec) Decode(data []byte) (interface{}, error) {
	var m Counters
	return &m, json.Unmarshal(data, &m)
}

func getValue(ctx goka.Context) *Counters {
	if v := ctx.Value(); v != nil {
		return v.(*Counters)
	}
	return &Counters{}
}

func detectSpammer(ctx goka.Context, c *Counters) {
	var (
		total = float64(c.Sent + c.Received)
		rate  = float64(c.Sent) / total
	)
	if total >= minMessages && rate >= maxRate {
		ctx.Emit(blocker.Stream, ctx.Key(), new(blocker.BlockEvent))
	}
}

func Run(brokers []string) {
	g := goka.DefineGroup(group,
		goka.Input(messaging.SentStream, new(messaging.MessageCodec), func(ctx goka.Context, msg interface{}) {
			c := getValue(ctx)
			c.Sent++
			ctx.SetValue(c)
			m := msg.(*messaging.Message)
			// Loop to receiver
			ctx.Loopback(m.To, m)
			// Check if sender is a spammer
			detectSpammer(ctx, c)
		}),
		goka.Loop(new(messaging.MessageCodec), func(ctx goka.Context, msg interface{}) {
			c := getValue(ctx)
			c.Received++
			ctx.SetValue(c)
			// Check if receiver is a spammer
			detectSpammer(ctx, c)
		}),
		goka.Output(blocker.Stream, new(blocker.BlockEventCodec)),
		goka.Persist(new(CountersCodec)),
	)
	if p, err := goka.NewProcessor(brokers, g); err != nil {
		panic(err)
	} else if err = p.Start(); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package loader_test

// This file enumerates all packages beneath $GOROOT, loads them, plus
// their external tests if any, runs the type checker on them, and
// prints some summary information.
//
// Run test with GOMAXPROCS=8.

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"io/ioutil"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"golang.org/x/tools/go/buildutil"
	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

// The set of packages that transitively depend on cmd/internal/objfile,
// which uses vendoring, which go/loader does not yet support.
// TODO(adonovan): add support for vendoring and delete this.
var skip = map[string]bool{
	"cmd/addr2line":        true,
	"cmd/internal/objfile": true,
	"cmd/nm":               true,
	"cmd/objdump":          true,
	"cmd/pprof":            true,
}

func TestStdlib(t *testing.T) {
	if runtime.GOOS == "android" {
		t.Skipf("incomplete std lib on %s", runtime.GOOS)
	}

	runtime.GC()
	t0 := time.Now()
	var memstats runtime.MemStats
	runtime.ReadMemStats(&memstats)
	alloc := memstats.Alloc

	// Load, parse and type-check the program.
	ctxt := build.Default // copy
	ctxt.GOPATH = ""      // disable GOPATH
	conf := loader.Config{Build: &ctxt}
	for _, path := range buildutil.AllPackages(conf.Build) {
		if skip[path] {
			continue
		}
		conf.ImportWithTests(path)
	}

	prog, err := conf.Load()
	if err != nil {
		t.Fatalf("Load failed: %v", err)
	}

	t1 := time.Now()
	runtime.GC()
	runtime.ReadMemStats(&memstats)

	numPkgs := len(prog.AllPackages)
	if want := 205; numPkgs < want {
		t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want)
	}

	// Dump package members.
	if false {
		for pkg := range prog.AllPackages {
			fmt.Printf("Package %s:\n", pkg.Path())
			scope := pkg.Scope()
			qualifier := types.RelativeTo(pkg)
			for _, name := range scope.Names() {
				if ast.IsExported(name) {
					fmt.Printf("\t%s\n", types.ObjectString(scope.Lookup(name), qualifier))
				}
			}
			fmt.Println()
		}
	}

	// Check that Test functions for io/ioutil, regexp and
	// compress/bzip2 are all simultaneously present.
	// (The apparent cycle formed when augmenting all three of
	// these packages by their tests was the original motivation
	// for reporting b/7114.)
	//
	// compress/bzip2.TestBitReader in bzip2_test.go    imports io/ioutil
	// io/ioutil.TestTempFile       in tempfile_test.go imports regexp
	// regexp.TestRE2Search         in exec_test.go     imports compress/bzip2
	for _, test := range []struct{ pkg, fn string }{
		{"io/ioutil", "TestTempFile"},
		{"regexp", "TestRE2Search"},
		{"compress/bzip2", "TestBitReader"},
	} {
		info := prog.Imported[test.pkg]
		if info == nil {
			t.Errorf("failed to load package %q", test.pkg)
			continue
		}
		obj, _ := info.Pkg.Scope().Lookup(test.fn).(*types.Func)
		if obj == nil {
			t.Errorf("package %q has no func %q", test.pkg, test.fn)
			continue
		}
	}

	// Dump some statistics.

	// determine line count
	var lineCount int
	prog.Fset.Iterate(func(f *token.File) bool {
		lineCount += f.LineCount()
		return true
	})

	t.Log("GOMAXPROCS:           ", runtime.GOMAXPROCS(0))
	t.Log("#Source lines:        ", lineCount)
	t.Log("Load/parse/typecheck: ", t1.Sub(t0))
	t.Log("#MB:                  ", int64(memstats.Alloc-alloc)/1000000)
}

func TestCgoOption(t *testing.T) {
	switch runtime.GOOS {
	// On these systems, the net and os/user packages don't use cgo
	// or the std library is incomplete (Android).
	case "android", "plan9", "solaris", "windows":
		t.Skipf("no cgo or incomplete std lib on %s", runtime.GOOS)
	}
	// In nocgo builds (e.g. linux-amd64-nocgo),
	// there is no "runtime/cgo" package,
	// so cgo-generated Go files will have a failing import.
	if !build.Default.CgoEnabled {
		return
	}
	// Test that we can load cgo-using packages with
	// CGO_ENABLED=[01], which causes go/build to select pure
	// Go/native implementations, respectively, based on build
	// tags.
	//
	// Each entry specifies a package-level object and the generic
	// file expected to define it when cgo is disabled.
	// When cgo is enabled, the exact file is not specified (since
	// it varies by platform), but must differ from the generic one.
	//
	// The test also loads the actual file to verify that the
	// object is indeed defined at that location.
	for _, test := range []struct {
		pkg, name, genericFile string
	}{
		{"net", "cgoLookupHost", "cgo_stub.go"},
		{"os/user", "lookupId", "lookup_stubs.go"},
	} {
		ctxt := build.Default
		for _, ctxt.CgoEnabled = range []bool{false, true} {
			conf := loader.Config{Build: &ctxt}
			conf.Import(test.pkg)
			prog, err := conf.Load()
			if err != nil {
				t.Errorf("Load failed: %v", err)
				continue
			}
			info := prog.Imported[test.pkg]
			if info == nil {
				t.Errorf("package %s not found", test.pkg)
				continue
			}
			obj := info.Pkg.Scope().Lookup(test.name)
			if obj == nil {
				t.Errorf("no object %s.%s", test.pkg, test.name)
				continue
			}
			posn := prog.Fset.Position(obj.Pos())
			t.Logf("%s: %s (CgoEnabled=%t)", posn, obj, ctxt.CgoEnabled)

			gotFile := filepath.Base(posn.Filename)
			filesMatch := gotFile == test.genericFile

			if ctxt.CgoEnabled && filesMatch {
				t.Errorf("CGO_ENABLED=1: %s found in %s, want native file",
					obj, gotFile)
			} else if !ctxt.CgoEnabled && !filesMatch {
				t.Errorf("CGO_ENABLED=0: %s found in %s, want %s",
					obj, gotFile, test.genericFile)
			}

			// Load the file and check the object is declared at the right place.
			b, err := ioutil.ReadFile(posn.Filename)
			if err != nil {
				t.Errorf("can't read %s: %s", posn.Filename, err)
				continue
			}
			line := string(bytes.Split(b, []byte("\n"))[posn.Line-1])
			ident := line[posn.Column-1:]
			if !strings.HasPrefix(ident, test.name) {
				t.Errorf("%s: %s not declared here (looking at %q)", posn, obj, ident)
			}
		}
	}
}
<commit_msg>go/loader: in stdlib_test, remove workaround for lack of vendoring<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package loader_test

// This file enumerates all packages beneath $GOROOT, loads them, plus
// their external tests if any, runs the type checker on them, and
// prints some summary information.

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"io/ioutil"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"golang.org/x/tools/go/buildutil"
	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

func TestStdlib(t *testing.T) {
	if runtime.GOOS == "android" {
		t.Skipf("incomplete std lib on %s", runtime.GOOS)
	}

	runtime.GC()
	t0 := time.Now()
	var memstats runtime.MemStats
	runtime.ReadMemStats(&memstats)
	alloc := memstats.Alloc

	// Load, parse and type-check the program.
	ctxt := build.Default // copy
	ctxt.GOPATH = ""      // disable GOPATH
	conf := loader.Config{Build: &ctxt}
	for _, path := range buildutil.AllPackages(conf.Build) {
		conf.ImportWithTests(path)
	}

	prog, err := conf.Load()
	if err != nil {
		t.Fatalf("Load failed: %v", err)
	}

	t1 := time.Now()
	runtime.GC()
	runtime.ReadMemStats(&memstats)

	numPkgs := len(prog.AllPackages)
	if want := 205; numPkgs < want {
		t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want)
	}

	// Dump package members.
	if false {
		for pkg := range prog.AllPackages {
			fmt.Printf("Package %s:\n", pkg.Path())
			scope := pkg.Scope()
			qualifier := types.RelativeTo(pkg)
			for _, name := range scope.Names() {
				if ast.IsExported(name) {
					fmt.Printf("\t%s\n", types.ObjectString(scope.Lookup(name), qualifier))
				}
			}
			fmt.Println()
		}
	}

	// Check that Test functions for io/ioutil, regexp and
	// compress/bzip2 are all simultaneously present.
	// (The apparent cycle formed when augmenting all three of
	// these packages by their tests was the original motivation
	// for reporting b/7114.)
	//
	// compress/bzip2.TestBitReader in bzip2_test.go    imports io/ioutil
	// io/ioutil.TestTempFile       in tempfile_test.go imports regexp
	// regexp.TestRE2Search         in exec_test.go     imports compress/bzip2
	for _, test := range []struct{ pkg, fn string }{
		{"io/ioutil", "TestTempFile"},
		{"regexp", "TestRE2Search"},
		{"compress/bzip2", "TestBitReader"},
	} {
		info := prog.Imported[test.pkg]
		if info == nil {
			t.Errorf("failed to load package %q", test.pkg)
			continue
		}
		obj, _ := info.Pkg.Scope().Lookup(test.fn).(*types.Func)
		if obj == nil {
			t.Errorf("package %q has no func %q", test.pkg, test.fn)
			continue
		}
	}

	// Dump some statistics.

	// determine line count
	var lineCount int
	prog.Fset.Iterate(func(f *token.File) bool {
		lineCount += f.LineCount()
		return true
	})

	t.Log("GOMAXPROCS:           ", runtime.GOMAXPROCS(0))
	t.Log("#Source lines:        ", lineCount)
	t.Log("Load/parse/typecheck: ", t1.Sub(t0))
	t.Log("#MB:                  ", int64(memstats.Alloc-alloc)/1000000)
}

func TestCgoOption(t *testing.T) {
	switch runtime.GOOS {
	// On these systems, the net and os/user packages don't use cgo
	// or the std library is incomplete (Android).
	case "android", "plan9", "solaris", "windows":
		t.Skipf("no cgo or incomplete std lib on %s", runtime.GOOS)
	}
	// In nocgo builds (e.g. linux-amd64-nocgo),
	// there is no "runtime/cgo" package,
	// so cgo-generated Go files will have a failing import.
	if !build.Default.CgoEnabled {
		return
	}
	// Test that we can load cgo-using packages with
	// CGO_ENABLED=[01], which causes go/build to select pure
	// Go/native implementations, respectively, based on build
	// tags.
	//
	// Each entry specifies a package-level object and the generic
	// file expected to define it when cgo is disabled.
	// When cgo is enabled, the exact file is not specified (since
	// it varies by platform), but must differ from the generic one.
	//
	// The test also loads the actual file to verify that the
	// object is indeed defined at that location.
	for _, test := range []struct {
		pkg, name, genericFile string
	}{
		{"net", "cgoLookupHost", "cgo_stub.go"},
		{"os/user", "lookupId", "lookup_stubs.go"},
	} {
		ctxt := build.Default
		for _, ctxt.CgoEnabled = range []bool{false, true} {
			conf := loader.Config{Build: &ctxt}
			conf.Import(test.pkg)
			prog, err := conf.Load()
			if err != nil {
				t.Errorf("Load failed: %v", err)
				continue
			}
			info := prog.Imported[test.pkg]
			if info == nil {
				t.Errorf("package %s not found", test.pkg)
				continue
			}
			obj := info.Pkg.Scope().Lookup(test.name)
			if obj == nil {
				t.Errorf("no object %s.%s", test.pkg, test.name)
				continue
			}
			posn := prog.Fset.Position(obj.Pos())
			t.Logf("%s: %s (CgoEnabled=%t)", posn, obj, ctxt.CgoEnabled)

			gotFile := filepath.Base(posn.Filename)
			filesMatch := gotFile == test.genericFile

			if ctxt.CgoEnabled && filesMatch {
				t.Errorf("CGO_ENABLED=1: %s found in %s, want native file",
					obj, gotFile)
			} else if !ctxt.CgoEnabled && !filesMatch {
				t.Errorf("CGO_ENABLED=0: %s found in %s, want %s",
					obj, gotFile, test.genericFile)
			}

			// Load the file and check the object is declared at the right place.
			b, err := ioutil.ReadFile(posn.Filename)
			if err != nil {
				t.Errorf("can't read %s: %s", posn.Filename, err)
				continue
			}
			line := string(bytes.Split(b, []byte("\n"))[posn.Line-1])
			ident := line[posn.Column-1:]
			if !strings.HasPrefix(ident, test.name) {
				t.Errorf("%s: %s not declared here (looking at %q)", posn, obj, ident)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
)

// Command represents a CLI command.
// For example, in the command `heroku apps:create` the command would be `create`.
// They must have a Topic name that links to a real topic's name.
type Command struct {
	Topic        string             `json:"topic"`
	Command      string             `json:"command,omitempty"`
	Plugin       string             `json:"plugin"`
	Usage        string             `json:"usage"`
	Description  string             `json:"description"`
	Help         string             `json:"help"`
	FullHelp     string             `json:"fullHelp"`
	Hidden       bool               `json:"hidden"`
	NeedsApp     bool               `json:"needsApp"`
	NeedsAuth    bool               `json:"needsAuth"`
	VariableArgs bool               `json:"variableArgs"`
	Args         []Arg              `json:"args"`
	Flags        []Flag             `json:"flags"`
	Run          func(ctx *Context) `json:"-"`
}

func (c *Command) String() string {
	if c.Command == "" {
		return c.Topic
	}
	return c.Topic + ":" + c.Command
}

func commandUsage(c *Command) string {
	return c.String() + argsString(c.Args)
}

func (c *Command) buildFullHelp() string {
	if len(c.Flags) == 0 {
		return c.Help
	}
	lines := make([]string, 0, len(c.Flags))
	for _, flag := range c.Flags {
		if flag.Hidden {
			continue
		}
		if flag.Description == "" {
			lines = append(lines, flag.String())
		} else {
			lines = append(lines, fmt.Sprintf("%-20s # %s", flag.String(), flag.Description))
		}
	}
	return strings.Join(lines, "\n") + "\n\n" + c.Help
}

// CommandSet is a slice of Command structs with some helper methods.
type CommandSet []*Command

// ByTopicAndCommand returns a command that matches the passed topic and command.
func (commands CommandSet) ByTopicAndCommand(topic, command string) *Command {
	for _, c := range commands {
		if c.Topic == topic && c.Command == command {
			return c
		}
	}
	return nil
}

func (commands CommandSet) loadUsages() {
	for _, c := range commands {
		if c.Usage == "" {
			c.Usage = commandUsage(c)
		}
	}
}

func (commands CommandSet) loadFullHelp() {
	for _, c := range commands {
		if c.FullHelp == "" {
			c.FullHelp = c.buildFullHelp()
		}
	}
}

// Arg defines an argument for a command.
// These will be parsed in Go and passed to the Run method in the Context struct.
type Arg struct {
	Name     string `json:"name"`
	Optional bool   `json:"optional"`
	Hidden   bool   `json:"hidden"`
}

func (a *Arg) String() string {
	if a.Optional {
		return "[" + strings.ToUpper(a.Name) + "]"
	}
	return strings.ToUpper(a.Name)
}

func argsString(args []Arg) string {
	var buffer bytes.Buffer
	for _, arg := range args {
		if arg.Hidden {
			continue
		}
		if arg.Optional {
			buffer.WriteString(" [" + strings.ToUpper(arg.Name) + "]")
		} else {
			buffer.WriteString(" " + strings.ToUpper(arg.Name))
		}
	}
	return buffer.String()
}

var commandsTopic = &Topic{
	Name:        "commands",
	Description: "list all commands",
	Hidden:      true,
}

var commandsListCmd = &Command{
	Topic:       "commands",
	Description: "list all commands",
	Flags:       []Flag{{Name: "json"}},
	Run: func(ctx *Context) {
		cli.LoadPlugins(GetPlugins())
		if ctx.Flags["json"] == true {
			cli.Commands.loadUsages()
			cli.Commands.loadFullHelp()
			doc := map[string]interface{}{"topics": cli.Topics, "commands": cli.Commands}
			s, _ := json.Marshal(doc)
			Println(string(s))
			return
		}
		for _, command := range cli.Commands {
			if command.Hidden {
				continue
			}
			if command.Command == "" {
				Printf("%s\n", command.Topic)
			} else {
				Printf("%s:%s\n", command.Topic, command.Command)
			}
		}
	},
}
<commit_msg>added default option<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
)

// Command represents a CLI command.
// For example, in the command `heroku apps:create` the command would be `create`.
// They must have a Topic name that links to a real topic's name.
type Command struct {
	Topic        string             `json:"topic"`
	Command      string             `json:"command,omitempty"`
	Plugin       string             `json:"plugin"`
	Usage        string             `json:"usage"`
	Description  string             `json:"description"`
	Default      bool               `json:"default"`
	Help         string             `json:"help"`
	FullHelp     string             `json:"fullHelp"`
	Hidden       bool               `json:"hidden"`
	NeedsApp     bool               `json:"needsApp"`
	NeedsAuth    bool               `json:"needsAuth"`
	VariableArgs bool               `json:"variableArgs"`
	Args         []Arg              `json:"args"`
	Flags        []Flag             `json:"flags"`
	Run          func(ctx *Context) `json:"-"`
}

func (c *Command) String() string {
	if c.Command == "" {
		return c.Topic
	}
	return c.Topic + ":" + c.Command
}

func commandUsage(c *Command) string {
	return c.String() + argsString(c.Args)
}

func (c *Command) buildFullHelp() string {
	if len(c.Flags) == 0 {
		return c.Help
	}
	lines := make([]string, 0, len(c.Flags))
	for _, flag := range c.Flags {
		if flag.Hidden {
			continue
		}
		if flag.Description == "" {
			lines = append(lines, flag.String())
		} else {
			lines = append(lines, fmt.Sprintf("%-20s # %s", flag.String(), flag.Description))
		}
	}
	return strings.Join(lines, "\n") + "\n\n" + c.Help
}

// CommandSet is a slice of Command structs with some helper methods.
type CommandSet []*Command

// ByTopicAndCommand returns a command that matches the passed topic and command.
func (commands CommandSet) ByTopicAndCommand(topic, command string) *Command {
	for _, c := range commands {
		if c.Topic == topic {
			if c.Command == command || c.Default && command == "" {
				return c
			}
		}
	}
	return nil
}

func (commands CommandSet) loadUsages() {
	for _, c := range commands {
		if c.Usage == "" {
			c.Usage = commandUsage(c)
		}
	}
}

func (commands CommandSet) loadFullHelp() {
	for _, c := range commands {
		if c.FullHelp == "" {
			c.FullHelp = c.buildFullHelp()
		}
	}
}

// Arg defines an argument for a command.
// These will be parsed in Go and passed to the Run method in the Context struct.
type Arg struct {
	Name     string `json:"name"`
	Optional bool   `json:"optional"`
	Hidden   bool   `json:"hidden"`
}

func (a *Arg) String() string {
	if a.Optional {
		return "[" + strings.ToUpper(a.Name) + "]"
	}
	return strings.ToUpper(a.Name)
}

func argsString(args []Arg) string {
	var buffer bytes.Buffer
	for _, arg := range args {
		if arg.Hidden {
			continue
		}
		if arg.Optional {
			buffer.WriteString(" [" + strings.ToUpper(arg.Name) + "]")
		} else {
			buffer.WriteString(" " + strings.ToUpper(arg.Name))
		}
	}
	return buffer.String()
}

var commandsTopic = &Topic{
	Name:        "commands",
	Description: "list all commands",
	Hidden:      true,
}

var commandsListCmd = &Command{
	Topic:       "commands",
	Description: "list all commands",
	Flags:       []Flag{{Name: "json"}},
	Run: func(ctx *Context) {
		cli.LoadPlugins(GetPlugins())
		if ctx.Flags["json"] == true {
			cli.Commands.loadUsages()
			cli.Commands.loadFullHelp()
			doc := map[string]interface{}{"topics": cli.Topics, "commands": cli.Commands}
			s, _ := json.Marshal(doc)
			Println(string(s))
			return
		}
		for _, command := range cli.Commands {
			if command.Hidden {
				continue
			}
			if command.Command == "" {
				Printf("%s\n", command.Topic)
			} else {
				Printf("%s:%s\n", command.Topic, command.Command)
			}
		}
	},
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package logger

import (
	"fmt"
	"runtime"
	"strings"
	"time"

	logging "github.com/keybase/go-logging"

	"golang.org/x/net/context"
)

// TestLogBackend is an interface for logging to a test object (i.e.,
// a *testing.T).  We define this in order to avoid pulling in the
// "testing" package in exported code.
type TestLogBackend interface {
	Error(args ...interface{})
	Errorf(format string, args ...interface{})
	Fatal(args ...interface{})
	Fatalf(format string, args ...interface{})
	Log(args ...interface{})
	Logf(format string, args ...interface{})
}

// TestLogger is a Logger that writes to a TestLogBackend.  All
// messages except Fatal are printed using Logf, to avoid failing a
// test that is trying to test an error condition.  No context tags
// are logged.
type TestLogger struct {
	log        TestLogBackend
	extraDepth int
}

func NewTestLogger(log TestLogBackend) *TestLogger {
	return &TestLogger{log: log}
}

// Verify TestLogger fully implements the Logger interface.
var _ Logger = (*TestLogger)(nil)

func prefixCaller(extraDepth int, lvl logging.Level, fmts string) string {
	// The testing library doesn't let us control the stack depth,
	// and it always prints out its own prefix, so use \r to clear
	// it out (at least on a terminal) and do our own formatting.
	_, file, line, _ := runtime.Caller(2 + extraDepth)
	elements := strings.Split(file, "/")
	return fmt.Sprintf("%s \r%s:%d: [%.1s] %s", time.Now(),
		elements[len(elements)-1], line, lvl, fmts)
}

func (log *TestLogger) Debug(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.DEBUG, fmts), arg...)
}

func (log *TestLogger) CDebugf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.DEBUG, fmts)), arg...)
}

func (log *TestLogger) Info(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.INFO, fmts), arg...)
}

func (log *TestLogger) CInfof(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.INFO, fmts)), arg...)
}

func (log *TestLogger) Notice(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.NOTICE, fmts), arg...)
}

func (log *TestLogger) CNoticef(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.NOTICE, fmts)), arg...)
}

func (log *TestLogger) Warning(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.WARNING, fmts), arg...)
}

func (log *TestLogger) CWarningf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.WARNING, fmts)), arg...)
}

func (log *TestLogger) Error(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.ERROR, fmts), arg...)
}

func (log *TestLogger) Errorf(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.ERROR, fmts), arg...)
}

func (log *TestLogger) CErrorf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.ERROR, fmts)), arg...)
}

func (log *TestLogger) Critical(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.CRITICAL, fmts), arg...)
}

func (log *TestLogger) CCriticalf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.CRITICAL, fmts)), arg...)
}

func (log *TestLogger) Fatalf(fmts string, arg ...interface{}) {
	log.log.Fatalf(prefixCaller(log.extraDepth, logging.CRITICAL, fmts), arg...)
}

func (log *TestLogger) CFatalf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Fatalf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.CRITICAL, fmts)), arg...)
}

func (log *TestLogger) Profile(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.CRITICAL, fmts), arg...)
}

func (log *TestLogger) Configure(style string, debug bool, filename string) {
	// no-op
}

func (log *TestLogger) RotateLogFile() error {
	// no-op
	return nil
}

func (log *TestLogger) CloneWithAddedDepth(depth int) Logger {
	clone := *log
	clone.extraDepth += depth
	return &clone
}

// no-op stubs to fulfill the Logger interface
func (log *TestLogger) SetExternalHandler(_ ExternalHandler) {}
<commit_msg>logger: move \r back to the front for test logs (#3777)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package logger

import (
	"fmt"
	"runtime"
	"strings"
	"time"

	logging "github.com/keybase/go-logging"

	"golang.org/x/net/context"
)

// TestLogBackend is an interface for logging to a test object (i.e.,
// a *testing.T).  We define this in order to avoid pulling in the
// "testing" package in exported code.
type TestLogBackend interface {
	Error(args ...interface{})
	Errorf(format string, args ...interface{})
	Fatal(args ...interface{})
	Fatalf(format string, args ...interface{})
	Log(args ...interface{})
	Logf(format string, args ...interface{})
}

// TestLogger is a Logger that writes to a TestLogBackend.  All
// messages except Fatal are printed using Logf, to avoid failing a
// test that is trying to test an error condition.  No context tags
// are logged.
type TestLogger struct {
	log        TestLogBackend
	extraDepth int
}

func NewTestLogger(log TestLogBackend) *TestLogger {
	return &TestLogger{log: log}
}

// Verify TestLogger fully implements the Logger interface.
var _ Logger = (*TestLogger)(nil)

func prefixCaller(extraDepth int, lvl logging.Level, fmts string) string {
	// The testing library doesn't let us control the stack depth,
	// and it always prints out its own prefix, so use \r to clear
	// it out (at least on a terminal) and do our own formatting.
	_, file, line, _ := runtime.Caller(2 + extraDepth)
	elements := strings.Split(file, "/")
	return fmt.Sprintf("\r%s %s:%d: [%.1s] %s", time.Now(),
		elements[len(elements)-1], line, lvl, fmts)
}

func (log *TestLogger) Debug(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.DEBUG, fmts), arg...)
}

func (log *TestLogger) CDebugf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.DEBUG, fmts)), arg...)
}

func (log *TestLogger) Info(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.INFO, fmts), arg...)
}

func (log *TestLogger) CInfof(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.INFO, fmts)), arg...)
}

func (log *TestLogger) Notice(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.NOTICE, fmts), arg...)
}

func (log *TestLogger) CNoticef(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.NOTICE, fmts)), arg...)
}

func (log *TestLogger) Warning(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.WARNING, fmts), arg...)
}

func (log *TestLogger) CWarningf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.WARNING, fmts)), arg...)
}

func (log *TestLogger) Error(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.ERROR, fmts), arg...)
}

func (log *TestLogger) Errorf(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.ERROR, fmts), arg...)
}

func (log *TestLogger) CErrorf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.ERROR, fmts)), arg...)
}

func (log *TestLogger) Critical(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.CRITICAL, fmts), arg...)
}

func (log *TestLogger) CCriticalf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Logf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.CRITICAL, fmts)), arg...)
}

func (log *TestLogger) Fatalf(fmts string, arg ...interface{}) {
	log.log.Fatalf(prefixCaller(log.extraDepth, logging.CRITICAL, fmts), arg...)
}

func (log *TestLogger) CFatalf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.log.Fatalf(prepareString(ctx,
		prefixCaller(log.extraDepth, logging.CRITICAL, fmts)), arg...)
}

func (log *TestLogger) Profile(fmts string, arg ...interface{}) {
	log.log.Logf(prefixCaller(log.extraDepth, logging.CRITICAL, fmts), arg...)
}

func (log *TestLogger) Configure(style string, debug bool, filename string) {
	// no-op
}

func (log *TestLogger) RotateLogFile() error {
	// no-op
	return nil
}

func (log *TestLogger) CloneWithAddedDepth(depth int) Logger {
	clone := *log
	clone.extraDepth += depth
	return &clone
}

// no-op stubs to fulfill the Logger interface
func (log *TestLogger) SetExternalHandler(_ ExternalHandler) {}
<|endoftext|>


<commit_before>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package logic

import (
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/agent"
	"github.com/outbrain/orchestrator/go/config"
	"github.com/outbrain/orchestrator/go/discovery"
	"github.com/outbrain/orchestrator/go/inst"
	ometrics "github.com/outbrain/orchestrator/go/metrics"
	"github.com/outbrain/orchestrator/go/process"
	"github.com/pmylund/go-cache"
	"github.com/rcrowley/go-metrics"
)

const (
	discoveryInstanceChannelCapacity = 5 // size here not critical as channel read by separate go routines.
)

// discoveryInstanceKeys is a channel of instanceKey-s that were requested for discovery.
// It can be continuously updated as discovery process progresses.
var discoveryInstanceKeys = make(chan inst.InstanceKey, discoveryInstanceChannelCapacity)

var discoveriesCounter = metrics.NewCounter()
var failedDiscoveriesCounter = metrics.NewCounter()
var discoveryQueueLengthGauge = metrics.NewGauge()
var discoveryRecentCountGauge = metrics.NewGauge()
var isElectedGauge = metrics.NewGauge()

var isElectedNode int64 = 0

var recentDiscoveryOperationKeys *cache.Cache

func init() {
	metrics.Register("discoveries.attempt", discoveriesCounter)
	metrics.Register("discoveries.fail", failedDiscoveriesCounter)
	metrics.Register("discoveries.queue_length", discoveryQueueLengthGauge)
	metrics.Register("discoveries.recent_count", discoveryRecentCountGauge)
	metrics.Register("elect.is_elected", isElectedGauge)

	ometrics.OnGraphiteTick(func() { discoveryQueueLengthGauge.Update(int64(len(discoveryInstanceKeys))) })
	ometrics.OnGraphiteTick(func() {
		if recentDiscoveryOperationKeys == nil {
			return
		}
		discoveryRecentCountGauge.Update(int64(recentDiscoveryOperationKeys.ItemCount()))
	})
	ometrics.OnGraphiteTick(func() { isElectedGauge.Update(int64(atomic.LoadInt64(&isElectedNode))) })
}

// acceptSignals registers for OS signals
func acceptSignals() {
	c := make(chan os.Signal, 1)

	signal.Notify(c, syscall.SIGHUP)
	go func() {
		for sig := range c {
			switch sig {
			case syscall.SIGHUP:
				log.Debugf("Received SIGHUP. Reloading configuration")
				config.Reload()
				inst.AuditOperation("reload-configuration", nil, "Triggered via SIGHUP")
			}
		}
	}()
}

// handleDiscoveryRequests iterates the discoveryInstanceKeys channel
// and calls upon instance discovery per entry. These requests will
// be handled by separate go routines so concurrency should be high.
func handleDiscoveryRequests() {
	log.Infof("handleDiscoveryRequests() DiscoveryMaxConcurrency: %v", config.Config.DiscoveryMaxConcurrency)
	if config.Config.DiscoveryMaxConcurrency > 0 {
		handleDiscoveryRequestsWithConcurrency(config.Config.DiscoveryMaxConcurrency)
	} else {
		handleDiscoveryRequestsWithoutConcurrencyLimits()
	}
}

// Handle the discovery requests with the maximum given concurrency
func handleDiscoveryRequestsWithConcurrency(maxConcurrency uint) {
	q := discovery.NewQueue(maxConcurrency, discoveryInstanceKeys, discoverInstance)
	q.HandleRequests()
}

// handleDiscoveryRequests iterates the discoveryInstanceKeys channel
// and calls upon instance discovery per entry.
func handleDiscoveryRequestsWithoutConcurrencyLimits() {
	for instanceKey := range discoveryInstanceKeys {
		// Possibly this used to be the elected node, but has
		// been demoted, while still the queue is full.
		if atomic.LoadInt64(&isElectedNode) != 1 {
			log.Debugf("Node apparently demoted. Skipping discovery of %+v. "+
				"Remaining queue size: %+v", instanceKey, len(discoveryInstanceKeys))
			continue
		}
		go discoverInstance(instanceKey)
	}
}

// discoverInstance will attempt to discover an instance (unless it is
// already up to date) and will list down its master and slaves (if any)
// for further discovery.
func discoverInstance(instanceKey inst.InstanceKey) {
	start := time.Now()

	instanceKey.Formalize()
	if !instanceKey.IsValid() {
		return
	}

	DiscoverStart(instanceKey)
	if existsInCacheError := recentDiscoveryOperationKeys.Add(instanceKey.DisplayString(), true, cache.DefaultExpiration); existsInCacheError != nil {
		DiscoverEnd(instanceKey)
		// Just recently attempted
		return
	}

	instance, found, err := inst.ReadInstance(&instanceKey)

	if found && instance.IsUpToDate && instance.IsLastCheckValid {
		recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
		DiscoverEnd(instanceKey)
		// we've already discovered this one. Skip!
		return
	}
	discoveriesCounter.Inc(1)
	// First we've ever heard of this instance. Continue investigation:
	instance, err = inst.ReadTopologyInstance(&instanceKey)
	// panic can occur (IO stuff). Therefore it may happen
	// that instance is nil. Check it.
	if instance == nil {
		recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
		DiscoverEnd(instanceKey)
		failedDiscoveriesCounter.Inc(1)
		log.Warningf("discoverInstance(%+v) instance is nil in %.3fs, error=%+v", instanceKey, time.Since(start).Seconds(), err)
		return
	}

	recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
	DiscoverEnd(instanceKey)
	log.Debugf("Discovered host: %+v, master: %+v, version: %+v in %.3fs", instance.Key, instance.MasterKey, instance.Version, time.Since(start).Seconds())

	if atomic.LoadInt64(&isElectedNode) == 0 {
		// Maybe this node was elected before, but isn't elected anymore.
		// If not elected, stop drilling up/down the topology
		return
	}

	// Investigate master and slaves asynchronously to ensure we
	// do not block processing other instances while adding these servers
	// to the queue.

	// Investigate slaves:
	for _, slaveKey := range instance.SlaveHosts.GetInstanceKeys() {
		slaveKey := slaveKey
		if slaveKey.IsValid() {
			discoveryInstanceKeys <- slaveKey
		}
	}
	// Investigate master:
	if instance.MasterKey.IsValid() {
		discoveryInstanceKeys <- instance.MasterKey
	}
}

// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
	if config.Config.DatabaselessMode__experimental {
		log.Fatal("Cannot execute continuous mode in databaseless mode")
	}

	log.Infof("Starting continuous discovery")
	recentDiscoveryOperationKeys = cache.New(time.Duration(config.Config.InstancePollSeconds)*time.Second, time.Second)

	inst.LoadHostnameResolveCache()
	go handleDiscoveryRequests()

	discoveryTick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
	instancePollTick := time.Tick(time.Duration(config.Config.InstancePollSeconds) * time.Second)
	caretakingTick := time.Tick(time.Minute)
	recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second)
	var snapshotTopologiesTick <-chan time.Time
	if config.Config.SnapshotTopologiesIntervalHours > 0 {
		snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour)
	}

	go ometrics.InitGraphiteMetrics()
	go acceptSignals()

	if *config.RuntimeCLIFlags.GrabElection {
		process.GrabElection()
	}
	for {
		select {
		case <-discoveryTick:
			go func() {
				wasAlreadyElected := atomic.LoadInt64(&isElectedNode)
				myIsElectedNode, err := process.AttemptElection()
				if err != nil {
					log.Errore(err)
				}
				if myIsElectedNode {
					atomic.StoreInt64(&isElectedNode, 1)
				} else {
					atomic.StoreInt64(&isElectedNode, 0)
				}

				if myIsElectedNode {
					instanceKeys, err := inst.ReadOutdatedInstanceKeys()
					if err != nil {
						log.Errore(err)
					}

					log.Debugf("outdated keys: %+v", instanceKeys)
					for _, instanceKey := range instanceKeys {
						instanceKey := instanceKey

						go func() {
							if instanceKey.IsValid() {
								discoveryInstanceKeys <- instanceKey
							}
						}()
					}
					if wasAlreadyElected == 0 {
						// Just turned to be leader!
						go process.RegisterNode("", "", false)
					}
				} else {
					hostname, token, _, err := process.ElectedNode()
					if err == nil {
						log.Debugf("Inactive node. Active node is: %v[%v]; polling", hostname, token)
					} else {
						log.Debugf("Inactive node. Unable to determine active node: %v; polling", err)
					}
				}
			}()
		case <-instancePollTick:
			go func() {
				// This tick does NOT do instance poll (these are handled by the oversmapling discoveryTick)
				// But rather should invoke such routinely operations that need to be as (or roughly as) frequent
				// as instance poll
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go inst.UpdateInstanceRecentRelaylogHistory()
					go inst.RecordInstanceCoordinatesHistory()
				}
			}()
		case <-caretakingTick:
			// Various periodic internal maintenance tasks
			go func() {
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go inst.RecordInstanceBinlogFileHistory()
					go inst.ForgetLongUnseenInstances()
					go inst.ForgetUnseenInstancesDifferentlyResolved()
					go inst.ForgetExpiredHostnameResolves()
					go inst.DeleteInvalidHostnameResolves()
					go inst.ReviewUnseenInstances()
					go inst.InjectUnseenMasters()
					go inst.ResolveUnknownMasterHostnameResolves()
					go inst.UpdateClusterAliases()
					go inst.ExpireMaintenance()
					go inst.ExpireDowntime()
					go inst.ExpireCandidateInstances()
					go inst.ExpireHostnameUnresolve()
					go inst.ExpireClusterDomainName()
					go inst.ExpireAudit()
					go inst.ExpireMasterPositionEquivalence()
					go inst.ExpirePoolInstances()
					go inst.FlushNontrivialResolveCacheToDatabase()
					go process.ExpireNodesHistory()
					go process.ExpireAccessTokens()
				} else {
					// Take this opportunity to refresh yourself
					go inst.LoadHostnameResolveCache()
				}
			}()
		case <-recoveryTick:
			go func() {
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go ClearActiveFailureDetections()
					go ClearActiveRecoveries()
					go ExpireBlockedRecoveries()
					go AcknowledgeCrashedRecoveries()
					go inst.ExpireInstanceAnalysisChangelog()
					go CheckAndRecover(nil, nil, false)
				}
			}()
		case <-snapshotTopologiesTick:
			go func() {
				go inst.SnapshotTopologies()
			}()
		}
	}
}

func pollAgent(hostname string) error {
	polledAgent, err := agent.GetAgent(hostname)
	agent.UpdateAgentLastChecked(hostname)

	if err != nil {
		return log.Errore(err)
	}

	err = agent.UpdateAgentInfo(hostname, polledAgent)
	if err != nil {
		return log.Errore(err)
	}

	return nil
}

// ContinuousAgentsPoll starts an asynchronuous infinite process where agents are
// periodically investigated and their status captured, and long since unseen agents are
// purged and forgotten.
func ContinuousAgentsPoll() {
	log.Infof("Starting continuous agents poll")

	go discoverSeededAgents()

	tick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
	caretakingTick := time.Tick(time.Hour)
	for range tick {
		agentsHosts, _ := agent.ReadOutdatedAgentsHosts()
		log.Debugf("outdated agents hosts: %+v", agentsHosts)
		for _, hostname := range agentsHosts {
			go pollAgent(hostname)
		}
		// See if we should also forget agents (lower frequency)
		select {
		case <-caretakingTick:
			agent.ForgetLongUnseenAgents()
			agent.FailStaleSeeds()
		default:
		}
	}
}

func discoverSeededAgents() {
	for seededAgent := range agent.SeededAgents {
		instanceKey := &inst.InstanceKey{Hostname: seededAgent.Hostname, Port: int(seededAgent.MySQLPort)}
		go inst.ReadTopologyInstance(instanceKey)
	}
}
<commit_msg>comment: rework discoveryInstanceKeys queueing later<commit_after>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package logic

import (
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/agent"
	"github.com/outbrain/orchestrator/go/config"
	"github.com/outbrain/orchestrator/go/discovery"
	"github.com/outbrain/orchestrator/go/inst"
	ometrics "github.com/outbrain/orchestrator/go/metrics"
	"github.com/outbrain/orchestrator/go/process"
	"github.com/pmylund/go-cache"
	"github.com/rcrowley/go-metrics"
)

const (
	discoveryInstanceChannelCapacity = 5 // size here not critical as channel read by separate go routines.
)

// discoveryInstanceKeys is a channel of instanceKey-s that were requested for discovery.
// It can be continuously updated as discovery process progresses.
var discoveryInstanceKeys = make(chan inst.InstanceKey, discoveryInstanceChannelCapacity)

var discoveriesCounter = metrics.NewCounter()
var failedDiscoveriesCounter = metrics.NewCounter()
var discoveryQueueLengthGauge = metrics.NewGauge()
var discoveryRecentCountGauge = metrics.NewGauge()
var isElectedGauge = metrics.NewGauge()

var isElectedNode int64 = 0

var recentDiscoveryOperationKeys *cache.Cache

func init() {
	metrics.Register("discoveries.attempt", discoveriesCounter)
	metrics.Register("discoveries.fail", failedDiscoveriesCounter)
	metrics.Register("discoveries.queue_length", discoveryQueueLengthGauge)
	metrics.Register("discoveries.recent_count", discoveryRecentCountGauge)
	metrics.Register("elect.is_elected", isElectedGauge)

	ometrics.OnGraphiteTick(func() { discoveryQueueLengthGauge.Update(int64(len(discoveryInstanceKeys))) })
	ometrics.OnGraphiteTick(func() {
		if recentDiscoveryOperationKeys == nil {
			return
		}
		discoveryRecentCountGauge.Update(int64(recentDiscoveryOperationKeys.ItemCount()))
	})
	ometrics.OnGraphiteTick(func() { isElectedGauge.Update(int64(atomic.LoadInt64(&isElectedNode))) })
}

// acceptSignals registers for OS signals
func acceptSignals() {
	c := make(chan os.Signal, 1)

	signal.Notify(c, syscall.SIGHUP)
	go func() {
		for sig := range c {
			switch sig {
			case syscall.SIGHUP:
				log.Debugf("Received SIGHUP. Reloading configuration")
				config.Reload()
				inst.AuditOperation("reload-configuration", nil, "Triggered via SIGHUP")
			}
		}
	}()
}

// handleDiscoveryRequests iterates the discoveryInstanceKeys channel
// and calls upon instance discovery per entry. These requests will
// be handled by separate go routines so concurrency should be high.
func handleDiscoveryRequests() {
	log.Infof("handleDiscoveryRequests() DiscoveryMaxConcurrency: %v", config.Config.DiscoveryMaxConcurrency)
	if config.Config.DiscoveryMaxConcurrency > 0 {
		handleDiscoveryRequestsWithConcurrency(config.Config.DiscoveryMaxConcurrency)
	} else {
		handleDiscoveryRequestsWithoutConcurrencyLimits()
	}
}

// Handle the discovery requests with the maximum given concurrency
func handleDiscoveryRequestsWithConcurrency(maxConcurrency uint) {
	q := discovery.NewQueue(maxConcurrency, discoveryInstanceKeys, discoverInstance)
	q.HandleRequests()
}

// handleDiscoveryRequests iterates the discoveryInstanceKeys channel
// and calls upon instance discovery per entry.
func handleDiscoveryRequestsWithoutConcurrencyLimits() {
	for instanceKey := range discoveryInstanceKeys {
		// Possibly this used to be the elected node, but has
		// been demoted, while still the queue is full.
		if atomic.LoadInt64(&isElectedNode) != 1 {
			log.Debugf("Node apparently demoted. Skipping discovery of %+v. "+
				"Remaining queue size: %+v", instanceKey, len(discoveryInstanceKeys))
			continue
		}
		go discoverInstance(instanceKey)
	}
}

// discoverInstance will attempt to discover an instance (unless it is
// already up to date) and will list down its master and slaves (if any)
// for further discovery.
func discoverInstance(instanceKey inst.InstanceKey) {
	start := time.Now()

	instanceKey.Formalize()
	if !instanceKey.IsValid() {
		return
	}

	DiscoverStart(instanceKey)
	if existsInCacheError := recentDiscoveryOperationKeys.Add(instanceKey.DisplayString(), true, cache.DefaultExpiration); existsInCacheError != nil {
		DiscoverEnd(instanceKey)
		// Just recently attempted
		return
	}

	instance, found, err := inst.ReadInstance(&instanceKey)
	if found && instance.IsUpToDate && instance.IsLastCheckValid {
		recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
		DiscoverEnd(instanceKey)
		// we've already discovered this one. Skip!
		return
	}

	discoveriesCounter.Inc(1)

	// First we've ever heard of this instance. Continue investigation:
	instance, err = inst.ReadTopologyInstance(&instanceKey)
	// panic can occur (IO stuff). Therefore it may happen
	// that instance is nil. Check it.
	if instance == nil {
		recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
		DiscoverEnd(instanceKey)
		failedDiscoveriesCounter.Inc(1)
		log.Warningf("discoverInstance(%+v) instance is nil in %.3fs, error=%+v", instanceKey, time.Since(start).Seconds(), err)
		return
	}

	recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
	DiscoverEnd(instanceKey)
	log.Debugf("Discovered host: %+v, master: %+v, version: %+v in %.3fs", instance.Key, instance.MasterKey, instance.Version, time.Since(start).Seconds())

	if atomic.LoadInt64(&isElectedNode) == 0 {
		// Maybe this node was elected before, but isn't elected anymore.
		// If not elected, stop drilling up/down the topology
		return
	}

	// Investigate master and slaves asynchronously to ensure we
	// do not block processing other instances while adding these servers
	// to the queue.

	// Investigate slaves:
	for _, slaveKey := range instance.SlaveHosts.GetInstanceKeys() {
		slaveKey := slaveKey
		if slaveKey.IsValid() {
			discoveryInstanceKeys <- slaveKey
		}
	}
	// Investigate master:
	if instance.MasterKey.IsValid() {
		discoveryInstanceKeys <- instance.MasterKey
	}
}

// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
	if config.Config.DatabaselessMode__experimental {
		log.Fatal("Cannot execute continuous mode in databaseless mode")
	}

	log.Infof("Starting continuous discovery")
	recentDiscoveryOperationKeys = cache.New(time.Duration(config.Config.InstancePollSeconds)*time.Second, time.Second)

	inst.LoadHostnameResolveCache()
	go handleDiscoveryRequests()

	discoveryTick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
	instancePollTick := time.Tick(time.Duration(config.Config.InstancePollSeconds) * time.Second)
	caretakingTick := time.Tick(time.Minute)
	recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second)
	var snapshotTopologiesTick <-chan time.Time
	if config.Config.SnapshotTopologiesIntervalHours > 0 {
		snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour)
	}

	go ometrics.InitGraphiteMetrics()
	go acceptSignals()

	if *config.RuntimeCLIFlags.GrabElection {
		process.GrabElection()
	}
	for {
		select {
		case <-discoveryTick:
			go func() {
				wasAlreadyElected := atomic.LoadInt64(&isElectedNode)
				myIsElectedNode, err := process.AttemptElection()
				if err != nil {
					log.Errore(err)
				}
				if myIsElectedNode {
					atomic.StoreInt64(&isElectedNode, 1)
				} else {
					atomic.StoreInt64(&isElectedNode, 0)
				}

				if myIsElectedNode {
					instanceKeys, err := inst.ReadOutdatedInstanceKeys()
					if err != nil {
						log.Errore(err)
					}

					log.Debugf("outdated keys: %+v", instanceKeys)
					for _, instanceKey := range instanceKeys {
						instanceKey := instanceKey

                        // XXX IMO forking a goroutine that is to be
                        //     blocked on a chan is a misuse of goroutines
						go func() {
							if instanceKey.IsValid() {
								discoveryInstanceKeys <- instanceKey
							}
						}()
					}
					if wasAlreadyElected == 0 {
						// Just turned to be leader!
						go process.RegisterNode("", "", false)
					}
				} else {
					hostname, token, _, err := process.ElectedNode()
					if err == nil {
						log.Debugf("Inactive node. Active node is: %v[%v]; polling", hostname, token)
					} else {
						log.Debugf("Inactive node. Unable to determine active node: %v; polling", err)
					}
				}
			}()
		case <-instancePollTick:
			go func() {
				// This tick does NOT do instance poll (these are handled by the oversmapling discoveryTick)
				// But rather should invoke such routinely operations that need to be as (or roughly as) frequent
				// as instance poll
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go inst.UpdateInstanceRecentRelaylogHistory()
					go inst.RecordInstanceCoordinatesHistory()
				}
			}()
		case <-caretakingTick:
			// Various periodic internal maintenance tasks
			go func() {
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go inst.RecordInstanceBinlogFileHistory()
					go inst.ForgetLongUnseenInstances()
					go inst.ForgetUnseenInstancesDifferentlyResolved()
					go inst.ForgetExpiredHostnameResolves()
					go inst.DeleteInvalidHostnameResolves()
					go inst.ReviewUnseenInstances()
					go inst.InjectUnseenMasters()
					go inst.ResolveUnknownMasterHostnameResolves()
					go inst.UpdateClusterAliases()
					go inst.ExpireMaintenance()
					go inst.ExpireDowntime()
					go inst.ExpireCandidateInstances()
					go inst.ExpireHostnameUnresolve()
					go inst.ExpireClusterDomainName()
					go inst.ExpireAudit()
					go inst.ExpireMasterPositionEquivalence()
					go inst.ExpirePoolInstances()
					go inst.FlushNontrivialResolveCacheToDatabase()
					go process.ExpireNodesHistory()
					go process.ExpireAccessTokens()
				} else {
					// Take this opportunity to refresh yourself
					go inst.LoadHostnameResolveCache()
				}
			}()
		case <-recoveryTick:
			go func() {
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go ClearActiveFailureDetections()
					go ClearActiveRecoveries()
					go ExpireBlockedRecoveries()
					go AcknowledgeCrashedRecoveries()
					go inst.ExpireInstanceAnalysisChangelog()
					go CheckAndRecover(nil, nil, false)
				}
			}()
		case <-snapshotTopologiesTick:
			go func() {
				go inst.SnapshotTopologies()
			}()
		}
	}
}

func pollAgent(hostname string) error {
	polledAgent, err := agent.GetAgent(hostname)
	agent.UpdateAgentLastChecked(hostname)

	if err != nil {
		return log.Errore(err)
	}

	err = agent.UpdateAgentInfo(hostname, polledAgent)
	if err != nil {
		return log.Errore(err)
	}

	return nil
}

// ContinuousAgentsPoll starts an asynchronuous infinite process where agents are
// periodically investigated and their status captured, and long since unseen agents are
// purged and forgotten.
func ContinuousAgentsPoll() {
	log.Infof("Starting continuous agents poll")

	go discoverSeededAgents()

	tick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
	caretakingTick := time.Tick(time.Hour)
	for range tick {
		agentsHosts, _ := agent.ReadOutdatedAgentsHosts()
		log.Debugf("outdated agents hosts: %+v", agentsHosts)
		for _, hostname := range agentsHosts {
			go pollAgent(hostname)
		}
		// See if we should also forget agents (lower frequency)
		select {
		case <-caretakingTick:
			agent.ForgetLongUnseenAgents()
			agent.FailStaleSeeds()
		default:
		}
	}
}

func discoverSeededAgents() {
	for seededAgent := range agent.SeededAgents {
		instanceKey := &inst.InstanceKey{Hostname: seededAgent.Hostname, Port: int(seededAgent.MySQLPort)}
		go inst.ReadTopologyInstance(instanceKey)
	}
}
<|endoftext|>


<commit_before>package goriak

// Command is the main query builder object
type Command struct {
	// Key information
	bucket     string
	bucketType string

	// Middleware
	runMiddleware []RunMiddleware
}

// Result contains your query result data from Run()
type Result struct {
	NotFound bool   // Wether or not the item was not found when using Get, GetJSON, or GetRaw.
	Key      string // Returns your automatically generated key when using Set, SetJSON, or SetRaw.
	Context  []byte // Returns the Riak Context used in map operations. Is set when using Get.
}

// Bucket specifies the bucket and bucket type that your following command will be performed on.
func Bucket(bucket, bucketType string) *Command {
	return &Command{
		bucket:     bucket,
		bucketType: bucketType,
	}
}

func (c *Command) RegisterRunMiddleware(middleware RunMiddleware) *Command {
	c.runMiddleware = append(c.runMiddleware, middleware)
	return c
}
<commit_msg>#46 - Comments<commit_after>package goriak

// Command is the main query builder object
type Command struct {
	// Key information
	bucket     string
	bucketType string

	// Middleware
	runMiddleware []RunMiddleware
}

// Result contains your query result data from Run()
type Result struct {
	NotFound bool   // Wether or not the item was not found when using Get, GetJSON, or GetRaw.
	Key      string // Returns your automatically generated key when using Set, SetJSON, or SetRaw.
	Context  []byte // Returns the Riak Context used in map operations. Is set when using Get.
}

// Bucket specifies the bucket and bucket type that your following command will be performed on.
func Bucket(bucket, bucketType string) *Command {
	return &Command{
		bucket:     bucket,
		bucketType: bucketType,
	}
}

// RegisterRunMiddleware adds a middleware function that will wrap the execution of the command.
// Is currently supported by Get, GetRaw, GetJSON, Set, SetRaw, and SetJSON
func (c *Command) RegisterRunMiddleware(middleware RunMiddleware) *Command {
	c.runMiddleware = append(c.runMiddleware, middleware)
	return c
}
<|endoftext|>


<commit_before>/*
   Copyright 2015 Shlomi Noach, courtesy Booking.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package process

import (
	"database/sql"
	"sync"
	"time"

	"github.com/github/orchestrator/go/config"
	"github.com/github/orchestrator/go/db"
	"github.com/openark/golib/log"
	"github.com/openark/golib/sqlutils"
)

const registrationPollSeconds = 10

type NodeHealth struct {
	Hostname        string
	Token           string
	AppVersion      string
	FirstSeenActive string
	LastSeenActive  string
}

type HealthStatus struct {
	Healthy        bool
	Hostname       string
	Token          string
	IsActiveNode   bool
	ActiveNode     NodeHealth
	Error          error
	AvailableNodes [](*NodeHealth)
}

type OrchestratorExecutionMode string

const (
	OrchestratorExecutionCliMode  OrchestratorExecutionMode = "CLIMode"
	OrchestratorExecutionHttpMode                           = "HttpMode"
)

var (
	continuousRegistrationInitiated bool       = false
	expiryMutex                     sync.Mutex // lock for ExpireAvailableNodes
	expiryRunning                   bool       // is ExpireAvailableNodes running
)

// RegisterNode writes down this node in the node_health table
func RegisterNode(extraInfo string, command string, firstTime bool) (sql.Result, error) {
	if firstTime {
		db.ExecOrchestrator(`
			insert ignore into node_health_history
				(hostname, token, first_seen_active, extra_info, command, app_version)
			values
				(?, ?, NOW(), ?, ?, ?)
			`,
			ThisHostname, ProcessToken.Hash, extraInfo, command,
			config.RuntimeCLIFlags.ConfiguredVersion,
		)
	}
	return db.ExecOrchestrator(`
			insert into node_health
				(hostname, token, first_seen_active, last_seen_active, extra_info, command, app_version)
			values
				(?, ?, now(), now(), ?, ?, ?)
			on duplicate key update
				token=values(token),
				last_seen_active=values(last_seen_active),
				extra_info=if(values(extra_info) != '', values(extra_info), extra_info),
				app_version=values(app_version)
			`,
		ThisHostname, ProcessToken.Hash, extraInfo, command,
		config.RuntimeCLIFlags.ConfiguredVersion,
	)
}

// HealthTest attempts to write to the backend database and get a result
func HealthTest() (*HealthStatus, error) {
	health := HealthStatus{Healthy: false, Hostname: ThisHostname, Token: ProcessToken.Hash}

	sqlResult, err := RegisterNode("", "", false)
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}
	rows, err := sqlResult.RowsAffected()
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}
	health.Healthy = (rows > 0)
	health.ActiveNode, health.IsActiveNode, err = ElectedNode()
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}

	health.AvailableNodes, err = ReadAvailableNodes(true)

	return &health, nil
}

// ContinuousRegistration will continuously update the node_health
// table showing that the current process is still running.
func ContinuousRegistration(extraInfo string, command string) {
	if continuousRegistrationInitiated {
		// This is a simple mechanism to make sure this function is not being called multiple times in the lifespan of this process.
		// It is not concurrency-protected.
		// Original use case: multiple instances as in "-i instance1,instance2,instance3" flag
		return
	}
	continuousRegistrationInitiated = true

	tickOperation := func(firstTime bool) {
		if _, err := RegisterNode(extraInfo, command, firstTime); err != nil {
			log.Errorf("ContinuousRegistration: RegisterNode failed: %+v", err)
		}
	}
	// First one is synchronous
	tickOperation(true)
	go func() {
		registrationTick := time.Tick(time.Duration(registrationPollSeconds) * time.Second)
		for range registrationTick {
			// We already run inside a go-routine so
			// do not do this asynchronously.  If we
			// get stuck then we don't want to fill up
			// the backend pool with connections running
			// this maintenance operation.
			tickOperation(false)
		}
	}()
}

// ExpireAvailableNodes is an aggressive purging method to remove
// node entries who have skipped their keepalive for two times.
func ExpireAvailableNodes() {
	// Ensure this is only running once in each process
	expiryMutex.Lock()
	if expiryRunning {
		expiryMutex.Unlock()
		log.Errorf("ExpireAvailableNodes: still running")
		return
	}
	expiryRunning = true
	expiryMutex.Unlock()

	_, err := db.ExecOrchestrator(`
			delete
				from node_health
			where
				last_seen_active < now() - interval ? second
			`,
		registrationPollSeconds*2,
	)
	if err != nil {
		log.Errorf("ExpireAvailableNodes: failed to remove old entries: %+v", err)
	}

	// show we're done
	expiryMutex.Lock()
	expiryRunning = false
	expiryMutex.Unlock()
}

// ExpireNodesHistory cleans up the nodes history and is run by
// the orchestrator active node.
func ExpireNodesHistory() error {
	_, err := db.ExecOrchestrator(`
			delete
				from node_health_history
			where
				first_seen_active < now() - interval ? hour
			`,
		config.Config.UnseenInstanceForgetHours,
	)
	return log.Errore(err)
}

func ReadAvailableNodes(onlyHttpNodes bool) (nodes [](*NodeHealth), err error) {
	extraInfo := ""
	if onlyHttpNodes {
		extraInfo = string(OrchestratorExecutionHttpMode)
	}
	query := `
		select
			hostname, token, app_version, first_seen_active, last_seen_active
		from
			node_health
		where
			last_seen_active > now() - interval ? second
			and ? in (extra_info, '')
		order by
			hostname
		`

	err = db.QueryOrchestrator(query, sqlutils.Args(registrationPollSeconds*2, extraInfo), func(m sqlutils.RowMap) error {
		nodeHealth := &NodeHealth{
			Hostname:        m.GetString("hostname"),
			Token:           m.GetString("token"),
			AppVersion:      m.GetString("app_version"),
			FirstSeenActive: m.GetString("first_seen_active"),
			LastSeenActive:  m.GetString("last_seen_active"),
		}
		nodes = append(nodes, nodeHealth)
		return nil
	})
	return nodes, log.Errore(err)
}

func TokenBelongsToHealthyHttpService(token string) (result bool, err error) {
	extraInfo := string(OrchestratorExecutionHttpMode)

	query := `
		select
			token
		from
			node_health
		where
			and token = ?
			and extra_info = ?
		`

	err = db.QueryOrchestrator(query, sqlutils.Args(token, extraInfo), func(m sqlutils.RowMap) error {
		// Row exists? We're happy
		result = true
		return nil
	})
	return result, log.Errore(err)
}

// Just check to make sure we can connect to the database
func SimpleHealthTest() (*HealthStatus, error) {
	health := HealthStatus{Healthy: false, Hostname: ThisHostname, Token: ProcessToken.Hash}

	db, err := db.OpenOrchestrator()
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}

	if err = db.Ping(); err != nil {
		health.Error = err
		return &health, log.Errore(err)
	} else {
		health.Healthy = true
		return &health, nil
	}
}
<commit_msg>ExpireAvailableNodes: remove checks that only running once<commit_after>/*
   Copyright 2015 Shlomi Noach, courtesy Booking.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package process

import (
	"database/sql"
	"sync"
	"time"

	"github.com/github/orchestrator/go/config"
	"github.com/github/orchestrator/go/db"
	"github.com/openark/golib/log"
	"github.com/openark/golib/sqlutils"
)

const registrationPollSeconds = 10

type NodeHealth struct {
	Hostname        string
	Token           string
	AppVersion      string
	FirstSeenActive string
	LastSeenActive  string
}

type HealthStatus struct {
	Healthy        bool
	Hostname       string
	Token          string
	IsActiveNode   bool
	ActiveNode     NodeHealth
	Error          error
	AvailableNodes [](*NodeHealth)
}

type OrchestratorExecutionMode string

const (
	OrchestratorExecutionCliMode  OrchestratorExecutionMode = "CLIMode"
	OrchestratorExecutionHttpMode                           = "HttpMode"
)

var continuousRegistrationInitiated bool = false

// RegisterNode writes down this node in the node_health table
func RegisterNode(extraInfo string, command string, firstTime bool) (sql.Result, error) {
	if firstTime {
		db.ExecOrchestrator(`
			insert ignore into node_health_history
				(hostname, token, first_seen_active, extra_info, command, app_version)
			values
				(?, ?, NOW(), ?, ?, ?)
			`,
			ThisHostname, ProcessToken.Hash, extraInfo, command,
			config.RuntimeCLIFlags.ConfiguredVersion,
		)
	}
	return db.ExecOrchestrator(`
			insert into node_health
				(hostname, token, first_seen_active, last_seen_active, extra_info, command, app_version)
			values
				(?, ?, now(), now(), ?, ?, ?)
			on duplicate key update
				token=values(token),
				last_seen_active=values(last_seen_active),
				extra_info=if(values(extra_info) != '', values(extra_info), extra_info),
				app_version=values(app_version)
			`,
		ThisHostname, ProcessToken.Hash, extraInfo, command,
		config.RuntimeCLIFlags.ConfiguredVersion,
	)
}

// HealthTest attempts to write to the backend database and get a result
func HealthTest() (*HealthStatus, error) {
	health := HealthStatus{Healthy: false, Hostname: ThisHostname, Token: ProcessToken.Hash}

	sqlResult, err := RegisterNode("", "", false)
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}
	rows, err := sqlResult.RowsAffected()
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}
	health.Healthy = (rows > 0)
	health.ActiveNode, health.IsActiveNode, err = ElectedNode()
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}

	health.AvailableNodes, err = ReadAvailableNodes(true)

	return &health, nil
}

// ContinuousRegistration will continuously update the node_health
// table showing that the current process is still running.
func ContinuousRegistration(extraInfo string, command string) {
	if continuousRegistrationInitiated {
		// This is a simple mechanism to make sure this function is not being called multiple times in the lifespan of this process.
		// It is not concurrency-protected.
		// Original use case: multiple instances as in "-i instance1,instance2,instance3" flag
		return
	}
	continuousRegistrationInitiated = true

	tickOperation := func(firstTime bool) {
		if _, err := RegisterNode(extraInfo, command, firstTime); err != nil {
			log.Errorf("ContinuousRegistration: RegisterNode failed: %+v", err)
		}
	}
	// First one is synchronous
	tickOperation(true)
	go func() {
		registrationTick := time.Tick(time.Duration(registrationPollSeconds) * time.Second)
		for range registrationTick {
			// We already run inside a go-routine so
			// do not do this asynchronously.  If we
			// get stuck then we don't want to fill up
			// the backend pool with connections running
			// this maintenance operation.
			tickOperation(false)
		}
	}()
}

// ExpireAvailableNodes is an aggressive purging method to remove
// node entries who have skipped their keepalive for two times.
func ExpireAvailableNodes() {
	_, err := db.ExecOrchestrator(`
			delete
				from node_health
			where
				last_seen_active < now() - interval ? second
			`,
		registrationPollSeconds*2,
	)
	if err != nil {
		log.Errorf("ExpireAvailableNodes: failed to remove old entries: %+v", err)
	}
}

// ExpireNodesHistory cleans up the nodes history and is run by
// the orchestrator active node.
func ExpireNodesHistory() error {
	_, err := db.ExecOrchestrator(`
			delete
				from node_health_history
			where
				first_seen_active < now() - interval ? hour
			`,
		config.Config.UnseenInstanceForgetHours,
	)
	return log.Errore(err)
}

func ReadAvailableNodes(onlyHttpNodes bool) (nodes [](*NodeHealth), err error) {
	extraInfo := ""
	if onlyHttpNodes {
		extraInfo = string(OrchestratorExecutionHttpMode)
	}
	query := `
		select
			hostname, token, app_version, first_seen_active, last_seen_active
		from
			node_health
		where
			last_seen_active > now() - interval ? second
			and ? in (extra_info, '')
		order by
			hostname
		`

	err = db.QueryOrchestrator(query, sqlutils.Args(registrationPollSeconds*2, extraInfo), func(m sqlutils.RowMap) error {
		nodeHealth := &NodeHealth{
			Hostname:        m.GetString("hostname"),
			Token:           m.GetString("token"),
			AppVersion:      m.GetString("app_version"),
			FirstSeenActive: m.GetString("first_seen_active"),
			LastSeenActive:  m.GetString("last_seen_active"),
		}
		nodes = append(nodes, nodeHealth)
		return nil
	})
	return nodes, log.Errore(err)
}

func TokenBelongsToHealthyHttpService(token string) (result bool, err error) {
	extraInfo := string(OrchestratorExecutionHttpMode)

	query := `
		select
			token
		from
			node_health
		where
			and token = ?
			and extra_info = ?
		`

	err = db.QueryOrchestrator(query, sqlutils.Args(token, extraInfo), func(m sqlutils.RowMap) error {
		// Row exists? We're happy
		result = true
		return nil
	})
	return result, log.Errore(err)
}

// Just check to make sure we can connect to the database
func SimpleHealthTest() (*HealthStatus, error) {
	health := HealthStatus{Healthy: false, Hostname: ThisHostname, Token: ProcessToken.Hash}

	db, err := db.OpenOrchestrator()
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}

	if err = db.Ping(); err != nil {
		health.Error = err
		return &health, log.Errore(err)
	} else {
		health.Healthy = true
		return &health, nil
	}
}
<|endoftext|>


<commit_before>package goarmorlogs

import (
	"regexp"
	"testing"
)

func TestDirectoryAndLogName(t *testing.T) {
	var a = []struct {
		in, dir, file string
	}{
		{"/var/log/app.log", "/var/log", "app.log"},
		{"/var/log", "/var/log", ""},
		{"/var/log/", "/var/log", ""},
		{"", ".", ""},
		{"/", "/", ""},
		{"//", "/", ""},
		{"///", "/", ""},
		{"///...", "/", ""},
		{"..///", "..", ""},
		{"...///", "...", ""},
		{"....///", "....", ""},
		{"...", ".", ""},
		{".", ".", ""},
	}

	var (
		x, y string
		r    *regexp.Regexp

		err error
	)

	r, err = regexp.Compile("^[.]+$")
	if err != nil {
		t.Fatal(err)
	}

	for _, v := range a {
		x, y, err = DirectoryAndLogName(v.in, r)
		if err != nil || x != v.dir || y != v.file {
			t.Errorf("directoryAndLogName(%q) => (%q, %q, %v) want (%q, %q, <nil>)",
				v.in, x, y, err, v.dir, v.file)
		}
	}
}

func TestPartitionedPathByUserID(t *testing.T) {
	var a = []struct {
		in   int64
		want string
	}{
		{1, "000/000/001"},
		{123456789, "123/456/789"},
	}

	var (
		s   string
		err error
	)

	for _, v := range a {
		s, err = PartitionedPathByUserID(v.in)
		if err != nil || s != v.want {
			t.Errorf("PartitionedPathByUserID(%d) => (%q, %v) want (%q, <nil>)",
				v.in, s, err, v.want)
		}
	}
}
<commit_msg>minor<commit_after>package goarmorlogs

import (
	"regexp"
	"testing"
)

func TestDirectoryAndLogName(t *testing.T) {
	var a = []struct {
		in, dir, file string
	}{
		{"/var/log/app.log", "/var/log", "app.log"},
		{"/var/log", "/var/log", ""},
		{"/var/log/", "/var/log", ""},
		{"", ".", ""},
		{"/", "/", ""},
		{"//", "/", ""},
		{"///", "/", ""},
		{"///...", "/", ""},
		{"..///", "..", ""},
		{"...///", "...", ""},
		{"....///", "....", ""},
		{"...", ".", ""},
		{".", ".", ""},
	}
	r, err := regexp.Compile("^[.]+$")
	if err != nil {
		t.Fatal(err)
	}
	var x, y string
	for _, v := range a {
		x, y, err = DirectoryAndLogName(v.in, r)
		if err != nil || x != v.dir || y != v.file {
			t.Errorf("directoryAndLogName(%q) => (%q, %q, %v) want (%q, %q, <nil>)",
				v.in, x, y, err, v.dir, v.file)
		}
	}
}

func TestPartitionedPathByUserID(t *testing.T) {
	var a = []struct {
		in   int64
		want string
	}{
		{1, "000/000/001"},
		{123456789, "123/456/789"},
	}
	var (
		s   string
		err error
	)
	for _, v := range a {
		s, err = PartitionedPathByUserID(v.in)
		if err != nil || s != v.want {
			t.Errorf("PartitionedPathByUserID(%d) => (%q, %v) want (%q, <nil>)",
				v.in, s, err, v.want)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/bagman/dpn"
	"os"
)

// dpn_sync copies updated records from the registries
// at other nodes into our local DPN registry.
func main() {
	configPath := "dpn/dpn_config.json"
	configName := parseCommandLine()
	dpnConfig, err := dpn.LoadConfig(configPath, configName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading dpn config file '%s': %v\n",
			configPath, err)
		os.Exit(2)
	}
	dpnSync, err := dpn.NewDPNSync(dpnConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "dpn_sync failed to start: %v\n", err)
		os.Exit(3)
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		fmt.Fprintf(os.Stderr, "dpn_sync could not get node list: %v\n", err)
		os.Exit(4)
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		// The sync code logs to a file called dpn_sync.log
		// in the log directory specified in the config file.
		syncResult := dpnSync.SyncEverythingFromNode(node)
		printErrors(syncResult, node)
	}

}

func parseCommandLine() (string) {
	config := flag.String("config", "", "DPN config name [dev|test|production]")
	flag.Parse()
	if config == nil || *config == "" {
		printUsage()
		fmt.Fprintln(os.Stderr, "You must specify a DPN config name (dev|test|production)")
		os.Exit(1)
	}
	return *config
}

func printUsage() {
	fmt.Println("Usage: dpn_sync -config=pathToDPNConfigFile")
	fmt.Println("Sync bag and transfer entries from remote nodes to our local registry.")
}

func printErrors(syncResult *dpn.SyncResult, node *dpn.DPNNode) {
	if syncResult.BagSyncError != nil {
		fmt.Fprintf(os.Stderr, "Error synching bags from %s: %v\n",
			node.Namespace, syncResult.BagSyncError)
	}
	if syncResult.ReplicationSyncError != nil {
		fmt.Fprintf(os.Stderr, "Error synching Replication Transfers from %s: %v\n",
			node.Namespace, syncResult.ReplicationSyncError)
	}
	if syncResult.RestoreSyncError != nil {
		fmt.Fprintf(os.Stderr, "Error synching Restore Transfers from %s: %v\n",
			node.Namespace, syncResult.RestoreSyncError)
	}
}
<commit_msg>Added some debug statements (currently commented out)<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/bagman/dpn"
	"os"
)

// dpn_sync copies updated records from the registries
// at other nodes into our local DPN registry.
func main() {
	configPath := "dpn/dpn_config.json"
	configName := parseCommandLine()
	dpnConfig, err := dpn.LoadConfig(configPath, configName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading dpn config file '%s': %v\n",
			configPath, err)
		os.Exit(2)
	}
	dpnSync, err := dpn.NewDPNSync(dpnConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "dpn_sync failed to start: %v\n", err)
		os.Exit(3)
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		fmt.Fprintf(os.Stderr, "dpn_sync could not get node list: %v\n", err)
		os.Exit(4)
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		// The sync code logs to a file called dpn_sync.log
		// in the log directory specified in the config file.
		syncResult := dpnSync.SyncEverythingFromNode(node)
		printErrors(syncResult, node)

		// -- DEBUG --
		// fmt.Printf("************* Sync Result for %s ************\n", node.Name)
		// fmt.Println("--- BAGS ---")
		// for _, bag := range syncResult.Bags {
		// 	fmt.Println(bag.UUID)
		// }
		// fmt.Printf("--- XFERS ---")
		// for _, xfer := range syncResult.ReplicationTransfers {
		// 	fmt.Println("%s : % -> %s\n", xfer.BagId, xfer.FromNode, xfer.ToNode)
		// }
		// fmt.Println("--- RESTORES ---")
		// for _, xfer := range syncResult.RestoreTransfers {
		// 	fmt.Printf("%s : % -> %s\n", xfer.BagId, xfer.ToNode, xfer.FromNode)
		// }
		// fmt.Printf(">>> Bag Sync Error: %s\n", syncResult.BagSyncError)
		// fmt.Printf(">>> Replication Sync Error: %s\n", syncResult.ReplicationSyncError)
		// fmt.Printf(">>> Restore Sync Error: %s\n", syncResult.RestoreSyncError)
	}

}

func parseCommandLine() (string) {
	config := flag.String("config", "", "DPN config name [dev|test|production]")
	flag.Parse()
	if config == nil || *config == "" {
		printUsage()
		fmt.Fprintln(os.Stderr, "You must specify a DPN config name (dev|test|production)")
		os.Exit(1)
	}
	return *config
}

func printUsage() {
	fmt.Println("Usage: dpn_sync -config=pathToDPNConfigFile")
	fmt.Println("Sync bag and transfer entries from remote nodes to our local registry.")
}

func printErrors(syncResult *dpn.SyncResult, node *dpn.DPNNode) {
	if syncResult.BagSyncError != nil {
		fmt.Fprintf(os.Stderr, "Error synching bags from %s: %v\n",
			node.Namespace, syncResult.BagSyncError)
	}
	if syncResult.ReplicationSyncError != nil {
		fmt.Fprintf(os.Stderr, "Error synching Replication Transfers from %s: %v\n",
			node.Namespace, syncResult.ReplicationSyncError)
	}
	if syncResult.RestoreSyncError != nil {
		fmt.Fprintf(os.Stderr, "Error synching Restore Transfers from %s: %v\n",
			node.Namespace, syncResult.RestoreSyncError)
	}
}
<|endoftext|>