<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rpc

import (
	"context"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/grpc/appstatus"
	"go.chromium.org/luci/server/auth"

	"go.chromium.org/luci/buildbucket/appengine/internal/perm"
	"go.chromium.org/luci/buildbucket/appengine/model"
	"go.chromium.org/luci/buildbucket/appengine/tasks"
	taskdefs "go.chromium.org/luci/buildbucket/appengine/tasks/defs"
	pb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/buildbucket/protoutil"
)

// validateCancel validates the given request.
func validateCancel(req *pb.CancelBuildRequest) error {
	var err error
	switch {
	case req.GetId() == 0:
		return errors.Reason("id is required").Err()
	case req.SummaryMarkdown == "":
		return errors.Reason("summary_markdown is required").Err()
	case teeErr(validateSummaryMarkdown(req.SummaryMarkdown), &err) != nil:
		return errors.Annotate(err, "summary_markdown").Err()
	}
	return nil
}

// CancelBuild handles a request to cancel a build. Implements pb.BuildsServer.
func (*Builds) CancelBuild(ctx context.Context, req *pb.CancelBuildRequest) (*pb.Build, error) {
	if err := validateCancel(req); err != nil {
		return nil, appstatus.BadRequest(err)
	}
	m, err := getFieldMask(req.Fields)
	if err != nil {
		return nil, appstatus.Errorf(codes.InvalidArgument, "invalid field mask")
	}

	bld, err := getBuild(ctx, req.Id)
	if err != nil {
		return nil, err
	}
	if err := perm.HasInBuilder(ctx, perm.BuildsCancel, bld.Proto.Builder); err != nil {
		return nil, err
	}

	// If the build has ended, there's nothing to cancel.
	if protoutil.IsEnded(bld.Proto.Status) {
		return bld.ToProto(ctx, m)
	}

	err = datastore.RunInTransaction(ctx, func(ctx context.Context) error {
		// TODO(crbug.com/1189937): Move outside transaction, do not clear fields in bld.
		bld = &model.Build{ID: req.Id}
		inf := &model.BuildInfra{Build: datastore.KeyForObj(ctx, bld)}
		stp := &model.BuildSteps{Build: inf.Build}

		if err := datastore.Get(ctx, bld, inf, stp); err != nil {
			switch merr, ok := err.(errors.MultiError); {
			case !ok:
				return errors.Annotate(err, "failed to fetch build: %d", bld.ID).Err()
			case merr[0] == datastore.ErrNoSuchEntity:
				return perm.NotFoundErr(ctx)
			case merr[0] != nil:
				return errors.Annotate(merr[0], "failed to fetch build: %d", bld.ID).Err()
			case merr[1] != nil && merr[1] != datastore.ErrNoSuchEntity:
				return errors.Annotate(merr[1], "failed to fetch build infra: %d", bld.ID).Err()
			case merr[2] != nil && merr[2] != datastore.ErrNoSuchEntity:
				return errors.Annotate(merr[2], "failed to fetch build steps: %d", bld.ID).Err()
			case protoutil.IsEnded(bld.Proto.Status):
				return nil
			case merr[2] == datastore.ErrNoSuchEntity:
				stp = nil
			}
		}
		if inf.Proto.Swarming.GetHostname() != "" && inf.Proto.Swarming.TaskId != "" {
			if err := tasks.CancelSwarmingTask(ctx, &taskdefs.CancelSwarmingTask{
				Hostname: inf.Proto.Swarming.Hostname,
				TaskId:   inf.Proto.Swarming.TaskId,
				Realm:    bld.Realm(),
			}); err != nil {
				return errors.Annotate(err, "failed to enqueue swarming task cancellation task: %d", bld.ID).Err()
			}
		}
		if inf.Proto.Resultdb.GetHostname() != "" && inf.Proto.Resultdb.Invocation != "" {
			if err := tasks.FinalizeResultDB(ctx, &taskdefs.FinalizeResultDB{
				BuildId: bld.ID,
			}); err != nil {
				return errors.Annotate(err, "failed to enqueue resultdb finalization task: %d", bld.ID).Err()
			}
		}
		if err := tasks.ExportBigQuery(ctx, &taskdefs.ExportBigQuery{
			BuildId: bld.ID,
		}); err != nil {
			return errors.Annotate(err, "failed to enqueue bigquery export task: %d", bld.ID).Err()
		}
		if err := notifyPubSub(ctx, bld); err != nil {
			return errors.Annotate(err, "failed to enqueue pubsub notification task: %d", bld.ID).Err()
		}

		bld.Leasee = nil
		bld.LeaseExpirationDate = time.Time{}
		bld.LeaseKey = 0
		bld.StatusChangedTime = clock.Now(ctx).UTC()

		bld.Proto.CanceledBy = string(auth.CurrentIdentity(ctx))
		bld.Proto.EndTime = timestamppb.New(bld.StatusChangedTime)
		bld.Proto.Status = pb.Status_CANCELED
		bld.Proto.SummaryMarkdown = req.SummaryMarkdown

		toPut := []interface{}{bld}

		if stp != nil {
			switch changed, err := stp.CancelIncomplete(ctx, bld.Proto.EndTime); {
			case err != nil:
				return errors.Annotate(err, "failed to fetch build steps: %d", bld.ID).Err()
			case changed:
				toPut = append(toPut, stp)
			}
		}

		if err := datastore.Put(ctx, toPut...); err != nil {
			return errors.Annotate(err, "failed to store build: %d", bld.ID).Err()
		}
		return nil
	}, nil)
	if err != nil {
		return nil, err
	}

	return bld.ToProto(ctx, m)
}
<commit_msg>[buildbucket] Don't nil entity inside transaction<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rpc

import (
	"context"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/grpc/appstatus"
	"go.chromium.org/luci/server/auth"

	"go.chromium.org/luci/buildbucket/appengine/internal/perm"
	"go.chromium.org/luci/buildbucket/appengine/model"
	"go.chromium.org/luci/buildbucket/appengine/tasks"
	taskdefs "go.chromium.org/luci/buildbucket/appengine/tasks/defs"
	pb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/buildbucket/protoutil"
)

// validateCancel validates the given request.
func validateCancel(req *pb.CancelBuildRequest) error {
	var err error
	switch {
	case req.GetId() == 0:
		return errors.Reason("id is required").Err()
	case req.SummaryMarkdown == "":
		return errors.Reason("summary_markdown is required").Err()
	case teeErr(validateSummaryMarkdown(req.SummaryMarkdown), &err) != nil:
		return errors.Annotate(err, "summary_markdown").Err()
	}
	return nil
}

// CancelBuild handles a request to cancel a build. Implements pb.BuildsServer.
func (*Builds) CancelBuild(ctx context.Context, req *pb.CancelBuildRequest) (*pb.Build, error) {
	if err := validateCancel(req); err != nil {
		return nil, appstatus.BadRequest(err)
	}
	m, err := getFieldMask(req.Fields)
	if err != nil {
		return nil, appstatus.Errorf(codes.InvalidArgument, "invalid field mask")
	}

	bld, err := getBuild(ctx, req.Id)
	if err != nil {
		return nil, err
	}
	if err := perm.HasInBuilder(ctx, perm.BuildsCancel, bld.Proto.Builder); err != nil {
		return nil, err
	}

	// If the build has ended, there's nothing to cancel.
	if protoutil.IsEnded(bld.Proto.Status) {
		return bld.ToProto(ctx, m)
	}

	err = datastore.RunInTransaction(ctx, func(ctx context.Context) error {
		// TODO(crbug.com/1189937): Move outside transaction, do not clear fields in bld.
		bld = &model.Build{ID: req.Id}
		inf := &model.BuildInfra{Build: datastore.KeyForObj(ctx, bld)}
		stp := &model.BuildSteps{Build: inf.Build}

		cancelSteps := true
		if err := datastore.Get(ctx, bld, inf, stp); err != nil {
			switch merr, ok := err.(errors.MultiError); {
			case !ok:
				return errors.Annotate(err, "failed to fetch build: %d", bld.ID).Err()
			case merr[0] == datastore.ErrNoSuchEntity:
				return perm.NotFoundErr(ctx)
			case merr[0] != nil:
				return errors.Annotate(merr[0], "failed to fetch build: %d", bld.ID).Err()
			case merr[1] != nil && merr[1] != datastore.ErrNoSuchEntity:
				return errors.Annotate(merr[1], "failed to fetch build infra: %d", bld.ID).Err()
			case merr[2] != nil && merr[2] != datastore.ErrNoSuchEntity:
				return errors.Annotate(merr[2], "failed to fetch build steps: %d", bld.ID).Err()
			case protoutil.IsEnded(bld.Proto.Status):
				return nil
			case merr[2] == datastore.ErrNoSuchEntity:
				cancelSteps = false
			}
		}
		if inf.Proto.Swarming.GetHostname() != "" && inf.Proto.Swarming.TaskId != "" {
			if err := tasks.CancelSwarmingTask(ctx, &taskdefs.CancelSwarmingTask{
				Hostname: inf.Proto.Swarming.Hostname,
				TaskId:   inf.Proto.Swarming.TaskId,
				Realm:    bld.Realm(),
			}); err != nil {
				return errors.Annotate(err, "failed to enqueue swarming task cancellation task: %d", bld.ID).Err()
			}
		}
		if inf.Proto.Resultdb.GetHostname() != "" && inf.Proto.Resultdb.Invocation != "" {
			if err := tasks.FinalizeResultDB(ctx, &taskdefs.FinalizeResultDB{
				BuildId: bld.ID,
			}); err != nil {
				return errors.Annotate(err, "failed to enqueue resultdb finalization task: %d", bld.ID).Err()
			}
		}
		if err := tasks.ExportBigQuery(ctx, &taskdefs.ExportBigQuery{
			BuildId: bld.ID,
		}); err != nil {
			return errors.Annotate(err, "failed to enqueue bigquery export task: %d", bld.ID).Err()
		}
		if err := notifyPubSub(ctx, bld); err != nil {
			return errors.Annotate(err, "failed to enqueue pubsub notification task: %d", bld.ID).Err()
		}

		bld.Leasee = nil
		bld.LeaseExpirationDate = time.Time{}
		bld.LeaseKey = 0
		bld.StatusChangedTime = clock.Now(ctx).UTC()

		bld.Proto.CanceledBy = string(auth.CurrentIdentity(ctx))
		bld.Proto.EndTime = timestamppb.New(bld.StatusChangedTime)
		bld.Proto.Status = pb.Status_CANCELED
		bld.Proto.SummaryMarkdown = req.SummaryMarkdown

		toPut := []interface{}{bld}

		if cancelSteps {
			switch changed, err := stp.CancelIncomplete(ctx, bld.Proto.EndTime); {
			case err != nil:
				return errors.Annotate(err, "failed to fetch build steps: %d", bld.ID).Err()
			case changed:
				toPut = append(toPut, stp)
			}
		}

		if err := datastore.Put(ctx, toPut...); err != nil {
			return errors.Annotate(err, "failed to store build: %d", bld.ID).Err()
		}
		return nil
	}, nil)
	if err != nil {
		return nil, err
	}

	return bld.ToProto(ctx, m)
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/serf/coordinate"
	"github.com/hashicorp/serf/serf"
)

const (
	// clientRPCCache controls how long we keep an idle connection
	// open to a server
	clientRPCCache = 30 * time.Second

	// clientMaxStreams controls how many idle streams we keep
	// open to a server
	clientMaxStreams = 32

	// serfEventBacklog is the maximum number of unprocessed Serf Events
	// that will be held in queue before new serf events block.  A
	// blocking serf event queue is a bad thing.
	serfEventBacklog = 256

	// serfEventBacklogWarning is the threshold at which point log
	// warnings will be emitted indicating a problem when processing serf
	// events.
	serfEventBacklogWarning = 200
)

// Interface is used to provide either a Client or Server,
// both of which can be used to perform certain common
// Consul methods
type Interface interface {
	RPC(method string, args interface{}, reply interface{}) error
	LANMembers() []serf.Member
	LocalMember() serf.Member
}

// Client is Consul client which uses RPC to communicate with the
// services for service discovery, health checking, and DC forwarding.
type Client struct {
	config *Config

	// Connection pool to consul servers
	connPool *ConnPool

	// serverConfig provides the necessary load/store semantics to
	// serverConfig
	serverConfigValue atomic.Value
	serverConfigMtx   sync.Mutex

	// consulServersCh is used to receive events related to the
	// maintenance of the list of consulServers
	consulServersCh chan consulServerEventTypes

	// eventCh is used to receive events from the
	// serf cluster in the datacenter
	eventCh chan serf.Event

	// Logger uses the provided LogOutput
	logger *log.Logger

	// serf is the Serf cluster maintained inside the DC
	// which contains all the DC nodes
	serf *serf.Serf

	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewClient is used to construct a new Consul client from the
// configuration, potentially returning an error
func NewClient(config *Config) (*Client, error) {
	// Check the protocol version
	if err := config.CheckVersion(); err != nil {
		return nil, err
	}

	// Check for a data directory!
	if config.DataDir == "" {
		return nil, fmt.Errorf("Config must provide a DataDir")
	}

	// Sanity check the ACLs
	if err := config.CheckACL(); err != nil {
		return nil, err
	}

	// Ensure we have a log output
	if config.LogOutput == nil {
		config.LogOutput = os.Stderr
	}

	// Create the tls Wrapper
	tlsWrap, err := config.tlsConfig().OutgoingTLSWrapper()
	if err != nil {
		return nil, err
	}

	// Create a logger
	logger := log.New(config.LogOutput, "", log.LstdFlags)

	// Create server
	c := &Client{
		config:     config,
		connPool:   NewPool(config.LogOutput, clientRPCConnMaxIdle, clientMaxStreams, tlsWrap),
		eventCh:    make(chan serf.Event, serfEventBacklog),
		logger:     logger,
		shutdownCh: make(chan struct{}),
	}

	// Create the initial serverConfig
	serverCfg := serverConfig{}
	c.serverConfigValue.Store(serverCfg)

	// Start consulServers maintenance
	go c.consulServersManager()

	// Start the Serf listeners to prevent a deadlock
	go c.lanEventHandler()

	// Initialize the lan Serf
	c.serf, err = c.setupSerf(config.SerfLANConfig,
		c.eventCh, serfLANSnapshot)
	if err != nil {
		c.Shutdown()
		return nil, fmt.Errorf("Failed to start lan serf: %v", err)
	}
	return c, nil
}

// setupSerf is used to setup and initialize a Serf
func (c *Client) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (*serf.Serf, error) {
	conf.Init()
	conf.NodeName = c.config.NodeName
	conf.Tags["role"] = "node"
	conf.Tags["dc"] = c.config.Datacenter
	conf.Tags["vsn"] = fmt.Sprintf("%d", c.config.ProtocolVersion)
	conf.Tags["vsn_min"] = fmt.Sprintf("%d", ProtocolVersionMin)
	conf.Tags["vsn_max"] = fmt.Sprintf("%d", ProtocolVersionMax)
	conf.Tags["build"] = c.config.Build
	conf.MemberlistConfig.LogOutput = c.config.LogOutput
	conf.LogOutput = c.config.LogOutput
	conf.EventCh = ch
	conf.SnapshotPath = filepath.Join(c.config.DataDir, path)
	conf.ProtocolVersion = protocolVersionMap[c.config.ProtocolVersion]
	conf.RejoinAfterLeave = c.config.RejoinAfterLeave
	conf.Merge = &lanMergeDelegate{dc: c.config.Datacenter}
	conf.DisableCoordinates = c.config.DisableCoordinates
	if err := ensurePath(conf.SnapshotPath, false); err != nil {
		return nil, err
	}
	return serf.Create(conf)
}

// Shutdown is used to shutdown the client
func (c *Client) Shutdown() error {
	c.logger.Printf("[INFO] consul: shutting down client")
	c.shutdownLock.Lock()
	defer c.shutdownLock.Unlock()

	if c.shutdown {
		return nil
	}

	c.shutdown = true
	close(c.shutdownCh)

	if c.serf != nil {
		c.serf.Shutdown()
	}

	// Close the connection pool
	c.connPool.Shutdown()
	return nil
}

// Leave is used to prepare for a graceful shutdown
func (c *Client) Leave() error {
	c.logger.Printf("[INFO] consul: client starting leave")

	// Leave the LAN pool
	if c.serf != nil {
		if err := c.serf.Leave(); err != nil {
			c.logger.Printf("[ERR] consul: Failed to leave LAN Serf cluster: %v", err)
		}
	}
	return nil
}

// JoinLAN is used to have Consul client join the inner-DC pool
// The target address should be another node inside the DC
// listening on the Serf LAN address
func (c *Client) JoinLAN(addrs []string) (int, error) {
	return c.serf.Join(addrs, true)
}

// LocalMember is used to return the local node
func (c *Client) LocalMember() serf.Member {
	return c.serf.LocalMember()
}

// LANMembers is used to return the members of the LAN cluster
func (c *Client) LANMembers() []serf.Member {
	return c.serf.Members()
}

// RemoveFailedNode is used to remove a failed node from the cluster
func (c *Client) RemoveFailedNode(node string) error {
	return c.serf.RemoveFailedNode(node)
}

// KeyManagerLAN returns the LAN Serf keyring manager
func (c *Client) KeyManagerLAN() *serf.KeyManager {
	return c.serf.KeyManager()
}

// Encrypted determines if gossip is encrypted
func (c *Client) Encrypted() bool {
	return c.serf.EncryptionEnabled()
}

// lanEventHandler is used to handle events from the lan Serf cluster
func (c *Client) lanEventHandler() {
	var numQueuedEvents int
	for {
		numQueuedEvents = len(c.eventCh)
		if numQueuedEvents > serfEventBacklogWarning {
			c.logger.Printf("[WARN] consul: number of queued serf events above warning threshold: %d/%d", numQueuedEvents, serfEventBacklogWarning)
		}

		select {
		case e := <-c.eventCh:
			switch e.EventType() {
			case serf.EventMemberJoin:
				c.nodeJoin(e.(serf.MemberEvent))
			case serf.EventMemberLeave, serf.EventMemberFailed:
				c.nodeFail(e.(serf.MemberEvent))
			case serf.EventUser:
				c.localEvent(e.(serf.UserEvent))
			case serf.EventMemberUpdate: // Ignore
			case serf.EventMemberReap: // Ignore
			case serf.EventQuery: // Ignore
			default:
				c.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-c.shutdownCh:
			return
		}
	}
}

// nodeJoin is used to handle join events on the serf cluster
func (c *Client) nodeJoin(me serf.MemberEvent) {
	for _, m := range me.Members {
		ok, parts := isConsulServer(m)
		if !ok {
			continue
		}
		if parts.Datacenter != c.config.Datacenter {
			c.logger.Printf("[WARN] consul: server %s for datacenter %s has joined wrong cluster",
				m.Name, parts.Datacenter)
			continue
		}
		c.logger.Printf("[INFO] consul: adding server %s", parts)
		c.AddServer(parts)

		// Trigger the callback
		if c.config.ServerUp != nil {
			c.config.ServerUp()
		}
	}
}

// nodeFail is used to handle fail events on the serf cluster
func (c *Client) nodeFail(me serf.MemberEvent) {
	for _, m := range me.Members {
		ok, parts := isConsulServer(m)
		if !ok {
			continue
		}
		c.logger.Printf("[INFO] consul: removing server %s", parts)
		c.RemoveServer(parts)
	}
}

// localEvent is called when we receive an event on the local Serf
func (c *Client) localEvent(event serf.UserEvent) {
	// Handle only consul events
	if !strings.HasPrefix(event.Name, "consul:") {
		return
	}

	switch name := event.Name; {
	case name == newLeaderEvent:
		c.logger.Printf("[INFO] consul: New leader elected: %s", event.Payload)

		// Trigger the callback
		if c.config.ServerUp != nil {
			c.config.ServerUp()
		}
	case isUserEvent(name):
		event.Name = rawUserEventName(name)
		c.logger.Printf("[DEBUG] consul: user event: %s", event.Name)

		// Trigger the callback
		if c.config.UserEventHandler != nil {
			c.config.UserEventHandler(event)
		}
	default:
		c.logger.Printf("[WARN] consul: Unhandled local event: %v", event)
	}
}

// RPC is used to forward an RPC call to a consul server, or fail if no servers
func (c *Client) RPC(method string, args interface{}, reply interface{}) error {
	serverCfgPtr := c.serverConfigValue.Load()
	if serverCfgPtr == nil {
		c.logger.Printf("[ERR] consul: Failed to load a server config")
		return structs.ErrNoServers
	}
	serverCfg := serverCfgPtr.(serverConfig)

	numServers := len(serverCfg.servers)
	if numServers == 0 {
		c.logger.Printf("[ERR] consul: No servers found in the server config")
		return structs.ErrNoServers
	}

	// Find the first non-failing server in the server list.  If this is
	// not the first server a prior RPC call marked the first server as
	// failed and we're waiting for the server management task to reorder
	// a working server to the front of the list.
	var server *serverParts
	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			server = serverCfg.servers[i]
			break
		}
	}

	// Forward to remote Consul
	if err := c.connPool.RPC(c.config.Datacenter, server.Addr, server.Version, method, args, reply); err != nil {
		atomic.AddUint64(&server.Disabled, 1)
		c.logger.Printf("[ERR] consul: RPC failed to server %s: %v", server.Addr, err)
		c.consulServersCh <- consulServersRPCError
		return err
	}

	return nil
}

// Stats is used to return statistics for debugging and insight
// for various sub-systems
func (c *Client) Stats() map[string]map[string]string {
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	toString := func(v uint64) string {
		return strconv.FormatUint(v, 10)
	}
	stats := map[string]map[string]string{
		"consul": map[string]string{
			"server":        "false",
			"known_servers": toString(uint64(len(serverCfg.servers))),
		},
		"serf_lan": c.serf.Stats(),
		"runtime":  runtimeStats(),
	}
	return stats
}

// GetCoordinate returns the network coordinate of the current node, as
// maintained by Serf.
func (c *Client) GetCoordinate() (*coordinate.Coordinate, error) {
	return c.serf.GetCoordinate()
}
<commit_msg>Handle the case where there are no healthy servers<commit_after>package consul

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/serf/coordinate"
	"github.com/hashicorp/serf/serf"
)

const (
	// clientRPCCache controls how long we keep an idle connection
	// open to a server
	clientRPCCache = 30 * time.Second

	// clientMaxStreams controls how many idle streams we keep
	// open to a server
	clientMaxStreams = 32

	// serfEventBacklog is the maximum number of unprocessed Serf Events
	// that will be held in queue before new serf events block.  A
	// blocking serf event queue is a bad thing.
	serfEventBacklog = 256

	// serfEventBacklogWarning is the threshold at which point log
	// warnings will be emitted indicating a problem when processing serf
	// events.
	serfEventBacklogWarning = 200
)

// Interface is used to provide either a Client or Server,
// both of which can be used to perform certain common
// Consul methods
type Interface interface {
	RPC(method string, args interface{}, reply interface{}) error
	LANMembers() []serf.Member
	LocalMember() serf.Member
}

// Client is Consul client which uses RPC to communicate with the
// services for service discovery, health checking, and DC forwarding.
type Client struct {
	config *Config

	// Connection pool to consul servers
	connPool *ConnPool

	// serverConfig provides the necessary load/store semantics to
	// serverConfig
	serverConfigValue atomic.Value
	serverConfigMtx   sync.Mutex

	// consulServersCh is used to receive events related to the
	// maintenance of the list of consulServers
	consulServersCh chan consulServerEventTypes

	// eventCh is used to receive events from the
	// serf cluster in the datacenter
	eventCh chan serf.Event

	// Logger uses the provided LogOutput
	logger *log.Logger

	// serf is the Serf cluster maintained inside the DC
	// which contains all the DC nodes
	serf *serf.Serf

	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewClient is used to construct a new Consul client from the
// configuration, potentially returning an error
func NewClient(config *Config) (*Client, error) {
	// Check the protocol version
	if err := config.CheckVersion(); err != nil {
		return nil, err
	}

	// Check for a data directory!
	if config.DataDir == "" {
		return nil, fmt.Errorf("Config must provide a DataDir")
	}

	// Sanity check the ACLs
	if err := config.CheckACL(); err != nil {
		return nil, err
	}

	// Ensure we have a log output
	if config.LogOutput == nil {
		config.LogOutput = os.Stderr
	}

	// Create the tls Wrapper
	tlsWrap, err := config.tlsConfig().OutgoingTLSWrapper()
	if err != nil {
		return nil, err
	}

	// Create a logger
	logger := log.New(config.LogOutput, "", log.LstdFlags)

	// Create server
	c := &Client{
		config:     config,
		connPool:   NewPool(config.LogOutput, clientRPCConnMaxIdle, clientMaxStreams, tlsWrap),
		eventCh:    make(chan serf.Event, serfEventBacklog),
		logger:     logger,
		shutdownCh: make(chan struct{}),
	}

	// Create the initial serverConfig
	serverCfg := serverConfig{}
	c.serverConfigValue.Store(serverCfg)

	// Start consulServers maintenance
	go c.consulServersManager()

	// Start the Serf listeners to prevent a deadlock
	go c.lanEventHandler()

	// Initialize the lan Serf
	c.serf, err = c.setupSerf(config.SerfLANConfig,
		c.eventCh, serfLANSnapshot)
	if err != nil {
		c.Shutdown()
		return nil, fmt.Errorf("Failed to start lan serf: %v", err)
	}
	return c, nil
}

// setupSerf is used to setup and initialize a Serf
func (c *Client) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (*serf.Serf, error) {
	conf.Init()
	conf.NodeName = c.config.NodeName
	conf.Tags["role"] = "node"
	conf.Tags["dc"] = c.config.Datacenter
	conf.Tags["vsn"] = fmt.Sprintf("%d", c.config.ProtocolVersion)
	conf.Tags["vsn_min"] = fmt.Sprintf("%d", ProtocolVersionMin)
	conf.Tags["vsn_max"] = fmt.Sprintf("%d", ProtocolVersionMax)
	conf.Tags["build"] = c.config.Build
	conf.MemberlistConfig.LogOutput = c.config.LogOutput
	conf.LogOutput = c.config.LogOutput
	conf.EventCh = ch
	conf.SnapshotPath = filepath.Join(c.config.DataDir, path)
	conf.ProtocolVersion = protocolVersionMap[c.config.ProtocolVersion]
	conf.RejoinAfterLeave = c.config.RejoinAfterLeave
	conf.Merge = &lanMergeDelegate{dc: c.config.Datacenter}
	conf.DisableCoordinates = c.config.DisableCoordinates
	if err := ensurePath(conf.SnapshotPath, false); err != nil {
		return nil, err
	}
	return serf.Create(conf)
}

// Shutdown is used to shutdown the client
func (c *Client) Shutdown() error {
	c.logger.Printf("[INFO] consul: shutting down client")
	c.shutdownLock.Lock()
	defer c.shutdownLock.Unlock()

	if c.shutdown {
		return nil
	}

	c.shutdown = true
	close(c.shutdownCh)

	if c.serf != nil {
		c.serf.Shutdown()
	}

	// Close the connection pool
	c.connPool.Shutdown()
	return nil
}

// Leave is used to prepare for a graceful shutdown
func (c *Client) Leave() error {
	c.logger.Printf("[INFO] consul: client starting leave")

	// Leave the LAN pool
	if c.serf != nil {
		if err := c.serf.Leave(); err != nil {
			c.logger.Printf("[ERR] consul: Failed to leave LAN Serf cluster: %v", err)
		}
	}
	return nil
}

// JoinLAN is used to have Consul client join the inner-DC pool
// The target address should be another node inside the DC
// listening on the Serf LAN address
func (c *Client) JoinLAN(addrs []string) (int, error) {
	return c.serf.Join(addrs, true)
}

// LocalMember is used to return the local node
func (c *Client) LocalMember() serf.Member {
	return c.serf.LocalMember()
}

// LANMembers is used to return the members of the LAN cluster
func (c *Client) LANMembers() []serf.Member {
	return c.serf.Members()
}

// RemoveFailedNode is used to remove a failed node from the cluster
func (c *Client) RemoveFailedNode(node string) error {
	return c.serf.RemoveFailedNode(node)
}

// KeyManagerLAN returns the LAN Serf keyring manager
func (c *Client) KeyManagerLAN() *serf.KeyManager {
	return c.serf.KeyManager()
}

// Encrypted determines if gossip is encrypted
func (c *Client) Encrypted() bool {
	return c.serf.EncryptionEnabled()
}

// lanEventHandler is used to handle events from the lan Serf cluster
func (c *Client) lanEventHandler() {
	var numQueuedEvents int
	for {
		numQueuedEvents = len(c.eventCh)
		if numQueuedEvents > serfEventBacklogWarning {
			c.logger.Printf("[WARN] consul: number of queued serf events above warning threshold: %d/%d", numQueuedEvents, serfEventBacklogWarning)
		}

		select {
		case e := <-c.eventCh:
			switch e.EventType() {
			case serf.EventMemberJoin:
				c.nodeJoin(e.(serf.MemberEvent))
			case serf.EventMemberLeave, serf.EventMemberFailed:
				c.nodeFail(e.(serf.MemberEvent))
			case serf.EventUser:
				c.localEvent(e.(serf.UserEvent))
			case serf.EventMemberUpdate: // Ignore
			case serf.EventMemberReap: // Ignore
			case serf.EventQuery: // Ignore
			default:
				c.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-c.shutdownCh:
			return
		}
	}
}

// nodeJoin is used to handle join events on the serf cluster
func (c *Client) nodeJoin(me serf.MemberEvent) {
	for _, m := range me.Members {
		ok, parts := isConsulServer(m)
		if !ok {
			continue
		}
		if parts.Datacenter != c.config.Datacenter {
			c.logger.Printf("[WARN] consul: server %s for datacenter %s has joined wrong cluster",
				m.Name, parts.Datacenter)
			continue
		}
		c.logger.Printf("[INFO] consul: adding server %s", parts)
		c.AddServer(parts)

		// Trigger the callback
		if c.config.ServerUp != nil {
			c.config.ServerUp()
		}
	}
}

// nodeFail is used to handle fail events on the serf cluster
func (c *Client) nodeFail(me serf.MemberEvent) {
	for _, m := range me.Members {
		ok, parts := isConsulServer(m)
		if !ok {
			continue
		}
		c.logger.Printf("[INFO] consul: removing server %s", parts)
		c.RemoveServer(parts)
	}
}

// localEvent is called when we receive an event on the local Serf
func (c *Client) localEvent(event serf.UserEvent) {
	// Handle only consul events
	if !strings.HasPrefix(event.Name, "consul:") {
		return
	}

	switch name := event.Name; {
	case name == newLeaderEvent:
		c.logger.Printf("[INFO] consul: New leader elected: %s", event.Payload)

		// Trigger the callback
		if c.config.ServerUp != nil {
			c.config.ServerUp()
		}
	case isUserEvent(name):
		event.Name = rawUserEventName(name)
		c.logger.Printf("[DEBUG] consul: user event: %s", event.Name)

		// Trigger the callback
		if c.config.UserEventHandler != nil {
			c.config.UserEventHandler(event)
		}
	default:
		c.logger.Printf("[WARN] consul: Unhandled local event: %v", event)
	}
}

// RPC is used to forward an RPC call to a consul server, or fail if no servers
func (c *Client) RPC(method string, args interface{}, reply interface{}) error {
	serverCfgPtr := c.serverConfigValue.Load()
	if serverCfgPtr == nil {
		c.logger.Printf("[ERR] consul: Failed to load a server config")
		return structs.ErrNoServers
	}
	serverCfg := serverCfgPtr.(serverConfig)

	numServers := len(serverCfg.servers)
	if numServers == 0 {
		c.logger.Printf("[ERR] consul: No servers found in the server config")
		return structs.ErrNoServers
	}

	// Find the first non-failing server in the server list.  If this is
	// not the first server a prior RPC call marked the first server as
	// failed and we're waiting for the server management task to reorder
	// a working server to the front of the list.
	var server *serverParts
	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			server = serverCfg.servers[i]
			break
		}
	}

	if server == nil {
		c.logger.Printf("[ERR] consul: No healthy servers found in the server config")
		return structs.ErrNoServers
	}

	// Forward to remote Consul
	if err := c.connPool.RPC(c.config.Datacenter, server.Addr, server.Version, method, args, reply); err != nil {
		atomic.AddUint64(&server.Disabled, 1)
		c.logger.Printf("[ERR] consul: RPC failed to server %s: %v", server.Addr, err)
		c.consulServersCh <- consulServersRPCError
		return err
	}

	return nil
}

// Stats is used to return statistics for debugging and insight
// for various sub-systems
func (c *Client) Stats() map[string]map[string]string {
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	toString := func(v uint64) string {
		return strconv.FormatUint(v, 10)
	}
	stats := map[string]map[string]string{
		"consul": map[string]string{
			"server":        "false",
			"known_servers": toString(uint64(len(serverCfg.servers))),
		},
		"serf_lan": c.serf.Stats(),
		"runtime":  runtimeStats(),
	}
	return stats
}

// GetCoordinate returns the network coordinate of the current node, as
// maintained by Serf.
func (c *Client) GetCoordinate() (*coordinate.Coordinate, error) {
	return c.serf.GetCoordinate()
}
<|endoftext|>


<commit_before>package apexovernsq

import (
	"fmt"
	"testing"

	"github.com/avct/apexovernsq/protobuf"

	alog "github.com/apex/log"
	"github.com/apex/log/handlers/memory"
	nsq "github.com/nsqio/go-nsq"
)

type entryList []*alog.Entry

// simulateMessageFromNSQ packs an Apex Log Entry into an NSQ Message
// and pushes it through the NSQApexLogHandler.  The NSQApexLogHandler
// will use the Apex Log memory handler to return the Apex Log Entry
// that would be logged.
func simulateMessageFromNSQ(sourceEntry *alog.Entry, filter *[]string) (*alog.Entry, error) {
	var handler *NSQApexLogHandler
	marshalledEntry, err := protobuf.Marshal(sourceEntry)
	if err != nil {
		return nil, fmt.Errorf("Couldn't marshal log entry: %s", err.Error())
	}

	msg := nsq.NewMessage(nsq.MessageID{'a', 'b', 'c'}, []byte(marshalledEntry))

	innerHandler := memory.New()
	if filter == nil {
		handler = NewNSQApexLogHandler(innerHandler, protobuf.Unmarshal)
	} else {
		filterHandler := NewApexLogServiceFilterHandler(innerHandler, filter)
		handler = NewNSQApexLogHandler(filterHandler, protobuf.Unmarshal)
	}

	handler.HandleMessage(msg)
	return innerHandler.Entries[0], nil
}

// simulateMessagesFromNSQ packs an entryList into a series of NSQ Messages and pushes them through the NSQApexLogHandler.  The NSQApexLogHandler will use the Apex Log memory handler to return an entryList that would be logged locally.
func simulateMessagesFromNSQ(sourceEntries *entryList, filter *[]string) (*entryList, error) {
	var marshalledEntry []byte
	var err error
	var msg *nsq.Message
	var handler nsq.Handler

	innerHandler := memory.New()
	if filter == nil {
		handler = NewNSQApexLogHandler(innerHandler, protobuf.Unmarshal)
	} else {
		filterHandler := NewApexLogServiceFilterHandler(innerHandler, filter)
		handler = NewNSQApexLogHandler(filterHandler, protobuf.Unmarshal)
	}

	for _, sourceEntry := range *sourceEntries {
		marshalledEntry, err = protobuf.Marshal(sourceEntry)
		if err != nil {
			return nil, fmt.Errorf("Couldn't marshal log entry: %s", err.Error())
		}

		msg = nsq.NewMessage(nsq.MessageID{'a', 'b', 'c'}, marshalledEntry)
		handler.HandleMessage(msg)
	}
	result := innerHandler.Entries[:]
	return (*entryList)(&result), nil

}

// simulateEntry returns a finalised Entry from the memory handler as
// it would appear in normal logging.
func simulateEntry(logger alog.Interface, fields alog.Fielder, msg string) *alog.Entry {
	memoryHandler := memory.New()
	alog.SetHandler(memoryHandler)
	if fields == nil {
		logger.Info(msg)
	} else {
		logger.WithFields(fields).Info(msg)
	}
	return memoryHandler.Entries[0]
}

// simulateEntries makes a stipulated number of finalised Apex Log
// Entry instances and stores them in a provided store (a pointer to
// an entryList), starting from a given offset. ⚠ Danger Will
// Robinson, here be side-effects. The provided offset and store will
// both be mutated. ⚠
func simulateEntries(ctx alog.Interface, count int, offset *int, store *entryList) {
	var m int
	for m = 0; m < count; m++ {
		entry := simulateEntry(ctx, nil, fmt.Sprintf("%d", *offset))
		(*store)[*offset] = entry
		*offset++
	}
}

func TestHandleMessage(t *testing.T) {
	logger := alog.Log
	sourceEntry := simulateEntry(logger, alog.Fields{
		"flavour": "pistachio",
		"scoops":  "2",
	}, "it's ice cream time!")

	entry, err := simulateMessageFromNSQ(sourceEntry, nil)
	if err != nil {
		t.Fatalf("%s", err.Error())
	}
	if entry.Message != sourceEntry.Message {
		t.Errorf("Expected %q, got %q", sourceEntry.Message, entry.Message)
	}
	if entry.Level != sourceEntry.Level {
		t.Errorf("Expected %s, got %s", sourceEntry.Level, entry.Level)
	}
	if entry.Timestamp != sourceEntry.Timestamp {
		t.Errorf("Expected %q, got %q", sourceEntry.Timestamp, entry.Timestamp)
	}
	expectedFieldCount := len(sourceEntry.Fields)
	actualFieldCount := len(entry.Fields)
	if expectedFieldCount != actualFieldCount {
		t.Errorf("Expected %d fields, but got %d fields", expectedFieldCount, actualFieldCount)
	}
	for fieldName, value := range sourceEntry.Fields {
		recieved := entry.Fields.Get(fieldName)
		if recieved != value {
			t.Errorf("Expected %s=%q, got %s=%q", fieldName, value, fieldName, recieved)
		}
	}
}

func TestFilterMessagesByService(t *testing.T) {
	var ctx alog.Interface
	var entries *entryList
	var err error
	var generatedMessages int
	var resultEntryCount int
	var sourceEntries entryList
	var totalMessages int

	var thisProcessFilter = &[]string{processName()}
	var otherProcessFilter = &[]string{"other"}
	var unknownProcessFilter = &[]string{"unknown"}

	var caseTable = []struct {
		serviceMessages      int
		otherServiceMessages int
		nonServiceMessages   int
		resultEntryCount     int
		filter               *[]string
		messages             []string
	}{
		{1, 1, 1, 3, nil, []string{"0", "1", "2"}},      // Default case, no filtering
		{1, 1, 1, 1, thisProcessFilter, []string{"0"}},  // Whitelist service messages
		{1, 1, 1, 1, otherProcessFilter, []string{"1"}}, // Whitelist other service messages
		{1, 1, 1, 0, unknownProcessFilter, []string{}},  // Whitelist service that isn't present

	}

	for caseNum, testCase := range caseTable {
		totalMessages = testCase.serviceMessages + testCase.nonServiceMessages + testCase.otherServiceMessages
		generatedMessages = 0
		sourceEntries = make([]*alog.Entry, totalMessages)

		// Generate service messages
		ctx = NewApexLogServiceContext()
		simulateEntries(ctx, testCase.serviceMessages, &generatedMessages, &sourceEntries)
		// Generate other service messages
		ctx = ctx.WithFields(alog.Fields{"service": "other"})
		simulateEntries(ctx, testCase.otherServiceMessages, &generatedMessages, &sourceEntries)

		// Generate non-service messages
		ctx = alog.Log
		simulateEntries(ctx, testCase.nonServiceMessages, &generatedMessages, &sourceEntries)

		entries, err = simulateMessagesFromNSQ(&sourceEntries, testCase.filter)
		if err != nil {
			t.Fatalf("Error in message simulation: %s", err.Error())
		}
		if entries == nil && testCase.resultEntryCount > 0 {
			t.Errorf("Expected %d entries from test case %d but got nil. Message counts: service %d, other-service %d, non-service %d.  Filter: %+v.", testCase.resultEntryCount, caseNum, testCase.serviceMessages, testCase.otherServiceMessages, testCase.nonServiceMessages, testCase.filter)
			continue
		}
		resultEntryCount = len(*entries)
		if resultEntryCount != testCase.resultEntryCount {
			t.Errorf("[Case %d] Expected %d entries, but got %d. Message counts: service %d, other-service %d, non-service %d.  Filter: %+v.", caseNum+1, testCase.resultEntryCount, resultEntryCount, testCase.serviceMessages, testCase.otherServiceMessages, testCase.nonServiceMessages, testCase.filter)
			continue
		}

		for idx, entry := range *entries {
			testMessage := testCase.messages[idx]
			if entry.Message != testMessage {
				t.Errorf("[Case %d] Expected Entry[%d] to have message %q, but got %q", caseNum, idx, testMessage, entry.Message)
			}
		}
	}
}
<commit_msg>Test multi-service filtering<commit_after>package apexovernsq

import (
	"fmt"
	"testing"

	"github.com/avct/apexovernsq/protobuf"

	alog "github.com/apex/log"
	"github.com/apex/log/handlers/memory"
	nsq "github.com/nsqio/go-nsq"
)

type entryList []*alog.Entry

// simulateMessageFromNSQ packs an Apex Log Entry into an NSQ Message
// and pushes it through the NSQApexLogHandler.  The NSQApexLogHandler
// will use the Apex Log memory handler to return the Apex Log Entry
// that would be logged.
func simulateMessageFromNSQ(sourceEntry *alog.Entry, filter *[]string) (*alog.Entry, error) {
	var handler *NSQApexLogHandler
	marshalledEntry, err := protobuf.Marshal(sourceEntry)
	if err != nil {
		return nil, fmt.Errorf("Couldn't marshal log entry: %s", err.Error())
	}

	msg := nsq.NewMessage(nsq.MessageID{'a', 'b', 'c'}, []byte(marshalledEntry))

	innerHandler := memory.New()
	if filter == nil {
		handler = NewNSQApexLogHandler(innerHandler, protobuf.Unmarshal)
	} else {
		filterHandler := NewApexLogServiceFilterHandler(innerHandler, filter)
		handler = NewNSQApexLogHandler(filterHandler, protobuf.Unmarshal)
	}

	handler.HandleMessage(msg)
	return innerHandler.Entries[0], nil
}

// simulateMessagesFromNSQ packs an entryList into a series of NSQ Messages and pushes them through the NSQApexLogHandler.  The NSQApexLogHandler will use the Apex Log memory handler to return an entryList that would be logged locally.
func simulateMessagesFromNSQ(sourceEntries *entryList, filter *[]string) (*entryList, error) {
	var marshalledEntry []byte
	var err error
	var msg *nsq.Message
	var handler nsq.Handler

	innerHandler := memory.New()
	if filter == nil {
		handler = NewNSQApexLogHandler(innerHandler, protobuf.Unmarshal)
	} else {
		filterHandler := NewApexLogServiceFilterHandler(innerHandler, filter)
		handler = NewNSQApexLogHandler(filterHandler, protobuf.Unmarshal)
	}

	for _, sourceEntry := range *sourceEntries {
		marshalledEntry, err = protobuf.Marshal(sourceEntry)
		if err != nil {
			return nil, fmt.Errorf("Couldn't marshal log entry: %s", err.Error())
		}

		msg = nsq.NewMessage(nsq.MessageID{'a', 'b', 'c'}, marshalledEntry)
		handler.HandleMessage(msg)
	}
	result := innerHandler.Entries[:]
	return (*entryList)(&result), nil

}

// simulateEntry returns a finalised Entry from the memory handler as
// it would appear in normal logging.
func simulateEntry(logger alog.Interface, fields alog.Fielder, msg string) *alog.Entry {
	memoryHandler := memory.New()
	alog.SetHandler(memoryHandler)
	if fields == nil {
		logger.Info(msg)
	} else {
		logger.WithFields(fields).Info(msg)
	}
	return memoryHandler.Entries[0]
}

// simulateEntries makes a stipulated number of finalised Apex Log
// Entry instances and stores them in a provided store (a pointer to
// an entryList), starting from a given offset. ⚠ Danger Will
// Robinson, here be side-effects. The provided offset and store will
// both be mutated. ⚠
func simulateEntries(ctx alog.Interface, count int, offset *int, store *entryList) {
	var m int
	for m = 0; m < count; m++ {
		entry := simulateEntry(ctx, nil, fmt.Sprintf("%d", *offset))
		(*store)[*offset] = entry
		*offset++
	}
}

func TestHandleMessage(t *testing.T) {
	logger := alog.Log
	sourceEntry := simulateEntry(logger, alog.Fields{
		"flavour": "pistachio",
		"scoops":  "2",
	}, "it's ice cream time!")

	entry, err := simulateMessageFromNSQ(sourceEntry, nil)
	if err != nil {
		t.Fatalf("%s", err.Error())
	}
	if entry.Message != sourceEntry.Message {
		t.Errorf("Expected %q, got %q", sourceEntry.Message, entry.Message)
	}
	if entry.Level != sourceEntry.Level {
		t.Errorf("Expected %s, got %s", sourceEntry.Level, entry.Level)
	}
	if entry.Timestamp != sourceEntry.Timestamp {
		t.Errorf("Expected %q, got %q", sourceEntry.Timestamp, entry.Timestamp)
	}
	expectedFieldCount := len(sourceEntry.Fields)
	actualFieldCount := len(entry.Fields)
	if expectedFieldCount != actualFieldCount {
		t.Errorf("Expected %d fields, but got %d fields", expectedFieldCount, actualFieldCount)
	}
	for fieldName, value := range sourceEntry.Fields {
		recieved := entry.Fields.Get(fieldName)
		if recieved != value {
			t.Errorf("Expected %s=%q, got %s=%q", fieldName, value, fieldName, recieved)
		}
	}
}

func TestFilterMessagesByService(t *testing.T) {
	var ctx alog.Interface
	var entries *entryList
	var err error
	var generatedMessages int
	var resultEntryCount int
	var sourceEntries entryList
	var totalMessages int

	var thisProcessFilter = &[]string{processName()}
	var otherProcessFilter = &[]string{"other"}
	var unknownProcessFilter = &[]string{"unknown"}
	var multiServiceFilter = &[]string{processName(), "other"}

	var caseTable = []struct {
		serviceMessages      int
		otherServiceMessages int
		nonServiceMessages   int
		resultEntryCount     int
		filter               *[]string
		messages             []string
	}{
		{1, 1, 1, 3, nil, []string{"0", "1", "2"}},           // Default case, no filtering
		{1, 1, 1, 1, thisProcessFilter, []string{"0"}},       // Whitelist service messages
		{1, 1, 1, 1, otherProcessFilter, []string{"1"}},      // Whitelist other service messages
		{1, 1, 1, 0, unknownProcessFilter, []string{}},       // Whitelist service that isn't present
		{1, 1, 1, 2, multiServiceFilter, []string{"0", "1"}}, // Whitelist both this service and "other" service

	}

	for caseNum, testCase := range caseTable {
		totalMessages = testCase.serviceMessages + testCase.nonServiceMessages + testCase.otherServiceMessages
		generatedMessages = 0
		sourceEntries = make([]*alog.Entry, totalMessages)

		// Generate service messages
		ctx = NewApexLogServiceContext()
		simulateEntries(ctx, testCase.serviceMessages, &generatedMessages, &sourceEntries)
		// Generate other service messages
		ctx = ctx.WithFields(alog.Fields{"service": "other"})
		simulateEntries(ctx, testCase.otherServiceMessages, &generatedMessages, &sourceEntries)

		// Generate non-service messages
		ctx = alog.Log
		simulateEntries(ctx, testCase.nonServiceMessages, &generatedMessages, &sourceEntries)

		entries, err = simulateMessagesFromNSQ(&sourceEntries, testCase.filter)
		if err != nil {
			t.Fatalf("Error in message simulation: %s", err.Error())
		}
		if entries == nil && testCase.resultEntryCount > 0 {
			t.Errorf("Expected %d entries from test case %d but got nil. Message counts: service %d, other-service %d, non-service %d.  Filter: %+v.", testCase.resultEntryCount, caseNum, testCase.serviceMessages, testCase.otherServiceMessages, testCase.nonServiceMessages, testCase.filter)
			continue
		}
		resultEntryCount = len(*entries)
		if resultEntryCount != testCase.resultEntryCount {
			t.Errorf("[Case %d] Expected %d entries, but got %d. Message counts: service %d, other-service %d, non-service %d.  Filter: %+v.", caseNum+1, testCase.resultEntryCount, resultEntryCount, testCase.serviceMessages, testCase.otherServiceMessages, testCase.nonServiceMessages, testCase.filter)
			continue
		}

		for idx, entry := range *entries {
			testMessage := testCase.messages[idx]
			if entry.Message != testMessage {
				t.Errorf("[Case %d] Expected Entry[%d] to have message %q, but got %q", caseNum, idx, testMessage, entry.Message)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"reflect"
	"testing"
)

var sut peekenv

func init() {
	log.SetOutput(ioutil.Discard)
}

var testSections = []Section{
	{title: `PATH`, lines: []string{`C:\Program Files\ConEmu`, `C:\Program Files\ConEmu\ConEmu`, `C:\Windows\SYSTEM32`, `C:\Windows`}},
	{title: `TEMP`, lines: []string{`%USERPROFILE%\AppData\Local\Temp`}},
	{title: `TMP`, lines: []string{`c:\temp`}},
}

func TestGetSections(t *testing.T) {
	sut = peekenv{registry: mock}
	expected := testSections
	sut.populateSectionsFrom(REG_KEY_USER)
	if !reflect.DeepEqual(expected, sut.sections) {
		t.Errorf("Expected: %q, was: %q", expected, sut.sections)
	}
}

func TestStringer(t *testing.T) {
	sut = peekenv{
		sections: testSections,
		registry: mock,
	}
	expected := `[PATH]
C:\Program Files\ConEmu
C:\Program Files\ConEmu\ConEmu
C:\Windows\SYSTEM32
C:\Windows

[TEMP]
%USERPROFILE%\AppData\Local\Temp

[TMP]
c:\temp

`
	assertEquals(t, expected, sut.String())
}

func assertEquals(t *testing.T, expected string, actual string) {
	if actual != expected {
		t.Errorf("Expected: %q, was: %q", expected, actual)
	}
}
<commit_msg>fix test<commit_after>package main

import (
	"io/ioutil"
	"log"
	"reflect"
	"testing"
)

var sut peekenv

func init() {
	log.SetOutput(ioutil.Discard)
}

var testSections = []Section{
	{title: `PATH`, lines: []string{`C:\Program Files\ConEmu`, `C:\Program Files\ConEmu\ConEmu`, `C:\Windows\SYSTEM32`, `C:\Windows`}},
	{title: `TEMP`, lines: []string{`%USERPROFILE%\AppData\Local\Temp`}},
	{title: `TMP`, lines: []string{`c:\temp`}},
}

func TestGetSections(t *testing.T) {
	sut = peekenv{registry: mock}
	expected := testSections
	sut.populateSectionsFrom(REG_KEY_USER)
	if !reflect.DeepEqual(expected, sut.sections) {
		t.Errorf("Expected: %q, was: %q", expected, sut.sections)
	}
}

func TestStringer(t *testing.T) {
	sut = peekenv{
		sections: testSections,
		registry: mock,
	}
	expected := `[PATH]
C:\Program Files\ConEmu
C:\Program Files\ConEmu\ConEmu
C:\Windows\SYSTEM32
C:\Windows

[TEMP]
%USERPROFILE%\AppData\Local\Temp

[TMP]
c:\temp
`
	assertEquals(t, expected, sut.String())
}

func assertEquals(t *testing.T, expected string, actual string) {
	if actual != expected {
		t.Errorf("Expected: %q, was: %q", expected, actual)
	}
}
<|endoftext|>


<commit_before>package types

import (
	"encoding/json"

	"github.com/asuleymanov/viz-go/encoding/transaction"
)

//ProposalObjects array of values of type ProposalObject.
type ProposalObjects []ProposalObject

//ProposalObject type from parameter JSON
type ProposalObject struct {
	Operation     Operation `json:"op"`
	OperationType OpType    `json:"-"`
}

type rawProposalObject struct {
	Operation *operationTuple `json:"op"`
}

//UnmarshalJSON unpacking the JSON parameter in the ProposalObject type.
func (op *ProposalObject) UnmarshalJSON(p []byte) error {
	var raw rawProposalObject
	if err := json.Unmarshal(p, &raw); err != nil {
		return err
	}

	op.Operation = raw.Operation.Data
	op.OperationType = raw.Operation.Type
	return nil
}

//MarshalJSON function for packing the ProposalObject type in JSON.
func (op *ProposalObject) MarshalJSON() ([]byte, error) {
	return JSONMarshal(&rawProposalObject{
		Operation: &operationTuple{op.Operation.Type(), op.Operation},
	})
}

//MarshalTransaction is a function of converting type ProposalObjects to bytes.
func (op ProposalObjects) MarshalTransaction(encoder *transaction.Encoder) error {
	enc := transaction.NewRollingEncoder(encoder)

	enc.EncodeUVarint(uint64(len(op)))
	for _, v := range op {
		enc.Encode(v.Operation)
	}

	return enc.Err()
}
<commit_msg>Update proposal_object.go<commit_after>package types

import (
	"encoding/json"

	"github.com/asuleymanov/golos-go/encoding/transaction"
)

//ProposalObjects array of values of type ProposalObject.
type ProposalObjects []ProposalObject

//ProposalObject type from parameter JSON
type ProposalObject struct {
	Operation     Operation `json:"op"`
	OperationType OpType    `json:"-"`
}

type rawProposalObject struct {
	Operation *operationTuple `json:"op"`
}

//UnmarshalJSON unpacking the JSON parameter in the ProposalObject type.
func (op *ProposalObject) UnmarshalJSON(p []byte) error {
	var raw rawProposalObject
	if err := json.Unmarshal(p, &raw); err != nil {
		return err
	}

	op.Operation = raw.Operation.Data
	op.OperationType = raw.Operation.Type
	return nil
}

//MarshalJSON function for packing the ProposalObject type in JSON.
func (op *ProposalObject) MarshalJSON() ([]byte, error) {
	return JSONMarshal(&rawProposalObject{
		Operation: &operationTuple{op.Operation.Type(), op.Operation},
	})
}

//MarshalTransaction is a function of converting type ProposalObjects to bytes.
func (op ProposalObjects) MarshalTransaction(encoder *transaction.Encoder) error {
	enc := transaction.NewRollingEncoder(encoder)

	enc.EncodeUVarint(uint64(len(op)))
	for _, v := range op {
		enc.Encode(v.Operation)
	}

	return enc.Err()
}
<|endoftext|>


<commit_before>// Copyright 2017-2020 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package graph

import (
	"errors"

	"github.com/OWASP/Amass/v3/graphdb"
	"github.com/OWASP/Amass/v3/queue"
	"github.com/OWASP/Amass/v3/stringset"
)

// Migrate copies the nodes and edges related to the Event identified by the uuid from the receiver Graph into another.
func (g *Graph) Migrate(uuid string, to *Graph) error {
	q := new(queue.Queue)
	ids := stringset.New()

	// Setup the Event associated with the UUID to start the migration
	event, err := g.db.ReadNode(uuid, "event")
	if err != nil {
		return err
	}

	toevent, newevent := g.migrateNode(event, ids, to)
	if toevent == nil || newevent == false {
		return errors.New("Graph: Migrate: Failed to migrate the event")
	}

	q.Append(event)
	for {
		element, ok := q.Next()
		if !ok {
			break
		}
		cur := element.(graphdb.Node)

		// Obtain the type for the node currently being worked on
		ntype := g.nodeToType(cur)
		if ntype == "" {
			continue
		}

		// Get the same node in the graph receiving the copies
		tocur, err := to.db.ReadNode(g.db.NodeToID(cur), ntype)
		if err != nil {
			continue
		}

		edges, err := g.db.ReadOutEdges(cur)
		if err != nil {
			continue
		}

		for _, edge := range edges {
			if !g.InEventScope(edge.To, uuid) {
				continue
			}

			node, newnode := g.migrateNode(edge.To, ids, to)
			if node == nil {
				continue
			}

			to.db.InsertEdge(&graphdb.Edge{
				Predicate: edge.Predicate,
				From:      tocur,
				To:        node,
			})

			if newnode {
				q.Append(edge.To)
			}
		}
	}

	return nil
}

func (g *Graph) migrateNode(node graphdb.Node, ids stringset.Set, to *Graph) (graphdb.Node, bool) {
	id := g.db.NodeToID(node)
	if id == "" {
		return nil, false
	}

	// Obtain the properties of the Node being migrated
	properties, err := g.db.ReadProperties(node)
	if err != nil || len(properties) == 0 {
		return nil, false
	}

	// Obtain the type of the Node being migrated
	ntype := g.getTypeFromProperties(properties)
	if ntype == "" {
		return nil, false
	}

	// Check that this Node has not already been created in the Graph
	if ids.Has(id) {
		if tonode, err := to.db.ReadNode(id, ntype); err == nil {
			return tonode, false
		}
		return nil, false
	}
	ids.Insert(id)

	// Create the Node in the Graph
	tonode, err := to.db.InsertNode(id, ntype)
	if err != nil {
		return nil, false
	}

	// Copy all the properties into the Node being migrated
	if err := g.migrateProperties(tonode, properties, to); err != nil {
		return nil, false
	}

	return tonode, true
}

func (g *Graph) nodeToType(node graphdb.Node) string {
	// Obtain the properties of the Node being migrated
	properties, err := g.db.ReadProperties(node)
	if err != nil || len(properties) == 0 {
		return ""
	}

	// Obtain the type of the Node being migrated
	return g.getTypeFromProperties(properties)
}

func (g *Graph) getTypeFromProperties(properties []*graphdb.Property) string {
	var ntype string

	for _, p := range properties {
		if p.Predicate == "type" {
			ntype = p.Value
			break
		}
	}

	return ntype
}

func (g *Graph) migrateProperties(node graphdb.Node, properties []*graphdb.Property, to *Graph) error {
	for _, p := range properties {
		if p.Predicate == "type" {
			continue
		}

		if err := to.db.InsertProperty(node, p.Predicate, p.Value); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>performance improvements to subgraph migration<commit_after>// Copyright 2017-2020 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package graph

import (
	"fmt"
	"sync"

	"github.com/OWASP/Amass/v3/graphdb"
	"github.com/OWASP/Amass/v3/queue"
	"github.com/OWASP/Amass/v3/semaphore"
)

// MigrateEvent copies the nodes and edges related to the Event identified by the uuid from the receiver Graph into another.
func (g *Graph) MigrateEvent(uuid string, to *Graph) error {
	q := new(queue.Queue)
	sem := semaphore.NewSimpleSemaphore(10)

	idToNode, err := newNodeMap(uuid, g, to)
	if err != nil {
		return fmt.Errorf("Graph: Migrate: Failed to setup the node map: %v", err)
	}

	event, err := g.db.ReadNode(uuid, "event")
	if err != nil {
		return fmt.Errorf("Graph: Migrate: Failed to read the event node for %s: %v", uuid, err)
	}

	if _, _, err := idToNode.getNode(uuid, event); err != nil {
		return fmt.Errorf("Graph: Migrate: %v", err)
	}

	q.Append(event)
	for {
		element, ok := q.Next()
		if !ok {
			break
		}
		cur := element.(graphdb.Node)

		curID := g.db.NodeToID(cur)
		// Get the same node that is in the graph receiving the copies
		tocur, found, err := idToNode.getNode(curID, cur)
		if err != nil || !found {
			return fmt.Errorf("Graph: Migrate: Failed to read the node %s", curID)
		}

		edges, err := g.db.ReadOutEdges(cur)
		if err != nil {
			continue
		}

		var wg sync.WaitGroup
		for _, edge := range edges {
			sem.Acquire(1)
			wg.Add(1)
			go g.migrateEdge(tocur, edge, to, q, idToNode, sem, &wg)
		}
		wg.Wait()
	}

	return nil
}

func (g *Graph) migrateEdge(cur graphdb.Node, edge *graphdb.Edge, to *Graph,
	q *queue.Queue, ids *nodeMap, sem semaphore.Semaphore, wg *sync.WaitGroup) {
	defer wg.Done()
	defer sem.Release(1)

	node, found, err := ids.getNode(g.db.NodeToID(edge.To), edge.To)
	if err != nil {
		return
	}

	if !found {
		q.Append(edge.To)
	}

	to.InsertEdge(&graphdb.Edge{
		Predicate: edge.Predicate,
		From:      cur,
		To:        node,
	})
}

func (g *Graph) migrateNode(node graphdb.Node, to *Graph) (graphdb.Node, error) {
	id := g.db.NodeToID(node)
	if id == "" {
		return nil, fmt.Errorf("migrateNode: Invalid %s node provided", g.String())
	}

	// Obtain the properties of the Node being migrated
	properties, err := g.db.ReadProperties(node)
	if err != nil || len(properties) == 0 {
		return nil, fmt.Errorf("migrateNode: Properties for the %s node %s could not be obtained: %v", g.String(), id, err)
	}

	// Obtain the type of the Node being migrated
	ntype := getTypeFromProperties(properties)
	if ntype == "" {
		return nil, fmt.Errorf("migrateNode: Unable to obtain the type for the %s node %s: %v", g.String(), id, err)
	}

	// Check if this node already exists in the 'to' graph
	tonode, err := to.db.ReadNode(id, ntype)
	if err == nil {
		return tonode, nil
	}

	// Create the Node in the Graph
	tonode, err = to.db.InsertNode(id, ntype)
	if err != nil {
		return nil, fmt.Errorf("migrateNode: Failed to insert the %s node %s: %v", g.String(), id, err)
	}

	// Copy all the properties into the Node being migrated
	if err := g.migrateProperties(tonode, properties, to); err != nil {
		return nil, fmt.Errorf("migrateNode: Failed to migrate properties for the %s node %s: %v", g.String(), id, err)
	}

	return tonode, nil
}

func (g *Graph) nodeToType(node graphdb.Node) string {
	// Obtain the properties of the Node being migrated
	properties, err := g.db.ReadProperties(node)
	if err != nil || len(properties) == 0 {
		return ""
	}

	// Obtain the type of the Node being migrated
	return getTypeFromProperties(properties)
}

func (g *Graph) migrateProperties(node graphdb.Node, properties []*graphdb.Property, to *Graph) error {
	for _, p := range properties {
		if p.Predicate == "type" {
			continue
		}

		if err := to.db.InsertProperty(node, p.Predicate, p.Value); err != nil {
			return err
		}
	}

	return nil
}

func getTypeFromProperties(properties []*graphdb.Property) string {
	var ntype string

	for _, p := range properties {
		if p.Predicate == "type" {
			ntype = p.Value
			break
		}
	}

	return ntype
}

type nodeMap struct {
	sync.Mutex
	uuid     string
	from, to *Graph
	nodes    map[string]graphdb.Node
}

func newNodeMap(uuid string, from, to *Graph) (*nodeMap, error) {
	// Setup the Event associated with the UUID to start the migration
	event, err := from.db.ReadNode(uuid, "event")
	if err != nil {
		return nil, err
	}

	m := &nodeMap{
		uuid:  uuid,
		from:  from,
		to:    to,
		nodes: make(map[string]graphdb.Node),
	}

	node, err := from.migrateNode(event, to)
	if err != nil {
		return nil, err
	}
	m.nodes[uuid] = node

	return m, nil
}

func (n *nodeMap) getNode(id string, fromNode graphdb.Node) (graphdb.Node, bool, error) {
	n.Lock()
	defer n.Unlock()

	node, found := n.nodes[id]
	if found {
		return node, found, nil
	}

	if !n.from.InEventScope(fromNode, n.uuid) {
		return nil, false, fmt.Errorf("Graph: Migrate: This node (%s) is out of scope", id)
	}

	var err error
	node, err = n.from.migrateNode(fromNode, n.to)
	if err == nil {
		n.nodes[id] = node
	}

	return node, false, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/ethchain-go"
	"github.com/ethereum/ethutil-go"
	"github.com/obscuren/secp256k1-go"
	"log"
	"os"
	"os/signal"
	"runtime"
)

const Debug = true

// Register interrupt handlers so we can stop the ethereum
func RegisterInterupts(s *eth.Ethereum) {
	// Buffered chan of one is enough
	c := make(chan os.Signal, 1)
	// Notify about interrupts for now
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Printf("Shutting down (%v) ... \n", sig)

			s.Stop()
		}
	}()
}

func CreateKeyPair(force bool) {
	data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
	if len(data) == 0 || force {
		log.Println("Generating new address and keypair")

		pub, prv := secp256k1.GenerateKeyPair()
		addr := ethutil.Sha3Bin(pub[1:])[12:]

		log.Printf("Your new address is %x\n", addr)

		keyRing := ethutil.NewValue([]interface{}{prv, addr, pub[1:]})
		ethutil.Config.Db.Put([]byte("KeyRing"), keyRing.Encode())
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	Init()

	ethchain.InitFees()
	ethutil.ReadConfig()

	// Instantiated a eth stack
	ethereum, err := eth.New(eth.CapDefault, UseUPnP)
	if err != nil {
		log.Println("eth start err:", err)
		return
	}

	if GenAddr {
		fmt.Println("This action overwrites your old private key. Are you sure? (y/n)")

		var r string
		fmt.Scanln(&r)
		for ; ; fmt.Scanln(&r) {
			if r == "n" || r == "y" {
				break
			} else {
				fmt.Printf("Yes or no?", r)
			}
		}

		if r == "y" {
			CreateKeyPair(true)
		}
		os.Exit(0)
	} else {
		CreateKeyPair(false)
	}

	if ShowGenesis {
		fmt.Println(ethereum.BlockManager.BlockChain().Genesis())
		os.Exit(0)
	}

	log.Printf("Starting Ethereum v%s\n", ethutil.Config.Ver)

	// Set the max peers
	ethereum.MaxPeers = MaxPeer

	if StartConsole {
		err := os.Mkdir(ethutil.Config.ExecPath, os.ModePerm)
		// Error is OK if the error is ErrExist
		if err != nil && !os.IsExist(err) {
			log.Panic("Unable to create EXECPATH:", err)
		}

		console := NewConsole(ethereum)
		go console.Start()
	}

	RegisterInterupts(ethereum)

	ethereum.Start()

	if StartMining {
		log.Printf("Miner started\n")

		// Fake block mining. It broadcasts a new block every 5 seconds
		go func() {
			pow := &ethchain.EasyPow{}
			data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
			keyRing := ethutil.NewValueFromBytes(data)
			addr := keyRing.Get(1).Bytes()

			for {
				txs := ethereum.TxPool.Flush()
				// Create a new block which we're going to mine
				block := ethereum.BlockManager.BlockChain().NewBlock(addr, txs)
				// Apply all transactions to the block
				ethereum.BlockManager.ApplyTransactions(block, block.Transactions())

				ethereum.BlockManager.AccumelateRewards(block, block)

				// Search the nonce
				block.Nonce = pow.Search(block)
				err := ethereum.BlockManager.ProcessBlock(block)
				if err != nil {
					log.Println(err)
				} else {
					log.Println("\n+++++++ MINED BLK +++++++\n", ethereum.BlockManager.BlockChain().CurrentBlock)
				}
			}
		}()
	}

	// Wait for shutdown
	ethereum.WaitForShutdown()
}
<commit_msg>Default to .ethereum<commit_after>package main

import (
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/ethchain-go"
	"github.com/ethereum/ethutil-go"
	"github.com/obscuren/secp256k1-go"
	"log"
	"os"
	"os/signal"
	"runtime"
)

const Debug = true

// Register interrupt handlers so we can stop the ethereum
func RegisterInterupts(s *eth.Ethereum) {
	// Buffered chan of one is enough
	c := make(chan os.Signal, 1)
	// Notify about interrupts for now
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Printf("Shutting down (%v) ... \n", sig)

			s.Stop()
		}
	}()
}

func CreateKeyPair(force bool) {
	data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
	if len(data) == 0 || force {
		log.Println("Generating new address and keypair")

		pub, prv := secp256k1.GenerateKeyPair()
		addr := ethutil.Sha3Bin(pub[1:])[12:]

		log.Printf("Your new address is %x\n", addr)

		keyRing := ethutil.NewValue([]interface{}{prv, addr, pub[1:]})
		ethutil.Config.Db.Put([]byte("KeyRing"), keyRing.Encode())
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	Init()

	ethchain.InitFees()
	ethutil.ReadConfig(".ethereum")
	ethutil.Config.Seed = UseSeed

	// Instantiated a eth stack
	ethereum, err := eth.New(eth.CapDefault, UseUPnP)
	if err != nil {
		log.Println("eth start err:", err)
		return
	}

	if GenAddr {
		fmt.Println("This action overwrites your old private key. Are you sure? (y/n)")

		var r string
		fmt.Scanln(&r)
		for ; ; fmt.Scanln(&r) {
			if r == "n" || r == "y" {
				break
			} else {
				fmt.Printf("Yes or no?", r)
			}
		}

		if r == "y" {
			CreateKeyPair(true)
		}
		os.Exit(0)
	} else {
		CreateKeyPair(false)
	}

	if ShowGenesis {
		fmt.Println(ethereum.BlockManager.BlockChain().Genesis())
		os.Exit(0)
	}

	log.Printf("Starting Ethereum v%s\n", ethutil.Config.Ver)

	// Set the max peers
	ethereum.MaxPeers = MaxPeer

	if StartConsole {
		err := os.Mkdir(ethutil.Config.ExecPath, os.ModePerm)
		// Error is OK if the error is ErrExist
		if err != nil && !os.IsExist(err) {
			log.Panic("Unable to create EXECPATH:", err)
		}

		console := NewConsole(ethereum)
		go console.Start()
	}

	RegisterInterupts(ethereum)

	ethereum.Start()

	if StartMining {
		log.Printf("Miner started\n")

		// Fake block mining. It broadcasts a new block every 5 seconds
		go func() {
			pow := &ethchain.EasyPow{}
			data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
			keyRing := ethutil.NewValueFromBytes(data)
			addr := keyRing.Get(1).Bytes()

			for {
				txs := ethereum.TxPool.Flush()
				// Create a new block which we're going to mine
				block := ethereum.BlockManager.BlockChain().NewBlock(addr, txs)
				// Apply all transactions to the block
				ethereum.BlockManager.ApplyTransactions(block, block.Transactions())

				ethereum.BlockManager.AccumelateRewards(block, block)

				// Search the nonce
				block.Nonce = pow.Search(block)
				err := ethereum.BlockManager.ProcessBlock(block)
				if err != nil {
					log.Println(err)
				} else {
					log.Println("\n+++++++ MINED BLK +++++++\n", ethereum.BlockManager.BlockChain().CurrentBlock)
				}
			}
		}()
	}

	// Wait for shutdown
	ethereum.WaitForShutdown()
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gcbench

import (
	"bytes"
	"fmt"
	"math"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"time"
)

type Benchmark struct {
	name string
	cfg  []config
	main func()
}

type config struct {
	k, v string
}

func NewBenchmark(name string, main func()) *Benchmark {
	return &Benchmark{name, nil, main}
}

func (b *Benchmark) Config(name string, value interface{}) *Benchmark {
	b.cfg = append(b.cfg, config{name, fmt.Sprint(value)})
	return b
}

func (b *Benchmark) FullName() string {
	buf := bytes.NewBufferString("Benchmark" + b.name)
	cpus := runtime.GOMAXPROCS(-1)
	if len(b.cfg) > 0 {
		for _, c := range b.cfg {
			fmt.Fprintf(buf, "/%s:%s", c.k, c.v)
		}
		if cpus != 1 {
			fmt.Fprintf(buf, "/gomaxprocs:%d", cpus)
		}
	} else if cpus != 1 {
		fmt.Fprintf(buf, "-%d", cpus)
	}
	return buf.String()
}

func (b *Benchmark) Run() {
	if gcbench := os.Getenv("GCBENCH"); gcbench != "" {
		if gcbench == b.FullName() {
			b.main()
		}
		os.Exit(0)
	}

	fmt.Printf("%s\t", b.FullName())

	cmd := exec.Command(os.Args[0], os.Args[1:]...)
	cmd.Env = append([]string{"GODEBUG=gctrace=1", "GCBENCH=" + b.FullName()}, os.Environ()...)
	startTime := time.Now()
	out, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to run %s: %s\n%s\n", os.Args[0], err, indent(string(out)))
		return
	}
	endTime := time.Now()

	// Parse the GC trace.
	gctrace, err := ParseGCTrace(string(out))
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to parse output of %s: %s\n%s\n", os.Args[0], err, indent(string(out)))
		return
	}

	run := RunInfo{Trace: gctrace, StartTime: startTime, EndTime: endTime}

	// Print metrics.
	fmt.Printf("%d", 1)
	// The \t's are a horrible hack to keep everything technically
	// on one line, but to make that line readable in a terminal
	// by putting each result on a separate visual line.
	align := "\t"
	if os.Getenv("TERM") != "dumb" {
		align = strings.Repeat("\t", 15) + " "
	}
	vals := make([]float64, len(metrics))
	for i, metric := range metrics {
		vals[i] = metric.Fn(run)
		if math.IsNaN(vals[i]) {
			continue
		}
		fmt.Printf("%s%10s %s", align, sigfigs(vals[i]), metric.Label)
	}
	fmt.Printf("\n")

	// Print warnings.
	for i, metric := range metrics {
		if metric.Check != nil && !math.IsNaN(vals[i]) {
			metric.Check(metric.Label, vals[i])
		}
	}

	// Print any non-GC output.
	nongc := []string{}
	for _, line := range strings.Split(strings.TrimRight(string(out), "\n"), "\n") {
		if !strings.HasPrefix(line, "gc ") {
			nongc = append(nongc, line)
		}
	}
	if len(nongc) > 0 {
		fmt.Fprintf(os.Stderr, "%s\n", strings.Join(nongc, "\n"))
	}
}

func indent(s string) string {
	return "    " + strings.Replace(s, "\n", "\n    ", -1)
}

// sigfigs formats v with three or more sigfigs.
func sigfigs(v float64) string {
	prec, m := 0, v
	for {
		if 99.5 <= m || m <= -99.5 || m*10 == m || m != m {
			return strconv.FormatFloat(v, 'f', prec, 64)
		}
		m *= 10
		prec++
	}
}
<commit_msg>gcbench: don't override GODEBUG<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gcbench

import (
	"bytes"
	"fmt"
	"math"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"time"
)

type Benchmark struct {
	name string
	cfg  []config
	main func()
}

type config struct {
	k, v string
}

func NewBenchmark(name string, main func()) *Benchmark {
	return &Benchmark{name, nil, main}
}

func (b *Benchmark) Config(name string, value interface{}) *Benchmark {
	b.cfg = append(b.cfg, config{name, fmt.Sprint(value)})
	return b
}

func (b *Benchmark) FullName() string {
	buf := bytes.NewBufferString("Benchmark" + b.name)
	cpus := runtime.GOMAXPROCS(-1)
	if len(b.cfg) > 0 {
		for _, c := range b.cfg {
			fmt.Fprintf(buf, "/%s:%s", c.k, c.v)
		}
		if cpus != 1 {
			fmt.Fprintf(buf, "/gomaxprocs:%d", cpus)
		}
	} else if cpus != 1 {
		fmt.Fprintf(buf, "-%d", cpus)
	}
	return buf.String()
}

func (b *Benchmark) Run() {
	// TODO: Have a flag or some easy way to print the GC trace of
	// the running benchmark.

	if gcbench := os.Getenv("GCBENCH"); gcbench != "" {
		if gcbench == b.FullName() {
			b.main()
		}
		os.Exit(0)
	}

	fmt.Printf("%s\t", b.FullName())

	godebug := os.Getenv("GODEBUG")
	if godebug != "" {
		godebug += ","
	}
	godebug += "gctrace=1"

	cmd := exec.Command(os.Args[0], os.Args[1:]...)
	cmd.Env = append([]string{"GODEBUG=" + godebug, "GCBENCH=" + b.FullName()}, os.Environ()...)
	startTime := time.Now()
	out, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to run %s: %s\n%s\n", os.Args[0], err, indent(string(out)))
		return
	}
	endTime := time.Now()

	// Parse the GC trace.
	gctrace, err := ParseGCTrace(string(out))
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to parse output of %s: %s\n%s\n", os.Args[0], err, indent(string(out)))
		return
	}

	run := RunInfo{Trace: gctrace, StartTime: startTime, EndTime: endTime}

	// Print metrics.
	fmt.Printf("%d", 1)
	// The \t's are a horrible hack to keep everything technically
	// on one line, but to make that line readable in a terminal
	// by putting each result on a separate visual line.
	align := "\t"
	if os.Getenv("TERM") != "dumb" {
		align = strings.Repeat("\t", 15) + " "
	}
	vals := make([]float64, len(metrics))
	for i, metric := range metrics {
		vals[i] = metric.Fn(run)
		if math.IsNaN(vals[i]) {
			continue
		}
		fmt.Printf("%s%10s %s", align, sigfigs(vals[i]), metric.Label)
	}
	fmt.Printf("\n")

	// Print warnings.
	for i, metric := range metrics {
		if metric.Check != nil && !math.IsNaN(vals[i]) {
			metric.Check(metric.Label, vals[i])
		}
	}

	// Print any non-GC output.
	nongc := []string{}
	for _, line := range strings.Split(strings.TrimRight(string(out), "\n"), "\n") {
		if !strings.HasPrefix(line, "gc ") {
			nongc = append(nongc, line)
		}
	}
	if len(nongc) > 0 {
		fmt.Fprintf(os.Stderr, "%s\n", strings.Join(nongc, "\n"))
	}
}

func indent(s string) string {
	return "    " + strings.Replace(s, "\n", "\n    ", -1)
}

// sigfigs formats v with three or more sigfigs.
func sigfigs(v float64) string {
	prec, m := 0, v
	for {
		if 99.5 <= m || m <= -99.5 || m*10 == m || m != m {
			return strconv.FormatFloat(v, 'f', prec, 64)
		}
		m *= 10
		prec++
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os"

	"gopkg.in/urfave/cli.v2"
)

var apiToken string
var baseURL = "https://app.rainforestqa.com/api/1"
var out io.Writer = os.Stdout

func main() {
	app := cli.NewApp()
	app.Name = "Rainforest CLI"
	app.Usage = "Command line utility for Rainforest QA"

	app.Flags = []cli.Flag{
		cli.StringSliceFlag{
			Name:  "tags",
			Usage: "Filter tests by tag",
		},
		cli.StringFlag{
			Name:  "token",
			Usage: "Rainforest API token",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "run",
			Usage: "Run your tests on Rainforest",
			Action: func(c *cli.Context) error {
				apiToken = c.String("token")
				createRun(c)
				return nil
			},
		},

		{
			Name:  "folders",
			Usage: "Retreive folders on Rainforest",
			Action: func(c *cli.Context) error {
				apiToken = c.String("token")
				printFolders()
				return nil
			},
		},
		//
		// {
		// 	Name:  "sites",
		// 	Usage: "Retreive sites on Rainforest",
		// 	Action: func(c *cli.Context) error {
		// 		apiToken = c.String("token")
		// 		printSites()
		// 		return nil
		// 	},
		// },
		//
		// {
		// 	Name:  "browsers",
		// 	Usage: "Retreive sites on Rainforest",
		// 	Action: func(c *cli.Context) error {
		// 		apiToken = c.String("token")
		// 		printBrowsers()
		// 		return nil
		// 	},
		// },
	}
	app.Run(os.Args)
}
<commit_msg>uncommented lines in cli<commit_after>package main

import (
	"io"
	"os"

	"gopkg.in/urfave/cli.v2"
)

var apiToken string
var baseURL = "https://app.rainforestqa.com/api/1"
var out io.Writer = os.Stdout

func main() {
	app := cli.NewApp()
	app.Name = "Rainforest CLI"
	app.Usage = "Command line utility for Rainforest QA"

	app.Flags = []cli.Flag{
		cli.StringSliceFlag{
			Name:  "tags",
			Usage: "Filter tests by tag",
		},
		cli.StringFlag{
			Name:  "token",
			Usage: "Rainforest API token",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "run",
			Usage: "Run your tests on Rainforest",
			Action: func(c *cli.Context) error {
				apiToken = c.String("token")
				createRun(c)
				return nil
			},
		},

		{
			Name:  "folders",
			Usage: "Retreive folders on Rainforest",
			Action: func(c *cli.Context) error {
				apiToken = c.String("token")
				printFolders()
				return nil
			},
		},
		{
			Name:  "sites",
			Usage: "Retreive sites on Rainforest",
			Action: func(c *cli.Context) error {
				apiToken = c.String("token")
				printSites()
				return nil
			},
		},

		{
			Name:  "browsers",
			Usage: "Retreive sites on Rainforest",
			Action: func(c *cli.Context) error {
				apiToken = c.String("token")
				printBrowsers()
				return nil
			},
		},
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>/*
tsuru is a command line tool for application developers.

It provides some commands that allow a developer to register himself/herself,
manage teams, apps and services.

Usage:

	% tsuru <command> [args]

The currently available commands are (grouped by subject):

	target            changes or retrive the current tsuru server
	version           displays current tsuru version

	user-create       creates a new user
	login             authenticates the user with tsuru server
	logout            finishes the session with tsuru server
	key-add           adds a public key to tsuru deploy server
	key-remove        removes a public key from tsuru deploy server

	team-create       creates a new team (adding the current user to it automatically)
	team-list         list teams that the user is member
	team-user-add     adds a user to a team
	team-user-remove  removes a user from a team

	app-create        creates an app
	app-remove        removes an app
	app-list          lists apps that the user has access (see app-grant and team-user-add)
	app-grant         allows a team to have access to an app
	app-revoke        revokes access to an app from a team
	log               shows log for an app
	run               runs a command in all units of an app
	restart           restarts the app's application server

	env-get           display environment variables for an app
	env-set           set environment variable(s) to an app
	env-unset         unset environment variable(s) from an app

	bind              binds an app to a service instance
	unbind            unbinds an app from a service instance

	service-list      list all services, and instances of each service
	service-add       creates a new instance of a service
	service-remove    removes a instance of a service
	service-status    checks the status of a service instance
	service-info      list instances of a service, and apps binded to each instance
	service-doc       displays documentation for a service

Use "tsuru help <command>" for more information about a command.

Change/retrieve remote tsuru server

Usage:

	% tsuru target [target]

This command should be used to set current tsuru target, or retrieve current
target.

The target is the tsuru server to which all operations will be directed to.

Check current version

Usage:

	% tsuru version

This command returns the current version of tsuru command.

Create a user

Usage:

	% tsuru user-create <email>

user-create creates a user within tsuru remote server. It will ask for the
password before issue the request.

Authenticate within remote tsuru server

Usage:

	% tsuru login <email>

Login will ask for the password and check if the user is successfully
authenticated. If so, the token generated by the tsuru server will be stored in
${HOME}/.tsuru_token.

All tsuru actions require the user to be authenticated (except login and
user-create, obviously).

Logout from remote tsuru server

Usage:

	% tsuru logout

Logout will delete the token file and terminate the session within tsuru
server.

Add SSH public key to tsuru's git server

Usage:

	% tsuru key-add [${HOME}/.ssh/id_rsa.pub]

key-add sends your public key to tsuru's git server. By default, it will try
send a public RSA key, located at ${HOME}/.ssh/id_rsa.pub. If you want to send
other file, you can call it with the path to the file. For example:

	% tsuru key-add /etc/my-keys/id_dsa.pub

The key will be added to the current logged in user.

Remove SSH public key from tsuru's git server

Usage:

	% tsuru key-remove [${HOME}/.ssh/id_rsa.pub]

key-remove removes your public key from tsuru's git server. By default, it will
try to remove a key that match you public RSA key located at
${HOME}/.ssh/id_rsa.pub. If you want to remove a key located somewhere else,
you can pass it as parameter to key-remove:

	% tsuru key-remove /etc/my-keys/id_dsa.pub

The key will be removed from the current logged in user.

Create a new team for the user

Usage:

	% tsuru team-create <teamname>

team-create will create a team for the user. Tsuru requires a user to be a
member of at least one team in order to create an app or a service instance.

When you create a team, you're automatically member of this team.

List teams that the user is member of

Usage:

	% tsuru team-list

team-list will list all teams that you are member of.

Add a user to a team

Usage:

	% tsuru team-user-add <teamname> <useremail>

team-user-add adds a user to a team. You need to be a member of the team to be
able to add another user to it.

Remove a user from a team

Usage:

	% tsuru team-user-remove <teamname> <useremail>

team-user-remove removes a user from a team. You need to be a member of the
team to be able to remove a user from it.

A team can never have 0 users. If you are the last member of a team, you can't
remove yourself from it.

Create an app

Usage:

	% tsuru app-create <appname> <platform>

app-create will create a new app using the given name and platform. For tsuru,
a platform is a Juju charm. To check the available platforms/charms, check this
URL: https://github.com/timeredbull/charms/tree/master/centos.

In order to create an app, you need to be member of at least one team. All
teams that you are member (see "tsuru team-list") will be able to access the
app.

Remove an app

Usage:

	% tsuru app-remove <appname>

app-remove removes an app. If the app is binded to any service instance, it
will be unbinded before be removed (see "tsuru unbind"). You need to be a
member of a team that has access to the app to be able to remove it (you are
able to remove any app that you see in "tsuru app-list").

List apps that the user has access to

Usage:

	% tsuru app-list

app-list will list all apps that you have access to. App access is controlled
by teams. If your team has access to an app, then you have access to it.

Allow a team to access an app

Usage:

	% tsuru app-grant <appname> <teamname>

app-grant will allow a team to access an app. You need to be a member of a team
that has access to the app to allow another team to access it.

Revoke from a team access to an app

Usage:

	% tsuru app-revoke <appname> <teamname>

app-revoke will revoke the permission to access an app from a team. You need to
have access to the app to revoke access from a team.

An app cannot be orphaned, so it will always have at least one authorized team.

See app's logs

Usage:

	% tsuru log <appname>

Log will show log entries for an app. These logs are not related to the code of
the app itself, but to actions of the app in tsuru server (deployments,
restarts, etc.).

Run an arbitrary command in the app machine

Usage:

	% tsuru run <appname> <command> [commandarg1] [commandarg2] ... [commandargn]

Run will run an arbitrary command in the app machine. Base directory for all
commands is the root of the app. For example, in a Django app, "tsuru run" may
show the following output:

	% tsuru run polls ls -l
	app.conf
	brogui
	deploy
	foo
	__init__.py
	__init__.pyc
	main.go
	manage.py
	settings.py
	settings.pyc
	templates
	urls.py
	urls.pyc

Restart the app's application server

Usage:

	% tsuru restart <appname>

Restart will call the restart hook from the app platform (the "restart" hook
from the Juju charm).
*/
package documentation
<commit_msg>cmd/crane, cmd/tsuru: fix head<commit_after>/*
tsuru is a command line tool for application developers.

It provides some commands that allow a developer to register himself/herself,
manage teams, apps and services.

Usage:

	% tsuru <command> [args]

The currently available commands are (grouped by subject):

	target            changes or retrive the current tsuru server
	version           displays current tsuru version

	user-create       creates a new user
	login             authenticates the user with tsuru server
	logout            finishes the session with tsuru server
	key-add           adds a public key to tsuru deploy server
	key-remove        removes a public key from tsuru deploy server

	team-create       creates a new team (adding the current user to it automatically)
	team-list         list teams that the user is member
	team-user-add     adds a user to a team
	team-user-remove  removes a user from a team

	app-create        creates an app
	app-remove        removes an app
	app-list          lists apps that the user has access (see app-grant and team-user-add)
	app-grant         allows a team to have access to an app
	app-revoke        revokes access to an app from a team
	log               shows log for an app
	run               runs a command in all units of an app
	restart           restarts the app's application server

	env-get           display environment variables for an app
	env-set           set environment variable(s) to an app
	env-unset         unset environment variable(s) from an app

	bind              binds an app to a service instance
	unbind            unbinds an app from a service instance

	service-list      list all services, and instances of each service
	service-add       creates a new instance of a service
	service-remove    removes a instance of a service
	service-status    checks the status of a service instance
	service-info      list instances of a service, and apps binded to each instance
	service-doc       displays documentation for a service

Use "tsuru help <command>" for more information about a command.

Change or retrieve remote tsuru server

Usage:

	% tsuru target [target]

This command should be used to set current tsuru target, or retrieve current
target.

The target is the tsuru server to which all operations will be directed to.

Check current version

Usage:

	% tsuru version

This command returns the current version of tsuru command.

Create a user

Usage:

	% tsuru user-create <email>

user-create creates a user within tsuru remote server. It will ask for the
password before issue the request.

Authenticate within remote tsuru server

Usage:

	% tsuru login <email>

Login will ask for the password and check if the user is successfully
authenticated. If so, the token generated by the tsuru server will be stored in
${HOME}/.tsuru_token.

All tsuru actions require the user to be authenticated (except login and
user-create, obviously).

Logout from remote tsuru server

Usage:

	% tsuru logout

Logout will delete the token file and terminate the session within tsuru
server.

Add SSH public key to tsuru's git server

Usage:

	% tsuru key-add [${HOME}/.ssh/id_rsa.pub]

key-add sends your public key to tsuru's git server. By default, it will try
send a public RSA key, located at ${HOME}/.ssh/id_rsa.pub. If you want to send
other file, you can call it with the path to the file. For example:

	% tsuru key-add /etc/my-keys/id_dsa.pub

The key will be added to the current logged in user.

Remove SSH public key from tsuru's git server

Usage:

	% tsuru key-remove [${HOME}/.ssh/id_rsa.pub]

key-remove removes your public key from tsuru's git server. By default, it will
try to remove a key that match you public RSA key located at
${HOME}/.ssh/id_rsa.pub. If you want to remove a key located somewhere else,
you can pass it as parameter to key-remove:

	% tsuru key-remove /etc/my-keys/id_dsa.pub

The key will be removed from the current logged in user.

Create a new team for the user

Usage:

	% tsuru team-create <teamname>

team-create will create a team for the user. Tsuru requires a user to be a
member of at least one team in order to create an app or a service instance.

When you create a team, you're automatically member of this team.

List teams that the user is member of

Usage:

	% tsuru team-list

team-list will list all teams that you are member of.

Add a user to a team

Usage:

	% tsuru team-user-add <teamname> <useremail>

team-user-add adds a user to a team. You need to be a member of the team to be
able to add another user to it.

Remove a user from a team

Usage:

	% tsuru team-user-remove <teamname> <useremail>

team-user-remove removes a user from a team. You need to be a member of the
team to be able to remove a user from it.

A team can never have 0 users. If you are the last member of a team, you can't
remove yourself from it.

Create an app

Usage:

	% tsuru app-create <appname> <platform>

app-create will create a new app using the given name and platform. For tsuru,
a platform is a Juju charm. To check the available platforms/charms, check this
URL: https://github.com/timeredbull/charms/tree/master/centos.

In order to create an app, you need to be member of at least one team. All
teams that you are member (see "tsuru team-list") will be able to access the
app.

Remove an app

Usage:

	% tsuru app-remove <appname>

app-remove removes an app. If the app is binded to any service instance, it
will be unbinded before be removed (see "tsuru unbind"). You need to be a
member of a team that has access to the app to be able to remove it (you are
able to remove any app that you see in "tsuru app-list").

List apps that the user has access to

Usage:

	% tsuru app-list

app-list will list all apps that you have access to. App access is controlled
by teams. If your team has access to an app, then you have access to it.

Allow a team to access an app

Usage:

	% tsuru app-grant <appname> <teamname>

app-grant will allow a team to access an app. You need to be a member of a team
that has access to the app to allow another team to access it.

Revoke from a team access to an app

Usage:

	% tsuru app-revoke <appname> <teamname>

app-revoke will revoke the permission to access an app from a team. You need to
have access to the app to revoke access from a team.

An app cannot be orphaned, so it will always have at least one authorized team.

See app's logs

Usage:

	% tsuru log <appname>

Log will show log entries for an app. These logs are not related to the code of
the app itself, but to actions of the app in tsuru server (deployments,
restarts, etc.).

Run an arbitrary command in the app machine

Usage:

	% tsuru run <appname> <command> [commandarg1] [commandarg2] ... [commandargn]

Run will run an arbitrary command in the app machine. Base directory for all
commands is the root of the app. For example, in a Django app, "tsuru run" may
show the following output:

	% tsuru run polls ls -l
	app.conf
	brogui
	deploy
	foo
	__init__.py
	__init__.pyc
	main.go
	manage.py
	settings.py
	settings.pyc
	templates
	urls.py
	urls.pyc

Restart the app's application server

Usage:

	% tsuru restart <appname>

Restart will call the restart hook from the app platform (the "restart" hook
from the Juju charm).
*/
package documentation
<|endoftext|>


<commit_before>/*

jwthelper is a Golang package that provides JWT(JSON Web Token) functions based on jwt-go.

*/

package jwthelper
<commit_msg>Remove blank line in doc.go<commit_after>/*

jwthelper is a Golang package that provides JWT(JSON Web Token) functions based on jwt-go.

*/
package jwthelper
<|endoftext|>