<commit_before>// Package ae provides tools to synchronize state between local and remote consul servers.
package ae

import (
	"errors"
	"log"
	"math"
	"sync"
	"time"

	"github.com/hashicorp/consul/lib"
)

// scaleThreshold is the number of nodes after which regular sync runs are
// spread out farther apart. The value should be a power of 2 since the
// scale function uses log2.
//
// When set to 128 nodes the delay between regular runs is doubled when the
// cluster is larger than 128 nodes. It doubles again when it passes 256
// nodes, and again at 512 nodes and so forth. At 8192 nodes, the delay
// factor is 8.
//
// If you update this, you may need to adjust the tuning of
// CoordinateUpdatePeriod and CoordinateUpdateMaxBatchSize.
const scaleThreshold = 128

// scaleFactor returns a factor by which the next sync run should be delayed to
// avoid saturation of the cluster. The larger the cluster grows the farther
// the sync runs should be spread apart.
//
// The current implementation uses a log2 scale which doubles the delay between
// runs every time the cluster doubles in size.
func scaleFactor(nodes int) int {
	if nodes <= scaleThreshold {
		return 1.0
	}
	return int(math.Ceil(math.Log2(float64(nodes))-math.Log2(float64(scaleThreshold))) + 1.0)
}

type State interface {
	SyncChanges() error
	SyncFull() error
}

// StateSyncer manages background synchronization of the given state.
//
// The state is synchronized on a regular basis or on demand when either
// the state has changed or a new Consul server has joined the cluster.
//
// The regular state sychronization provides a self-healing mechanism
// for the cluster which is also called anti-entropy.
type StateSyncer struct {
	// State contains the data that needs to be synchronized.
	State State

	// Interval is the time between two full sync runs.
	Interval time.Duration

	// ShutdownCh is closed when the application is shutting down.
	ShutdownCh chan struct{}

	// Logger is the logger.
	Logger *log.Logger

	// ClusterSize returns the number of members in the cluster to
	// allow staggering the sync runs based on cluster size.
	// This needs to be set before Run() is called.
	ClusterSize func() int

	// SyncFull allows triggering an immediate but staggered full sync
	// in a non-blocking way.
	SyncFull *Trigger

	// SyncChanges allows triggering an immediate partial sync
	// in a non-blocking way.
	SyncChanges *Trigger

	// paused stores whether sync runs are temporarily disabled.
	pauseLock sync.Mutex
	paused    int

	// stagger randomly picks a duration between 0s and the given duration.
	stagger func(time.Duration) time.Duration

	// serverUpInterval is the max time after which a full sync is
	// performed when a server has been added to the cluster.
	serverUpInterval time.Duration

	// retryFailInterval is the time after which a failed full sync is retried.
	retryFailInterval time.Duration
}

const (
	// serverUpIntv is the max time to wait before a sync is triggered
	// when a consul server has been added to the cluster.
	serverUpIntv = 3 * time.Second

	// retryFailIntv is the min time to wait before a failed sync is retried.
	retryFailIntv = 15 * time.Second
)

func NewStateSyncer(state State, intv time.Duration, shutdownCh chan struct{}, logger *log.Logger) *StateSyncer {
	s := &StateSyncer{
		State:             state,
		Interval:          intv,
		ShutdownCh:        shutdownCh,
		Logger:            logger,
		SyncFull:          NewTrigger(),
		SyncChanges:       NewTrigger(),
		serverUpInterval:  serverUpIntv,
		retryFailInterval: retryFailIntv,
	}
	s.stagger = func(d time.Duration) time.Duration {
		f := scaleFactor(s.ClusterSize())
		return lib.RandomStagger(time.Duration(f) * d)
	}
	return s
}

var errPaused = errors.New("paused")

// Run is the long running method to perform state synchronization
// between local and remote servers.
func (s *StateSyncer) Run() {
	if s.ClusterSize == nil {
		panic("ClusterSize not set")
	}

FullSync:
	for {
		// attempt a full sync
		err := s.ifNotPausedRun(s.State.SyncFull)
		if err != nil {
			if err != errPaused {
				s.Logger.Printf("[ERR] agent: failed to sync remote state: %v", err)
			}

			select {
			// trigger a full sync immediately.
			// this is usually called when a consul server was added to the cluster.
			// stagger the delay to avoid a thundering herd.
			case <-s.SyncFull.Notif():
				select {
				case <-time.After(s.stagger(s.serverUpInterval)):
				case <-s.ShutdownCh:
					return
				}

			// retry full sync after some time
			// todo(fs): why don't we use s.Interval here?
			case <-time.After(s.retryFailInterval + s.stagger(s.retryFailInterval)):

			case <-s.ShutdownCh:
				return
			}

			continue
		}

		// do partial syncs until it is time for a full sync again
		for {
			select {
			// trigger a full sync immediately
			// this is usually called when a consul server was added to the cluster.
			// stagger the delay to avoid a thundering herd.
			case <-s.SyncFull.Notif():
				select {
				case <-time.After(s.stagger(s.serverUpInterval)):
					continue FullSync
				case <-s.ShutdownCh:
					return
				}

			// time for a full sync again
			case <-time.After(s.Interval + s.stagger(s.Interval)):
				continue FullSync

			// do partial syncs on demand
			case <-s.SyncChanges.Notif():
				err := s.ifNotPausedRun(s.State.SyncChanges)
				if err != nil && err != errPaused {
					s.Logger.Printf("[ERR] agent: failed to sync changes: %v", err)
				}

			case <-s.ShutdownCh:
				return
			}
		}
	}
}

func (s *StateSyncer) ifNotPausedRun(f func() error) error {
	s.pauseLock.Lock()
	defer s.pauseLock.Unlock()
	if s.paused != 0 {
		return errPaused
	}
	return f()
}

// Pause temporarily disables sync runs.
func (s *StateSyncer) Pause() {
	s.pauseLock.Lock()
	s.paused++
	s.pauseLock.Unlock()
}

// Paused returns whether sync runs are temporarily disabled.
func (s *StateSyncer) Paused() bool {
	s.pauseLock.Lock()
	defer s.pauseLock.Unlock()
	return s.paused != 0
}

// Resume re-enables sync runs.
func (s *StateSyncer) Resume() {
	s.pauseLock.Lock()
	s.paused--
	if s.paused < 0 {
		panic("unbalanced pause/resume")
	}
	if s.paused == 0 {
		s.SyncChanges.Trigger()
	}
	s.pauseLock.Unlock()
}
<commit_msg>ae: make control flow more explicit<commit_after>// Package ae provides tools to synchronize state between local and remote consul servers.
package ae

import (
	"errors"
	"log"
	"math"
	"sync"
	"time"

	"github.com/hashicorp/consul/lib"
)

// scaleThreshold is the number of nodes after which regular sync runs are
// spread out farther apart. The value should be a power of 2 since the
// scale function uses log2.
//
// When set to 128 nodes the delay between regular runs is doubled when the
// cluster is larger than 128 nodes. It doubles again when it passes 256
// nodes, and again at 512 nodes and so forth. At 8192 nodes, the delay
// factor is 8.
//
// If you update this, you may need to adjust the tuning of
// CoordinateUpdatePeriod and CoordinateUpdateMaxBatchSize.
const scaleThreshold = 128

// scaleFactor returns a factor by which the next sync run should be delayed to
// avoid saturation of the cluster. The larger the cluster grows the farther
// the sync runs should be spread apart.
//
// The current implementation uses a log2 scale which doubles the delay between
// runs every time the cluster doubles in size.
func scaleFactor(nodes int) int {
	if nodes <= scaleThreshold {
		return 1.0
	}
	return int(math.Ceil(math.Log2(float64(nodes))-math.Log2(float64(scaleThreshold))) + 1.0)
}

type State interface {
	SyncChanges() error
	SyncFull() error
}

// StateSyncer manages background synchronization of the given state.
//
// The state is synchronized on a regular basis or on demand when either
// the state has changed or a new Consul server has joined the cluster.
//
// The regular state sychronization provides a self-healing mechanism
// for the cluster which is also called anti-entropy.
type StateSyncer struct {
	// State contains the data that needs to be synchronized.
	State State

	// Interval is the time between two full sync runs.
	Interval time.Duration

	// ShutdownCh is closed when the application is shutting down.
	ShutdownCh chan struct{}

	// Logger is the logger.
	Logger *log.Logger

	// ClusterSize returns the number of members in the cluster to
	// allow staggering the sync runs based on cluster size.
	// This needs to be set before Run() is called.
	ClusterSize func() int

	// SyncFull allows triggering an immediate but staggered full sync
	// in a non-blocking way.
	SyncFull *Trigger

	// SyncChanges allows triggering an immediate partial sync
	// in a non-blocking way.
	SyncChanges *Trigger

	// paused stores whether sync runs are temporarily disabled.
	pauseLock sync.Mutex
	paused    int

	// stagger randomly picks a duration between 0s and the given duration.
	stagger func(time.Duration) time.Duration

	// serverUpInterval is the max time after which a full sync is
	// performed when a server has been added to the cluster.
	serverUpInterval time.Duration

	// retryFailInterval is the time after which a failed full sync is retried.
	retryFailInterval time.Duration
}

const (
	// serverUpIntv is the max time to wait before a sync is triggered
	// when a consul server has been added to the cluster.
	serverUpIntv = 3 * time.Second

	// retryFailIntv is the min time to wait before a failed sync is retried.
	retryFailIntv = 15 * time.Second
)

func NewStateSyncer(state State, intv time.Duration, shutdownCh chan struct{}, logger *log.Logger) *StateSyncer {
	s := &StateSyncer{
		State:             state,
		Interval:          intv,
		ShutdownCh:        shutdownCh,
		Logger:            logger,
		SyncFull:          NewTrigger(),
		SyncChanges:       NewTrigger(),
		serverUpInterval:  serverUpIntv,
		retryFailInterval: retryFailIntv,
	}
	s.stagger = func(d time.Duration) time.Duration {
		f := scaleFactor(s.ClusterSize())
		return lib.RandomStagger(time.Duration(f) * d)
	}
	return s
}

var errPaused = errors.New("paused")

// Run is the long running method to perform state synchronization
// between local and remote servers.
func (s *StateSyncer) Run() {
	if s.ClusterSize == nil {
		panic("ClusterSize not set")
	}

FullSync:
	for {
		// attempt a full sync
		err := s.ifNotPausedRun(s.State.SyncFull)
		if err != nil {
			if err != errPaused {
				s.Logger.Printf("[ERR] agent: failed to sync remote state: %v", err)
			}

			select {
			// trigger a full sync immediately.
			// this is usually called when a consul server was added to the cluster.
			// stagger the delay to avoid a thundering herd.
			case <-s.SyncFull.Notif():
				select {
				case <-time.After(s.stagger(s.serverUpInterval)):
					continue FullSync
				case <-s.ShutdownCh:
					return
				}

			// retry full sync after some time
			// todo(fs): why don't we use s.Interval here?
			case <-time.After(s.retryFailInterval + s.stagger(s.retryFailInterval)):
				continue FullSync

			case <-s.ShutdownCh:
				return
			}
		}

		// do partial syncs until it is time for a full sync again
		for {
			select {
			// trigger a full sync immediately
			// this is usually called when a consul server was added to the cluster.
			// stagger the delay to avoid a thundering herd.
			case <-s.SyncFull.Notif():
				select {
				case <-time.After(s.stagger(s.serverUpInterval)):
					continue FullSync
				case <-s.ShutdownCh:
					return
				}

			// time for a full sync again
			case <-time.After(s.Interval + s.stagger(s.Interval)):
				continue FullSync

			// do partial syncs on demand
			case <-s.SyncChanges.Notif():
				err := s.ifNotPausedRun(s.State.SyncChanges)
				if err != nil && err != errPaused {
					s.Logger.Printf("[ERR] agent: failed to sync changes: %v", err)
				}

			case <-s.ShutdownCh:
				return
			}
		}
	}
}

func (s *StateSyncer) ifNotPausedRun(f func() error) error {
	s.pauseLock.Lock()
	defer s.pauseLock.Unlock()
	if s.paused != 0 {
		return errPaused
	}
	return f()
}

// Pause temporarily disables sync runs.
func (s *StateSyncer) Pause() {
	s.pauseLock.Lock()
	s.paused++
	s.pauseLock.Unlock()
}

// Paused returns whether sync runs are temporarily disabled.
func (s *StateSyncer) Paused() bool {
	s.pauseLock.Lock()
	defer s.pauseLock.Unlock()
	return s.paused != 0
}

// Resume re-enables sync runs.
func (s *StateSyncer) Resume() {
	s.pauseLock.Lock()
	s.paused--
	if s.paused < 0 {
		panic("unbalanced pause/resume")
	}
	if s.paused == 0 {
		s.SyncChanges.Trigger()
	}
	s.pauseLock.Unlock()
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package agent is a Romana service which provides networking functions on the host.
package agent

import (
	"github.com/romana/core/common"
	"github.com/romana/core/common/log/trace"
	enforcer "github.com/romana/core/pkg/policy/enforcer"
	"github.com/romana/core/pkg/util/iptsave"
	policyCache "github.com/romana/core/pkg/util/policy/cache"
	tenantCache "github.com/romana/core/pkg/util/tenant/cache"
	log "github.com/romana/rlog"
)

// Agent provides access to configuration and helper functions, shared across
// all the threads.
// Types Config, Leasefile and Firewall are designed to be loosely coupled
// so they could later be separated into packages and used independently.
type Agent struct {
	// Discovered run-time configuration.
	networkConfig *NetworkConfig

	config common.ServiceConfig
	// Leasefile is a type that manages DHCP leases in the file
	leaseFile *LeaseFile

	// Helper here is a type that organizes swappable interfaces for 3rd
	// party libraries (e.g. os.exec), and some functions that are using
	// those interfaces directly. Main purpose is to support unit testing.
	// Choice of having Helper as a field of an Agent is made to
	// support multiple instances of an Agent running at same time.
	// We like this approach, since it gives us flexibility as the agent evolves in the future.
	// Should this flexibility not be required, a suitable alternative is to re-implement the
	// Agent structure as a set of global variables.
	Helper *Helper

	waitForIfaceTry int

	// Whether this is running in test mode.
	TestMode bool

	// Agent store to keep records about managed resources.
	store agentStore

	client *common.RestClient

	policyEnabled bool

	// Manages iptables rules to reflect romana policies.
	enforcer enforcer.Interface

	// Stops policy enforcer.
	policyStop chan struct{}
}

// SetConfig implements SetConfig function of the Service interface.
func (a *Agent) SetConfig(config common.ServiceConfig) error {
	var err error
	log.Trace(trace.Public, config)
	a.config = config
	leaseFileName := config.ServiceSpecific["lease_file"].(string)
	lf := NewLeaseFile(leaseFileName, a)
	a.leaseFile = &lf

	var ok bool
	a.policyEnabled, ok = config.ServiceSpecific["policy_enabled"].(bool)
	if !ok {
		a.policyEnabled = false
	}
	a.waitForIfaceTry = int(config.ServiceSpecific["wait_for_iface_try"].(float64))
	a.networkConfig = &NetworkConfig{}
	store, err := NewStore(config)
	if err != nil {
		return err
	}
	a.store = *store
	log.Trace(trace.Inside, "Agent.SetConfig() finished.")
	return nil
}

// Routes implements Routes function of Service interface.
func (a *Agent) Routes() common.Routes {
	routes := common.Routes{
		common.Route{
			Method:  "GET",
			Pattern: "/",
			Handler: a.statusHandler,
		},
		common.Route{
			Method:  "POST",
			Pattern: "/vm",
			Handler: a.vmUpHandler,
			MakeMessage: func() interface{} {
				return &NetIf{}
			},
			UseRequestToken: false,
		},
		common.Route{
			Method:  "DELETE",
			Pattern: "/vm",
			Handler: a.vmDownHandler,
			MakeMessage: func() interface{} {
				return &NetIf{}
			},
			UseRequestToken: false,
		},
		common.Route{
			Method:  "POST",
			Pattern: "/pod",
			Handler: a.podUpHandler,
			MakeMessage: func() interface{} {
				return &NetworkRequest{}
			},
			// TODO this is for the future so we ensure idempotence.
			UseRequestToken: true,
		},
		common.Route{
			Method:  "DELETE",
			Pattern: "/pod",
			Handler: a.podDownHandler,
			MakeMessage: func() interface{} {
				return &NetworkRequest{}
			},
		},
		common.Route{
			Method:  "POST",
			Pattern: "/policies",
			Handler: a.addPolicy,
			MakeMessage: func() interface{} {
				return &common.Policy{}
			},
			UseRequestToken: false,
		},
		common.Route{
			Method:  "DELETE",
			Pattern: "/policies",
			MakeMessage: func() interface{} {
				return &common.Policy{}
			},
			Handler: a.deletePolicy,
		},
		common.Route{
			Method:  "GET",
			Pattern: "/policies",
			Handler: a.listPolicies,
		},
	}
	return routes
}

// Name implements method of Service interface.
func (a *Agent) Name() string {
	return "agent"
}

const (
	// defaultCacheTickTime controls how oftend storage cache is updated.
	defaultCacheTickTime = 5

	// defaultPolicyRefreshSeconds controls how often policy agent checks
	// storage caches for updates and applies changes if any.
	defaultPolicyRefreshSeconds = 2

	// defaultRouteRefreshSeconds controls how often agent checks
	// for route update and applies changes if any.
	defaultRouteRefreshSeconds = 120
)

// Initialize implements the Initialize method of common.Service
// interface.
func (a *Agent) Initialize(client *common.RestClient) error {
	log.Trace(trace.Public, "Entering Agent.Initialize()")
	a.client = client

	err := a.store.Connect()
	if err != nil {
		log.Error("Agent.Initialize() : Failed to connect to database.")
		return err
	}

	// identifyCurrentHost() needs to be called at-least once before
	// calling createRomanaGW() to populate romanaGW and romanaGWMask.
	log.Info("Agent: Attempting to identify current host.")
	if err := a.identifyCurrentHost(); err != nil {
		log.Error("Agent: Failed to identify current host: ", agentError(err))
		return agentError(err)
	}

	// Create Romana Gateway and bring up the necessary config for it
	// for example: assign IP Address to it, etc.
	if err := a.createRomanaGW(); err != nil {
		log.Error("Agent: Failed to create Romana Gateway on the node:", err)
		return err
	}

	// Enable default kernel settings needed by romana, for example:
	// ip forward, proxy arp, etc
	if err := a.enableRomanaKernelDefaults(); err != nil {
		log.Error("Agent: Failed to enable Romana kernel defaults on the node:", err)
		return err
	}

	// Will try to refresh routes every routeRefreshSeconds.
	var routeRefreshSeconds int
	if a.config.ServiceSpecific["route_refresh_seconds"] != nil {
		routeRefreshSeconds = int(a.config.ServiceSpecific["route_refresh_seconds"].(float64))
	} else {
		routeRefreshSeconds = defaultRouteRefreshSeconds
	}

	// Channel for stopping route update mechanism.
	stopRouteUpdater := make(chan struct{})

	// a.RouteUpdater updates routes on the current node for
	// the newly added or removed nodes in romana cluster.
	if err := a.routeUpdater(stopRouteUpdater, routeRefreshSeconds); err != nil {
		log.Errorf("Agent: Failed to start route updater on the node: %s", err)
		return err
	}

	if checkFeatureSnatEnabled() {
		iptables := &iptsave.IPtables{}
		featureSnat(iptables, a.Helper.Executor, a.networkConfig)
		if err := enforcer.ApplyIPtables(iptables, a.Helper.Executor); err != nil {
			log.Errorf("Filed to install rules supporting FEATURE_SNAT iptables-restore call failed %s", err)
		}

	}

	if a.policyEnabled {
		// Tenant and Policy cache will poll backend storage every cacheTickTime seconds.
		var cacheTickTime int
		if a.config.ServiceSpecific["cache_tick_time"] != nil {
			cacheTickTime = int(a.config.ServiceSpecific["cache_tick_time"].(float64))
		} else {
			cacheTickTime = defaultCacheTickTime
		}

		// Will try to refresh policies every policyRefreshSeconds.
		var policyRefreshSeconds int
		if a.config.ServiceSpecific["policy_refresh_seconds"] != nil {
			policyRefreshSeconds = int(a.config.ServiceSpecific["policy_refresh_seconds"].(float64))
		} else {
			policyRefreshSeconds = defaultPolicyRefreshSeconds
		}

		a.policyStop = make(chan struct{})
		tenantCache := tenantCache.New(a.client, tenantCache.Config{CacheTickSeconds: cacheTickTime})
		policyCache := policyCache.New(a.client, policyCache.Config{CacheTickSeconds: cacheTickTime})
		a.enforcer = enforcer.New(tenantCache, policyCache, a.networkConfig, a.Helper.Executor, policyRefreshSeconds)
		a.enforcer.Run(a.policyStop)
	}

	return nil
}

// CreateSchema creates database schema.
func (a *Agent) CreateSchema(overwrite bool) error {
	return a.store.CreateSchema(overwrite)
}
<commit_msg>Enabled policies by default<commit_after>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package agent is a Romana service which provides networking functions on the host.
package agent

import (
	"github.com/romana/core/common"
	"github.com/romana/core/common/log/trace"
	enforcer "github.com/romana/core/pkg/policy/enforcer"
	"github.com/romana/core/pkg/util/iptsave"
	policyCache "github.com/romana/core/pkg/util/policy/cache"
	tenantCache "github.com/romana/core/pkg/util/tenant/cache"
	log "github.com/romana/rlog"
)

// Agent provides access to configuration and helper functions, shared across
// all the threads.
// Types Config, Leasefile and Firewall are designed to be loosely coupled
// so they could later be separated into packages and used independently.
type Agent struct {
	// Discovered run-time configuration.
	networkConfig *NetworkConfig

	config common.ServiceConfig
	// Leasefile is a type that manages DHCP leases in the file
	leaseFile *LeaseFile

	// Helper here is a type that organizes swappable interfaces for 3rd
	// party libraries (e.g. os.exec), and some functions that are using
	// those interfaces directly. Main purpose is to support unit testing.
	// Choice of having Helper as a field of an Agent is made to
	// support multiple instances of an Agent running at same time.
	// We like this approach, since it gives us flexibility as the agent evolves in the future.
	// Should this flexibility not be required, a suitable alternative is to re-implement the
	// Agent structure as a set of global variables.
	Helper *Helper

	waitForIfaceTry int

	// Whether this is running in test mode.
	TestMode bool

	// Agent store to keep records about managed resources.
	store agentStore

	client *common.RestClient

	policyEnabled bool

	// Manages iptables rules to reflect romana policies.
	enforcer enforcer.Interface

	// Stops policy enforcer.
	policyStop chan struct{}
}

// SetConfig implements SetConfig function of the Service interface.
func (a *Agent) SetConfig(config common.ServiceConfig) error {
	var err error
	log.Trace(trace.Public, config)
	a.config = config
	leaseFileName := config.ServiceSpecific["lease_file"].(string)
	lf := NewLeaseFile(leaseFileName, a)
	a.leaseFile = &lf

	var ok bool
	a.policyEnabled, ok = config.ServiceSpecific["policy_enabled"].(bool)
	if !ok {
		a.policyEnabled = true
	}
	a.waitForIfaceTry = int(config.ServiceSpecific["wait_for_iface_try"].(float64))
	a.networkConfig = &NetworkConfig{}
	store, err := NewStore(config)
	if err != nil {
		return err
	}
	a.store = *store
	log.Trace(trace.Inside, "Agent.SetConfig() finished.")
	return nil
}

// Routes implements Routes function of Service interface.
func (a *Agent) Routes() common.Routes {
	routes := common.Routes{
		common.Route{
			Method:  "GET",
			Pattern: "/",
			Handler: a.statusHandler,
		},
		common.Route{
			Method:  "POST",
			Pattern: "/vm",
			Handler: a.vmUpHandler,
			MakeMessage: func() interface{} {
				return &NetIf{}
			},
			UseRequestToken: false,
		},
		common.Route{
			Method:  "DELETE",
			Pattern: "/vm",
			Handler: a.vmDownHandler,
			MakeMessage: func() interface{} {
				return &NetIf{}
			},
			UseRequestToken: false,
		},
		common.Route{
			Method:  "POST",
			Pattern: "/pod",
			Handler: a.podUpHandler,
			MakeMessage: func() interface{} {
				return &NetworkRequest{}
			},
			// TODO this is for the future so we ensure idempotence.
			UseRequestToken: true,
		},
		common.Route{
			Method:  "DELETE",
			Pattern: "/pod",
			Handler: a.podDownHandler,
			MakeMessage: func() interface{} {
				return &NetworkRequest{}
			},
		},
		common.Route{
			Method:  "POST",
			Pattern: "/policies",
			Handler: a.addPolicy,
			MakeMessage: func() interface{} {
				return &common.Policy{}
			},
			UseRequestToken: false,
		},
		common.Route{
			Method:  "DELETE",
			Pattern: "/policies",
			MakeMessage: func() interface{} {
				return &common.Policy{}
			},
			Handler: a.deletePolicy,
		},
		common.Route{
			Method:  "GET",
			Pattern: "/policies",
			Handler: a.listPolicies,
		},
	}
	return routes
}

// Name implements method of Service interface.
func (a *Agent) Name() string {
	return "agent"
}

const (
	// defaultCacheTickTime controls how oftend storage cache is updated.
	defaultCacheTickTime = 5

	// defaultPolicyRefreshSeconds controls how often policy agent checks
	// storage caches for updates and applies changes if any.
	defaultPolicyRefreshSeconds = 2

	// defaultRouteRefreshSeconds controls how often agent checks
	// for route update and applies changes if any.
	defaultRouteRefreshSeconds = 120
)

// Initialize implements the Initialize method of common.Service
// interface.
func (a *Agent) Initialize(client *common.RestClient) error {
	log.Trace(trace.Public, "Entering Agent.Initialize()")
	a.client = client

	err := a.store.Connect()
	if err != nil {
		log.Error("Agent.Initialize() : Failed to connect to database.")
		return err
	}

	// identifyCurrentHost() needs to be called at-least once before
	// calling createRomanaGW() to populate romanaGW and romanaGWMask.
	log.Info("Agent: Attempting to identify current host.")
	if err := a.identifyCurrentHost(); err != nil {
		log.Error("Agent: Failed to identify current host: ", agentError(err))
		return agentError(err)
	}

	// Create Romana Gateway and bring up the necessary config for it
	// for example: assign IP Address to it, etc.
	if err := a.createRomanaGW(); err != nil {
		log.Error("Agent: Failed to create Romana Gateway on the node:", err)
		return err
	}

	// Enable default kernel settings needed by romana, for example:
	// ip forward, proxy arp, etc
	if err := a.enableRomanaKernelDefaults(); err != nil {
		log.Error("Agent: Failed to enable Romana kernel defaults on the node:", err)
		return err
	}

	// Will try to refresh routes every routeRefreshSeconds.
	var routeRefreshSeconds int
	if a.config.ServiceSpecific["route_refresh_seconds"] != nil {
		routeRefreshSeconds = int(a.config.ServiceSpecific["route_refresh_seconds"].(float64))
	} else {
		routeRefreshSeconds = defaultRouteRefreshSeconds
	}

	// Channel for stopping route update mechanism.
	stopRouteUpdater := make(chan struct{})

	// a.RouteUpdater updates routes on the current node for
	// the newly added or removed nodes in romana cluster.
	if err := a.routeUpdater(stopRouteUpdater, routeRefreshSeconds); err != nil {
		log.Errorf("Agent: Failed to start route updater on the node: %s", err)
		return err
	}

	if checkFeatureSnatEnabled() {
		iptables := &iptsave.IPtables{}
		featureSnat(iptables, a.Helper.Executor, a.networkConfig)
		if err := enforcer.ApplyIPtables(iptables, a.Helper.Executor); err != nil {
			log.Errorf("Filed to install rules supporting FEATURE_SNAT iptables-restore call failed %s", err)
		}

	}

	if a.policyEnabled {
		// Tenant and Policy cache will poll backend storage every cacheTickTime seconds.
		var cacheTickTime int
		if a.config.ServiceSpecific["cache_tick_time"] != nil {
			cacheTickTime = int(a.config.ServiceSpecific["cache_tick_time"].(float64))
		} else {
			cacheTickTime = defaultCacheTickTime
		}

		// Will try to refresh policies every policyRefreshSeconds.
		var policyRefreshSeconds int
		if a.config.ServiceSpecific["policy_refresh_seconds"] != nil {
			policyRefreshSeconds = int(a.config.ServiceSpecific["policy_refresh_seconds"].(float64))
		} else {
			policyRefreshSeconds = defaultPolicyRefreshSeconds
		}

		a.policyStop = make(chan struct{})
		tenantCache := tenantCache.New(a.client, tenantCache.Config{CacheTickSeconds: cacheTickTime})
		policyCache := policyCache.New(a.client, policyCache.Config{CacheTickSeconds: cacheTickTime})
		a.enforcer = enforcer.New(tenantCache, policyCache, a.networkConfig, a.Helper.Executor, policyRefreshSeconds)
		a.enforcer.Run(a.policyStop)
	}

	return nil
}

// CreateSchema creates database schema.
func (a *Agent) CreateSchema(overwrite bool) error {
	return a.store.CreateSchema(overwrite)
}
<|endoftext|>


<commit_before>package xslt

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

// Simple naive test; primarily exists as a canary in case test helpers break
func TestNaive(t *testing.T) {
	xslFile := "testdata/test.xsl"
	inputXml := "testdata/test.xml"
	outputXml := "testdata/test.out"

	runXslTest(t, xslFile, inputXml, outputXml)
}

func runXslTest(t *testing.T, xslFile, inputXmlFile, outputXmlFile string) bool {
	style, _ := xmlReadFile(xslFile)
	input, _ := xmlReadFile(inputXmlFile)
	outData, _ := ioutil.ReadFile(outputXmlFile)
	expected := string(outData)
	stylesheet, _ := ParseStylesheet(style, xslFile)
	testOptions := StylesheetOptions{false, nil}
	output, _ := stylesheet.Process(input, testOptions)
	if output != expected {
		t.Error(xslFile, "failed")
		fmt.Println("---- EXPECTED  ", xslFile, "----")
		fmt.Println(expected)
		fmt.Println("---- ACTUAL  ", xslFile, "----")
		fmt.Println(output)
		return false
	}
	return true
}

func visit(path string, f os.FileInfo, err error) error {
	fmt.Printf("Visited: %s\n", path)
	return nil
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// Runs the tests derived from the XSLT 1.0 specification examples
func TestXsltREC(t *testing.T) {
	var passed []string
	d, _ := os.Open("testdata/REC")
	fi, _ := d.Readdir(-1)
	for _, f := range fi {
		if f.Mode().IsRegular() && path.Ext(f.Name()) == ".xsl" {
			xslname := path.Join("testdata/REC", f.Name())
			b := xslname[0 : len(xslname)-4]
			inName := b + ".xml"
			outName := b + ".out"
			ii, _ := exists(inName)
			oo, _ := exists(outName)
			if ii && oo {
				ok := runXslTest(t, xslname, inName, outName)
				if ok {
					passed = append(passed, xslname)
				}
			} else {
				fmt.Println("Cannot test", xslname)
			}
		}
	}
	//for _, p := range passed {
	//	fmt.Println("PASSED", p)
	//}
	fmt.Println("passed", len(passed), "tests")
}

// Tests the first full example presented in the XSLT 1.0 spec
func TestXsltRECexample1(t *testing.T) {
	xslFile := "testdata/REC1/doc.xsl"
	inputXml := "testdata/REC1/doc.xml"
	outputXml := "testdata/REC1/result.xml"

	runXslTest(t, xslFile, inputXml, outputXml)
}

// Tests the second full example presented in the XSLT 1.0 spec
func TestXsltRECexample2(t *testing.T) {
	inputXml := "testdata/REC2/data.xml"

	runXslTest(t, "testdata/REC2/html.xsl", inputXml, "testdata/REC2/html.xml")
	runXslTest(t, "testdata/REC2/svg.xsl", inputXml, "testdata/REC2/svg.xml")
	runXslTest(t, "testdata/REC2/vrml.xsl", inputXml, "testdata/REC2/vrml.xml")
}

//convenience function to fix up the paths before running a test
func runGeneralXslTest(t *testing.T, xslFile string) bool {
	xslf := fmt.Sprintf("testdata/general/%v.xsl", xslFile)
	ii := fmt.Sprintf("testdata/docs/%v.xml", xslFile)
	oo := fmt.Sprintf("testdata/general/%v.out", xslFile)
	return runXslTest(t, xslf, ii, oo)
}

// Runs the general suite of libxslt regression tests
func TestXsltGeneral(t *testing.T) {
	//runGeneralXslTest(t, "array") // document('') needs to tweak XPath context somehow
	runGeneralXslTest(t, "bug-1-")
	runGeneralXslTest(t, "bug-2-")
	runGeneralXslTest(t, "bug-3-")
	runGeneralXslTest(t, "bug-4-")
	//runGeneralXslTest(t, "bug-5-") //seems to be an issue with for-each select=$ACTIONgrid
	runGeneralXslTest(t, "bug-6-")
	runGeneralXslTest(t, "bug-7-")
	//runGeneralXslTest(t, "bug-8-")
	runGeneralXslTest(t, "bug-9-")
	runGeneralXslTest(t, "bug-10-")
	runGeneralXslTest(t, "bug-11-")
	runGeneralXslTest(t, "bug-12-")
	runGeneralXslTest(t, "bug-13-")
	runGeneralXslTest(t, "bug-14-")
	runGeneralXslTest(t, "bug-15-")
	runGeneralXslTest(t, "bug-16-")
	runGeneralXslTest(t, "bug-17-")
	runGeneralXslTest(t, "bug-18-")
	runGeneralXslTest(t, "bug-19-")
	//runGeneralXslTest(t, "bug-100") // libxslt:test extension element
	runGeneralXslTest(t, "bug-101") // xsl:element with default namespace
	//runGeneralXslTest(t, "bug-102") // imported xsl:attribute-set
	//runGeneralXslTest(t, "bug-103") //copy-of needs to explcitly set empty namespace when needed
	//runGeneralXslTest(t, "bug-104") //copy-of should preserve attr prefix if plausible
	runGeneralXslTest(t, "bug-105")
	runGeneralXslTest(t, "bug-106") //copy-of
}
<commit_msg>Enable some more xsltproc tests<commit_after>package xslt

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

// Simple naive test; primarily exists as a canary in case test helpers break
func TestNaive(t *testing.T) {
	xslFile := "testdata/test.xsl"
	inputXml := "testdata/test.xml"
	outputXml := "testdata/test.out"

	runXslTest(t, xslFile, inputXml, outputXml)
}

func runXslTest(t *testing.T, xslFile, inputXmlFile, outputXmlFile string) bool {
	style, _ := xmlReadFile(xslFile)
	input, _ := xmlReadFile(inputXmlFile)
	outData, _ := ioutil.ReadFile(outputXmlFile)
	expected := string(outData)
	stylesheet, _ := ParseStylesheet(style, xslFile)
	testOptions := StylesheetOptions{false, nil}
	output, _ := stylesheet.Process(input, testOptions)
	if output != expected {
		t.Error(xslFile, "failed")
		fmt.Println("---- EXPECTED  ", xslFile, "----")
		fmt.Println(expected)
		fmt.Println("---- ACTUAL  ", xslFile, "----")
		fmt.Println(output)
		return false
	}
	return true
}

func visit(path string, f os.FileInfo, err error) error {
	fmt.Printf("Visited: %s\n", path)
	return nil
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// Runs the tests derived from the XSLT 1.0 specification examples
func TestXsltREC(t *testing.T) {
	var passed []string
	d, _ := os.Open("testdata/REC")
	fi, _ := d.Readdir(-1)
	for _, f := range fi {
		if f.Mode().IsRegular() && path.Ext(f.Name()) == ".xsl" {
			xslname := path.Join("testdata/REC", f.Name())
			b := xslname[0 : len(xslname)-4]
			inName := b + ".xml"
			outName := b + ".out"
			ii, _ := exists(inName)
			oo, _ := exists(outName)
			if ii && oo {
				ok := runXslTest(t, xslname, inName, outName)
				if ok {
					passed = append(passed, xslname)
				}
			} else {
				fmt.Println("Cannot test", xslname)
			}
		}
	}
	//for _, p := range passed {
	//	fmt.Println("PASSED", p)
	//}
	fmt.Println("passed", len(passed), "tests")
}

// Tests the first full example presented in the XSLT 1.0 spec
func TestXsltRECexample1(t *testing.T) {
	xslFile := "testdata/REC1/doc.xsl"
	inputXml := "testdata/REC1/doc.xml"
	outputXml := "testdata/REC1/result.xml"

	runXslTest(t, xslFile, inputXml, outputXml)
}

// Tests the second full example presented in the XSLT 1.0 spec
func TestXsltRECexample2(t *testing.T) {
	inputXml := "testdata/REC2/data.xml"

	runXslTest(t, "testdata/REC2/html.xsl", inputXml, "testdata/REC2/html.xml")
	runXslTest(t, "testdata/REC2/svg.xsl", inputXml, "testdata/REC2/svg.xml")
	runXslTest(t, "testdata/REC2/vrml.xsl", inputXml, "testdata/REC2/vrml.xml")
}

//convenience function to fix up the paths before running a test
func runGeneralXslTest(t *testing.T, xslFile string) bool {
	xslf := fmt.Sprintf("testdata/general/%v.xsl", xslFile)
	ii := fmt.Sprintf("testdata/docs/%v.xml", xslFile)
	oo := fmt.Sprintf("testdata/general/%v.out", xslFile)
	return runXslTest(t, xslf, ii, oo)
}

// Runs the general suite of libxslt regression tests
func TestXsltGeneral(t *testing.T) {
	//runGeneralXslTest(t, "array") // document('') needs to tweak XPath context somehow
	runGeneralXslTest(t, "bug-1-")
	runGeneralXslTest(t, "bug-2-")
	runGeneralXslTest(t, "bug-3-")
	runGeneralXslTest(t, "bug-4-")
	//runGeneralXslTest(t, "bug-5-") //seems to be an issue with for-each select=$ACTIONgrid
	runGeneralXslTest(t, "bug-6-")
	runGeneralXslTest(t, "bug-7-")
	//runGeneralXslTest(t, "bug-8-")
	runGeneralXslTest(t, "bug-9-")
	runGeneralXslTest(t, "bug-10-")
	runGeneralXslTest(t, "bug-11-")
	runGeneralXslTest(t, "bug-12-")
	runGeneralXslTest(t, "bug-13-")
	runGeneralXslTest(t, "bug-14-")
	runGeneralXslTest(t, "bug-15-")
	runGeneralXslTest(t, "bug-16-")
	runGeneralXslTest(t, "bug-17-")
	runGeneralXslTest(t, "bug-18-")
	runGeneralXslTest(t, "bug-19-")
	runGeneralXslTest(t, "bug-20-")
	//runGeneralXslTest(t, "bug-21-") // unparsed-entity-uri()
	runGeneralXslTest(t, "bug-22-")
	runGeneralXslTest(t, "bug-23-")
	runGeneralXslTest(t, "bug-24-")
	//runGeneralXslTest(t, "bug-25-") // encoding attr when UTF-8 explictly specified in doc
	runGeneralXslTest(t, "bug-26-")
	runGeneralXslTest(t, "bug-27-")
	runGeneralXslTest(t, "bug-28-")
	// runGeneralXslTest(t, "bug-29-") // document('href'); need to resolve to new source document
	//runGeneralXslTest(t, "bug-100") // libxslt:test extension element
	runGeneralXslTest(t, "bug-101") // xsl:element with default namespace
	//runGeneralXslTest(t, "bug-102") // imported xsl:attribute-set
	//runGeneralXslTest(t, "bug-103") //copy-of needs to explcitly set empty namespace when needed
	//runGeneralXslTest(t, "bug-104") //copy-of should preserve attr prefix if plausible
	runGeneralXslTest(t, "bug-105")
	runGeneralXslTest(t, "bug-106") //copy-of
}
<|endoftext|>


<commit_before>package torrent

import (
	"github.com/anacrolix/missinggo/pubsub"

	"github.com/anacrolix/torrent/metainfo"
)

// This file contains Torrent, until I decide where the private, lower-case
// "torrent" type belongs. That type is currently mostly in torrent.go.

// The public handle to a live torrent within a Client.
type Torrent struct {
	cl      *Client
	torrent *torrent
}

// The torrent's infohash. This is fixed and cannot change. It uniquely
// identifies a torrent.
func (t Torrent) InfoHash() InfoHash {
	return t.torrent.InfoHash
}

// Closed when the info (.Info()) for the torrent has become available. Using
// features of Torrent that require the info before it is available will have
// undefined behaviour.
func (t Torrent) GotInfo() <-chan struct{} {
	return t.torrent.gotMetainfo
}

// Returns the metainfo info dictionary, or nil if it's not yet available.
func (t Torrent) Info() *metainfo.Info {
	return t.torrent.Info
}

// Returns a Reader bound to the torrent's data. All read calls block until
// the data requested is actually available. Priorities are set to ensure the
// data requested will be downloaded as soon as possible.
func (t Torrent) NewReader() (ret *Reader) {
	ret = &Reader{
		t:         &t,
		readahead: 5 * 1024 * 1024,
	}
	t.addReader(ret)
	return
}

// Returns the state of pieces of the torrent. They are grouped into runs of
// same state. The sum of the state run lengths is the number of pieces
// in the torrent.
func (t Torrent) PieceStateRuns() []PieceStateRun {
	t.torrent.stateMu.Lock()
	defer t.torrent.stateMu.Unlock()
	return t.torrent.pieceStateRuns()
}

// The number of pieces in the torrent. This requires that the info has been
// obtained first.
func (t Torrent) NumPieces() int {
	return t.torrent.numPieces()
}

// Drop the torrent from the client, and close it.
func (t Torrent) Drop() {
	t.cl.mu.Lock()
	t.cl.dropTorrent(t.torrent.InfoHash)
	t.cl.mu.Unlock()
}

// Number of bytes of the entire torrent we have completed.
func (t Torrent) BytesCompleted() int64 {
	t.cl.mu.RLock()
	defer t.cl.mu.RUnlock()
	return t.torrent.bytesCompleted()
}

// The subscription emits as (int) the index of pieces as their state changes.
// A state change is when the PieceState for a piece alters in value.
func (t Torrent) SubscribePieceStateChanges() *pubsub.Subscription {
	return t.torrent.pieceStateChanges.Subscribe()
}

// Returns true if the torrent is currently being seeded. This occurs when the
// client is willing to upload without wanting anything in return.
func (t Torrent) Seeding() bool {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	return t.cl.seeding(t.torrent)
}

// Clobbers the torrent display name. The display name is used as the torrent
// name if the metainfo is not available.
func (t Torrent) SetDisplayName(dn string) {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	t.torrent.setDisplayName(dn)
}

// The current working name for the torrent. Either the name in the info dict,
// or a display name given such as by the dn value in a magnet link, or "".
func (t Torrent) Name() string {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	return t.torrent.Name()
}

func (t Torrent) Length() int64 {
	select {
	case <-t.GotInfo():
		return t.torrent.length
	default:
		return -1
	}
}

// Returns a run-time generated metainfo for the torrent that includes the
// info bytes and announce-list as currently known to the client.
func (t Torrent) MetaInfo() *metainfo.MetaInfo {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	return t.torrent.MetaInfo()
}

func (t Torrent) addReader(r *Reader) {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	if t.torrent.readers == nil {
		t.torrent.readers = make(map[*Reader]struct{})
	}
	t.torrent.readers[r] = struct{}{}
	t.torrent.readersChanged(t.cl)
}

func (t Torrent) deleteReader(r *Reader) {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	delete(t.torrent.readers, r)
	t.torrent.readersChanged(t.cl)
}

func (t Torrent) DownloadPieces(begin, end int) {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	for i := begin; i < end; i++ {
		t.torrent.pendPiece(i, t.cl)
	}
}
<commit_msg>Remove obsolete note about priorities<commit_after>package torrent

import (
	"github.com/anacrolix/missinggo/pubsub"

	"github.com/anacrolix/torrent/metainfo"
)

// This file contains Torrent, until I decide where the private, lower-case
// "torrent" type belongs. That type is currently mostly in torrent.go.

// The public handle to a live torrent within a Client.
type Torrent struct {
	cl      *Client
	torrent *torrent
}

// The torrent's infohash. This is fixed and cannot change. It uniquely
// identifies a torrent.
func (t Torrent) InfoHash() InfoHash {
	return t.torrent.InfoHash
}

// Closed when the info (.Info()) for the torrent has become available. Using
// features of Torrent that require the info before it is available will have
// undefined behaviour.
func (t Torrent) GotInfo() <-chan struct{} {
	return t.torrent.gotMetainfo
}

// Returns the metainfo info dictionary, or nil if it's not yet available.
func (t Torrent) Info() *metainfo.Info {
	return t.torrent.Info
}

// Returns a Reader bound to the torrent's data. All read calls block until
// the data requested is actually available.
func (t Torrent) NewReader() (ret *Reader) {
	ret = &Reader{
		t:         &t,
		readahead: 5 * 1024 * 1024,
	}
	t.addReader(ret)
	return
}

// Returns the state of pieces of the torrent. They are grouped into runs of
// same state. The sum of the state run lengths is the number of pieces
// in the torrent.
func (t Torrent) PieceStateRuns() []PieceStateRun {
	t.torrent.stateMu.Lock()
	defer t.torrent.stateMu.Unlock()
	return t.torrent.pieceStateRuns()
}

// The number of pieces in the torrent. This requires that the info has been
// obtained first.
func (t Torrent) NumPieces() int {
	return t.torrent.numPieces()
}

// Drop the torrent from the client, and close it.
func (t Torrent) Drop() {
	t.cl.mu.Lock()
	t.cl.dropTorrent(t.torrent.InfoHash)
	t.cl.mu.Unlock()
}

// Number of bytes of the entire torrent we have completed.
func (t Torrent) BytesCompleted() int64 {
	t.cl.mu.RLock()
	defer t.cl.mu.RUnlock()
	return t.torrent.bytesCompleted()
}

// The subscription emits as (int) the index of pieces as their state changes.
// A state change is when the PieceState for a piece alters in value.
func (t Torrent) SubscribePieceStateChanges() *pubsub.Subscription {
	return t.torrent.pieceStateChanges.Subscribe()
}

// Returns true if the torrent is currently being seeded. This occurs when the
// client is willing to upload without wanting anything in return.
func (t Torrent) Seeding() bool {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	return t.cl.seeding(t.torrent)
}

// Clobbers the torrent display name. The display name is used as the torrent
// name if the metainfo is not available.
func (t Torrent) SetDisplayName(dn string) {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	t.torrent.setDisplayName(dn)
}

// The current working name for the torrent. Either the name in the info dict,
// or a display name given such as by the dn value in a magnet link, or "".
func (t Torrent) Name() string {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	return t.torrent.Name()
}

func (t Torrent) Length() int64 {
	select {
	case <-t.GotInfo():
		return t.torrent.length
	default:
		return -1
	}
}

// Returns a run-time generated metainfo for the torrent that includes the
// info bytes and announce-list as currently known to the client.
func (t Torrent) MetaInfo() *metainfo.MetaInfo {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	return t.torrent.MetaInfo()
}

func (t Torrent) addReader(r *Reader) {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	if t.torrent.readers == nil {
		t.torrent.readers = make(map[*Reader]struct{})
	}
	t.torrent.readers[r] = struct{}{}
	t.torrent.readersChanged(t.cl)
}

func (t Torrent) deleteReader(r *Reader) {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	delete(t.torrent.readers, r)
	t.torrent.readersChanged(t.cl)
}

func (t Torrent) DownloadPieces(begin, end int) {
	t.cl.mu.Lock()
	defer t.cl.mu.Unlock()
	for i := begin; i < end; i++ {
		t.torrent.pendPiece(i, t.cl)
	}
}
<|endoftext|>


<commit_before>package logger

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"syscall"

	"github.com/creack/pty"
	"github.com/mattn/go-isatty"
	"github.com/omakoto/go-common/src/shell"
	"github.com/omakoto/go-common/src/textio"
	"github.com/omakoto/go-common/src/utils"
	"github.com/omakoto/zenlog/zenlog/config"
	"github.com/omakoto/zenlog/zenlog/envs"
	"github.com/omakoto/zenlog/zenlog/logfiles"
	"github.com/omakoto/zenlog/zenlog/util"
	"github.com/pkg/term"
)

type Logger struct {
	Config *config.Config

	child *exec.Cmd

	OuterTty  string
	master    *os.File
	stdinTerm *term.Term

	ForwardPipe *os.File
	ReversePipe *os.File

	startRequest        *StartRequest
	logFiles            *logfiles.LogFiles
	numLines            int
	hasDanglingLastLine bool

	sanitizer *textio.Sanitizer

	clock utils.Clock
}

func mustMakeFifo(config *config.Config, suffix string) *os.File {
	filename := fmt.Sprintf("%szenlog.%d%s.pipe", config.TempDir, config.ZenlogPid, suffix)
	os.Remove(filename)

	util.Debugf("Making fifo '%s'...", filename)
	err := syscall.Mkfifo(filename, 0600)
	util.Check(err, "Makefifo failed for '%s'", filename)

	file, err := os.OpenFile(filename, os.O_RDWR, 0600)
	util.Check(err, "OpenFile failed for '%s'", filename)
	return file
}

func NewLogger(config *config.Config) *Logger {
	if !isatty.IsTerminal(os.Stdin.Fd()) {
		util.Fatalf("Stdin must be terminal.")
	}

	l := Logger{Config: config}

	l.sanitizer = textio.NewSanitizer()

	l.OuterTty = util.Ttyname(os.Stdin.Fd())
	stdinTerm, err := term.Open(l.OuterTty)
	util.Check(err, "Cannot open tty '%s'", l.OuterTty)

	util.Debugf("stdinTerm=%+v", stdinTerm)
	l.stdinTerm = stdinTerm

	err = l.stdinTerm.SetRaw()
	util.Check(err, "SetRaw failed")
	util.SetOutputIsRaw(true)

	// Make the pipes.
	l.ForwardPipe = mustMakeFifo(config, "f")
	l.ReversePipe = mustMakeFifo(config, "r")

	l.clock = utils.NewClock()

	// Update config with the pipe names.
	config.LoggerIn = l.ForwardPipe.Name()
	config.LoggerOut = l.ReversePipe.Name()
	config.OuterTty = util.Tty()

	util.Dump("Logger=", l)

	return &l
}

func (l *Logger) exportEnviron() {
	os.Setenv(envs.ZenlogBin, util.FindZenlogBin())
	os.Setenv(envs.ZenlogSourceDir, config.ZenlogSrcTopDir())
	os.Setenv(envs.ZenlogBinCtime, strconv.FormatInt(util.ZenlogBinCtime().Unix(), 10))

	os.Setenv(envs.ZenlogDir, l.Config.LogDir)
	os.Setenv(envs.ZenlogPid, strconv.Itoa(l.Config.ZenlogPid))
	os.Setenv(envs.ZenlogOuterTty, l.OuterTty)
	os.Setenv(envs.ZenlogLoggerIn, l.ForwardPipe.Name())
	os.Setenv(envs.ZenlogLoggerOut, l.ReversePipe.Name())
}

// StartChild starts a child process in a new PTY.
func (l *Logger) StartChild() {
	l.exportEnviron()

	// Create a pty and start the child command.
	util.Debugf("Executing: %s", l.Config.StartCommand)
	l.child = exec.Command("/bin/sh", "-c",
		envs.ZenlogSignature+
			fmt.Sprintf("=\"$(tty)\":%s ", shell.Escape(Signature()))+
			l.Config.StartCommand)
	var err error
	l.master, err = pty.Start(l.child)
	util.Check(err, "Unable to create pty or execute /bin/sh")

	util.PropagateTerminalSize(os.Stdin, l.master)
}

// Child returns the child process.
func (l *Logger) Child() *exec.Cmd {
	return l.child
}

// Child returns the master tty.
func (l *Logger) Master() *os.File {
	return l.master
}

func (l *Logger) startForwarders() {
	m := l.master

	if l.Config.UseSplice {
		// Forward the input from stdin to the l.
		go forward(os.Stdin, m)

		// Read the output, and write to the STDOUT, and also to the pipe.
		go tee(m, l.ForwardPipe, os.Stdout)
	} else {
		go forwardSimple(os.Stdin, m)
		go teeSimple(m, l.ForwardPipe, os.Stdout)
	}
}

func (l *Logger) CleanUp() {
	if l.master != nil {
		l.master.Close()
	}

	l.stdinTerm.Restore()
	util.SetOutputIsRaw(false)

	l.ForwardPipe.Close()
	l.ReversePipe.Close()

	util.Warn(os.Remove(l.ForwardPipe.Name()), "Remove failed")
	util.Warn(os.Remove(l.ReversePipe.Name()), "Remove failed")
}

func (l *Logger) MustReply(config *config.Config, vals []string) {
	reply := util.Encode(vals)
	util.Debugf("Replying: %v", vals)
	_, err := l.ReversePipe.WriteString(reply)
	util.Check(err, "Failed to reply from logger")
}

func (l *Logger) isOpen() bool {
	return l.logFiles != nil
}

func (l *Logger) SendCloseRequest() {
	util.WriteToFile(l.Config.LoggerIn, utils.StringSlice(CloseSessionCommand))
}

func (l *Logger) SendFlushRequest() {
	util.WriteToFile(l.Config.LoggerIn, utils.StringSlice(FlushCommand))
}

// Open log files.
func (l *Logger) openLogs(request *StartRequest) {
	// If the previous log is still open, close it.
	l.closeLogs(nil)

	// Re-init the fields for the command.
	l.startRequest = request
	l.logFiles = &request.LogFiles

	l.logFiles.Open(false)

	l.write([]byte("$ " + request.Command.CommandLine + "\n"))

	l.numLines = 0 // Don't count the first line. Start with 0 here.
	l.hasDanglingLastLine = false

	// Check nolog.
	if request.Command.NoLog {
		l.write([]byte("[reducted]"))
		l.closeLogs(nil)

		// HACK: This is to update the injected clock even for reducted commands.
		util.GetInjectedNow(l.clock)
	}
}

// Close log files.
func (l *Logger) closeLogs(req *StopRequest) {
	if !l.isOpen() {
		return
	}
	if req != nil {
		l.logFiles.WriteFinishToEnv(req.ExitStatus, l.startRequest.StartTime, util.GetInjectedNow(l.clock))
	}
	l.logFiles.Close()

	l.startRequest = nil
	l.logFiles = nil

	// If the last line didn't finish with NL, then add one line.
	if l.hasDanglingLastLine {
		l.numLines++
		l.hasDanglingLastLine = false
	}
}

// Write a log line.
func (l *Logger) write(line []byte) {
	if !l.isOpen() || len(line) == 0 {
		return
	}
	_, err := l.logFiles.Raw.Write(line)
	util.Warn(err, "Write failed")

	_, err = l.logFiles.San.Write(l.sanitizer.Sanitize(line))
	util.Warn(err, "Write failed")

	l.numLines += util.NumLines(line)
	l.hasDanglingLastLine = line[len(line)-1] != '\n'

	if l.Config.AutoFlush {
		l.flush()
	}
}
func (l *Logger) flush() {
	if l.isOpen() {
		l.logFiles.San.Flush()
		l.logFiles.Raw.Flush()
	}
}

// Start the forwarders, and do the main loop.
func (l *Logger) DoLogger() {
	l.startForwarders()

	bout := bufio.NewReader(l.ForwardPipe)
	for {
		line, err := bout.ReadBytes('\n')
		if len(line) != 0 {
			decoded, pre, args := util.TryDecodeBytes(line)
			if !decoded {
				l.write(line)
			} else {
				l.write(pre)

				if len(args) == 0 {
					util.Say("Received empty command.")
					continue
				}
				switch args[0] {
				case CloseSessionCommand:
					l.closeLogs(nil)
					return

				case FlushCommand:
					l.flush()
					continue

				case CommandStartCommand:
					if len(args) != 2 {
						util.Say("Invalid number of args (%d) for %s.", len(args), CommandStartCommand)
						continue
					}
					// Parse request.

					req := StartRequest{}
					if !util.TryUnmarshal(args[1], &req) {
						continue
					}
					util.Dump("StartRequest=", req)

					// Open log.
					l.openLogs(&req)
					continue
				case CommandEndCommand:
					if len(args) != 3 {
						util.Say("Invalid number of args (%d) for %s.", len(args), CommandEndCommand)
						continue
					}
					fingerprint := args[1]

					// Parse request.
					req := StopRequest{}
					if !util.TryUnmarshal(args[2], &req) {
						continue
					}
					util.Dump("StopRequest=", req)

					// Close log.
					l.closeLogs(&req)

					// Send reply.
					l.MustReply(l.Config, utils.StringSlice(CommandEndCommand, fingerprint, util.MustMarshal(StopReply{l.numLines})))
					continue
				}
			}
		}
		if err != nil {
			if err == io.EOF {
				break
			}
			util.Fatalf("ReadString failed: %s", err)
		}
	}
}

func (l *Logger) OnChildDied() {
	l.SendCloseRequest()
}
<commit_msg>Make sure to flush after command line<commit_after>package logger

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"syscall"

	"github.com/creack/pty"
	"github.com/mattn/go-isatty"
	"github.com/omakoto/go-common/src/shell"
	"github.com/omakoto/go-common/src/textio"
	"github.com/omakoto/go-common/src/utils"
	"github.com/omakoto/zenlog/zenlog/config"
	"github.com/omakoto/zenlog/zenlog/envs"
	"github.com/omakoto/zenlog/zenlog/logfiles"
	"github.com/omakoto/zenlog/zenlog/util"
	"github.com/pkg/term"
)

type Logger struct {
	Config *config.Config

	child *exec.Cmd

	OuterTty  string
	master    *os.File
	stdinTerm *term.Term

	ForwardPipe *os.File
	ReversePipe *os.File

	startRequest        *StartRequest
	logFiles            *logfiles.LogFiles
	numLines            int
	hasDanglingLastLine bool

	sanitizer *textio.Sanitizer

	clock utils.Clock
}

func mustMakeFifo(config *config.Config, suffix string) *os.File {
	filename := fmt.Sprintf("%szenlog.%d%s.pipe", config.TempDir, config.ZenlogPid, suffix)
	os.Remove(filename)

	util.Debugf("Making fifo '%s'...", filename)
	err := syscall.Mkfifo(filename, 0600)
	util.Check(err, "Makefifo failed for '%s'", filename)

	file, err := os.OpenFile(filename, os.O_RDWR, 0600)
	util.Check(err, "OpenFile failed for '%s'", filename)
	return file
}

func NewLogger(config *config.Config) *Logger {
	if !isatty.IsTerminal(os.Stdin.Fd()) {
		util.Fatalf("Stdin must be terminal.")
	}

	l := Logger{Config: config}

	l.sanitizer = textio.NewSanitizer()

	l.OuterTty = util.Ttyname(os.Stdin.Fd())
	stdinTerm, err := term.Open(l.OuterTty)
	util.Check(err, "Cannot open tty '%s'", l.OuterTty)

	util.Debugf("stdinTerm=%+v", stdinTerm)
	l.stdinTerm = stdinTerm

	err = l.stdinTerm.SetRaw()
	util.Check(err, "SetRaw failed")
	util.SetOutputIsRaw(true)

	// Make the pipes.
	l.ForwardPipe = mustMakeFifo(config, "f")
	l.ReversePipe = mustMakeFifo(config, "r")

	l.clock = utils.NewClock()

	// Update config with the pipe names.
	config.LoggerIn = l.ForwardPipe.Name()
	config.LoggerOut = l.ReversePipe.Name()
	config.OuterTty = util.Tty()

	util.Dump("Logger=", l)

	return &l
}

func (l *Logger) exportEnviron() {
	os.Setenv(envs.ZenlogBin, util.FindZenlogBin())
	os.Setenv(envs.ZenlogSourceDir, config.ZenlogSrcTopDir())
	os.Setenv(envs.ZenlogBinCtime, strconv.FormatInt(util.ZenlogBinCtime().Unix(), 10))

	os.Setenv(envs.ZenlogDir, l.Config.LogDir)
	os.Setenv(envs.ZenlogPid, strconv.Itoa(l.Config.ZenlogPid))
	os.Setenv(envs.ZenlogOuterTty, l.OuterTty)
	os.Setenv(envs.ZenlogLoggerIn, l.ForwardPipe.Name())
	os.Setenv(envs.ZenlogLoggerOut, l.ReversePipe.Name())
}

// StartChild starts a child process in a new PTY.
func (l *Logger) StartChild() {
	l.exportEnviron()

	// Create a pty and start the child command.
	util.Debugf("Executing: %s", l.Config.StartCommand)
	l.child = exec.Command("/bin/sh", "-c",
		envs.ZenlogSignature+
			fmt.Sprintf("=\"$(tty)\":%s ", shell.Escape(Signature()))+
			l.Config.StartCommand)
	var err error
	l.master, err = pty.Start(l.child)
	util.Check(err, "Unable to create pty or execute /bin/sh")

	util.PropagateTerminalSize(os.Stdin, l.master)
}

// Child returns the child process.
func (l *Logger) Child() *exec.Cmd {
	return l.child
}

// Child returns the master tty.
func (l *Logger) Master() *os.File {
	return l.master
}

func (l *Logger) startForwarders() {
	m := l.master

	if l.Config.UseSplice {
		// Forward the input from stdin to the l.
		go forward(os.Stdin, m)

		// Read the output, and write to the STDOUT, and also to the pipe.
		go tee(m, l.ForwardPipe, os.Stdout)
	} else {
		go forwardSimple(os.Stdin, m)
		go teeSimple(m, l.ForwardPipe, os.Stdout)
	}
}

func (l *Logger) CleanUp() {
	if l.master != nil {
		l.master.Close()
	}

	l.stdinTerm.Restore()
	util.SetOutputIsRaw(false)

	l.ForwardPipe.Close()
	l.ReversePipe.Close()

	util.Warn(os.Remove(l.ForwardPipe.Name()), "Remove failed")
	util.Warn(os.Remove(l.ReversePipe.Name()), "Remove failed")
}

func (l *Logger) MustReply(config *config.Config, vals []string) {
	reply := util.Encode(vals)
	util.Debugf("Replying: %v", vals)
	_, err := l.ReversePipe.WriteString(reply)
	util.Check(err, "Failed to reply from logger")
}

func (l *Logger) isOpen() bool {
	return l.logFiles != nil
}

func (l *Logger) SendCloseRequest() {
	util.WriteToFile(l.Config.LoggerIn, utils.StringSlice(CloseSessionCommand))
}

func (l *Logger) SendFlushRequest() {
	util.WriteToFile(l.Config.LoggerIn, utils.StringSlice(FlushCommand))
}

// Open log files.
func (l *Logger) openLogs(request *StartRequest) {
	// If the previous log is still open, close it.
	l.closeLogs(nil)

	// Re-init the fields for the command.
	l.startRequest = request
	l.logFiles = &request.LogFiles

	l.logFiles.Open(false)

	l.write([]byte("$ " + request.Command.CommandLine + "\n"))
	l.flush()

	l.numLines = 0 // Don't count the first line. Start with 0 here.
	l.hasDanglingLastLine = false

	// Check nolog.
	if request.Command.NoLog {
		l.write([]byte("[reducted]"))
		l.closeLogs(nil)

		// HACK: This is to update the injected clock even for reducted commands.
		util.GetInjectedNow(l.clock)
	}
}

// Close log files.
func (l *Logger) closeLogs(req *StopRequest) {
	if !l.isOpen() {
		return
	}
	if req != nil {
		l.logFiles.WriteFinishToEnv(req.ExitStatus, l.startRequest.StartTime, util.GetInjectedNow(l.clock))
	}
	l.logFiles.Close()

	l.startRequest = nil
	l.logFiles = nil

	// If the last line didn't finish with NL, then add one line.
	if l.hasDanglingLastLine {
		l.numLines++
		l.hasDanglingLastLine = false
	}
}

// Write a log line.
func (l *Logger) write(line []byte) {
	if !l.isOpen() || len(line) == 0 {
		return
	}
	_, err := l.logFiles.Raw.Write(line)
	util.Warn(err, "Write failed")

	_, err = l.logFiles.San.Write(l.sanitizer.Sanitize(line))
	util.Warn(err, "Write failed")

	l.numLines += util.NumLines(line)
	l.hasDanglingLastLine = line[len(line)-1] != '\n'

	if l.Config.AutoFlush {
		l.flush()
	}
}
func (l *Logger) flush() {
	if l.isOpen() {
		l.logFiles.San.Flush()
		l.logFiles.Raw.Flush()
	}
}

// Start the forwarders, and do the main loop.
func (l *Logger) DoLogger() {
	l.startForwarders()

	bout := bufio.NewReader(l.ForwardPipe)
	for {
		line, err := bout.ReadBytes('\n')
		if len(line) != 0 {
			decoded, pre, args := util.TryDecodeBytes(line)
			if !decoded {
				l.write(line)
			} else {
				l.write(pre)

				if len(args) == 0 {
					util.Say("Received empty command.")
					continue
				}
				switch args[0] {
				case CloseSessionCommand:
					l.closeLogs(nil)
					return

				case FlushCommand:
					l.flush()
					continue

				case CommandStartCommand:
					if len(args) != 2 {
						util.Say("Invalid number of args (%d) for %s.", len(args), CommandStartCommand)
						continue
					}
					// Parse request.

					req := StartRequest{}
					if !util.TryUnmarshal(args[1], &req) {
						continue
					}
					util.Dump("StartRequest=", req)

					// Open log.
					l.openLogs(&req)
					continue
				case CommandEndCommand:
					if len(args) != 3 {
						util.Say("Invalid number of args (%d) for %s.", len(args), CommandEndCommand)
						continue
					}
					fingerprint := args[1]

					// Parse request.
					req := StopRequest{}
					if !util.TryUnmarshal(args[2], &req) {
						continue
					}
					util.Dump("StopRequest=", req)

					// Close log.
					l.closeLogs(&req)

					// Send reply.
					l.MustReply(l.Config, utils.StringSlice(CommandEndCommand, fingerprint, util.MustMarshal(StopReply{l.numLines})))
					continue
				}
			}
		}
		if err != nil {
			if err == io.EOF {
				break
			}
			util.Fatalf("ReadString failed: %s", err)
		}
	}
}

func (l *Logger) OnChildDied() {
	l.SendCloseRequest()
}
<|endoftext|>


<commit_before>package core

import "time"

type AvailablePlugin interface {
	Name() string
	Version() int
	HitCount() int
	LastHit() time.Time
	TypeName() string
	ID() int
}

// the public interface for a plugin
// this should be the contract for
// how mgmt modules know a plugin
type CatalogedPlugin interface {
	Name() string
	Version() int
	TypeName() string
	Status() string
	LoadedTimestamp() int64
}

// the collection of cataloged plugins used
// by mgmt modules
type PluginCatalog []CatalogedPlugin

type PluginType int

const (
	// List of plugin type
	CollectorPluginType PluginType = iota
	PublisherPluginType
	ProcessorPluginType
)

type Plugin interface {
	Name() string
	Version() int
}
<commit_msg>Simpler linking by embedding interface<commit_after>package core

import "time"

type AvailablePlugin interface {
	Name() string
	Version() int
	HitCount() int
	LastHit() time.Time
	TypeName() string
	ID() int
}

// the public interface for a plugin
// this should be the contract for
// how mgmt modules know a plugin
type CatalogedPlugin interface {
	Plugin
	TypeName() string
	Status() string
	LoadedTimestamp() int64
}

// the collection of cataloged plugins used
// by mgmt modules
type PluginCatalog []CatalogedPlugin

type PluginType int

const (
	// List of plugin type
	CollectorPluginType PluginType = iota
	PublisherPluginType
	ProcessorPluginType
)

type Plugin interface {
	Name() string
	Version() int
}
<|endoftext|>


<commit_before>// +build linux

// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package app

import (
	"os"
	"os/signal"
	"syscall"

	log "github.com/cihub/seelog"
	"golang.org/x/net/context"
)

// startSigchldHandler registers a channel to receiev SIGCHLD signals for
// the agent process. On receiving SIGCHLD, it invokes the wait4 syscall
// so that child processes are appropriately cleaned up.
func (agent *ecsAgent) startSigchldHandler(ctx context.Context) {
	signals := make(chan os.Signal)
	signal.Notify(signals, syscall.SIGCHLD)
	go processSignal(ctx, signals)
}

func processSignal(ctx context.Context, signals <-chan os.Signal) {
	log.Info("Starting the SIGCHLD handler")
	for {
		select {
		case s := <-signals:
			log.Debugf("Received SIGCHLD: %s", s.String())
			go wait()
		case <-ctx.Done():
			log.Info("Stopping the SIGCHLD handler")
			return
		}
	}
}

// wait wraps the Wait4 syscall and returns the error if any
func wait() {
	var ws syscall.WaitStatus
	var ru syscall.Rusage
	// More information on wait4 syscall can be found in manual pages
	// via `man 2 wait4` command. The syscall is used to wait for state
	// changes in the child processes of the Agent. Examples of such
	// processes include CNI plugins and the dhclient processes started
	// by these plugins. As per man pages, if a wait is not performed,
	// then the terminated child remains in a "zombie" state, which is
	// especially problematic for the dhclient process as the network
	// resources, including the namespace that holds the ENI would not
	// be properly cleaned up.
	//
	// wait4(pid, status, options, rusage) is equivalent to
	// the waitpid(pid, status, options) syscall. The value of -1 for
	// the pid field means that we wait for any child process. The
	// handles for waitstatus and rusage will be populated and can be
	// optionall used to infer the status of the child process when
	// needed. The options field is set to 0 as we are not setting any
	// additional options on the wait4 syscall.
	if _, err := syscall.Wait4(-1, &ws, 0, &ru); err != nil {
		log.Debugf("Error waiting for state change of the child process: %v", err)
		return
	}
	log.Debug("Wait for state change of the child process complete")
}
<commit_msg>agent/app: stop receiving sigchld when the context is cancelled<commit_after>// +build linux

// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package app

import (
	"os"
	"os/signal"
	"syscall"

	log "github.com/cihub/seelog"
	"golang.org/x/net/context"
)

const (
	// waitForAnyPID is used by the wait4(2) syscall to wait for any
	// process.
	waitForAnyPID = -1
	// wait4Options specifies the options set when invoking the
	// wait4(2) syscall
	wait4Options = 0
)

// startSigchldHandler registers a channel to receiev SIGCHLD signals for
// the agent process. On receiving SIGCHLD, it invokes the wait4(2) syscall
// so that child processes are appropriately cleaned up.
func (agent *ecsAgent) startSigchldHandler(ctx context.Context) {
	// As per https://golang.org/pkg/os/signal/#Notify,
	// Package signal will not block sending to c. For a channel used for
	// notification of just one signal value, a buffer of size
	// 1 is sufficient.
	sigChldReceiver := make(chan os.Signal, 1)
	signal.Notify(sigChldReceiver, syscall.SIGCHLD)
	go processSignals(ctx, sigChldReceiver)
}

func processSignals(ctx context.Context, sigChldReceiver chan os.Signal) {
	log.Info("Starting the SIGCHLD handler")
	for {
		select {
		case s := <-sigChldReceiver:
			log.Debugf("Received SIGCHLD: %s", s.String())
			go wait()
		case <-ctx.Done():
			log.Info("Stopping the SIGCHLD handler")
			signal.Stop(sigChldReceiver)
			return
		}
	}
}

// wait wraps the Wait4 syscall and returns the error if any
func wait() {
	var waitStatus syscall.WaitStatus
	var rusage syscall.Rusage
	// More information on wait4 syscall can be found in manual pages
	// via `man 2 wait4` command. The syscall is used to wait for state
	// changes in the child processes of the Agent. Examples of such
	// processes include CNI plugins and the dhclient processes started
	// by these plugins. As per man pages, if a wait is not performed,
	// then the terminated child remains in a "zombie" state, which is
	// especially problematic for the dhclient process as the network
	// resources, including the namespace that holds the ENI would not
	// be properly cleaned up.
	//
	// wait4(pid, status, options, rusage) is equivalent to
	// the waitpid(pid, status, options) syscall. The value of -1 for
	// the pid field means that we wait for any child process. The
	// handles for waitstatus and rusage will be populated and can be
	// optionally used to infer the status of the child process when
	// needed. The options field is set to 0 as we are not setting any
	// additional options on the wait4 syscall.
	if _, err := syscall.Wait4(waitForAnyPID, &waitStatus, wait4Options, &rusage); err != nil {
		log.Debugf("Error waiting for state change of the child process: %v", err)
		return
	}
	log.Debug("Wait for state change of the child process complete")
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package searcher

import (
	"math"

	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/search"
)

type PhraseSearcher struct {
	indexReader  index.IndexReader
	mustSearcher *ConjunctionSearcher
	queryNorm    float64
	currMust     *search.DocumentMatch
	slop         int
	terms        []string
	initialized  bool
}

func NewPhraseSearcher(indexReader index.IndexReader, mustSearcher *ConjunctionSearcher, terms []string) (*PhraseSearcher, error) {
	// build our searcher
	rv := PhraseSearcher{
		indexReader:  indexReader,
		mustSearcher: mustSearcher,
		terms:        terms,
	}
	rv.computeQueryNorm()
	return &rv, nil
}

func (s *PhraseSearcher) computeQueryNorm() {
	// first calculate sum of squared weights
	sumOfSquaredWeights := 0.0
	if s.mustSearcher != nil {
		sumOfSquaredWeights += s.mustSearcher.Weight()
	}

	// now compute query norm from this
	s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)
	// finally tell all the downstream searchers the norm
	if s.mustSearcher != nil {
		s.mustSearcher.SetQueryNorm(s.queryNorm)
	}
}

func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error {
	err := s.advanceNextMust(ctx)
	if err != nil {
		return err
	}

	s.initialized = true
	return nil
}

func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error {
	var err error

	if s.mustSearcher != nil {
		s.currMust, err = s.mustSearcher.Next(ctx)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *PhraseSearcher) Weight() float64 {
	return s.mustSearcher.Weight()
}

func (s *PhraseSearcher) SetQueryNorm(qnorm float64) {
	s.mustSearcher.SetQueryNorm(qnorm)
}

func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
	if !s.initialized {
		err := s.initSearchers(ctx)
		if err != nil {
			return nil, err
		}
	}

	var rv *search.DocumentMatch
	for s.currMust != nil {
		rvftlm := make(search.FieldTermLocationMap, 0)
		freq := 0
		firstTerm := s.terms[0]
		for field, termLocMap := range s.currMust.Locations {
			rvtlm := make(search.TermLocationMap, 0)
			locations, ok := termLocMap[firstTerm]
			if !ok {
				continue
			}
		OUTER:
			for _, location := range locations {
				crvtlm := make(search.TermLocationMap, 0)
				crvtlm.AddLocation(firstTerm, location)
			INNER:
				for i := 1; i < len(s.terms); i++ {
					nextTerm := s.terms[i]
					if nextTerm == "" {
						continue
					}
					// look through all these term locations
					// to try and find the correct offsets
					nextLocations, ok := termLocMap[nextTerm]
					if !ok {
						continue OUTER
					}
					for _, nextLocation := range nextLocations {
						if nextLocation.Pos == location.Pos+float64(i) && nextLocation.SameArrayElement(location) {
							// found a location match for this term
							crvtlm.AddLocation(nextTerm, nextLocation)
							continue INNER
						}
					}
					// if we got here we didn't find a location match for this term
					continue OUTER
				}
				// if we got here all the terms matched
				freq++
				search.MergeTermLocationMaps(rvtlm, crvtlm)
				rvftlm[field] = rvtlm
			}
		}

		if freq > 0 {
			// return match
			rv = s.currMust
			rv.Locations = rvftlm
			err := s.advanceNextMust(ctx)
			if err != nil {
				return nil, err
			}
			return rv, nil
		}

		err := s.advanceNextMust(ctx)
		if err != nil {
			return nil, err
		}
	}

	return nil, nil
}

func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
	if !s.initialized {
		err := s.initSearchers(ctx)
		if err != nil {
			return nil, err
		}
	}
	var err error
	s.currMust, err = s.mustSearcher.Advance(ctx, ID)
	if err != nil {
		return nil, err
	}
	return s.Next(ctx)
}

func (s *PhraseSearcher) Count() uint64 {
	// for now return a worst case
	return s.mustSearcher.Count()
}

func (s *PhraseSearcher) Close() error {
	if s.mustSearcher != nil {
		err := s.mustSearcher.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *PhraseSearcher) Min() int {
	return 0
}

func (s *PhraseSearcher) DocumentMatchPoolSize() int {
	return s.mustSearcher.DocumentMatchPoolSize() + 1
}
<commit_msg>Initialize the return value as late as possible<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package searcher

import (
	"math"

	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/search"
)

type PhraseSearcher struct {
	indexReader  index.IndexReader
	mustSearcher *ConjunctionSearcher
	queryNorm    float64
	currMust     *search.DocumentMatch
	slop         int
	terms        []string
	initialized  bool
}

func NewPhraseSearcher(indexReader index.IndexReader, mustSearcher *ConjunctionSearcher, terms []string) (*PhraseSearcher, error) {
	// build our searcher
	rv := PhraseSearcher{
		indexReader:  indexReader,
		mustSearcher: mustSearcher,
		terms:        terms,
	}
	rv.computeQueryNorm()
	return &rv, nil
}

func (s *PhraseSearcher) computeQueryNorm() {
	// first calculate sum of squared weights
	sumOfSquaredWeights := 0.0
	if s.mustSearcher != nil {
		sumOfSquaredWeights += s.mustSearcher.Weight()
	}

	// now compute query norm from this
	s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)
	// finally tell all the downstream searchers the norm
	if s.mustSearcher != nil {
		s.mustSearcher.SetQueryNorm(s.queryNorm)
	}
}

func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error {
	err := s.advanceNextMust(ctx)
	if err != nil {
		return err
	}

	s.initialized = true
	return nil
}

func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error {
	var err error

	if s.mustSearcher != nil {
		s.currMust, err = s.mustSearcher.Next(ctx)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *PhraseSearcher) Weight() float64 {
	return s.mustSearcher.Weight()
}

func (s *PhraseSearcher) SetQueryNorm(qnorm float64) {
	s.mustSearcher.SetQueryNorm(qnorm)
}

func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
	if !s.initialized {
		err := s.initSearchers(ctx)
		if err != nil {
			return nil, err
		}
	}

	var rv *search.DocumentMatch
	for s.currMust != nil {
		rvftlm := make(search.FieldTermLocationMap, 0)
		freq := 0
		firstTerm := s.terms[0]
		for field, termLocMap := range s.currMust.Locations {
			locations, ok := termLocMap[firstTerm]
			if !ok {
				continue
			}

			rvtlm := make(search.TermLocationMap, 0)

		OUTER:
			for _, location := range locations {
				crvtlm := make(search.TermLocationMap, 0)
				crvtlm.AddLocation(firstTerm, location)
			INNER:
				for i := 1; i < len(s.terms); i++ {
					nextTerm := s.terms[i]
					if nextTerm == "" {
						continue
					}
					// look through all these term locations
					// to try and find the correct offsets
					nextLocations, ok := termLocMap[nextTerm]
					if !ok {
						continue OUTER
					}
					for _, nextLocation := range nextLocations {
						if nextLocation.Pos == location.Pos+float64(i) && nextLocation.SameArrayElement(location) {
							// found a location match for this term
							crvtlm.AddLocation(nextTerm, nextLocation)
							continue INNER
						}
					}
					// if we got here we didn't find a location match for this term
					continue OUTER
				}
				// if we got here all the terms matched
				freq++
				search.MergeTermLocationMaps(rvtlm, crvtlm)
				rvftlm[field] = rvtlm
			}
		}

		if freq > 0 {
			// return match
			rv = s.currMust
			rv.Locations = rvftlm
			err := s.advanceNextMust(ctx)
			if err != nil {
				return nil, err
			}
			return rv, nil
		}

		err := s.advanceNextMust(ctx)
		if err != nil {
			return nil, err
		}
	}

	return nil, nil
}

func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
	if !s.initialized {
		err := s.initSearchers(ctx)
		if err != nil {
			return nil, err
		}
	}
	var err error
	s.currMust, err = s.mustSearcher.Advance(ctx, ID)
	if err != nil {
		return nil, err
	}
	return s.Next(ctx)
}

func (s *PhraseSearcher) Count() uint64 {
	// for now return a worst case
	return s.mustSearcher.Count()
}

func (s *PhraseSearcher) Close() error {
	if s.mustSearcher != nil {
		err := s.mustSearcher.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *PhraseSearcher) Min() int {
	return 0
}

func (s *PhraseSearcher) DocumentMatchPoolSize() int {
	return s.mustSearcher.DocumentMatchPoolSize() + 1
}
<|endoftext|>