<commit_before>package cluster

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"strconv"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/crypto/scrypt"

	"github.com/lxc/lxd/lxd/config"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/shared/validate"
)

// Config holds cluster-wide configuration values.
type Config struct {
	tx *db.ClusterTx // DB transaction the values in this config are bound to.
	m  config.Map    // Low-level map holding the config values.
}

// ConfigLoad loads a new Config object with the current cluster configuration
// values fetched from the database.
func ConfigLoad(tx *db.ClusterTx) (*Config, error) {
	// Load current raw values from the database, any error is fatal.
	values, err := tx.Config()
	if err != nil {
		return nil, fmt.Errorf("cannot fetch node config from database: %v", err)
	}

	m, err := config.SafeLoad(ConfigSchema, values)
	if err != nil {
		return nil, fmt.Errorf("failed to load node config: %v", err)
	}

	return &Config{tx: tx, m: m}, nil
}

// HTTPSAllowedHeaders returns the relevant CORS setting.
func (c *Config) HTTPSAllowedHeaders() string {
	return c.m.GetString("core.https_allowed_headers")
}

// HTTPSAllowedMethods returns the relevant CORS setting.
func (c *Config) HTTPSAllowedMethods() string {
	return c.m.GetString("core.https_allowed_methods")
}

// HTTPSAllowedOrigin returns the relevant CORS setting.
func (c *Config) HTTPSAllowedOrigin() string {
	return c.m.GetString("core.https_allowed_origin")
}

// HTTPSAllowedCredentials returns the relevant CORS setting.
func (c *Config) HTTPSAllowedCredentials() bool {
	return c.m.GetBool("core.https_allowed_credentials")
}

// TrustPassword returns the LXD trust password for authenticating clients.
func (c *Config) TrustPassword() string {
	return c.m.GetString("core.trust_password")
}

// TrustCACertificates returns whether client certificates are checked
// against a CA.
func (c *Config) TrustCACertificates() bool {
	return c.m.GetBool("core.trust_ca_certificates")
}

// CandidServer returns all the Candid settings needed to connect to a server.
func (c *Config) CandidServer() (string, string, int64, string) {
	return c.m.GetString("candid.api.url"),
		c.m.GetString("candid.api.key"),
		c.m.GetInt64("candid.expiry"),
		c.m.GetString("candid.domains")
}

// RBACServer returns all the Candid settings needed to connect to a server.
func (c *Config) RBACServer() (string, string, int64, string, string, string, string) {
	return c.m.GetString("rbac.api.url"),
		c.m.GetString("rbac.api.key"),
		c.m.GetInt64("rbac.expiry"),
		c.m.GetString("rbac.agent.url"),
		c.m.GetString("rbac.agent.username"),
		c.m.GetString("rbac.agent.private_key"),
		c.m.GetString("rbac.agent.public_key")
}

// ProxyHTTPS returns the configured HTTPS proxy, if any.
func (c *Config) ProxyHTTPS() string {
	return c.m.GetString("core.proxy_https")
}

// ProxyHTTP returns the configured HTTP proxy, if any.
func (c *Config) ProxyHTTP() string {
	return c.m.GetString("core.proxy_http")
}

// ProxyIgnoreHosts returns the configured ignore-hosts proxy setting, if any.
func (c *Config) ProxyIgnoreHosts() string {
	return c.m.GetString("core.proxy_ignore_hosts")
}

// MAASController the configured MAAS url and key, if any.
func (c *Config) MAASController() (string, string) {
	url := c.m.GetString("maas.api.url")
	key := c.m.GetString("maas.api.key")
	return url, key
}

// OfflineThreshold returns the configured heartbeat threshold, i.e. the
// number of seconds before after which an unresponsive node is considered
// offline..
func (c *Config) OfflineThreshold() time.Duration {
	n := c.m.GetInt64("cluster.offline_threshold")
	return time.Duration(n) * time.Second
}

// ImagesMinimalReplica returns the numbers of nodes for cluster images replication
func (c *Config) ImagesMinimalReplica() int64 {
	return c.m.GetInt64("cluster.images_minimal_replica")
}

// MaxVoters returns the maximum number of members in a cluster that will be
// assigned the voter role.
func (c *Config) MaxVoters() int64 {
	return c.m.GetInt64("cluster.max_voters")
}

// MaxStandBy returns the maximum number of standby members in a cluster that
// will be assigned the stand-by role.
func (c *Config) MaxStandBy() int64 {
	return c.m.GetInt64("cluster.max_standby")
}

// Dump current configuration keys and their values. Keys with values matching
// their defaults are omitted.
func (c *Config) Dump() map[string]interface{} {
	return c.m.Dump()
}

// Replace the current configuration with the given values.
//
// Return what has actually changed.
func (c *Config) Replace(values map[string]interface{}) (map[string]string, error) {
	return c.update(values)
}

// Patch changes only the configuration keys in the given map.
//
// Return what has actually changed.
func (c *Config) Patch(patch map[string]interface{}) (map[string]string, error) {
	values := c.Dump() // Use current values as defaults
	for name, value := range patch {
		values[name] = value
	}
	return c.update(values)
}

func (c *Config) update(values map[string]interface{}) (map[string]string, error) {
	changed, err := c.m.Change(values)
	if err != nil {
		return nil, err
	}

	err = c.tx.UpdateConfig(changed)
	if err != nil {
		return nil, errors.Wrap(err, "cannot persist configuration changes: %v")
	}

	return changed, nil
}

// ConfigGetString is a convenience for loading the cluster configuration and
// returning the value of a particular key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetString(cluster *db.Cluster, key string) (string, error) {
	config, err := configGet(cluster)
	if err != nil {
		return "", err
	}
	return config.m.GetString(key), nil
}

// ConfigGetBool is a convenience for loading the cluster configuration and
// returning the value of a particular boolean key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetBool(cluster *db.Cluster, key string) (bool, error) {
	config, err := configGet(cluster)
	if err != nil {
		return false, err
	}
	return config.m.GetBool(key), nil
}

// ConfigGetInt64 is a convenience for loading the cluster configuration and
// returning the value of a particular key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetInt64(cluster *db.Cluster, key string) (int64, error) {
	config, err := configGet(cluster)
	if err != nil {
		return 0, err
	}
	return config.m.GetInt64(key), nil
}

func configGet(cluster *db.Cluster) (*Config, error) {
	var config *Config
	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		config, err = ConfigLoad(tx)
		return err
	})
	return config, err
}

// ConfigSchema defines available server configuration keys.
var ConfigSchema = config.Schema{
	"backups.compression_algorithm":  {Default: "gzip", Validator: validate.IsCompressionAlgorithm},
	"cluster.offline_threshold":      {Type: config.Int64, Default: offlineThresholdDefault(), Validator: offlineThresholdValidator},
	"cluster.images_minimal_replica": {Type: config.Int64, Default: "3", Validator: imageMinimalReplicaValidator},
	"cluster.max_voters":             {Type: config.Int64, Default: "3", Validator: maxVotersValidator},
	"cluster.max_standby":            {Type: config.Int64, Default: "2", Validator: maxStandByValidator},
	"core.https_allowed_headers":     {},
	"core.https_allowed_methods":     {},
	"core.https_allowed_origin":      {},
	"core.https_allowed_credentials": {Type: config.Bool},
	"core.proxy_http":                {},
	"core.proxy_https":               {},
	"core.proxy_ignore_hosts":        {},
	"core.trust_password":            {Hidden: true, Setter: passwordSetter},
	"core.trust_ca_certificates":     {Type: config.Bool},
	"candid.api.key":                 {},
	"candid.api.url":                 {},
	"candid.domains":                 {},
	"candid.expiry":                  {Type: config.Int64, Default: "3600"},
	"images.auto_update_cached":      {Type: config.Bool, Default: "true"},
	"images.auto_update_interval":    {Type: config.Int64, Default: "6"},
	"images.compression_algorithm":   {Default: "gzip", Validator: validate.IsCompressionAlgorithm},
	"images.remote_cache_expiry":     {Type: config.Int64, Default: "10"},
	"maas.api.key":                   {},
	"maas.api.url":                   {},
	"rbac.agent.url":                 {},
	"rbac.agent.username":            {},
	"rbac.agent.private_key":         {},
	"rbac.agent.public_key":          {},
	"rbac.api.expiry":                {Type: config.Int64, Default: "3600"},
	"rbac.api.key":                   {},
	"rbac.api.url":                   {},
	"rbac.expiry":                    {Type: config.Int64, Default: "3600"},

	// Keys deprecated since the implementation of the storage api.
	"storage.lvm_fstype":           {Setter: deprecatedStorage, Default: "ext4"},
	"storage.lvm_mount_options":    {Setter: deprecatedStorage, Default: "discard"},
	"storage.lvm_thinpool_name":    {Setter: deprecatedStorage, Default: "LXDThinPool"},
	"storage.lvm_vg_name":          {Setter: deprecatedStorage},
	"storage.lvm_volume_size":      {Setter: deprecatedStorage, Default: "10GiB"},
	"storage.zfs_pool_name":        {Setter: deprecatedStorage},
	"storage.zfs_remove_snapshots": {Setter: deprecatedStorage, Type: config.Bool},
	"storage.zfs_use_refquota":     {Setter: deprecatedStorage, Type: config.Bool},

	// OVN networking global keys.
	"network.ovn.integration_bridge":    {Default: "br-int"},
	"network.ovn.northbound_connection": {Default: "unix:/var/run/ovn/ovnnb_db.sock"},
}

func offlineThresholdDefault() string {
	return strconv.Itoa(db.DefaultOfflineThreshold)
}

func offlineThresholdValidator(value string) error {
	// Ensure that the given value is greater than the heartbeat interval,
	// which is the lower bound granularity of the offline check.
	threshold, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Offline threshold is not a number")
	}

	if threshold <= heartbeatInterval {
		return fmt.Errorf("Value must be greater than '%d'", heartbeatInterval)
	}

	return nil
}

func imageMinimalReplicaValidator(value string) error {
	count, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Minimal image replica count is not a number")
	}

	if count < 1 && count != -1 {
		return fmt.Errorf("Invalid value for image replica count")
	}

	return nil
}

func maxVotersValidator(value string) error {
	n, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Value is not a number")
	}

	if n < 3 || n%2 != 1 {
		return fmt.Errorf("Value must be an odd number equal to or higher than 3")
	}

	return nil
}

func maxStandByValidator(value string) error {
	n, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Value is not a number")
	}

	if n < 0 || n > 5 {
		return fmt.Errorf("Value must be between 0 and 5")
	}

	return nil
}

func passwordSetter(value string) (string, error) {
	// Nothing to do on unset
	if value == "" {
		return value, nil
	}

	// Hash the password
	buf := make([]byte, 32)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", err
	}

	hash, err := scrypt.Key([]byte(value), buf, 1<<14, 8, 1, 64)
	if err != nil {
		return "", err
	}

	buf = append(buf, hash...)
	value = hex.EncodeToString(buf)

	return value, nil
}

func deprecatedStorage(value string) (string, error) {
	if value == "" {
		return "", nil
	}
	return "", fmt.Errorf("deprecated: use storage pool configuration")
}
<commit_msg>lxd/cluster: Add images.default_architecture<commit_after>package cluster

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"strconv"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/crypto/scrypt"

	"github.com/lxc/lxd/lxd/config"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/shared/validate"
)

// Config holds cluster-wide configuration values.
type Config struct {
	tx *db.ClusterTx // DB transaction the values in this config are bound to.
	m  config.Map    // Low-level map holding the config values.
}

// ConfigLoad loads a new Config object with the current cluster configuration
// values fetched from the database.
func ConfigLoad(tx *db.ClusterTx) (*Config, error) {
	// Load current raw values from the database, any error is fatal.
	values, err := tx.Config()
	if err != nil {
		return nil, fmt.Errorf("cannot fetch node config from database: %v", err)
	}

	m, err := config.SafeLoad(ConfigSchema, values)
	if err != nil {
		return nil, fmt.Errorf("failed to load node config: %v", err)
	}

	return &Config{tx: tx, m: m}, nil
}

// HTTPSAllowedHeaders returns the relevant CORS setting.
func (c *Config) HTTPSAllowedHeaders() string {
	return c.m.GetString("core.https_allowed_headers")
}

// HTTPSAllowedMethods returns the relevant CORS setting.
func (c *Config) HTTPSAllowedMethods() string {
	return c.m.GetString("core.https_allowed_methods")
}

// HTTPSAllowedOrigin returns the relevant CORS setting.
func (c *Config) HTTPSAllowedOrigin() string {
	return c.m.GetString("core.https_allowed_origin")
}

// HTTPSAllowedCredentials returns the relevant CORS setting.
func (c *Config) HTTPSAllowedCredentials() bool {
	return c.m.GetBool("core.https_allowed_credentials")
}

// TrustPassword returns the LXD trust password for authenticating clients.
func (c *Config) TrustPassword() string {
	return c.m.GetString("core.trust_password")
}

// TrustCACertificates returns whether client certificates are checked
// against a CA.
func (c *Config) TrustCACertificates() bool {
	return c.m.GetBool("core.trust_ca_certificates")
}

// CandidServer returns all the Candid settings needed to connect to a server.
func (c *Config) CandidServer() (string, string, int64, string) {
	return c.m.GetString("candid.api.url"),
		c.m.GetString("candid.api.key"),
		c.m.GetInt64("candid.expiry"),
		c.m.GetString("candid.domains")
}

// RBACServer returns all the Candid settings needed to connect to a server.
func (c *Config) RBACServer() (string, string, int64, string, string, string, string) {
	return c.m.GetString("rbac.api.url"),
		c.m.GetString("rbac.api.key"),
		c.m.GetInt64("rbac.expiry"),
		c.m.GetString("rbac.agent.url"),
		c.m.GetString("rbac.agent.username"),
		c.m.GetString("rbac.agent.private_key"),
		c.m.GetString("rbac.agent.public_key")
}

// ProxyHTTPS returns the configured HTTPS proxy, if any.
func (c *Config) ProxyHTTPS() string {
	return c.m.GetString("core.proxy_https")
}

// ProxyHTTP returns the configured HTTP proxy, if any.
func (c *Config) ProxyHTTP() string {
	return c.m.GetString("core.proxy_http")
}

// ProxyIgnoreHosts returns the configured ignore-hosts proxy setting, if any.
func (c *Config) ProxyIgnoreHosts() string {
	return c.m.GetString("core.proxy_ignore_hosts")
}

// MAASController the configured MAAS url and key, if any.
func (c *Config) MAASController() (string, string) {
	url := c.m.GetString("maas.api.url")
	key := c.m.GetString("maas.api.key")
	return url, key
}

// OfflineThreshold returns the configured heartbeat threshold, i.e. the
// number of seconds before after which an unresponsive node is considered
// offline..
func (c *Config) OfflineThreshold() time.Duration {
	n := c.m.GetInt64("cluster.offline_threshold")
	return time.Duration(n) * time.Second
}

// ImagesMinimalReplica returns the numbers of nodes for cluster images replication
func (c *Config) ImagesMinimalReplica() int64 {
	return c.m.GetInt64("cluster.images_minimal_replica")
}

// MaxVoters returns the maximum number of members in a cluster that will be
// assigned the voter role.
func (c *Config) MaxVoters() int64 {
	return c.m.GetInt64("cluster.max_voters")
}

// MaxStandBy returns the maximum number of standby members in a cluster that
// will be assigned the stand-by role.
func (c *Config) MaxStandBy() int64 {
	return c.m.GetInt64("cluster.max_standby")
}

// Dump current configuration keys and their values. Keys with values matching
// their defaults are omitted.
func (c *Config) Dump() map[string]interface{} {
	return c.m.Dump()
}

// Replace the current configuration with the given values.
//
// Return what has actually changed.
func (c *Config) Replace(values map[string]interface{}) (map[string]string, error) {
	return c.update(values)
}

// Patch changes only the configuration keys in the given map.
//
// Return what has actually changed.
func (c *Config) Patch(patch map[string]interface{}) (map[string]string, error) {
	values := c.Dump() // Use current values as defaults
	for name, value := range patch {
		values[name] = value
	}
	return c.update(values)
}

func (c *Config) update(values map[string]interface{}) (map[string]string, error) {
	changed, err := c.m.Change(values)
	if err != nil {
		return nil, err
	}

	err = c.tx.UpdateConfig(changed)
	if err != nil {
		return nil, errors.Wrap(err, "cannot persist configuration changes: %v")
	}

	return changed, nil
}

// ConfigGetString is a convenience for loading the cluster configuration and
// returning the value of a particular key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetString(cluster *db.Cluster, key string) (string, error) {
	config, err := configGet(cluster)
	if err != nil {
		return "", err
	}
	return config.m.GetString(key), nil
}

// ConfigGetBool is a convenience for loading the cluster configuration and
// returning the value of a particular boolean key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetBool(cluster *db.Cluster, key string) (bool, error) {
	config, err := configGet(cluster)
	if err != nil {
		return false, err
	}
	return config.m.GetBool(key), nil
}

// ConfigGetInt64 is a convenience for loading the cluster configuration and
// returning the value of a particular key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetInt64(cluster *db.Cluster, key string) (int64, error) {
	config, err := configGet(cluster)
	if err != nil {
		return 0, err
	}
	return config.m.GetInt64(key), nil
}

func configGet(cluster *db.Cluster) (*Config, error) {
	var config *Config
	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		config, err = ConfigLoad(tx)
		return err
	})
	return config, err
}

// ConfigSchema defines available server configuration keys.
var ConfigSchema = config.Schema{
	"backups.compression_algorithm":  {Default: "gzip", Validator: validate.IsCompressionAlgorithm},
	"cluster.offline_threshold":      {Type: config.Int64, Default: offlineThresholdDefault(), Validator: offlineThresholdValidator},
	"cluster.images_minimal_replica": {Type: config.Int64, Default: "3", Validator: imageMinimalReplicaValidator},
	"cluster.max_voters":             {Type: config.Int64, Default: "3", Validator: maxVotersValidator},
	"cluster.max_standby":            {Type: config.Int64, Default: "2", Validator: maxStandByValidator},
	"core.https_allowed_headers":     {},
	"core.https_allowed_methods":     {},
	"core.https_allowed_origin":      {},
	"core.https_allowed_credentials": {Type: config.Bool},
	"core.proxy_http":                {},
	"core.proxy_https":               {},
	"core.proxy_ignore_hosts":        {},
	"core.trust_password":            {Hidden: true, Setter: passwordSetter},
	"core.trust_ca_certificates":     {Type: config.Bool},
	"candid.api.key":                 {},
	"candid.api.url":                 {},
	"candid.domains":                 {},
	"candid.expiry":                  {Type: config.Int64, Default: "3600"},
	"images.auto_update_cached":      {Type: config.Bool, Default: "true"},
	"images.auto_update_interval":    {Type: config.Int64, Default: "6"},
	"images.compression_algorithm":   {Default: "gzip", Validator: validate.IsCompressionAlgorithm},
	"images.default_architecture":    {Validator: validate.IsArchitecture},
	"images.remote_cache_expiry":     {Type: config.Int64, Default: "10"},
	"maas.api.key":                   {},
	"maas.api.url":                   {},
	"rbac.agent.url":                 {},
	"rbac.agent.username":            {},
	"rbac.agent.private_key":         {},
	"rbac.agent.public_key":          {},
	"rbac.api.expiry":                {Type: config.Int64, Default: "3600"},
	"rbac.api.key":                   {},
	"rbac.api.url":                   {},
	"rbac.expiry":                    {Type: config.Int64, Default: "3600"},

	// Keys deprecated since the implementation of the storage api.
	"storage.lvm_fstype":           {Setter: deprecatedStorage, Default: "ext4"},
	"storage.lvm_mount_options":    {Setter: deprecatedStorage, Default: "discard"},
	"storage.lvm_thinpool_name":    {Setter: deprecatedStorage, Default: "LXDThinPool"},
	"storage.lvm_vg_name":          {Setter: deprecatedStorage},
	"storage.lvm_volume_size":      {Setter: deprecatedStorage, Default: "10GiB"},
	"storage.zfs_pool_name":        {Setter: deprecatedStorage},
	"storage.zfs_remove_snapshots": {Setter: deprecatedStorage, Type: config.Bool},
	"storage.zfs_use_refquota":     {Setter: deprecatedStorage, Type: config.Bool},

	// OVN networking global keys.
	"network.ovn.integration_bridge":    {Default: "br-int"},
	"network.ovn.northbound_connection": {Default: "unix:/var/run/ovn/ovnnb_db.sock"},
}

func offlineThresholdDefault() string {
	return strconv.Itoa(db.DefaultOfflineThreshold)
}

func offlineThresholdValidator(value string) error {
	// Ensure that the given value is greater than the heartbeat interval,
	// which is the lower bound granularity of the offline check.
	threshold, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Offline threshold is not a number")
	}

	if threshold <= heartbeatInterval {
		return fmt.Errorf("Value must be greater than '%d'", heartbeatInterval)
	}

	return nil
}

func imageMinimalReplicaValidator(value string) error {
	count, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Minimal image replica count is not a number")
	}

	if count < 1 && count != -1 {
		return fmt.Errorf("Invalid value for image replica count")
	}

	return nil
}

func maxVotersValidator(value string) error {
	n, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Value is not a number")
	}

	if n < 3 || n%2 != 1 {
		return fmt.Errorf("Value must be an odd number equal to or higher than 3")
	}

	return nil
}

func maxStandByValidator(value string) error {
	n, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Value is not a number")
	}

	if n < 0 || n > 5 {
		return fmt.Errorf("Value must be between 0 and 5")
	}

	return nil
}

func passwordSetter(value string) (string, error) {
	// Nothing to do on unset
	if value == "" {
		return value, nil
	}

	// Hash the password
	buf := make([]byte, 32)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", err
	}

	hash, err := scrypt.Key([]byte(value), buf, 1<<14, 8, 1, 64)
	if err != nil {
		return "", err
	}

	buf = append(buf, hash...)
	value = hex.EncodeToString(buf)

	return value, nil
}

func deprecatedStorage(value string) (string, error) {
	if value == "" {
		return "", nil
	}
	return "", fmt.Errorf("deprecated: use storage pool configuration")
}
<|endoftext|>


<commit_before>package cluster

import (
	"context"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/events"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// eventHubMinHosts is the minimum number of members that must have the event-hub role to trigger switching into
// event-hub mode (where cluster members will only connect to event-hub members rather than all members when
// operating in the normal full-mesh mode).
const eventHubMinHosts = 2

// EventMode indicates the event distribution mode.
type EventMode string

// EventModeFullMesh is when every cluster member connects to every other cluster member to pull events.
const EventModeFullMesh EventMode = "full-mesh"

// EventModeHubServer is when the cluster is operating in event-hub mode and this server is designated as a hub
// server, meaning that it will only connect to the other event-hub members and not other members.
const EventModeHubServer EventMode = "hub-server"

// EventModeHubClient is when the cluster is operating in event-hub mode and this member is designated as a hub
// client, meaning that it is expected to connect to the event-hub members.
const EventModeHubClient EventMode = "hub-client"

// eventListenerClient stores both the event listener and its associated client.
type eventListenerClient struct {
	*lxd.EventListener

	client        lxd.InstanceServer
	hubPushCancel context.CancelFunc
}

// Disconnect disconnects both the listener and the client.
func (lc *eventListenerClient) Disconnect() {
	if lc.hubPushCancel != nil {
		lc.hubPushCancel()
	}

	lc.EventListener.Disconnect()
	lc.client.Disconnect()
}

// SetEventMode applies the specified eventMode of the local server to the listener.
// If the eventMode is EventModeHubClient then a go routine is started that consumes events from eventHubPushCh and
// pushes them to the remote server. If the eventMode is anything else then the go routine is stopped if running.
func (lc *eventListenerClient) SetEventMode(eventMode EventMode, eventHubPushCh chan api.Event) {
	if eventMode == EventModeHubClient {
		if lc.hubPushCancel != nil || !lc.IsActive() {
			return
		}

		ctx, cancel := context.WithCancel(context.Background())

		go func() {
			lc.hubPushCancel = cancel
			info, _ := lc.client.GetConnectionInfo()
			logger.Info("Event hub client started", log.Ctx{"remote": info.URL})
			defer logger.Info("Event hub client stopped", log.Ctx{"remote": info.URL})
			defer func() {
				cancel()
				lc.hubPushCancel = nil
			}()

			for {
				select {
				case event, more := <-eventHubPushCh:
					if !more {
						return
					}

					err := lc.client.SendEvent(event)
					if err != nil {
						//  Send failed, something is wrong with this hub server.
						lc.Disconnect() // Disconnect listener and client.

						// Try and put event back onto event hub push queue for consumption
						// by another consumer.
						ctx, cancel := context.WithTimeout(context.Background(), eventHubPushChTimeout)
						defer cancel()

						select {
						case eventHubPushCh <- event:
						case <-ctx.Done(): // Don't block if all consumers are slow/down.
						}

						return
					}
				case <-ctx.Done():
					return
				}
			}
		}()
	} else if lc.hubPushCancel != nil {
		lc.hubPushCancel()
		lc.hubPushCancel = nil
	}

	return
}

var eventMode EventMode = EventModeFullMesh
var eventHubAddresses []string
var eventHubPushCh = make(chan api.Event, 10) // Buffer size to accommodate slow consumers before dropping events.
var eventHubPushChTimeout = time.Duration(time.Second)
var listeners = map[string]*eventListenerClient{}
var listenersNotify = map[chan struct{}][]string{}
var listenersLock sync.Mutex
var listenersUpdateLock sync.Mutex

// ServerEventMode returns the event distribution mode that this local server is operating in.
func ServerEventMode() EventMode {
	listenersLock.Lock()
	defer listenersLock.Unlock()

	return eventMode
}

// RoleInSlice returns whether or not the rule is within the roles list.
func RoleInSlice(role db.ClusterRole, roles []db.ClusterRole) bool {
	for _, r := range roles {
		if r == role {
			return true
		}
	}

	return false
}

// EventListenerWait waits for there to be listener connected to the specified address, or one of the event hubs
// if operating in event hub mode.
func EventListenerWait(ctx context.Context, address string) error {
	// Check if there is already a listener.
	listenersLock.Lock()
	listener, found := listeners[address]
	if found && listener.IsActive() {
		listenersLock.Unlock()
		return nil
	}

	listenAddresses := []string{address}

	// Check if operating in event hub mode and if one of the event hub connections is available.
	// If so then we are ready to receive events from all members.
	if eventMode != EventModeFullMesh {
		for _, eventHubAddress := range eventHubAddresses {
			listener, found := listeners[eventHubAddress]
			if found && listener.IsActive() {
				listenersLock.Unlock()
				return nil
			}

			listenAddresses = append(listenAddresses, eventHubAddress)
		}
	}

	// If not setup a notification for when the desired address or any of the event hubs connect.
	connected := make(chan struct{})
	listenersNotify[connected] = listenAddresses
	listenersLock.Unlock()

	defer func() {
		listenersLock.Lock()
		delete(listenersNotify, connected)
		listenersLock.Unlock()
	}()

	// Wait for the connected channel to be closed (indicating a new listener has been connected), and return.
	select {
	case <-connected:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// hubAddresses returns the addresses of members with event-hub role, and the event mode of the server.
// The event mode will only be hub-server or hub-client if at least eventHubMinHosts have an event-hub role.
// Otherwise the mode will be full-mesh.
func hubAddresses(localAddress string, members map[int64]APIHeartbeatMember) ([]string, EventMode) {
	var hubAddresses []string
	var localHasHubRole bool

	// Do a first pass of members to count the members with event-hub role, and whether we are a hub server.
	for _, member := range members {
		if RoleInSlice(db.ClusterRoleEventHub, member.Roles) {
			hubAddresses = append(hubAddresses, member.Address)

			if member.Address == localAddress {
				localHasHubRole = true
			}
		}
	}

	eventMode := EventModeFullMesh
	if len(hubAddresses) >= eventHubMinHosts {
		if localHasHubRole {
			eventMode = EventModeHubServer
		} else {
			eventMode = EventModeHubClient
		}
	}

	return hubAddresses, eventMode
}

// EventsUpdateListeners refreshes the cluster event listener connections.
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, inject events.InjectFunc) {
	listenersUpdateLock.Lock()
	defer listenersUpdateLock.Unlock()

	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
				Roles:         dbMember.Roles,
			}
		}
	}

	localAddress := endpoints.NetworkAddress()
	hubAddresses, localEventMode := hubAddresses(localAddress, members)

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == localAddress || !member.Online {
			continue
		}

		if localEventMode != EventModeFullMesh && !RoleInSlice(db.ClusterRoleEventHub, member.Roles) {
			continue // Skip non-event-hub members if we are operating in event-hub mode.
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listener.IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listener.SetEventMode(localEventMode, eventHubPushCh)
				listenersLock.Unlock()
				continue
			}

			// Disconnect and delete listener, but don't delete any listenersNotify entry as there
			// might be something waiting for a future connection.
			listener.Disconnect()
			delete(listeners, member.Address)
			listenersLock.Unlock()

			// Log after releasing listenersLock to avoid deadlock on listenersLock with EventHubPush.
			logger.Info("Removed inactive member event listener client", log.Ctx{"local": localAddress, "remote": member.Address})
		} else {
			listenersLock.Unlock()
		}

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			logger := logging.AddContext(logger.Log, log.Ctx{"local": localAddress, "remote": m.Address})

			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener client", log.Ctx{"err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) {
				// Inject event received via pull as forwarded so that its not forwarded again
				// onto other members.
				inject(event, events.EventSourcePull)
			})

			listener.SetEventMode(localEventMode, eventHubPushCh)

			listenersLock.Lock()
			listeners[m.Address] = listener

			// Indicate to any notifiers waiting for this member's address that it is connected.
			for connected, notifyAddresses := range listenersNotify {
				if shared.StringInSlice(m.Address, notifyAddresses) {
					close(connected)
					delete(listenersNotify, connected)
				}
			}

			listenersLock.Unlock()

			// Log after releasing listenersLock to avoid deadlock on listenersLock with EventHubPush.
			logger.Info("Added member event listener client")
		}(member)
	}
	wg.Wait()

	// Disconnect and delete any out of date listeners and their notifiers.
	var removedAddresses []string

	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)

			// Record address removed, but don't log it here as this could cause a deadlock on
			// listenersLock with EventHubPush
			removedAddresses = append(removedAddresses, address)
		}
	}

	// Store event hub addresses in global slice late in the function after all event connections have been
	// opened above. This way the reported state by this server won't be updated until its ready.
	eventHubAddresses = hubAddresses
	eventMode = localEventMode

	listenersLock.Unlock()

	// Log the listeners removed after releasing listenersLock.
	for _, removedAddress := range removedAddresses {
		logger.Info("Removed old member event listener client", log.Ctx{"local": localAddress, "remote": removedAddress})
	}

	if len(members) > 1 && len(keepListeners) <= 0 {
		logger.Error("No active cluster event listener clients")
	}
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*eventListenerClient, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	revert := revert.New()
	revert.Add(func() {
		client.Disconnect()
	})

	listener, err := client.GetEventsAllProjects()
	if err != nil {
		return nil, err
	}

	revert.Success()

	lc := &eventListenerClient{
		EventListener: listener,
		client:        client,
	}

	return lc, nil
}

// EventHubPush pushes the event to the event hub members if local server is an event-hub client.
func EventHubPush(event api.Event) {
	listenersLock.Lock()
	// If the local server isn't an event-hub client, then we don't need to push messages as the other
	// members should be connected to us via a pull event listener and so will receive the event that way.
	// Also if there are no listeners available then there's no point in pushing to the eventHubPushCh as it
	// will have no consumers reading from it (this allows somewhat graceful handling of the situation where
	// all event-hub members are down by dropping events rather than slowing down the local system).
	if eventMode != EventModeHubClient || len(listeners) <= 0 {
		listenersLock.Unlock()
		return
	}
	listenersLock.Unlock()

	// Run in a go routine so as not to delay caller of this function as we try and deliver it.
	go func() {
		ctx, cancel := context.WithTimeout(context.Background(), eventHubPushChTimeout)
		defer cancel()

		select {
		case eventHubPushCh <- event:
		case <-ctx.Done(): // Don't block if all consumers are slow/down.
		}
	}()
}
<commit_msg>lxd/cluster/events: log No active cluster event listeners<commit_after>package cluster

import (
	"context"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/events"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// eventHubMinHosts is the minimum number of members that must have the event-hub role to trigger switching into
// event-hub mode (where cluster members will only connect to event-hub members rather than all members when
// operating in the normal full-mesh mode).
const eventHubMinHosts = 2

// EventMode indicates the event distribution mode.
type EventMode string

// EventModeFullMesh is when every cluster member connects to every other cluster member to pull events.
const EventModeFullMesh EventMode = "full-mesh"

// EventModeHubServer is when the cluster is operating in event-hub mode and this server is designated as a hub
// server, meaning that it will only connect to the other event-hub members and not other members.
const EventModeHubServer EventMode = "hub-server"

// EventModeHubClient is when the cluster is operating in event-hub mode and this member is designated as a hub
// client, meaning that it is expected to connect to the event-hub members.
const EventModeHubClient EventMode = "hub-client"

// eventListenerClient stores both the event listener and its associated client.
type eventListenerClient struct {
	*lxd.EventListener

	client        lxd.InstanceServer
	hubPushCancel context.CancelFunc
}

// Disconnect disconnects both the listener and the client.
func (lc *eventListenerClient) Disconnect() {
	if lc.hubPushCancel != nil {
		lc.hubPushCancel()
	}

	lc.EventListener.Disconnect()
	lc.client.Disconnect()
}

// SetEventMode applies the specified eventMode of the local server to the listener.
// If the eventMode is EventModeHubClient then a go routine is started that consumes events from eventHubPushCh and
// pushes them to the remote server. If the eventMode is anything else then the go routine is stopped if running.
func (lc *eventListenerClient) SetEventMode(eventMode EventMode, eventHubPushCh chan api.Event) {
	if eventMode == EventModeHubClient {
		if lc.hubPushCancel != nil || !lc.IsActive() {
			return
		}

		ctx, cancel := context.WithCancel(context.Background())

		go func() {
			lc.hubPushCancel = cancel
			info, _ := lc.client.GetConnectionInfo()
			logger.Info("Event hub client started", log.Ctx{"remote": info.URL})
			defer logger.Info("Event hub client stopped", log.Ctx{"remote": info.URL})
			defer func() {
				cancel()
				lc.hubPushCancel = nil
			}()

			for {
				select {
				case event, more := <-eventHubPushCh:
					if !more {
						return
					}

					err := lc.client.SendEvent(event)
					if err != nil {
						//  Send failed, something is wrong with this hub server.
						lc.Disconnect() // Disconnect listener and client.

						// Try and put event back onto event hub push queue for consumption
						// by another consumer.
						ctx, cancel := context.WithTimeout(context.Background(), eventHubPushChTimeout)
						defer cancel()

						select {
						case eventHubPushCh <- event:
						case <-ctx.Done(): // Don't block if all consumers are slow/down.
						}

						return
					}
				case <-ctx.Done():
					return
				}
			}
		}()
	} else if lc.hubPushCancel != nil {
		lc.hubPushCancel()
		lc.hubPushCancel = nil
	}

	return
}

var eventMode EventMode = EventModeFullMesh
var eventHubAddresses []string
var eventHubPushCh = make(chan api.Event, 10) // Buffer size to accommodate slow consumers before dropping events.
var eventHubPushChTimeout = time.Duration(time.Second)
var listeners = map[string]*eventListenerClient{}
var listenersNotify = map[chan struct{}][]string{}
var listenersLock sync.Mutex
var listenersUpdateLock sync.Mutex

// ServerEventMode returns the event distribution mode that this local server is operating in.
func ServerEventMode() EventMode {
	listenersLock.Lock()
	defer listenersLock.Unlock()

	return eventMode
}

// RoleInSlice returns whether or not the rule is within the roles list.
func RoleInSlice(role db.ClusterRole, roles []db.ClusterRole) bool {
	for _, r := range roles {
		if r == role {
			return true
		}
	}

	return false
}

// EventListenerWait waits for there to be listener connected to the specified address, or one of the event hubs
// if operating in event hub mode.
func EventListenerWait(ctx context.Context, address string) error {
	// Check if there is already a listener.
	listenersLock.Lock()
	listener, found := listeners[address]
	if found && listener.IsActive() {
		listenersLock.Unlock()
		return nil
	}

	listenAddresses := []string{address}

	// Check if operating in event hub mode and if one of the event hub connections is available.
	// If so then we are ready to receive events from all members.
	if eventMode != EventModeFullMesh {
		for _, eventHubAddress := range eventHubAddresses {
			listener, found := listeners[eventHubAddress]
			if found && listener.IsActive() {
				listenersLock.Unlock()
				return nil
			}

			listenAddresses = append(listenAddresses, eventHubAddress)
		}
	}

	// If not setup a notification for when the desired address or any of the event hubs connect.
	connected := make(chan struct{})
	listenersNotify[connected] = listenAddresses
	listenersLock.Unlock()

	defer func() {
		listenersLock.Lock()
		delete(listenersNotify, connected)
		listenersLock.Unlock()
	}()

	// Wait for the connected channel to be closed (indicating a new listener has been connected), and return.
	select {
	case <-connected:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// hubAddresses returns the addresses of members with event-hub role, and the event mode of the server.
// The event mode will only be hub-server or hub-client if at least eventHubMinHosts have an event-hub role.
// Otherwise the mode will be full-mesh.
func hubAddresses(localAddress string, members map[int64]APIHeartbeatMember) ([]string, EventMode) {
	var hubAddresses []string
	var localHasHubRole bool

	// Do a first pass of members to count the members with event-hub role, and whether we are a hub server.
	for _, member := range members {
		if RoleInSlice(db.ClusterRoleEventHub, member.Roles) {
			hubAddresses = append(hubAddresses, member.Address)

			if member.Address == localAddress {
				localHasHubRole = true
			}
		}
	}

	eventMode := EventModeFullMesh
	if len(hubAddresses) >= eventHubMinHosts {
		if localHasHubRole {
			eventMode = EventModeHubServer
		} else {
			eventMode = EventModeHubClient
		}
	}

	return hubAddresses, eventMode
}

// EventsUpdateListeners refreshes the cluster event listener connections.
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, inject events.InjectFunc) {
	listenersUpdateLock.Lock()
	defer listenersUpdateLock.Unlock()

	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
				Roles:         dbMember.Roles,
			}
		}
	}

	localAddress := endpoints.NetworkAddress()
	hubAddresses, localEventMode := hubAddresses(localAddress, members)

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == localAddress || !member.Online {
			continue
		}

		if localEventMode != EventModeFullMesh && !RoleInSlice(db.ClusterRoleEventHub, member.Roles) {
			continue // Skip non-event-hub members if we are operating in event-hub mode.
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listener.IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listener.SetEventMode(localEventMode, eventHubPushCh)
				listenersLock.Unlock()
				continue
			}

			// Disconnect and delete listener, but don't delete any listenersNotify entry as there
			// might be something waiting for a future connection.
			listener.Disconnect()
			delete(listeners, member.Address)
			listenersLock.Unlock()

			// Log after releasing listenersLock to avoid deadlock on listenersLock with EventHubPush.
			logger.Info("Removed inactive member event listener client", log.Ctx{"local": localAddress, "remote": member.Address})
		} else {
			listenersLock.Unlock()
		}

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			logger := logging.AddContext(logger.Log, log.Ctx{"local": localAddress, "remote": m.Address})

			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener client", log.Ctx{"err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) {
				// Inject event received via pull as forwarded so that its not forwarded again
				// onto other members.
				inject(event, events.EventSourcePull)
			})

			listener.SetEventMode(localEventMode, eventHubPushCh)

			listenersLock.Lock()
			listeners[m.Address] = listener

			// Indicate to any notifiers waiting for this member's address that it is connected.
			for connected, notifyAddresses := range listenersNotify {
				if shared.StringInSlice(m.Address, notifyAddresses) {
					close(connected)
					delete(listenersNotify, connected)
				}
			}

			listenersLock.Unlock()

			// Log after releasing listenersLock to avoid deadlock on listenersLock with EventHubPush.
			logger.Info("Added member event listener client")
		}(member)
	}
	wg.Wait()

	// Disconnect and delete any out of date listeners and their notifiers.
	var removedAddresses []string

	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)

			// Record address removed, but don't log it here as this could cause a deadlock on
			// listenersLock with EventHubPush
			removedAddresses = append(removedAddresses, address)
		}
	}

	// Store event hub addresses in global slice late in the function after all event connections have been
	// opened above. This way the reported state by this server won't be updated until its ready.
	eventHubAddresses = hubAddresses
	eventMode = localEventMode

	listenersLock.Unlock()

	// Log the listeners removed after releasing listenersLock.
	for _, removedAddress := range removedAddresses {
		logger.Info("Removed old member event listener client", log.Ctx{"local": localAddress, "remote": removedAddress})
	}

	if len(members) > 1 && len(keepListeners) <= 0 {
		logger.Error("No active cluster event listener clients", log.Ctx{"local": localAddress})
	}
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*eventListenerClient, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	revert := revert.New()
	revert.Add(func() {
		client.Disconnect()
	})

	listener, err := client.GetEventsAllProjects()
	if err != nil {
		return nil, err
	}

	revert.Success()

	lc := &eventListenerClient{
		EventListener: listener,
		client:        client,
	}

	return lc, nil
}

// EventHubPush pushes the event to the event hub members if local server is an event-hub client.
func EventHubPush(event api.Event) {
	listenersLock.Lock()
	// If the local server isn't an event-hub client, then we don't need to push messages as the other
	// members should be connected to us via a pull event listener and so will receive the event that way.
	// Also if there are no listeners available then there's no point in pushing to the eventHubPushCh as it
	// will have no consumers reading from it (this allows somewhat graceful handling of the situation where
	// all event-hub members are down by dropping events rather than slowing down the local system).
	if eventMode != EventModeHubClient || len(listeners) <= 0 {
		listenersLock.Unlock()
		return
	}
	listenersLock.Unlock()

	// Run in a go routine so as not to delay caller of this function as we try and deliver it.
	go func() {
		ctx, cancel := context.WithTimeout(context.Background(), eventHubPushChTimeout)
		defer cancel()

		select {
		case eventHubPushCh <- event:
		case <-ctx.Done(): // Don't block if all consumers are slow/down.
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

func readUint(path string) (uint64, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, err
	}

	value, err := strconv.ParseUint(strings.TrimSpace(string(content)), 10, 64)
	if err != nil {
		return 0, err
	}

	return value, nil
}

func networkAutoAttach(cluster *db.Cluster, devName string) error {
	_, dbInfo, err := cluster.GetNetworkWithInterface(devName)
	if err != nil {
		// No match found, move on
		return nil
	}

	return network.AttachInterface(dbInfo.Name, devName)
}

func networkGetInterfaces(cluster *db.Cluster) ([]string, error) {
	networks, err := cluster.GetNetworks()
	if err != nil {
		return nil, err
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		// Ignore veth pairs (for performance reasons)
		if strings.HasPrefix(iface.Name, "veth") {
			continue
		}

		// Append to the list
		if !shared.StringInSlice(iface.Name, networks) {
			networks = append(networks, iface.Name)
		}
	}

	return networks, nil
}

// networkUpdateForkdnsServersTask runs every 30s and refreshes the forkdns servers list.
func networkUpdateForkdnsServersTask(s *state.State, heartbeatData *cluster.APIHeartbeat) error {
	// Get a list of managed networks
	networks, err := s.Cluster.GetNonPendingNetworks()
	if err != nil {
		return err
	}

	for _, name := range networks {
		n, err := network.LoadByName(s, name)
		if err != nil {
			return err
		}

		if n.Type() == "bridge" && n.Config()["bridge.mode"] == "fan" {
			err := n.HandleHeartbeat(heartbeatData)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func networkGetState(netIf net.Interface) api.NetworkState {
	netState := "down"
	netType := "unknown"

	if netIf.Flags&net.FlagBroadcast > 0 {
		netType = "broadcast"
	}

	if netIf.Flags&net.FlagPointToPoint > 0 {
		netType = "point-to-point"
	}

	if netIf.Flags&net.FlagLoopback > 0 {
		netType = "loopback"
	}

	if netIf.Flags&net.FlagUp > 0 {
		netState = "up"
	}

	network := api.NetworkState{
		Addresses: []api.NetworkStateAddress{},
		Counters:  api.NetworkStateCounters{},
		Hwaddr:    netIf.HardwareAddr.String(),
		Mtu:       netIf.MTU,
		State:     netState,
		Type:      netType,
	}

	// Populate address information.
	addrs, err := netIf.Addrs()
	if err == nil {
		for _, addr := range addrs {
			fields := strings.SplitN(addr.String(), "/", 2)
			if len(fields) != 2 {
				continue
			}

			family := "inet"
			if strings.Contains(fields[0], ":") {
				family = "inet6"
			}

			scope := "global"
			if strings.HasPrefix(fields[0], "127") {
				scope = "local"
			}

			if fields[0] == "::1" {
				scope = "local"
			}

			if strings.HasPrefix(fields[0], "169.254") {
				scope = "link"
			}

			if strings.HasPrefix(fields[0], "fe80:") {
				scope = "link"
			}

			address := api.NetworkStateAddress{}
			address.Family = family
			address.Address = fields[0]
			address.Netmask = fields[1]
			address.Scope = scope

			network.Addresses = append(network.Addresses, address)
		}
	}

	// Populate bond details.
	bondPath := fmt.Sprintf("/sys/class/net/%s/bonding", netIf.Name)
	if shared.PathExists(bondPath) {
		bonding := api.NetworkStateBond{}

		// Bond mode.
		strValue, err := ioutil.ReadFile(filepath.Join(bondPath, "mode"))
		if err == nil {
			bonding.Mode = strings.Split(strings.TrimSpace(string(strValue)), " ")[0]
		}

		// Bond transmit policy.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "xmit_hash_policy"))
		if err == nil {
			bonding.TransmitPolicy = strings.Split(strings.TrimSpace(string(strValue)), " ")[0]
		}

		// Up delay.
		uintValue, err := readUint(filepath.Join(bondPath, "updelay"))
		if err == nil {
			bonding.UpDelay = uintValue
		}

		// Down delay.
		uintValue, err = readUint(filepath.Join(bondPath, "downdelay"))
		if err == nil {
			bonding.DownDelay = uintValue
		}

		// MII frequency.
		uintValue, err = readUint(filepath.Join(bondPath, "miimon"))
		if err == nil {
			bonding.MIIFrequency = uintValue
		}

		// MII state.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "mii_status"))
		if err == nil {
			bonding.MIIState = strings.TrimSpace(string(strValue))
		}

		// Lower devices.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "slaves"))
		if err == nil {
			bonding.LowerDevices = strings.Split(strings.TrimSpace(string(strValue)), " ")
		}

		network.Bond = &bonding
	}

	// Populate bridge details.
	bridgePath := fmt.Sprintf("/sys/class/net/%s/bridge", netIf.Name)
	if shared.PathExists(bridgePath) {
		bridge := api.NetworkStateBridge{}

		// Bridge ID.
		strValue, err := ioutil.ReadFile(filepath.Join(bridgePath, "bridge_id"))
		if err == nil {
			bridge.ID = strings.TrimSpace(string(strValue))
		}

		// Bridge STP.
		uintValue, err := readUint(filepath.Join(bridgePath, "stp_state"))
		if err == nil {
			bridge.STP = uintValue == 1
		}

		// Bridge forward delay.
		uintValue, err = readUint(filepath.Join(bridgePath, "forward_delay"))
		if err == nil {
			bridge.ForwardDelay = uintValue
		}

		// Bridge default VLAN.
		uintValue, err = readUint(filepath.Join(bridgePath, "default_pvid"))
		if err == nil {
			bridge.VLANDefault = uintValue
		}

		// Bridge VLAN filtering.
		uintValue, err = readUint(filepath.Join(bridgePath, "vlan_filtering"))
		if err == nil {
			bridge.VLANFiltering = uintValue == 1
		}

		// Upper devices.
		bridgeIfPath := fmt.Sprintf("/sys/class/net/%s/brif", netIf.Name)
		if shared.PathExists(bridgeIfPath) {
			entries, err := ioutil.ReadDir(bridgeIfPath)
			if err == nil {
				bridge.UpperDevices = []string{}
				for _, entry := range entries {
					bridge.UpperDevices = append(bridge.UpperDevices, entry.Name())
				}
			}
		}

		network.Bridge = &bridge
	}

	// Get counters.
	network.Counters = shared.NetworkGetCounters(netIf.Name)
	return network
}
<commit_msg>lxd/networks/utils: Skip network load error in networkUpdateForkdnsServersTask<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

func readUint(path string) (uint64, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, err
	}

	value, err := strconv.ParseUint(strings.TrimSpace(string(content)), 10, 64)
	if err != nil {
		return 0, err
	}

	return value, nil
}

func networkAutoAttach(cluster *db.Cluster, devName string) error {
	_, dbInfo, err := cluster.GetNetworkWithInterface(devName)
	if err != nil {
		// No match found, move on
		return nil
	}

	return network.AttachInterface(dbInfo.Name, devName)
}

func networkGetInterfaces(cluster *db.Cluster) ([]string, error) {
	networks, err := cluster.GetNetworks()
	if err != nil {
		return nil, err
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		// Ignore veth pairs (for performance reasons)
		if strings.HasPrefix(iface.Name, "veth") {
			continue
		}

		// Append to the list
		if !shared.StringInSlice(iface.Name, networks) {
			networks = append(networks, iface.Name)
		}
	}

	return networks, nil
}

// networkUpdateForkdnsServersTask runs every 30s and refreshes the forkdns servers list.
func networkUpdateForkdnsServersTask(s *state.State, heartbeatData *cluster.APIHeartbeat) error {
	// Get a list of managed networks
	networks, err := s.Cluster.GetNonPendingNetworks()
	if err != nil {
		return err
	}

	for _, name := range networks {
		n, err := network.LoadByName(s, name)
		if err != nil {
			logger.Errorf("Failed to load network %q for heartbeat", name)
			continue
		}

		if n.Type() == "bridge" && n.Config()["bridge.mode"] == "fan" {
			err := n.HandleHeartbeat(heartbeatData)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func networkGetState(netIf net.Interface) api.NetworkState {
	netState := "down"
	netType := "unknown"

	if netIf.Flags&net.FlagBroadcast > 0 {
		netType = "broadcast"
	}

	if netIf.Flags&net.FlagPointToPoint > 0 {
		netType = "point-to-point"
	}

	if netIf.Flags&net.FlagLoopback > 0 {
		netType = "loopback"
	}

	if netIf.Flags&net.FlagUp > 0 {
		netState = "up"
	}

	network := api.NetworkState{
		Addresses: []api.NetworkStateAddress{},
		Counters:  api.NetworkStateCounters{},
		Hwaddr:    netIf.HardwareAddr.String(),
		Mtu:       netIf.MTU,
		State:     netState,
		Type:      netType,
	}

	// Populate address information.
	addrs, err := netIf.Addrs()
	if err == nil {
		for _, addr := range addrs {
			fields := strings.SplitN(addr.String(), "/", 2)
			if len(fields) != 2 {
				continue
			}

			family := "inet"
			if strings.Contains(fields[0], ":") {
				family = "inet6"
			}

			scope := "global"
			if strings.HasPrefix(fields[0], "127") {
				scope = "local"
			}

			if fields[0] == "::1" {
				scope = "local"
			}

			if strings.HasPrefix(fields[0], "169.254") {
				scope = "link"
			}

			if strings.HasPrefix(fields[0], "fe80:") {
				scope = "link"
			}

			address := api.NetworkStateAddress{}
			address.Family = family
			address.Address = fields[0]
			address.Netmask = fields[1]
			address.Scope = scope

			network.Addresses = append(network.Addresses, address)
		}
	}

	// Populate bond details.
	bondPath := fmt.Sprintf("/sys/class/net/%s/bonding", netIf.Name)
	if shared.PathExists(bondPath) {
		bonding := api.NetworkStateBond{}

		// Bond mode.
		strValue, err := ioutil.ReadFile(filepath.Join(bondPath, "mode"))
		if err == nil {
			bonding.Mode = strings.Split(strings.TrimSpace(string(strValue)), " ")[0]
		}

		// Bond transmit policy.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "xmit_hash_policy"))
		if err == nil {
			bonding.TransmitPolicy = strings.Split(strings.TrimSpace(string(strValue)), " ")[0]
		}

		// Up delay.
		uintValue, err := readUint(filepath.Join(bondPath, "updelay"))
		if err == nil {
			bonding.UpDelay = uintValue
		}

		// Down delay.
		uintValue, err = readUint(filepath.Join(bondPath, "downdelay"))
		if err == nil {
			bonding.DownDelay = uintValue
		}

		// MII frequency.
		uintValue, err = readUint(filepath.Join(bondPath, "miimon"))
		if err == nil {
			bonding.MIIFrequency = uintValue
		}

		// MII state.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "mii_status"))
		if err == nil {
			bonding.MIIState = strings.TrimSpace(string(strValue))
		}

		// Lower devices.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "slaves"))
		if err == nil {
			bonding.LowerDevices = strings.Split(strings.TrimSpace(string(strValue)), " ")
		}

		network.Bond = &bonding
	}

	// Populate bridge details.
	bridgePath := fmt.Sprintf("/sys/class/net/%s/bridge", netIf.Name)
	if shared.PathExists(bridgePath) {
		bridge := api.NetworkStateBridge{}

		// Bridge ID.
		strValue, err := ioutil.ReadFile(filepath.Join(bridgePath, "bridge_id"))
		if err == nil {
			bridge.ID = strings.TrimSpace(string(strValue))
		}

		// Bridge STP.
		uintValue, err := readUint(filepath.Join(bridgePath, "stp_state"))
		if err == nil {
			bridge.STP = uintValue == 1
		}

		// Bridge forward delay.
		uintValue, err = readUint(filepath.Join(bridgePath, "forward_delay"))
		if err == nil {
			bridge.ForwardDelay = uintValue
		}

		// Bridge default VLAN.
		uintValue, err = readUint(filepath.Join(bridgePath, "default_pvid"))
		if err == nil {
			bridge.VLANDefault = uintValue
		}

		// Bridge VLAN filtering.
		uintValue, err = readUint(filepath.Join(bridgePath, "vlan_filtering"))
		if err == nil {
			bridge.VLANFiltering = uintValue == 1
		}

		// Upper devices.
		bridgeIfPath := fmt.Sprintf("/sys/class/net/%s/brif", netIf.Name)
		if shared.PathExists(bridgeIfPath) {
			entries, err := ioutil.ReadDir(bridgeIfPath)
			if err == nil {
				bridge.UpperDevices = []string{}
				for _, entry := range entries {
					bridge.UpperDevices = append(bridge.UpperDevices, entry.Name())
				}
			}
		}

		network.Bridge = &bridge
	}

	// Get counters.
	network.Counters = shared.NetworkGetCounters(netIf.Name)
	return network
}
<|endoftext|>


<commit_before>package stan_output

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/google/uuid"
	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/outputs"
	"github.com/mitchellh/mapstructure"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/stan.go"
	"github.com/openconfig/gnmi/proto/gnmi"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/reflect/protoreflect"
)

const (
	stanDefaultTimeout      = 10
	stanDefaultPingInterval = 5
	stanDefaultPingRetry    = 2

	defaultSubjectName = "gnmic-telemetry"
)

func init() {
	outputs.Register("stan", func() outputs.Output {
		return &StanOutput{
			Cfg: &Config{},
		}
	})
}

// StanOutput //
type StanOutput struct {
	Cfg      *Config
	conn     stan.Conn
	metrics  []prometheus.Collector
	logger   *log.Logger
	stopChan chan struct{}
}

// Config //
type Config struct {
	Name          string `mapstructure:"name,omitempty"`
	Address       string `mapstructure:"address,omitempty"`
	SubjectPrefix string `mapstructure:"subject-prefix,omitempty"`
	Subject       string `mapstructure:"subject,omitempty"`
	Username      string `mapstructure:"username,omitempty"`
	Password      string `mapstructure:"password,omitempty"`
	ClusterName   string `mapstructure:"cluster-name,omitempty"`
	Timeout       int    `mapstructure:"timeout,omitempty"`
	PingInterval  int    `mapstructure:"ping-interval,omitempty"`
	PingRetry     int    `mapstructure:"ping-retry,omitempty"`
	Format        string `mapstructure:"format,omitempty"`
}

func (s *StanOutput) String() string {
	b, err := json.Marshal(s)
	if err != nil {
		return ""
	}
	return string(b)
}

// Init //
func (s *StanOutput) Init(cfg map[string]interface{}, logger *log.Logger) error {
	err := mapstructure.Decode(cfg, s.Cfg)
	if err != nil {
		return err
	}
	if s.Cfg.Name == "" {
		s.Cfg.Name = "gnmic-" + uuid.New().String()
	}
	if s.Cfg.ClusterName == "" {
		return fmt.Errorf("clusterName is mandatory")
	}
	if s.Cfg.Subject == "" && s.Cfg.SubjectPrefix == "" {
		s.Cfg.Subject = defaultSubjectName
	}
	s.conn, err = s.createSTANConn(s.Cfg)
	if err != nil {
		return err
	}
	s.logger = log.New(os.Stderr, "stan_output ", log.LstdFlags|log.Lmicroseconds)
	if logger != nil {
		s.logger.SetOutput(logger.Writer())
		s.logger.SetFlags(logger.Flags())
	}
	if s.Cfg.Format == "" {
		s.Cfg.Format = "event"
	}
	if !(s.Cfg.Format == "event" || s.Cfg.Format == "json" || s.Cfg.Format == "proto") {
		return fmt.Errorf("unsupported output format: %s", s.Cfg.Format)
	}
	s.stopChan = make(chan struct{})
	s.logger.Printf("initialized stan producer: %s", s.String())
	return nil
}

// Write //
func (s *StanOutput) Write(rsp protoreflect.ProtoMessage, meta outputs.Meta) {
	if rsp == nil {
		return
	}
	if format, ok := meta["format"]; ok {
		if format == "textproto" {
			return
		}
	}
	ssb := strings.Builder{}
	ssb.WriteString(s.Cfg.SubjectPrefix)
	if s.Cfg.SubjectPrefix != "" {
		if s, ok := meta["source"]; ok {
			source := strings.ReplaceAll(fmt.Sprintf("%s", s), ".", "-")
			source = strings.ReplaceAll(source, " ", "_")
			ssb.WriteString(".")
			ssb.WriteString(source)
		}
		if subname, ok := meta["subscription-name"]; ok {
			ssb.WriteString(".")
			ssb.WriteString(fmt.Sprintf("%s", subname))
		}
	} else if s.Cfg.Subject != "" {
		ssb.WriteString(s.Cfg.Subject)
	}
	subject := strings.ReplaceAll(ssb.String(), " ", "_")
	b := make([]byte, 0)
	var err error
	switch s.Cfg.Format {
	case "proto":
		b, err = proto.Marshal(rsp)
	case "json":
		b, err = protojson.MarshalOptions{Multiline: true, Indent: "  "}.Marshal(rsp)
	case "event":
		switch sub := rsp.ProtoReflect().Interface().(type) {
		case *gnmi.SubscribeResponse:
			var subscriptionName string
			var ok bool
			if subscriptionName, ok = meta["subscription-name"]; !ok {
				subscriptionName = "default"
			}
			switch sub.Response.(type) {
			case *gnmi.SubscribeResponse_Update:
				events, err := collector.ResponseToEventMsgs(subscriptionName, sub, meta)
				if err != nil {
					s.logger.Printf("failed converting response to events: %v", err)
					return
				}
				b, err = json.MarshalIndent(events, "", "  ")
				if err != nil {
					s.logger.Printf("failed marshaling events: %v", err)
					return
				}
			case *gnmi.SubscribeResponse_SyncResponse:
				s.logger.Printf("received subscribe syncResponse with %v", meta)
			case *gnmi.SubscribeResponse_Error:
				gnmiErr := sub.GetError()
				s.logger.Printf("received subscribe response error with %v, code=%d, message=%v, data=%v ",
					meta, gnmiErr.Code, gnmiErr.Message, gnmiErr.Data)
			}
		}
	}
	if err != nil {
		s.logger.Printf("failed marshaling event: %v", err)
		return
	}
	err = s.conn.Publish(subject, b)
	if err != nil {
		log.Printf("failed to write to stan subject '%s': %v", subject, err)
		return
	}
	//s.logger.Printf("wrote %d bytes to stan_subject=%s", len(b), s.Cfg.Subject)
}

// Metrics //
func (s *StanOutput) Metrics() []prometheus.Collector { return s.metrics }

// Close //
func (s *StanOutput) Close() error {
	close(s.stopChan)
	return s.conn.Close()
}

func (s *StanOutput) createSTANConn(c *Config) (stan.Conn, error) {
	if c.Timeout == 0 {
		c.Timeout = stanDefaultTimeout
	}
	if c.PingInterval == 0 {
		c.PingInterval = stanDefaultPingInterval
	}
	if c.PingRetry == 0 {
		c.PingRetry = stanDefaultPingRetry
	}
	opts := []nats.Option{
		nats.Name(c.Name),
		// nats.Timeout(time.Duration(c.Timeout) * time.Second),
		// nats.PingInterval(time.Duration(c.PingInterval) * time.Second),
		// nats.MaxPingsOutstanding(c.PingRetry),
	}
	if c.Username != "" && c.Password != "" {
		opts = append(opts, nats.UserInfo(c.Username, c.Password))
	}
	nc, err := nats.Connect(c.Address, opts...)
	if err != nil {
		return nil, err
	}
	sc, err := stan.Connect(c.ClusterName, c.Name,
		stan.NatsConn(nc),
		stan.Pings(c.PingInterval, c.PingRetry),
		stan.SetConnectionLostHandler(func(_ stan.Conn, err error) {
			log.Fatalf("STAN connection lost, reason: %v", err)
		}),
	)
	if err != nil {
		return nil, fmt.Errorf("cannot connect to %s: %v", c.Address, err)
	}
	return sc, nil
}
<commit_msg>remove indentation from stan output<commit_after>package stan_output

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/google/uuid"
	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/outputs"
	"github.com/mitchellh/mapstructure"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/stan.go"
	"github.com/openconfig/gnmi/proto/gnmi"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/reflect/protoreflect"
)

const (
	stanDefaultTimeout      = 10
	stanDefaultPingInterval = 5
	stanDefaultPingRetry    = 2

	defaultSubjectName = "gnmic-telemetry"
)

func init() {
	outputs.Register("stan", func() outputs.Output {
		return &StanOutput{
			Cfg: &Config{},
		}
	})
}

// StanOutput //
type StanOutput struct {
	Cfg      *Config
	conn     stan.Conn
	metrics  []prometheus.Collector
	logger   *log.Logger
	stopChan chan struct{}
}

// Config //
type Config struct {
	Name          string `mapstructure:"name,omitempty"`
	Address       string `mapstructure:"address,omitempty"`
	SubjectPrefix string `mapstructure:"subject-prefix,omitempty"`
	Subject       string `mapstructure:"subject,omitempty"`
	Username      string `mapstructure:"username,omitempty"`
	Password      string `mapstructure:"password,omitempty"`
	ClusterName   string `mapstructure:"cluster-name,omitempty"`
	Timeout       int    `mapstructure:"timeout,omitempty"`
	PingInterval  int    `mapstructure:"ping-interval,omitempty"`
	PingRetry     int    `mapstructure:"ping-retry,omitempty"`
	Format        string `mapstructure:"format,omitempty"`
}

func (s *StanOutput) String() string {
	b, err := json.Marshal(s)
	if err != nil {
		return ""
	}
	return string(b)
}

// Init //
func (s *StanOutput) Init(cfg map[string]interface{}, logger *log.Logger) error {
	err := mapstructure.Decode(cfg, s.Cfg)
	if err != nil {
		return err
	}
	if s.Cfg.Name == "" {
		s.Cfg.Name = "gnmic-" + uuid.New().String()
	}
	if s.Cfg.ClusterName == "" {
		return fmt.Errorf("clusterName is mandatory")
	}
	if s.Cfg.Subject == "" && s.Cfg.SubjectPrefix == "" {
		s.Cfg.Subject = defaultSubjectName
	}
	s.conn, err = s.createSTANConn(s.Cfg)
	if err != nil {
		return err
	}
	s.logger = log.New(os.Stderr, "stan_output ", log.LstdFlags|log.Lmicroseconds)
	if logger != nil {
		s.logger.SetOutput(logger.Writer())
		s.logger.SetFlags(logger.Flags())
	}
	if s.Cfg.Format == "" {
		s.Cfg.Format = "event"
	}
	if !(s.Cfg.Format == "event" || s.Cfg.Format == "json" || s.Cfg.Format == "proto") {
		return fmt.Errorf("unsupported output format: %s", s.Cfg.Format)
	}
	s.stopChan = make(chan struct{})
	s.logger.Printf("initialized stan producer: %s", s.String())
	return nil
}

// Write //
func (s *StanOutput) Write(rsp protoreflect.ProtoMessage, meta outputs.Meta) {
	if rsp == nil {
		return
	}
	if format, ok := meta["format"]; ok {
		if format == "textproto" {
			return
		}
	}
	ssb := strings.Builder{}
	ssb.WriteString(s.Cfg.SubjectPrefix)
	if s.Cfg.SubjectPrefix != "" {
		if s, ok := meta["source"]; ok {
			source := strings.ReplaceAll(fmt.Sprintf("%s", s), ".", "-")
			source = strings.ReplaceAll(source, " ", "_")
			ssb.WriteString(".")
			ssb.WriteString(source)
		}
		if subname, ok := meta["subscription-name"]; ok {
			ssb.WriteString(".")
			ssb.WriteString(fmt.Sprintf("%s", subname))
		}
	} else if s.Cfg.Subject != "" {
		ssb.WriteString(s.Cfg.Subject)
	}
	subject := strings.ReplaceAll(ssb.String(), " ", "_")
	b := make([]byte, 0)
	var err error
	switch s.Cfg.Format {
	case "proto":
		b, err = proto.Marshal(rsp)
	case "json":
		b, err = protojson.Marshal(rsp)
	case "event":
		switch sub := rsp.ProtoReflect().Interface().(type) {
		case *gnmi.SubscribeResponse:
			var subscriptionName string
			var ok bool
			if subscriptionName, ok = meta["subscription-name"]; !ok {
				subscriptionName = "default"
			}
			switch sub.Response.(type) {
			case *gnmi.SubscribeResponse_Update:
				events, err := collector.ResponseToEventMsgs(subscriptionName, sub, meta)
				if err != nil {
					s.logger.Printf("failed converting response to events: %v", err)
					return
				}
				b, err = json.Marshal(events)
				if err != nil {
					s.logger.Printf("failed marshaling events: %v", err)
					return
				}
			case *gnmi.SubscribeResponse_SyncResponse:
				s.logger.Printf("received subscribe syncResponse with %v", meta)
			case *gnmi.SubscribeResponse_Error:
				gnmiErr := sub.GetError()
				s.logger.Printf("received subscribe response error with %v, code=%d, message=%v, data=%v ",
					meta, gnmiErr.Code, gnmiErr.Message, gnmiErr.Data)
			}
		}
	}
	if err != nil {
		s.logger.Printf("failed marshaling event: %v", err)
		return
	}
	err = s.conn.Publish(subject, b)
	if err != nil {
		log.Printf("failed to write to stan subject '%s': %v", subject, err)
		return
	}
	//s.logger.Printf("wrote %d bytes to stan_subject=%s", len(b), s.Cfg.Subject)
}

// Metrics //
func (s *StanOutput) Metrics() []prometheus.Collector { return s.metrics }

// Close //
func (s *StanOutput) Close() error {
	close(s.stopChan)
	return s.conn.Close()
}

func (s *StanOutput) createSTANConn(c *Config) (stan.Conn, error) {
	if c.Timeout == 0 {
		c.Timeout = stanDefaultTimeout
	}
	if c.PingInterval == 0 {
		c.PingInterval = stanDefaultPingInterval
	}
	if c.PingRetry == 0 {
		c.PingRetry = stanDefaultPingRetry
	}
	opts := []nats.Option{
		nats.Name(c.Name),
		// nats.Timeout(time.Duration(c.Timeout) * time.Second),
		// nats.PingInterval(time.Duration(c.PingInterval) * time.Second),
		// nats.MaxPingsOutstanding(c.PingRetry),
	}
	if c.Username != "" && c.Password != "" {
		opts = append(opts, nats.UserInfo(c.Username, c.Password))
	}
	nc, err := nats.Connect(c.Address, opts...)
	if err != nil {
		return nil, err
	}
	sc, err := stan.Connect(c.ClusterName, c.Name,
		stan.NatsConn(nc),
		stan.Pings(c.PingInterval, c.PingRetry),
		stan.SetConnectionLostHandler(func(_ stan.Conn, err error) {
			log.Fatalf("STAN connection lost, reason: %v", err)
		}),
	)
	if err != nil {
		return nil, fmt.Errorf("cannot connect to %s: %v", c.Address, err)
	}
	return sc, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"go/build"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

func runTest(t *testing.T, path string) {
	exitCode = 0

	*recursive = false
	if suffix := ".go"; strings.HasSuffix(path, suffix) {
		// single file
		path = filepath.Join(runtime.GOROOT(), "src/pkg", path)
		path, file := filepath.Split(path)
		*pkgName = file[:len(file)-len(suffix)]
		processFiles([]string{path}, true)
	} else {
		// package directory
		// TODO(gri) gotype should use the build package instead
		pkg, err := build.Import(path, "", 0)
		if err != nil {
			t.Errorf("build.Import error for path = %s: %s", path, err)
			return
		}
		// TODO(gri) there ought to be a more direct way using the build package...
		files := make([]string, len(pkg.GoFiles))
		for i, file := range pkg.GoFiles {
			files[i] = filepath.Join(pkg.Dir, file)
		}
		*pkgName = pkg.Name
		processFiles(files, true)
	}

	if exitCode != 0 {
		t.Errorf("processing %s failed: exitCode = %d", path, exitCode)
	}
}

var tests = []string{
	// individual files
	"exp/gotype/testdata/test1.go",

	// directories
	// Note: packages that don't typecheck yet are commented out
	"archive/tar",
	"archive/zip",

	"bufio",
	"bytes",

	"compress/bzip2",
	"compress/flate",
	"compress/gzip",
	"compress/lzw",
	"compress/zlib",

	"container/heap",
	"container/list",
	"container/ring",

	"crypto",
	"crypto/aes",
	"crypto/cipher",
	"crypto/des",
	"crypto/dsa",
	"crypto/ecdsa",
	"crypto/elliptic",
	"crypto/hmac",
	"crypto/md5",
	"crypto/rand",
	"crypto/rc4",
	// "crypto/rsa", // src/pkg/crypto/rsa/pkcs1v15.go:21:27: undeclared name: io
	"crypto/sha1",
	"crypto/sha256",
	"crypto/sha512",
	"crypto/subtle",
	"crypto/tls",
	// "crypto/x509", // src/pkg/crypto/x509/root.go:15:10: undeclared name: initSystemRoots
	"crypto/x509/pkix",

	"database/sql",
	"database/sql/driver",

	"debug/dwarf",
	"debug/elf",
	"debug/gosym",
	"debug/macho",
	"debug/pe",

	"encoding/ascii85",
	"encoding/asn1",
	"encoding/base32",
	"encoding/base64",
	"encoding/binary",
	"encoding/csv",
	"encoding/gob",
	"encoding/hex",
	"encoding/json",
	"encoding/pem",
	"encoding/xml",

	"errors",
	"expvar",
	"flag",
	"fmt",

	"exp/types",
	"exp/gotype",

	"go/ast",
	"go/build",
	"go/doc",
	"go/format",
	"go/parser",
	"go/printer",
	"go/scanner",
	"go/token",

	"hash/adler32",
	"hash/crc32",
	"hash/crc64",
	"hash/fnv",

	"image",
	"image/color",
	"image/draw",
	"image/gif",
	"image/jpeg",
	"image/png",

	"index/suffixarray",

	"io",
	"io/ioutil",

	"log",
	"log/syslog",

	"math",
	"math/big",
	"math/cmplx",
	"math/rand",

	"mime",
	"mime/multipart",

	// "net", // src/pkg/net/lookup_unix.go:56:20: undeclared name: cgoLookupHost
	"net/http",
	"net/http/cgi",
	"net/http/fcgi",
	"net/http/httptest",
	"net/http/httputil",
	"net/http/pprof",
	"net/mail",
	"net/rpc",
	"net/rpc/jsonrpc",
	"net/smtp",
	"net/textproto",
	"net/url",

	"path",
	"path/filepath",

	// "reflect", // unsafe.Sizeof must return size > 0 for pointer types

	"regexp",
	"regexp/syntax",

	"runtime",
	"runtime/cgo",
	"runtime/debug",
	"runtime/pprof",

	"sort",
	// "strconv", // bug in switch case duplicate detection
	"strings",

	"sync",
	"sync/atomic",

	"syscall",

	"testing",
	"testing/iotest",
	"testing/quick",

	"text/scanner",
	"text/tabwriter",
	"text/template",
	"text/template/parse",

	// "time", // local const decls without initialization expressions
	"unicode",
	"unicode/utf16",
	"unicode/utf8",
}

func Test(t *testing.T) {
	for _, test := range tests {
		runTest(t, test)
	}
}
<commit_msg>exp/gotype: disable failing tests and add a few more<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"go/build"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

func runTest(t *testing.T, path string) {
	exitCode = 0

	*recursive = false
	if suffix := ".go"; strings.HasSuffix(path, suffix) {
		// single file
		path = filepath.Join(runtime.GOROOT(), "src/pkg", path)
		path, file := filepath.Split(path)
		*pkgName = file[:len(file)-len(suffix)]
		processFiles([]string{path}, true)
	} else {
		// package directory
		// TODO(gri) gotype should use the build package instead
		ctxt := build.Default
		ctxt.CgoEnabled = false
		pkg, err := ctxt.Import(path, "", 0)
		if err != nil {
			t.Errorf("build.Import error for path = %s: %s", path, err)
			return
		}
		// TODO(gri) there ought to be a more direct way using the build package...
		files := make([]string, len(pkg.GoFiles))
		for i, file := range pkg.GoFiles {
			files[i] = filepath.Join(pkg.Dir, file)
		}
		*pkgName = pkg.Name
		processFiles(files, true)
	}

	if exitCode != 0 {
		t.Errorf("processing %s failed: exitCode = %d", path, exitCode)
	}
}

var tests = []string{
	// individual files
	"exp/gotype/testdata/test1.go",

	// directories
	// Note: packages that don't typecheck yet are commented out
	"archive/tar",
	"archive/zip",

	"bufio",
	"bytes",

	"compress/bzip2",
	"compress/flate",
	"compress/gzip",
	"compress/lzw",
	"compress/zlib",

	"container/heap",
	"container/list",
	"container/ring",

	"crypto",
	"crypto/aes",
	"crypto/cipher",
	"crypto/des",
	"crypto/dsa",
	"crypto/ecdsa",
	"crypto/elliptic",
	"crypto/hmac",
	"crypto/md5",
	"crypto/rand",
	"crypto/rc4",
	// "crypto/rsa", // intermittent failure: /home/gri/go2/src/pkg/crypto/rsa/pkcs1v15.go:21:27: undeclared name: io
	"crypto/sha1",
	"crypto/sha256",
	"crypto/sha512",
	"crypto/subtle",
	"crypto/tls",
	"crypto/x509",
	"crypto/x509/pkix",

	"database/sql",
	"database/sql/driver",

	"debug/dwarf",
	"debug/elf",
	"debug/gosym",
	"debug/macho",
	"debug/pe",

	"encoding/ascii85",
	"encoding/asn1",
	"encoding/base32",
	"encoding/base64",
	"encoding/binary",
	"encoding/csv",
	"encoding/gob",
	"encoding/hex",
	"encoding/json",
	"encoding/pem",
	"encoding/xml",

	"errors",
	"expvar",
	"flag",
	"fmt",

	"exp/types",
	"exp/gotype",

	"go/ast",
	"go/build",
	"go/doc",
	"go/format",
	"go/parser",
	"go/printer",
	"go/scanner",
	"go/token",

	"hash/adler32",
	"hash/crc32",
	"hash/crc64",
	"hash/fnv",

	"image",
	"image/color",
	"image/draw",
	"image/gif",
	"image/jpeg",
	"image/png",

	"index/suffixarray",

	"io",
	"io/ioutil",

	"log",
	"log/syslog",

	"math",
	"math/big",
	"math/cmplx",
	"math/rand",

	"mime",
	"mime/multipart",

	// "net", // c:\go\root\src\pkg\net\interface_windows.go:54:13: invalid operation: division by zero
	"net/http",
	"net/http/cgi",
	"net/http/fcgi",
	"net/http/httptest",
	"net/http/httputil",
	"net/http/pprof",
	"net/mail",
	"net/rpc",
	"net/rpc/jsonrpc",
	"net/smtp",
	"net/textproto",
	"net/url",

	"path",
	"path/filepath",

	// "reflect", // unsafe.Sizeof must return size > 0 for pointer types

	"regexp",
	"regexp/syntax",

	"runtime",
	"runtime/cgo",
	"runtime/debug",
	"runtime/pprof",

	"sort",
	// "strconv", // bug in switch case duplicate detection
	"strings",

	"sync",
	"sync/atomic",

	// "syscall", c:\go\root\src\pkg\syscall\syscall_windows.go:35:16: cannot convert EINVAL (constant 536870951) to error

	"testing",
	"testing/iotest",
	"testing/quick",

	"text/scanner",
	"text/tabwriter",
	"text/template",
	"text/template/parse",

	// "time", // local const decls without initialization expressions
	"unicode",
	"unicode/utf16",
	"unicode/utf8",
}

func Test(t *testing.T) {
	for _, test := range tests {
		runTest(t, test)
	}
}
<|endoftext|>


<commit_before>package outbox_test

import (
	"encoding/json"
	"errors"
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fake_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/cloudfoundry-incubator/stager/outbox"
	"github.com/cloudfoundry-incubator/stager/stager"
	"github.com/cloudfoundry/yagnats"
	"github.com/cloudfoundry/yagnats/fakeyagnats"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

var _ = Describe("Outbox", func() {
	var (
		fakenats  *fakeyagnats.FakeYagnats
		logger    lager.Logger
		task      models.Task
		bbs       *fake_bbs.FakeStagerBBS
		published <-chan []byte
		appId     string
		taskId    string

		completedTasks chan models.Task
		watchStopChan  chan bool
		watchErrChan   chan error

		outbox ifrit.Process
	)

	BeforeEach(func() {
		fakenats = fakeyagnats.New()
		logger = lager.NewLogger("fakelogger")
		appId = "my_app_id"
		taskId = "do_this"
		annotationJson, _ := json.Marshal(models.StagingTaskAnnotation{
			AppId:  appId,
			TaskId: taskId,
		})

		task = models.Task{
			Guid:       "some-task-id",
			Result:     "{}",
			Annotation: string(annotationJson),
			Domain:     stager.TaskDomain,
		}

		completedTasks = make(chan models.Task)
		watchStopChan = make(chan bool)
		watchErrChan = make(chan error)
		bbs = &fake_bbs.FakeStagerBBS{}
		bbs.WatchForCompletedTaskReturns(completedTasks, watchStopChan, watchErrChan)

		publishedCallback := make(chan []byte)

		published = publishedCallback

		fakenats.Subscribe(DiegoStageFinishedSubject, func(msg *yagnats.Message) {
			publishedCallback <- msg.Payload
		})
	})

	JustBeforeEach(func() {
		outbox = ifrit.Envoke(New(bbs, fakenats, logger))
	})

	AfterEach(func() {
		outbox.Signal(syscall.SIGTERM)
		Eventually(outbox.Wait()).Should(Receive())
	})

	Context("when a completed Task appears in the outbox", func() {
		BeforeEach(func() {
			task.Result = `{
				"buildpack_key":"buildpack-key",
				"detected_buildpack":"Some Buildpack",
				"detected_start_command":"./some-start-command"
			}`
		})

		It("resolves the completed task, publishes its result and then marks the Task as resolved", func() {
			completedTasks <- task

			Eventually(bbs.ResolvingTaskCallCount).Should(Equal(1))

			var receivedPayload []byte
			Eventually(published).Should(Receive(&receivedPayload))
			Ω(receivedPayload).Should(MatchJSON(fmt.Sprintf(`{
				"buildpack_key":"buildpack-key",
				"detected_buildpack":"Some Buildpack",
				"detected_start_command":"./some-start-command",
				"app_id": "%s",
				"task_id": "%s"
			}`, appId, taskId)))

			Eventually(bbs.ResolveTaskCallCount).Should(Equal(1))
			Ω(bbs.ResolveTaskArgsForCall(0)).Should(Equal(task.Guid))
		})

		Context("when the task is not a staging task", func() {
			BeforeEach(func() {
				task.Domain = "some-random-domain"
			})

			It("Should not resolve the completed task ", func() {
				completedTasks <- task
				Consistently(bbs.ResolvingTaskCallCount).Should(BeZero())
			})
		})

		Context("when the response fails to go out", func() {
			BeforeEach(func() {
				fakenats.WhenPublishing(DiegoStageFinishedSubject, func(msg *yagnats.Message) error {
					return errors.New("kaboom!")
				})
			})

			It("does not attempt to resolve the Task", func() {
				completedTasks <- task

				Consistently(bbs.ResolveTaskCallCount).Should(Equal(0))
			})
		})
	})

	Context("when a failed Task appears in the outbox", func() {
		BeforeEach(func() {
			task.Failed = true
			task.FailureReason = "because i said so"
		})

		It("publishes its reason as an error and then marks the Task as completed", func() {
			completedTasks <- task

			var receivedPayload []byte
			Eventually(published).Should(Receive(&receivedPayload))
			Ω(receivedPayload).Should(MatchJSON(fmt.Sprintf(`{
				"error":"because i said so",
				"app_id":"%s",
				"task_id":"%s"
			}`, appId, taskId)))

			Eventually(bbs.ResolveTaskCallCount).Should(Equal(1))
			Ω(bbs.ResolveTaskArgsForCall(0)).Should(Equal(task.Guid))
		})
	})

	Context("when ResolvingTask fails", func() {
		BeforeEach(func() {
			bbs.ResolvingTaskReturns(errors.New("oops"))
		})

		It("does not send a response to the requester, because another stager probably resolved it", func() {
			completedTasks <- task

			Consistently(published).ShouldNot(Receive())
			Consistently(bbs.ResolveTaskCallCount).Should(Equal(0))
		})
	})

	Context("when an error is seen while watching", func() {
		It("starts watching again", func() {
			sinceStart := time.Now()

			watchErrChan <- errors.New("oh no!")

			Eventually(bbs.WatchForCompletedTaskCallCount, 4).Should(Equal(2))

			Ω(time.Since(sinceStart)).Should(BeNumerically("~", 3*time.Second, 200*time.Millisecond))

			completedTasks <- task
			Eventually(published).Should(Receive())
			Eventually(bbs.ResolveTaskCallCount).Should(Equal(1))
			Ω(bbs.ResolveTaskArgsForCall(0)).Should(Equal(task.Guid))
		})
	})

	Describe("asynchronous message processing", func() {
		It("can accept new Completed Tasks before it's done processing existing Tasks in the queue", func() {
			task.Result = `{"detected_buildpack":"Some Buildpack"}`

			completedTasks <- task
			completedTasks <- task

			var receivedPayload []byte

			Eventually(published).Should(Receive(&receivedPayload))
			Ω(receivedPayload).Should(MatchJSON(fmt.Sprintf(`{
				"detected_buildpack":"Some Buildpack",
				"app_id": "%s",
				"task_id": "%s"
			}`, appId, taskId)))

			Eventually(published).Should(Receive(&receivedPayload))
			Ω(receivedPayload).Should(MatchJSON(fmt.Sprintf(`{
				"detected_buildpack":"Some Buildpack",
				"app_id": "%s",
				"task_id": "%s"
			}`, appId, taskId)))
		})
	})
})
<commit_msg>S -> s<commit_after>package outbox_test

import (
	"encoding/json"
	"errors"
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fake_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/cloudfoundry-incubator/stager/outbox"
	"github.com/cloudfoundry-incubator/stager/stager"
	"github.com/cloudfoundry/yagnats"
	"github.com/cloudfoundry/yagnats/fakeyagnats"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

var _ = Describe("Outbox", func() {
	var (
		fakenats  *fakeyagnats.FakeYagnats
		logger    lager.Logger
		task      models.Task
		bbs       *fake_bbs.FakeStagerBBS
		published <-chan []byte
		appId     string
		taskId    string

		completedTasks chan models.Task
		watchStopChan  chan bool
		watchErrChan   chan error

		outbox ifrit.Process
	)

	BeforeEach(func() {
		fakenats = fakeyagnats.New()
		logger = lager.NewLogger("fakelogger")
		appId = "my_app_id"
		taskId = "do_this"
		annotationJson, _ := json.Marshal(models.StagingTaskAnnotation{
			AppId:  appId,
			TaskId: taskId,
		})

		task = models.Task{
			Guid:       "some-task-id",
			Result:     "{}",
			Annotation: string(annotationJson),
			Domain:     stager.TaskDomain,
		}

		completedTasks = make(chan models.Task)
		watchStopChan = make(chan bool)
		watchErrChan = make(chan error)
		bbs = &fake_bbs.FakeStagerBBS{}
		bbs.WatchForCompletedTaskReturns(completedTasks, watchStopChan, watchErrChan)

		publishedCallback := make(chan []byte)

		published = publishedCallback

		fakenats.Subscribe(DiegoStageFinishedSubject, func(msg *yagnats.Message) {
			publishedCallback <- msg.Payload
		})
	})

	JustBeforeEach(func() {
		outbox = ifrit.Envoke(New(bbs, fakenats, logger))
	})

	AfterEach(func() {
		outbox.Signal(syscall.SIGTERM)
		Eventually(outbox.Wait()).Should(Receive())
	})

	Context("when a completed Task appears in the outbox", func() {
		BeforeEach(func() {
			task.Result = `{
				"buildpack_key":"buildpack-key",
				"detected_buildpack":"Some Buildpack",
				"detected_start_command":"./some-start-command"
			}`
		})

		It("resolves the completed task, publishes its result and then marks the Task as resolved", func() {
			completedTasks <- task

			Eventually(bbs.ResolvingTaskCallCount).Should(Equal(1))

			var receivedPayload []byte
			Eventually(published).Should(Receive(&receivedPayload))
			Ω(receivedPayload).Should(MatchJSON(fmt.Sprintf(`{
				"buildpack_key":"buildpack-key",
				"detected_buildpack":"Some Buildpack",
				"detected_start_command":"./some-start-command",
				"app_id": "%s",
				"task_id": "%s"
			}`, appId, taskId)))

			Eventually(bbs.ResolveTaskCallCount).Should(Equal(1))
			Ω(bbs.ResolveTaskArgsForCall(0)).Should(Equal(task.Guid))
		})

		Context("when the task is not a staging task", func() {
			BeforeEach(func() {
				task.Domain = "some-random-domain"
			})

			It("should not resolve the completed task ", func() {
				completedTasks <- task
				Consistently(bbs.ResolvingTaskCallCount).Should(BeZero())
			})
		})

		Context("when the response fails to go out", func() {
			BeforeEach(func() {
				fakenats.WhenPublishing(DiegoStageFinishedSubject, func(msg *yagnats.Message) error {
					return errors.New("kaboom!")
				})
			})

			It("does not attempt to resolve the Task", func() {
				completedTasks <- task

				Consistently(bbs.ResolveTaskCallCount).Should(Equal(0))
			})
		})
	})

	Context("when a failed Task appears in the outbox", func() {
		BeforeEach(func() {
			task.Failed = true
			task.FailureReason = "because i said so"
		})

		It("publishes its reason as an error and then marks the Task as completed", func() {
			completedTasks <- task

			var receivedPayload []byte
			Eventually(published).Should(Receive(&receivedPayload))
			Ω(receivedPayload).Should(MatchJSON(fmt.Sprintf(`{
				"error":"because i said so",
				"app_id":"%s",
				"task_id":"%s"
			}`, appId, taskId)))

			Eventually(bbs.ResolveTaskCallCount).Should(Equal(1))
			Ω(bbs.ResolveTaskArgsForCall(0)).Should(Equal(task.Guid))
		})
	})

	Context("when ResolvingTask fails", func() {
		BeforeEach(func() {
			bbs.ResolvingTaskReturns(errors.New("oops"))
		})

		It("does not send a response to the requester, because another stager probably resolved it", func() {
			completedTasks <- task

			Consistently(published).ShouldNot(Receive())
			Consistently(bbs.ResolveTaskCallCount).Should(Equal(0))
		})
	})

	Context("when an error is seen while watching", func() {
		It("starts watching again", func() {
			sinceStart := time.Now()

			watchErrChan <- errors.New("oh no!")

			Eventually(bbs.WatchForCompletedTaskCallCount, 4).Should(Equal(2))

			Ω(time.Since(sinceStart)).Should(BeNumerically("~", 3*time.Second, 200*time.Millisecond))

			completedTasks <- task
			Eventually(published).Should(Receive())
			Eventually(bbs.ResolveTaskCallCount).Should(Equal(1))
			Ω(bbs.ResolveTaskArgsForCall(0)).Should(Equal(task.Guid))
		})
	})

	Describe("asynchronous message processing", func() {
		It("can accept new Completed Tasks before it's done processing existing Tasks in the queue", func() {
			task.Result = `{"detected_buildpack":"Some Buildpack"}`

			completedTasks <- task
			completedTasks <- task

			var receivedPayload []byte

			Eventually(published).Should(Receive(&receivedPayload))
			Ω(receivedPayload).Should(MatchJSON(fmt.Sprintf(`{
				"detected_buildpack":"Some Buildpack",
				"app_id": "%s",
				"task_id": "%s"
			}`, appId, taskId)))

			Eventually(published).Should(Receive(&receivedPayload))
			Ω(receivedPayload).Should(MatchJSON(fmt.Sprintf(`{
				"detected_buildpack":"Some Buildpack",
				"app_id": "%s",
				"task_id": "%s"
			}`, appId, taskId)))
		})
	})
})
<|endoftext|>