<commit_before>package cluster

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"strconv"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/crypto/scrypt"

	"github.com/lxc/lxd/lxd/config"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/shared/validate"
)

// Config holds cluster-wide configuration values.
type Config struct {
	tx *db.ClusterTx // DB transaction the values in this config are bound to.
	m  config.Map    // Low-level map holding the config values.
}

// ConfigLoad loads a new Config object with the current cluster configuration
// values fetched from the database.
func ConfigLoad(tx *db.ClusterTx) (*Config, error) {
	// Load current raw values from the database, any error is fatal.
	values, err := tx.Config()
	if err != nil {
		return nil, fmt.Errorf("cannot fetch node config from database: %v", err)
	}

	m, err := config.SafeLoad(ConfigSchema, values)
	if err != nil {
		return nil, fmt.Errorf("failed to load node config: %v", err)
	}

	return &Config{tx: tx, m: m}, nil
}

// HTTPSAllowedHeaders returns the relevant CORS setting.
func (c *Config) HTTPSAllowedHeaders() string {
	return c.m.GetString("core.https_allowed_headers")
}

// HTTPSAllowedMethods returns the relevant CORS setting.
func (c *Config) HTTPSAllowedMethods() string {
	return c.m.GetString("core.https_allowed_methods")
}

// HTTPSAllowedOrigin returns the relevant CORS setting.
func (c *Config) HTTPSAllowedOrigin() string {
	return c.m.GetString("core.https_allowed_origin")
}

// HTTPSAllowedCredentials returns the relevant CORS setting.
func (c *Config) HTTPSAllowedCredentials() bool {
	return c.m.GetBool("core.https_allowed_credentials")
}

// TrustPassword returns the LXD trust password for authenticating clients.
func (c *Config) TrustPassword() string {
	return c.m.GetString("core.trust_password")
}

// TrustCACertificates returns whether client certificates are checked
// against a CA.
func (c *Config) TrustCACertificates() bool {
	return c.m.GetBool("core.trust_ca_certificates")
}

// CandidServer returns all the Candid settings needed to connect to a server.
func (c *Config) CandidServer() (string, string, int64, string) {
	return c.m.GetString("candid.api.url"),
		c.m.GetString("candid.api.key"),
		c.m.GetInt64("candid.expiry"),
		c.m.GetString("candid.domains")
}

// RBACServer returns all the Candid settings needed to connect to a server.
func (c *Config) RBACServer() (string, string, int64, string, string, string, string) {
	return c.m.GetString("rbac.api.url"),
		c.m.GetString("rbac.api.key"),
		c.m.GetInt64("rbac.expiry"),
		c.m.GetString("rbac.agent.url"),
		c.m.GetString("rbac.agent.username"),
		c.m.GetString("rbac.agent.private_key"),
		c.m.GetString("rbac.agent.public_key")
}

// ProxyHTTPS returns the configured HTTPS proxy, if any.
func (c *Config) ProxyHTTPS() string {
	return c.m.GetString("core.proxy_https")
}

// ProxyHTTP returns the configured HTTP proxy, if any.
func (c *Config) ProxyHTTP() string {
	return c.m.GetString("core.proxy_http")
}

// ProxyIgnoreHosts returns the configured ignore-hosts proxy setting, if any.
func (c *Config) ProxyIgnoreHosts() string {
	return c.m.GetString("core.proxy_ignore_hosts")
}

// HTTPSTrustedProxy returns the configured HTTPS trusted proxy setting, if any.
func (c *Config) HTTPSTrustedProxy() string {
	return c.m.GetString("core.https_trusted_proxy")
}

// MAASController the configured MAAS url and key, if any.
func (c *Config) MAASController() (string, string) {
	url := c.m.GetString("maas.api.url")
	key := c.m.GetString("maas.api.key")
	return url, key
}

// OfflineThreshold returns the configured heartbeat threshold, i.e. the
// number of seconds before after which an unresponsive node is considered
// offline..
func (c *Config) OfflineThreshold() time.Duration {
	n := c.m.GetInt64("cluster.offline_threshold")
	return time.Duration(n) * time.Second
}

// ImagesMinimalReplica returns the numbers of nodes for cluster images replication
func (c *Config) ImagesMinimalReplica() int64 {
	return c.m.GetInt64("cluster.images_minimal_replica")
}

// MaxVoters returns the maximum number of members in a cluster that will be
// assigned the voter role.
func (c *Config) MaxVoters() int64 {
	return c.m.GetInt64("cluster.max_voters")
}

// MaxStandBy returns the maximum number of standby members in a cluster that
// will be assigned the stand-by role.
func (c *Config) MaxStandBy() int64 {
	return c.m.GetInt64("cluster.max_standby")
}

// ShutdownTimeout returns the number of minutes to wait for running operation to complete
// before LXD server shut down
func (c *Config) ShutdownTimeout() time.Duration {
	n := c.m.GetInt64("core.shutdown_timeout")
	return time.Duration(n) * time.Minute
}

// Dump current configuration keys and their values. Keys with values matching
// their defaults are omitted.
func (c *Config) Dump() map[string]interface{} {
	return c.m.Dump()
}

// Replace the current configuration with the given values.
//
// Return what has actually changed.
func (c *Config) Replace(values map[string]interface{}) (map[string]string, error) {
	return c.update(values)
}

// Patch changes only the configuration keys in the given map.
//
// Return what has actually changed.
func (c *Config) Patch(patch map[string]interface{}) (map[string]string, error) {
	values := c.Dump() // Use current values as defaults
	for name, value := range patch {
		values[name] = value
	}
	return c.update(values)
}

func (c *Config) update(values map[string]interface{}) (map[string]string, error) {
	changed, err := c.m.Change(values)
	if err != nil {
		return nil, err
	}

	err = c.tx.UpdateConfig(changed)
	if err != nil {
		return nil, errors.Wrap(err, "cannot persist configuration changes: %v")
	}

	return changed, nil
}

// ConfigGetString is a convenience for loading the cluster configuration and
// returning the value of a particular key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetString(cluster *db.Cluster, key string) (string, error) {
	config, err := configGet(cluster)
	if err != nil {
		return "", err
	}
	return config.m.GetString(key), nil
}

// ConfigGetBool is a convenience for loading the cluster configuration and
// returning the value of a particular boolean key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetBool(cluster *db.Cluster, key string) (bool, error) {
	config, err := configGet(cluster)
	if err != nil {
		return false, err
	}
	return config.m.GetBool(key), nil
}

// ConfigGetInt64 is a convenience for loading the cluster configuration and
// returning the value of a particular key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetInt64(cluster *db.Cluster, key string) (int64, error) {
	config, err := configGet(cluster)
	if err != nil {
		return 0, err
	}
	return config.m.GetInt64(key), nil
}

func configGet(cluster *db.Cluster) (*Config, error) {
	var config *Config
	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		config, err = ConfigLoad(tx)
		return err
	})
	return config, err
}

// ConfigSchema defines available server configuration keys.
var ConfigSchema = config.Schema{
	"backups.compression_algorithm":  {Default: "gzip", Validator: validate.IsCompressionAlgorithm},
	"cluster.offline_threshold":      {Type: config.Int64, Default: offlineThresholdDefault(), Validator: offlineThresholdValidator},
	"cluster.images_minimal_replica": {Type: config.Int64, Default: "3", Validator: imageMinimalReplicaValidator},
	"cluster.max_voters":             {Type: config.Int64, Default: "3", Validator: maxVotersValidator},
	"cluster.max_standby":            {Type: config.Int64, Default: "2", Validator: maxStandByValidator},
	"core.https_allowed_headers":     {},
	"core.https_allowed_methods":     {},
	"core.https_allowed_origin":      {},
	"core.https_allowed_credentials": {Type: config.Bool},
	"core.https_trusted_proxy":       {},
	"core.proxy_http":                {},
	"core.proxy_https":               {},
	"core.proxy_ignore_hosts":        {},
	"core.shutdown_timeout":          {Type: config.Int64, Default: "5"},
	"core.trust_password":            {Hidden: true, Setter: passwordSetter},
	"core.trust_ca_certificates":     {Type: config.Bool},
	"candid.api.key":                 {},
	"candid.api.url":                 {},
	"candid.domains":                 {},
	"candid.expiry":                  {Type: config.Int64, Default: "3600"},
	"images.auto_update_cached":      {Type: config.Bool, Default: "true"},
	"images.auto_update_interval":    {Type: config.Int64, Default: "6"},
	"images.compression_algorithm":   {Default: "gzip", Validator: validate.IsCompressionAlgorithm},
	"images.default_architecture":    {Validator: validate.IsArchitecture},
	"images.remote_cache_expiry":     {Type: config.Int64, Default: "10"},
	"maas.api.key":                   {},
	"maas.api.url":                   {},
	"rbac.agent.url":                 {},
	"rbac.agent.username":            {},
	"rbac.agent.private_key":         {},
	"rbac.agent.public_key":          {},
	"rbac.api.expiry":                {Type: config.Int64, Default: "3600"},
	"rbac.api.key":                   {},
	"rbac.api.url":                   {},
	"rbac.expiry":                    {Type: config.Int64, Default: "3600"},

	// Keys deprecated since the implementation of the storage api.
	"storage.lvm_fstype":           {Setter: deprecatedStorage, Default: "ext4"},
	"storage.lvm_mount_options":    {Setter: deprecatedStorage, Default: "discard"},
	"storage.lvm_thinpool_name":    {Setter: deprecatedStorage, Default: "LXDThinPool"},
	"storage.lvm_vg_name":          {Setter: deprecatedStorage},
	"storage.lvm_volume_size":      {Setter: deprecatedStorage, Default: "10GiB"},
	"storage.zfs_pool_name":        {Setter: deprecatedStorage},
	"storage.zfs_remove_snapshots": {Setter: deprecatedStorage, Type: config.Bool},
	"storage.zfs_use_refquota":     {Setter: deprecatedStorage, Type: config.Bool},

	// OVN networking global keys.
	"network.ovn.integration_bridge":    {Default: "br-int"},
	"network.ovn.northbound_connection": {Default: "unix:/var/run/ovn/ovnnb_db.sock"},
}

func offlineThresholdDefault() string {
	return strconv.Itoa(db.DefaultOfflineThreshold)
}

func offlineThresholdValidator(value string) error {
	minThreshold := 10

	// Ensure that the given value is greater than the heartbeat interval,
	// which is the lower bound granularity of the offline check.
	threshold, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Offline threshold is not a number")
	}

	if threshold <= minThreshold {
		return fmt.Errorf("Value must be greater than '%d'", minThreshold)
	}

	return nil
}

func imageMinimalReplicaValidator(value string) error {
	count, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Minimal image replica count is not a number")
	}

	if count < 1 && count != -1 {
		return fmt.Errorf("Invalid value for image replica count")
	}

	return nil
}

func maxVotersValidator(value string) error {
	n, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Value is not a number")
	}

	if n < 3 || n%2 != 1 {
		return fmt.Errorf("Value must be an odd number equal to or higher than 3")
	}

	return nil
}

func maxStandByValidator(value string) error {
	n, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Value is not a number")
	}

	if n < 0 || n > 5 {
		return fmt.Errorf("Value must be between 0 and 5")
	}

	return nil
}

func passwordSetter(value string) (string, error) {
	// Nothing to do on unset
	if value == "" {
		return value, nil
	}

	// Hash the password
	buf := make([]byte, 32)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", err
	}

	hash, err := scrypt.Key([]byte(value), buf, 1<<14, 8, 1, 64)
	if err != nil {
		return "", err
	}

	buf = append(buf, hash...)
	value = hex.EncodeToString(buf)

	return value, nil
}

func deprecatedStorage(value string) (string, error) {
	if value == "" {
		return "", nil
	}
	return "", fmt.Errorf("deprecated: use storage pool configuration")
}
<commit_msg>lxd/cluster/config: Wraps images.default_architecture with validate.Optional due to IsOneOf change in IsArchitecture<commit_after>package cluster

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"strconv"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/crypto/scrypt"

	"github.com/lxc/lxd/lxd/config"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/shared/validate"
)

// Config holds cluster-wide configuration values.
type Config struct {
	tx *db.ClusterTx // DB transaction the values in this config are bound to.
	m  config.Map    // Low-level map holding the config values.
}

// ConfigLoad loads a new Config object with the current cluster configuration
// values fetched from the database.
func ConfigLoad(tx *db.ClusterTx) (*Config, error) {
	// Load current raw values from the database, any error is fatal.
	values, err := tx.Config()
	if err != nil {
		return nil, fmt.Errorf("cannot fetch node config from database: %v", err)
	}

	m, err := config.SafeLoad(ConfigSchema, values)
	if err != nil {
		return nil, fmt.Errorf("failed to load node config: %v", err)
	}

	return &Config{tx: tx, m: m}, nil
}

// HTTPSAllowedHeaders returns the relevant CORS setting.
func (c *Config) HTTPSAllowedHeaders() string {
	return c.m.GetString("core.https_allowed_headers")
}

// HTTPSAllowedMethods returns the relevant CORS setting.
func (c *Config) HTTPSAllowedMethods() string {
	return c.m.GetString("core.https_allowed_methods")
}

// HTTPSAllowedOrigin returns the relevant CORS setting.
func (c *Config) HTTPSAllowedOrigin() string {
	return c.m.GetString("core.https_allowed_origin")
}

// HTTPSAllowedCredentials returns the relevant CORS setting.
func (c *Config) HTTPSAllowedCredentials() bool {
	return c.m.GetBool("core.https_allowed_credentials")
}

// TrustPassword returns the LXD trust password for authenticating clients.
func (c *Config) TrustPassword() string {
	return c.m.GetString("core.trust_password")
}

// TrustCACertificates returns whether client certificates are checked
// against a CA.
func (c *Config) TrustCACertificates() bool {
	return c.m.GetBool("core.trust_ca_certificates")
}

// CandidServer returns all the Candid settings needed to connect to a server.
func (c *Config) CandidServer() (string, string, int64, string) {
	return c.m.GetString("candid.api.url"),
		c.m.GetString("candid.api.key"),
		c.m.GetInt64("candid.expiry"),
		c.m.GetString("candid.domains")
}

// RBACServer returns all the Candid settings needed to connect to a server.
func (c *Config) RBACServer() (string, string, int64, string, string, string, string) {
	return c.m.GetString("rbac.api.url"),
		c.m.GetString("rbac.api.key"),
		c.m.GetInt64("rbac.expiry"),
		c.m.GetString("rbac.agent.url"),
		c.m.GetString("rbac.agent.username"),
		c.m.GetString("rbac.agent.private_key"),
		c.m.GetString("rbac.agent.public_key")
}

// ProxyHTTPS returns the configured HTTPS proxy, if any.
func (c *Config) ProxyHTTPS() string {
	return c.m.GetString("core.proxy_https")
}

// ProxyHTTP returns the configured HTTP proxy, if any.
func (c *Config) ProxyHTTP() string {
	return c.m.GetString("core.proxy_http")
}

// ProxyIgnoreHosts returns the configured ignore-hosts proxy setting, if any.
func (c *Config) ProxyIgnoreHosts() string {
	return c.m.GetString("core.proxy_ignore_hosts")
}

// HTTPSTrustedProxy returns the configured HTTPS trusted proxy setting, if any.
func (c *Config) HTTPSTrustedProxy() string {
	return c.m.GetString("core.https_trusted_proxy")
}

// MAASController the configured MAAS url and key, if any.
func (c *Config) MAASController() (string, string) {
	url := c.m.GetString("maas.api.url")
	key := c.m.GetString("maas.api.key")
	return url, key
}

// OfflineThreshold returns the configured heartbeat threshold, i.e. the
// number of seconds before after which an unresponsive node is considered
// offline..
func (c *Config) OfflineThreshold() time.Duration {
	n := c.m.GetInt64("cluster.offline_threshold")
	return time.Duration(n) * time.Second
}

// ImagesMinimalReplica returns the numbers of nodes for cluster images replication
func (c *Config) ImagesMinimalReplica() int64 {
	return c.m.GetInt64("cluster.images_minimal_replica")
}

// MaxVoters returns the maximum number of members in a cluster that will be
// assigned the voter role.
func (c *Config) MaxVoters() int64 {
	return c.m.GetInt64("cluster.max_voters")
}

// MaxStandBy returns the maximum number of standby members in a cluster that
// will be assigned the stand-by role.
func (c *Config) MaxStandBy() int64 {
	return c.m.GetInt64("cluster.max_standby")
}

// ShutdownTimeout returns the number of minutes to wait for running operation to complete
// before LXD server shut down
func (c *Config) ShutdownTimeout() time.Duration {
	n := c.m.GetInt64("core.shutdown_timeout")
	return time.Duration(n) * time.Minute
}

// Dump current configuration keys and their values. Keys with values matching
// their defaults are omitted.
func (c *Config) Dump() map[string]interface{} {
	return c.m.Dump()
}

// Replace the current configuration with the given values.
//
// Return what has actually changed.
func (c *Config) Replace(values map[string]interface{}) (map[string]string, error) {
	return c.update(values)
}

// Patch changes only the configuration keys in the given map.
//
// Return what has actually changed.
func (c *Config) Patch(patch map[string]interface{}) (map[string]string, error) {
	values := c.Dump() // Use current values as defaults
	for name, value := range patch {
		values[name] = value
	}
	return c.update(values)
}

func (c *Config) update(values map[string]interface{}) (map[string]string, error) {
	changed, err := c.m.Change(values)
	if err != nil {
		return nil, err
	}

	err = c.tx.UpdateConfig(changed)
	if err != nil {
		return nil, errors.Wrap(err, "cannot persist configuration changes: %v")
	}

	return changed, nil
}

// ConfigGetString is a convenience for loading the cluster configuration and
// returning the value of a particular key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetString(cluster *db.Cluster, key string) (string, error) {
	config, err := configGet(cluster)
	if err != nil {
		return "", err
	}
	return config.m.GetString(key), nil
}

// ConfigGetBool is a convenience for loading the cluster configuration and
// returning the value of a particular boolean key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetBool(cluster *db.Cluster, key string) (bool, error) {
	config, err := configGet(cluster)
	if err != nil {
		return false, err
	}
	return config.m.GetBool(key), nil
}

// ConfigGetInt64 is a convenience for loading the cluster configuration and
// returning the value of a particular key.
//
// It's a deprecated API meant to be used by call sites that are not
// interacting with the database in a transactional way.
func ConfigGetInt64(cluster *db.Cluster, key string) (int64, error) {
	config, err := configGet(cluster)
	if err != nil {
		return 0, err
	}
	return config.m.GetInt64(key), nil
}

func configGet(cluster *db.Cluster) (*Config, error) {
	var config *Config
	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		config, err = ConfigLoad(tx)
		return err
	})
	return config, err
}

// ConfigSchema defines available server configuration keys.
var ConfigSchema = config.Schema{
	"backups.compression_algorithm":  {Default: "gzip", Validator: validate.IsCompressionAlgorithm},
	"cluster.offline_threshold":      {Type: config.Int64, Default: offlineThresholdDefault(), Validator: offlineThresholdValidator},
	"cluster.images_minimal_replica": {Type: config.Int64, Default: "3", Validator: imageMinimalReplicaValidator},
	"cluster.max_voters":             {Type: config.Int64, Default: "3", Validator: maxVotersValidator},
	"cluster.max_standby":            {Type: config.Int64, Default: "2", Validator: maxStandByValidator},
	"core.https_allowed_headers":     {},
	"core.https_allowed_methods":     {},
	"core.https_allowed_origin":      {},
	"core.https_allowed_credentials": {Type: config.Bool},
	"core.https_trusted_proxy":       {},
	"core.proxy_http":                {},
	"core.proxy_https":               {},
	"core.proxy_ignore_hosts":        {},
	"core.shutdown_timeout":          {Type: config.Int64, Default: "5"},
	"core.trust_password":            {Hidden: true, Setter: passwordSetter},
	"core.trust_ca_certificates":     {Type: config.Bool},
	"candid.api.key":                 {},
	"candid.api.url":                 {},
	"candid.domains":                 {},
	"candid.expiry":                  {Type: config.Int64, Default: "3600"},
	"images.auto_update_cached":      {Type: config.Bool, Default: "true"},
	"images.auto_update_interval":    {Type: config.Int64, Default: "6"},
	"images.compression_algorithm":   {Default: "gzip", Validator: validate.IsCompressionAlgorithm},
	"images.default_architecture":    {Validator: validate.Optional(validate.IsArchitecture)},
	"images.remote_cache_expiry":     {Type: config.Int64, Default: "10"},
	"maas.api.key":                   {},
	"maas.api.url":                   {},
	"rbac.agent.url":                 {},
	"rbac.agent.username":            {},
	"rbac.agent.private_key":         {},
	"rbac.agent.public_key":          {},
	"rbac.api.expiry":                {Type: config.Int64, Default: "3600"},
	"rbac.api.key":                   {},
	"rbac.api.url":                   {},
	"rbac.expiry":                    {Type: config.Int64, Default: "3600"},

	// Keys deprecated since the implementation of the storage api.
	"storage.lvm_fstype":           {Setter: deprecatedStorage, Default: "ext4"},
	"storage.lvm_mount_options":    {Setter: deprecatedStorage, Default: "discard"},
	"storage.lvm_thinpool_name":    {Setter: deprecatedStorage, Default: "LXDThinPool"},
	"storage.lvm_vg_name":          {Setter: deprecatedStorage},
	"storage.lvm_volume_size":      {Setter: deprecatedStorage, Default: "10GiB"},
	"storage.zfs_pool_name":        {Setter: deprecatedStorage},
	"storage.zfs_remove_snapshots": {Setter: deprecatedStorage, Type: config.Bool},
	"storage.zfs_use_refquota":     {Setter: deprecatedStorage, Type: config.Bool},

	// OVN networking global keys.
	"network.ovn.integration_bridge":    {Default: "br-int"},
	"network.ovn.northbound_connection": {Default: "unix:/var/run/ovn/ovnnb_db.sock"},
}

func offlineThresholdDefault() string {
	return strconv.Itoa(db.DefaultOfflineThreshold)
}

func offlineThresholdValidator(value string) error {
	minThreshold := 10

	// Ensure that the given value is greater than the heartbeat interval,
	// which is the lower bound granularity of the offline check.
	threshold, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Offline threshold is not a number")
	}

	if threshold <= minThreshold {
		return fmt.Errorf("Value must be greater than '%d'", minThreshold)
	}

	return nil
}

func imageMinimalReplicaValidator(value string) error {
	count, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Minimal image replica count is not a number")
	}

	if count < 1 && count != -1 {
		return fmt.Errorf("Invalid value for image replica count")
	}

	return nil
}

func maxVotersValidator(value string) error {
	n, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Value is not a number")
	}

	if n < 3 || n%2 != 1 {
		return fmt.Errorf("Value must be an odd number equal to or higher than 3")
	}

	return nil
}

func maxStandByValidator(value string) error {
	n, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("Value is not a number")
	}

	if n < 0 || n > 5 {
		return fmt.Errorf("Value must be between 0 and 5")
	}

	return nil
}

func passwordSetter(value string) (string, error) {
	// Nothing to do on unset
	if value == "" {
		return value, nil
	}

	// Hash the password
	buf := make([]byte, 32)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", err
	}

	hash, err := scrypt.Key([]byte(value), buf, 1<<14, 8, 1, 64)
	if err != nil {
		return "", err
	}

	buf = append(buf, hash...)
	value = hex.EncodeToString(buf)

	return value, nil
}

func deprecatedStorage(value string) (string, error) {
	if value == "" {
		return "", nil
	}
	return "", fmt.Errorf("deprecated: use storage pool configuration")
}
<|endoftext|>


<commit_before>package cluster

import (
	"context"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// eventHubMinHosts is the minimum number of members that must have the event-hub role to trigger switching into
// event-hub mode (where cluster members will only connect to event-hub members rather than all members when
// operating in the normal full-mesh mode).
const eventHubMinHosts = 2

// EventMode indicates the event distribution mode.
type EventMode string

// EventModeFullMesh is when every cluster member connects to every other cluster member to pull events.
const EventModeFullMesh EventMode = "full-mesh"

// EventModeHubServer is when the cluster is operating in event-hub mode and this server is designated as a hub
// server, meaning that it will only connect to the other event-hub members and not other members.
const EventModeHubServer EventMode = "hub-server"

// EventModeHubClient is when the cluster is operating in event-hub mode and this member is designated as a hub
// client, meaning that it is expected to connect to the event-hub members.
const EventModeHubClient EventMode = "hub-client"

var listeners = map[string]*lxd.EventListener{}
var listenersNotify = map[chan struct{}][]string{}
var listenersLock sync.Mutex
var listenersUpdateLock sync.Mutex

// ServerEventMode returns the event distribution mode that this local server is operating in.
func ServerEventMode() EventMode {
	listenersLock.Lock()
	defer listenersLock.Unlock()

	return eventMode
}

// RoleInSlice returns whether or not the rule is within the roles list.
func RoleInSlice(role db.ClusterRole, roles []db.ClusterRole) bool {
	for _, r := range roles {
		if r == role {
			return true
		}
	}

	return false
}

// EventListenerWait waits for there to be listener connected to the specified address, or one of the event hubs
// if operating in event hub mode.
func EventListenerWait(ctx context.Context, address string) error {
	// Check if there is already a listener.
	listenersLock.Lock()
	listener, found := listeners[address]
	if found && listener.IsActive() {
		listenersLock.Unlock()
		return nil
	}

	listenAddresses := []string{address}

	// If not setup a notification for when the desired address or any of the event hubs connect.
	connected := make(chan struct{})
	listenersNotify[connected] = listenAddresses
	listenersLock.Unlock()

	defer func() {
		listenersLock.Lock()
		delete(listenersNotify, connected)
		listenersLock.Unlock()
	}()

	// Wait for the connected channel to be closed (indicating a new listener has been connected), and return.
	select {
	case <-connected:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// EventsUpdateListeners refreshes the cluster event listener connections.
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, f func(int64, api.Event)) {
	listenersUpdateLock.Lock()
	defer listenersUpdateLock.Unlock()

	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
				Roles:         dbMember.Roles,
			}
		}
	}

	networkAddress := endpoints.NetworkAddress()

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == networkAddress || !member.Online {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listener.IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listenersLock.Unlock()
				continue
			}

			// Disconnect and delete listener, but don't delete any listenersNotify entry as there
			// might be something waiting for a future connection.
			listener.Disconnect()
			delete(listeners, member.Address)
			logger.Info("Removed inactive member event listener client", log.Ctx{"local": networkAddress, "remote": member.Address})
		}
		listenersLock.Unlock()

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener client", log.Ctx{"local": networkAddress, "remote": m.Address, "err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) { f(m.ID, event) })

			listenersLock.Lock()
			listeners[m.Address] = listener

			// Indicate to any notifiers waiting for this member's address that it is connected.
			for connected, notifyAddresses := range listenersNotify {
				if shared.StringInSlice(m.Address, notifyAddresses) {
					close(connected)
					delete(listenersNotify, connected)
				}
			}

			logger.Info("Added member event listener client", log.Ctx{"local": networkAddress, "remote": m.Address})
			listenersLock.Unlock()
		}(member)
	}

	wg.Wait()

	// Disconnect and delete any out of date listeners and their notifiers.
	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)
			logger.Info("Removed old member event listener client", log.Ctx{"local": networkAddress, "remote": address})
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	revert := revert.New()
	revert.Add(func() {
		client.Disconnect()
	})

	listener, err := client.GetEventsAllProjects()
	if err != nil {
		return nil, err
	}

	revert.Success()
	return listener, nil
}
<commit_msg>lxd/cluster/events: Updates EventsUpdateListeners to only connect to event-hub servers<commit_after>package cluster

import (
	"context"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// eventHubMinHosts is the minimum number of members that must have the event-hub role to trigger switching into
// event-hub mode (where cluster members will only connect to event-hub members rather than all members when
// operating in the normal full-mesh mode).
const eventHubMinHosts = 2

// EventMode indicates the event distribution mode.
type EventMode string

// EventModeFullMesh is when every cluster member connects to every other cluster member to pull events.
const EventModeFullMesh EventMode = "full-mesh"

// EventModeHubServer is when the cluster is operating in event-hub mode and this server is designated as a hub
// server, meaning that it will only connect to the other event-hub members and not other members.
const EventModeHubServer EventMode = "hub-server"

// EventModeHubClient is when the cluster is operating in event-hub mode and this member is designated as a hub
// client, meaning that it is expected to connect to the event-hub members.
const EventModeHubClient EventMode = "hub-client"

var eventMode EventMode = EventModeFullMesh
var eventHubAddresses []string
var listeners = map[string]*lxd.EventListener{}
var listenersNotify = map[chan struct{}][]string{}
var listenersLock sync.Mutex
var listenersUpdateLock sync.Mutex

// ServerEventMode returns the event distribution mode that this local server is operating in.
func ServerEventMode() EventMode {
	listenersLock.Lock()
	defer listenersLock.Unlock()

	return eventMode
}

// RoleInSlice returns whether or not the rule is within the roles list.
func RoleInSlice(role db.ClusterRole, roles []db.ClusterRole) bool {
	for _, r := range roles {
		if r == role {
			return true
		}
	}

	return false
}

// EventListenerWait waits for there to be listener connected to the specified address, or one of the event hubs
// if operating in event hub mode.
func EventListenerWait(ctx context.Context, address string) error {
	// Check if there is already a listener.
	listenersLock.Lock()
	listener, found := listeners[address]
	if found && listener.IsActive() {
		listenersLock.Unlock()
		return nil
	}

	listenAddresses := []string{address}

	// If not setup a notification for when the desired address or any of the event hubs connect.
	connected := make(chan struct{})
	listenersNotify[connected] = listenAddresses
	listenersLock.Unlock()

	defer func() {
		listenersLock.Lock()
		delete(listenersNotify, connected)
		listenersLock.Unlock()
	}()

	// Wait for the connected channel to be closed (indicating a new listener has been connected), and return.
	select {
	case <-connected:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// hubAddresses returns the addresses of members with event-hub role, and the event mode of the server.
// The event mode will only be hub-server or hub-client if at least eventHubMinHosts have an event-hub role.
// Otherwise the mode will be full-mesh.
func hubAddresses(localAddress string, members map[int64]APIHeartbeatMember) ([]string, EventMode) {
	var hubAddresses []string
	var localHasHubRole bool

	// Do a first pass of members to count the members with event-hub role, and whether we are a hub server.
	for _, member := range members {
		if RoleInSlice(db.ClusterRoleEventHub, member.Roles) {
			hubAddresses = append(hubAddresses, member.Address)

			if member.Address == localAddress {
				localHasHubRole = true
			}
		}
	}

	eventMode := EventModeFullMesh
	if len(hubAddresses) >= eventHubMinHosts {
		if localHasHubRole {
			eventMode = EventModeHubServer
		} else {
			eventMode = EventModeHubClient
		}
	}

	return hubAddresses, eventMode
}

// EventsUpdateListeners refreshes the cluster event listener connections.
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, f func(int64, api.Event)) {
	listenersUpdateLock.Lock()
	defer listenersUpdateLock.Unlock()

	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
				Roles:         dbMember.Roles,
			}
		}
	}

	networkAddress := endpoints.NetworkAddress()
	hubAddresses, localEventMode := hubAddresses(networkAddress, members)

	// Store event hub addresses in global slice.
	listenersLock.Lock()
	eventHubAddresses = hubAddresses
	eventMode = localEventMode
	listenersLock.Unlock()

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == networkAddress || !member.Online {
			continue
		}

		if localEventMode != EventModeFullMesh && !RoleInSlice(db.ClusterRoleEventHub, member.Roles) {
			continue // Skip non-event-hub members if we are operating in event-hub mode.
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listener.IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listenersLock.Unlock()
				continue
			}

			// Disconnect and delete listener, but don't delete any listenersNotify entry as there
			// might be something waiting for a future connection.
			listener.Disconnect()
			delete(listeners, member.Address)
			logger.Info("Removed inactive member event listener client", log.Ctx{"local": networkAddress, "remote": member.Address})
		}
		listenersLock.Unlock()

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener client", log.Ctx{"local": networkAddress, "remote": m.Address, "err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) { f(m.ID, event) })

			listenersLock.Lock()
			listeners[m.Address] = listener

			// Indicate to any notifiers waiting for this member's address that it is connected.
			for connected, notifyAddresses := range listenersNotify {
				if shared.StringInSlice(m.Address, notifyAddresses) {
					close(connected)
					delete(listenersNotify, connected)
				}
			}

			logger.Info("Added member event listener client", log.Ctx{"local": networkAddress, "remote": m.Address})
			listenersLock.Unlock()
		}(member)
	}

	wg.Wait()

	// Disconnect and delete any out of date listeners and their notifiers.
	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)
			logger.Info("Removed old member event listener client", log.Ctx{"local": networkAddress, "remote": address})
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	revert := revert.New()
	revert.Add(func() {
		client.Disconnect()
	})

	listener, err := client.GetEventsAllProjects()
	if err != nil {
		return nil, err
	}

	revert.Success()
	return listener, nil
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"

	_ "github.com/mattn/go-sqlite3"
)

type Backup struct {
	ID       int    `json:"id"`
	Name     string `json:"name"`
	Started  string `json:"started"`
	Finished string `json:"finished"`
	Duration string `json:"duration"`
	Status   string `json:"status"`
}

type BackupCollection struct {
	Backups []Backup `json:"items"`
}

func GetBackups(db *sql.DB) BackupCollection {
	sql := "SELECT * FROM backups"
	rows, err := db.Query(sql)

	if err != nil {
		panic(err)
	}
	// Cleanup if exit
	defer rows.Close()

	result := BackupCollection{}

	for rows.Next() {
		backup := Backup{}
		err2 := rows.Scan(&backup.ID, &backup.Name)

		if err2 != nil {
			panic(err2)
		}

		result.Backups = append(result.Backups, backup)
	}
	return result
}

func PutBackup(db *sql.DB, name string, starting string, finished string, duration string, status string) (int64, error) {
	sql := "INSERT INTO backups(name, starting, finished, duration, status) VALUES(?,?,?,?,?)"

	// Create prepared sql statement
	stmt, err := db.Prepare(sql)
	if err != nil {
		panic(err)
	}

	defer stmt.Close()

	result, err2 := stmt.Exec(name)

	if err2 != nil {
		panic(err)
	}

	return result.LastInsertId()

}
<commit_msg>🐛 Fix PutBackup and GetBackup on Backup Model<commit_after>package models

import (
	"database/sql"

	_ "github.com/mattn/go-sqlite3"
)

type Backup struct {
	ID       int    `json:"id"`
	Name     string `json:"name"`
	Started  string `json:"started"`
	Finished string `json:"finished"`
	Duration string `json:"duration"`
	Status   string `json:"status"`
}

type BackupCollection struct {
	Backups []Backup `json:"items"`
}

func GetBackups(db *sql.DB) BackupCollection {
	sql := "SELECT * FROM backups"
	rows, err := db.Query(sql)

	if err != nil {
		panic(err)
	}
	// Cleanup if exit
	defer rows.Close()

	result := BackupCollection{}

	for rows.Next() {
		backup := Backup{}
		err2 := rows.Scan(&backup.ID, &backup.Name, &backup.Started, &backup.Finished, &backup.Duration, &backup.Status)

		if err2 != nil {
			panic(err2)
		}

		result.Backups = append(result.Backups, backup)
	}
	return result
}

func PutBackup(db *sql.DB, name string, starting string, finished string, duration string, status string) (int64, error) {
	sql := "INSERT INTO backups(name, starting, finished, duration, status) VALUES(?, ?, ?, ?, ?)"

	// Create prepared sql statement
	stmt, err := db.Prepare(sql)
	if err != nil {
		panic(err)
	}

	defer stmt.Close()

	result, err2 := stmt.Exec(name, starting, finished, duration, status)

	if err2 != nil {
		panic(err)
	}

	return result.LastInsertId()

}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"github.com/Shopify/sarama"
)

var (
	DefaultMessageList *MessageManager
	kafkaTopic      = "franz"
	kafkaBroker     = "localhost:9092"
)

type Message struct {
	ID    int64  // Unique identifier
	Title string // Description
	Done  bool   // Is this Message done?
}

// NewMessage creates a new message given a title, that can't be empty.
func NewMessage(title string) (*Message, error) {
	if title == "" {
		return nil, fmt.Errorf("empty title")
	}
	return &Message{0, title, false}, nil
}

// MessageManager manages a list of messages in memory.
type MessageManager struct {
	messages  []*Message
	lastID int64
}

// NewMessageManager returns an empty MessageManager.
func NewMessageManager() *MessageManager {
	return &MessageManager{}
}

// Save saves the given Message in the MessageManager.
func (m *MessageManager) Save(message *Message) error {
	if message.ID == 0 {
		m.lastID++
		message.ID = m.lastID
		m.messages = append(m.messages, cloneMessage(message))
		return nil
	}

	for i, t := range m.messages {
		if t.ID == message.ID {
			m.messages[i] = cloneMessage(message)
			return nil
		}
	}
	return fmt.Errorf("unknown message")
}

// cloneMessage creates and returns a deep copy of the given Message.
func cloneMessage(t *Message) *Message {
	c := *t
	return &c
}

// All returns the list of all the Messages in the MessageManager.
func (m *MessageManager) All() []*Message {
	return m.messages
}

// Find returns the Message with the given id in the MessageManager and a boolean
// indicating if the id was found.
func (m *MessageManager) Find(ID int64) (*Message, bool) {
	for _, t := range m.messages {
		if t.ID == ID {
			return t, true
		}
	}
	return nil, false
}

func (m *MessageManager) Send(message *Message) error {
	kafkaMessage := &sarama.ProducerMessage{
		Topic: kafkaTopic,
		Key:   nil,
		Value: sarama.StringEncoder(message.Title),
	}

	kafkaProducer, err := sarama.NewSyncProducer([]string{kafkaBroker}, nil)
	if err != nil {
		//TODO Cosmin
	}
	defer func() {
		if errClose := kafkaProducer.Close(); errClose != nil {
			//TODO Cosmin
		}
	}()

	_, _, errSend := kafkaProducer.SendMessage(kafkaMessage)
	return errSend
}

func init() {
	DefaultMessageList = NewMessageManager()
}
<commit_msg>adding error handling at message sending<commit_after>package models

import (
	"fmt"
	"github.com/Shopify/sarama"
	"github.com/astaxie/beego"
)

var (
	DefaultMessageList *MessageManager
	kafkaTopic      = "franz"
	kafkaBroker     = "localhost:9092"
)

type Message struct {
	ID    int64  // Unique identifier
	Title string // Description
	Done  bool   // Is this Message done?
}

// NewMessage creates a new message given a title, that can't be empty.
func NewMessage(title string) (*Message, error) {
	if title == "" {
		return nil, fmt.Errorf("empty title")
	}
	return &Message{0, title, false}, nil
}

// MessageManager manages a list of messages in memory.
type MessageManager struct {
	messages  []*Message
	lastID int64
}

// NewMessageManager returns an empty MessageManager.
func NewMessageManager() *MessageManager {
	return &MessageManager{}
}

// Save saves the given Message in the MessageManager.
func (m *MessageManager) Save(message *Message) error {
	if message.ID == 0 {
		m.lastID++
		message.ID = m.lastID
		m.messages = append(m.messages, cloneMessage(message))
		return nil
	}

	for i, t := range m.messages {
		if t.ID == message.ID {
			m.messages[i] = cloneMessage(message)
			return nil
		}
	}
	return fmt.Errorf("unknown message")
}

// cloneMessage creates and returns a deep copy of the given Message.
func cloneMessage(t *Message) *Message {
	c := *t
	return &c
}

// All returns the list of all the Messages in the MessageManager.
func (m *MessageManager) All() []*Message {
	return m.messages
}

// Find returns the Message with the given id in the MessageManager and a boolean
// indicating if the id was found.
func (m *MessageManager) Find(ID int64) (*Message, bool) {
	for _, t := range m.messages {
		if t.ID == ID {
			return t, true
		}
	}
	return nil, false
}

func (m *MessageManager) Send(message *Message) error {
	kafkaMessage := &sarama.ProducerMessage{
		Topic: kafkaTopic,
		Key:   nil,
		Value: sarama.StringEncoder(message.Title),
	}

	kafkaProducer, err := sarama.NewSyncProducer([]string{kafkaBroker}, nil)
	if err != nil {
		beego.Error("error when creating Kafka SyncProducer", err)
	}
	defer func() {
		if errClose := kafkaProducer.Close(); errClose != nil {
			beego.Error("error when closing Kafka SyncProducer", errClose)
		}
	}()

	_, _, errSend := kafkaProducer.SendMessage(kafkaMessage)
	return errSend
}

func init() {
	DefaultMessageList = NewMessageManager()
}
<|endoftext|>


<commit_before>// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package models

import (
	"errors"
	// "fmt"
	"os"
	"path"
	"strings"
	"time"

	"github.com/Unknwon/com"

	"github.com/gpmgo/switch/modules/archive"
	"github.com/gpmgo/switch/modules/log"
	"github.com/gpmgo/switch/modules/qiniu"
	"github.com/gpmgo/switch/modules/setting"
)

var (
	ErrRevisionIsLocal  = errors.New("Revision archive is in local")
	ErrPackageNotExist  = errors.New("Package does not exist")
	ErrRevisionNotExist = errors.New("Revision does not exist")
)

type Storage int

const (
	LOCAL Storage = iota
	QINIU
)

// Revision represents a revision of a Go package.
type Revision struct {
	Id       int64
	PkgId    int64    `xorm:"UNIQUE(s)"`
	Pkg      *Package `xorm:"-"`
	Revision string   `xorm:"UNIQUE(s)"`
	Storage
	Size    int64
	Updated time.Time `xorm:"UPDATED"`
}

func (r *Revision) GetPackage() (err error) {
	if r.Pkg != nil {
		return nil
	}
	r.Pkg, err = GetPakcageById(r.PkgId)
	return err
}

// KeyName returns QiNiu key name.
func (r *Revision) KeyName() (string, error) {
	if r.Storage == LOCAL {
		return "", ErrRevisionIsLocal
	}
	if err := r.GetPackage(); err != nil {
		return "", err
	}
	return r.Pkg.ImportPath + "-" + r.Revision + archive.GetExtension(r.Pkg.ImportPath), nil
}

// GetRevision returns revision by given pakcage ID and revision.
func GetRevision(pkgId int64, rev string) (*Revision, error) {
	r := &Revision{
		PkgId:    pkgId,
		Revision: rev,
	}
	has, err := x.Get(r)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrRevisionNotExist
	}
	return r, nil
}

// UpdateRevision updates revision information.
func UpdateRevision(rev *Revision) error {
	_, err := x.Id(rev.Id).Update(rev)
	return err
}

// DeleteRevisionById delete revision by given ID.
func DeleteRevisionById(revId int64) error {
	_, err := x.Id(revId).Delete(new(Revision))
	return err
}

// GetLocalRevisions returns all revisions that archives are saved locally.
func GetLocalRevisions() ([]*Revision, error) {
	revs := make([]*Revision, 0, 10)
	err := x.Where("storage=0").Find(&revs)
	return revs, err
}

// GetRevisionsByPkgId returns a list of revisions of given package ID.
func GetRevisionsByPkgId(pkgId int64) ([]*Revision, error) {
	revs := make([]*Revision, 0, 10)
	err := x.Where("pkg_id=?", pkgId).Find(&revs)
	return revs, err
}

// Package represents a Go package.
type Package struct {
	Id             int64
	ImportPath     string `xorm:"UNIQUE"`
	Description    string
	Homepage       string
	Issues         string
	DownloadCount  int64
	RecentDownload int64
	IsValidated    bool      `xorm:"DEFAULT 0"`
	Created        time.Time `xorm:"CREATED"`
}

func (pkg *Package) GetRevisions() ([]*Revision, error) {
	return GetRevisionsByPkgId(pkg.Id)
}

// NewPackage creates
func NewPackage(importPath string) (*Package, error) {
	pkg := &Package{
		ImportPath: importPath,
	}
	if _, err := x.Insert(pkg); err != nil {
		return nil, err
	}
	return pkg, nil
}

// GetPakcageById returns a package by given ID.
func GetPakcageById(pkgId int64) (*Package, error) {
	pkg := &Package{}
	has, err := x.Id(pkgId).Get(pkg)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrPackageNotExist
	}
	return pkg, nil
}

// GetPakcageByPath returns a package by given import path.
func GetPakcageByPath(importPath string) (*Package, error) {
	pkg := &Package{
		ImportPath: importPath,
	}
	has, err := x.Get(pkg)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrPackageNotExist
	}
	return pkg, nil
}

// CheckPkg checks if versioned package is in records, and download it when needed.
func CheckPkg(importPath, rev string) (*Revision, error) {
	// Check package record.
	pkg, err := GetPakcageByPath(importPath)
	if err != nil {
		if err != ErrPackageNotExist {
			return nil, err
		}
		blocked, blockErr, err := IsPackageBlocked(importPath)
		if err != nil {
			return nil, err
		} else if blocked {
			return nil, blockErr
		}
	}

	n := archive.NewNode(importPath, rev)

	// Get and check revision record.
	if err = n.GetRevision(); err != nil {
		return nil, err
	}

	var r *Revision
	if pkg != nil {
		r, err = GetRevision(pkg.Id, n.Revision)
		if err != nil && err != ErrRevisionNotExist {
			return nil, err
		}
	}

	// return nil, fmt.Errorf("Revision: %s", n.Revision)

	if r == nil || (r.Storage == LOCAL && !com.IsFile(n.ArchivePath)) {
		if err := n.Download(); err != nil {
			return nil, err
		}
	}

	if pkg == nil {
		pkg, err = NewPackage(n.ImportPath)
		if err != nil {
			return nil, err
		}
	}

	if r == nil {
		r = &Revision{
			PkgId:    pkg.Id,
			Revision: n.Revision,
		}
		_, err = x.Insert(r)
	} else {
		_, err = x.Id(r.Id).Update(r)
	}
	return r, nil
}

// IncreasePackageDownloadCount increase package download count by 1.
func IncreasePackageDownloadCount(importPath string) error {
	pkg, err := GetPakcageByPath(importPath)
	if err != nil {
		return err
	}
	pkg.DownloadCount++
	pkg.RecentDownload++
	_, err = x.Id(pkg.Id).Update(pkg)
	return err
}

// SearchPackages searchs packages by given keyword.
func SearchPackages(keys string) ([]*Package, error) {
	keys = strings.TrimSpace(keys)
	if len(keys) == 0 {
		return nil, nil
	}
	key := strings.Split(keys, " ")[0]
	if len(key) == 0 {
		return nil, nil
	}

	pkgs := make([]*Package, 0, 50)
	err := x.Limit(50).Where("name like '%" + keys + "%'").Find(&pkgs)
	return pkgs, err
}

const _EXPIRE_DURATION = -1 * 24 * 30 * 3 * time.Hour

func cleanExpireRevesions() {
	if err := x.Where("updated<?", time.Now().Add(_EXPIRE_DURATION)).
		Iterate(new(Revision), func(idx int, bean interface{}) (err error) {
		rev := bean.(*Revision)
		if err = rev.GetPackage(); err != nil {
			return err
		}

		if _, err = x.Id(rev.Id).Delete(new(Revision)); err != nil {
			return err
		}

		ext := archive.GetExtension(rev.Pkg.ImportPath)
		fpath := path.Join(setting.ArchivePath, rev.Pkg.ImportPath, rev.Revision+ext)

		switch rev.Storage {
		case LOCAL:
			os.Remove(fpath)
			log.Info("Revision deleted: %s", fpath)
			return nil
		case QINIU:
			key, err := rev.KeyName()
			if err != nil {
				return err
			}
			if setting.ProdMode {
				if err = qiniu.DeleteArchive(key); err != nil {
					return err
				}
			}
			log.Info("Revision deleted: %s", key)
			return nil
		default:
			return nil
		}

		return nil
	}); err != nil {
		log.Error(4, "Fail to clean expire revisions: %v", err)
	}
}
<commit_msg>models/package.go: fix log depth<commit_after>// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package models

import (
	"errors"
	// "fmt"
	"os"
	"path"
	"strings"
	"time"

	"github.com/Unknwon/com"

	"github.com/gpmgo/switch/modules/archive"
	"github.com/gpmgo/switch/modules/log"
	"github.com/gpmgo/switch/modules/qiniu"
	"github.com/gpmgo/switch/modules/setting"
)

var (
	ErrRevisionIsLocal  = errors.New("Revision archive is in local")
	ErrPackageNotExist  = errors.New("Package does not exist")
	ErrRevisionNotExist = errors.New("Revision does not exist")
)

type Storage int

const (
	LOCAL Storage = iota
	QINIU
)

// Revision represents a revision of a Go package.
type Revision struct {
	Id       int64
	PkgId    int64    `xorm:"UNIQUE(s)"`
	Pkg      *Package `xorm:"-"`
	Revision string   `xorm:"UNIQUE(s)"`
	Storage
	Size    int64
	Updated time.Time `xorm:"UPDATED"`
}

func (r *Revision) GetPackage() (err error) {
	if r.Pkg != nil {
		return nil
	}
	r.Pkg, err = GetPakcageById(r.PkgId)
	return err
}

// KeyName returns QiNiu key name.
func (r *Revision) KeyName() (string, error) {
	if r.Storage == LOCAL {
		return "", ErrRevisionIsLocal
	}
	if err := r.GetPackage(); err != nil {
		return "", err
	}
	return r.Pkg.ImportPath + "-" + r.Revision + archive.GetExtension(r.Pkg.ImportPath), nil
}

// GetRevision returns revision by given pakcage ID and revision.
func GetRevision(pkgId int64, rev string) (*Revision, error) {
	r := &Revision{
		PkgId:    pkgId,
		Revision: rev,
	}
	has, err := x.Get(r)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrRevisionNotExist
	}
	return r, nil
}

// UpdateRevision updates revision information.
func UpdateRevision(rev *Revision) error {
	_, err := x.Id(rev.Id).Update(rev)
	return err
}

// DeleteRevisionById delete revision by given ID.
func DeleteRevisionById(revId int64) error {
	_, err := x.Id(revId).Delete(new(Revision))
	return err
}

// GetLocalRevisions returns all revisions that archives are saved locally.
func GetLocalRevisions() ([]*Revision, error) {
	revs := make([]*Revision, 0, 10)
	err := x.Where("storage=0").Find(&revs)
	return revs, err
}

// GetRevisionsByPkgId returns a list of revisions of given package ID.
func GetRevisionsByPkgId(pkgId int64) ([]*Revision, error) {
	revs := make([]*Revision, 0, 10)
	err := x.Where("pkg_id=?", pkgId).Find(&revs)
	return revs, err
}

// Package represents a Go package.
type Package struct {
	Id             int64
	ImportPath     string `xorm:"UNIQUE"`
	Description    string
	Homepage       string
	Issues         string
	DownloadCount  int64
	RecentDownload int64
	IsValidated    bool      `xorm:"DEFAULT 0"`
	Created        time.Time `xorm:"CREATED"`
}

func (pkg *Package) GetRevisions() ([]*Revision, error) {
	return GetRevisionsByPkgId(pkg.Id)
}

// NewPackage creates
func NewPackage(importPath string) (*Package, error) {
	pkg := &Package{
		ImportPath: importPath,
	}
	if _, err := x.Insert(pkg); err != nil {
		return nil, err
	}
	return pkg, nil
}

// GetPakcageById returns a package by given ID.
func GetPakcageById(pkgId int64) (*Package, error) {
	pkg := &Package{}
	has, err := x.Id(pkgId).Get(pkg)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrPackageNotExist
	}
	return pkg, nil
}

// GetPakcageByPath returns a package by given import path.
func GetPakcageByPath(importPath string) (*Package, error) {
	pkg := &Package{
		ImportPath: importPath,
	}
	has, err := x.Get(pkg)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrPackageNotExist
	}
	return pkg, nil
}

// CheckPkg checks if versioned package is in records, and download it when needed.
func CheckPkg(importPath, rev string) (*Revision, error) {
	// Check package record.
	pkg, err := GetPakcageByPath(importPath)
	if err != nil {
		if err != ErrPackageNotExist {
			return nil, err
		}
		blocked, blockErr, err := IsPackageBlocked(importPath)
		if err != nil {
			return nil, err
		} else if blocked {
			return nil, blockErr
		}
	}

	n := archive.NewNode(importPath, rev)

	// Get and check revision record.
	if err = n.GetRevision(); err != nil {
		return nil, err
	}

	var r *Revision
	if pkg != nil {
		r, err = GetRevision(pkg.Id, n.Revision)
		if err != nil && err != ErrRevisionNotExist {
			return nil, err
		}
	}

	// return nil, fmt.Errorf("Revision: %s", n.Revision)

	if r == nil || (r.Storage == LOCAL && !com.IsFile(n.ArchivePath)) {
		if err := n.Download(); err != nil {
			return nil, err
		}
	}

	if pkg == nil {
		pkg, err = NewPackage(n.ImportPath)
		if err != nil {
			return nil, err
		}
	}

	if r == nil {
		r = &Revision{
			PkgId:    pkg.Id,
			Revision: n.Revision,
		}
		_, err = x.Insert(r)
	} else {
		_, err = x.Id(r.Id).Update(r)
	}
	return r, nil
}

// IncreasePackageDownloadCount increase package download count by 1.
func IncreasePackageDownloadCount(importPath string) error {
	pkg, err := GetPakcageByPath(importPath)
	if err != nil {
		return err
	}
	pkg.DownloadCount++
	pkg.RecentDownload++
	_, err = x.Id(pkg.Id).Update(pkg)
	return err
}

// SearchPackages searchs packages by given keyword.
func SearchPackages(keys string) ([]*Package, error) {
	keys = strings.TrimSpace(keys)
	if len(keys) == 0 {
		return nil, nil
	}
	key := strings.Split(keys, " ")[0]
	if len(key) == 0 {
		return nil, nil
	}

	pkgs := make([]*Package, 0, 50)
	err := x.Limit(50).Where("name like '%" + keys + "%'").Find(&pkgs)
	return pkgs, err
}

const _EXPIRE_DURATION = -1 * 24 * 30 * 3 * time.Hour

func cleanExpireRevesions() {
	if err := x.Where("updated<?", time.Now().Add(_EXPIRE_DURATION)).
		Iterate(new(Revision), func(idx int, bean interface{}) (err error) {
		rev := bean.(*Revision)
		if err = rev.GetPackage(); err != nil {
			return err
		}

		if _, err = x.Id(rev.Id).Delete(new(Revision)); err != nil {
			return err
		}

		ext := archive.GetExtension(rev.Pkg.ImportPath)
		fpath := path.Join(setting.ArchivePath, rev.Pkg.ImportPath, rev.Revision+ext)

		switch rev.Storage {
		case LOCAL:
			os.Remove(fpath)
			log.Info("Revision deleted: %s", fpath)
			return nil
		case QINIU:
			key, err := rev.KeyName()
			if err != nil {
				return err
			}
			if setting.ProdMode {
				if err = qiniu.DeleteArchive(key); err != nil {
					return err
				}
			}
			log.Info("Revision deleted: %s", key)
			return nil
		default:
			return nil
		}

		return nil
	}); err != nil {
		log.Error(3, "Fail to clean expire revisions: %v", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
A trivial example of wrapping a C library in Go.
For a more complex example and explanation,
see ../gmp/gmp.go.
*/

package stdio

/*
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <errno.h>

char* greeting = "hello, world";
*/
import "C"
import "unsafe"

type File C.FILE

var Stdout = (*File)(C.stdout)
var Stderr = (*File)(C.stderr)

// Test reference to library symbol.
// Stdout and stderr are too special to be a reliable test.
var myerr = C.sys_errlist

func (f *File) WriteString(s string) {
	p := C.CString(s)
	C.fputs(p, (*C.FILE)(f))
	C.free(unsafe.Pointer(p))
	f.Flush()
}

func (f *File) Flush() {
	C.fflush((*C.FILE)(f))
}

var Greeting = C.GoString(C.greeting)
<commit_msg>go/build: fix windows build by commenting out references to stdout and stderr in cgotest<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
A trivial example of wrapping a C library in Go.
For a more complex example and explanation,
see ../gmp/gmp.go.
*/

package stdio

/*
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <errno.h>

char* greeting = "hello, world";
*/
import "C"
import "unsafe"

type File C.FILE

// TODO(brainman): uncomment once stdout and stderr references are working on Windows.
//var Stdout = (*File)(C.stdout)
//var Stderr = (*File)(C.stderr)

// Test reference to library symbol.
// Stdout and stderr are too special to be a reliable test.
var myerr = C.sys_errlist

func (f *File) WriteString(s string) {
	p := C.CString(s)
	C.fputs(p, (*C.FILE)(f))
	C.free(unsafe.Pointer(p))
	f.Flush()
}

func (f *File) Flush() {
	C.fflush((*C.FILE)(f))
}

var Greeting = C.GoString(C.greeting)
<|endoftext|>


<commit_before>package mpeg4file

type mdat struct{
	size uint32
	largeSize uint64
	boxtype uint32
	data []byte
}

func NewMdat (s uint32, payload []byte) *mdat{
	newMdat:=new(mdat)
	newMdat.size=s
	newMdat.data = payload
	return newMdat
}

func NewMdatLargeSize (s uint64, payload []byte) *mdat{
	newMdat:=new(mdat)
	newMdat.size=1
	newMdat.largeSize = s
	newMdat.data = payload
	return newMdat
}

func (m *mdat) SetSize (s uint64){
	if s>4294967295 {
		m.size = uint32(s)
	}else{
		m.size = 1
		m.largeSize = s
	}
}<commit_msg>updated mdat<commit_after>package mpeg4file

type mdat struct{
	size uint32
	largeSize uint64
	boxtype uint32
	data []byte
}

func NewMdat (s uint64, payload []byte) *mdat{
	newMdat:=new(mdat)
	newMdat.SetSize(s)
	newMdat.data = payload
	return newMdat
}

func (m *mdat) SetSize (s uint64){
	if s==0{
		m.size=0
	} else {
		if s>4294967295 {
			m.size = uint32(s)
		}else{
			m.size = 1
			m.largeSize = s
		}
	}
}<|endoftext|>


<commit_before>package gonameparts

import (
	"fmt"
	"testing"
)

func TestLooksCorporate(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Sprockets Inc"}

	res := n.looksCorporate()

	if res != true {
		t.Errorf("Expected true.  Actual %v", res)
	}

}

func TestSearchParts(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Mr. James Polera"}

	res := n.searchParts(&salutations)

	if res != 0 {
		t.Errorf("Expected true.  Actual %v", res)
	}

}

func TestClean(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Mr. James Polera"}

	res := n.cleaned()

	if res[0] != "Mr" {
		t.Errorf("Expected 'Mr'.  Actual %v", res[0])
	}

}

func TestLocateSalutation(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Mr. James Polera"}

	res := n.find("salutation")

	if res != 0 {
		t.Errorf("Expected 0.  Actual %v", res)
	}
}

func TestHasComma(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Polera, James"}
	res := n.hasComma()

	if res != true {
		t.Errorf("Expected true.  Actual %v", res)
	}

}

func TestNormalize(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Polera, James"}
	res := n.normalize()

	if res[0] != "James" {
		t.Errorf("Expected James.  Actual %v", res[0])
	}

	if res[1] != "Polera" {
		t.Errorf("Expected Polera.  Actual %v", res[1])
	}

}

func TestParseAllFields(t *testing.T) {
	t.Parallel()
	res := Parse("Mr. James J. Polera Jr. Esq.")

	if res.Salutation != "Mr." {
		t.Errorf("Expected 'Mr.'.  Actual %v", res.Salutation)
	}

	if res.FirstName != "James" {
		t.Errorf("Expected 'James'.  Actual %v", res.FirstName)
	}

	if res.MiddleName != "J." {
		t.Errorf("Expected 'J.'.  Actual %v", res.MiddleName)
	}

	if res.LastName != "Polera" {
		t.Errorf("Expected 'Polera'.  Actual %v", res.LastName)
	}

	if res.Generation != "Jr." {
		t.Errorf("Expected 'Jr.'.  Actual %v", res.Generation)
	}

	if res.Suffix != "Esq." {
		t.Errorf("Expected 'Esq.'.  Actual %v", res.Suffix)
	}
}

func TestParseFirstLast(t *testing.T) {
	t.Parallel()

	res := Parse("James Polera")
	if res.FirstName != "James" {
		t.Errorf("Expected 'James'.  Actual %v", res.FirstName)
	}

	if res.LastName != "Polera" {
		t.Errorf("Expected 'Polera'.  Actual %v", res.LastName)
	}
}

func TestLastNamePrefix(t *testing.T) {
	t.Parallel()

	res := Parse("Otto von Bismark")

	if res.FirstName != "Otto" {
		t.Errorf("Expected 'Otto'.  Actual %v", res.FirstName)
	}

	if res.LastName != "von Bismark" {
		t.Errorf("Expected 'von Bismark'.  Actual %v", res.LastName)
	}

}

func TestAliases(t *testing.T) {
	t.Parallel()

	res := Parse("James Polera a/k/a Batman")

	if res.Aliases[0].FirstName != "Batman" {
		t.Errorf("Expected 'Batman'.  Actual: %v", res.Aliases[0].FirstName)
	}

}

func TestNickname(t *testing.T) {
	t.Parallel()

	res := Parse("Philip Francis 'The Scooter' Rizzuto")

	if res.Nickname != "'The Scooter'" {
		t.Errorf("Expected 'The Scooter'.  Actual: %v", res.Nickname)
	}
}

func TestStripSupplemental(t *testing.T) {
	t.Parallel()

	res := Parse("Philip Francis 'The Scooter' Rizzuto, deceased")

	if res.FirstName != "Philip" {
		t.Errorf("Expected 'Philip'.  Actual: %v", res.FirstName)
	}

	if res.MiddleName != "Francis" {
		t.Errorf("Expected 'Francis'.  Actual: %v", res.MiddleName)
	}

	if res.Nickname != "'The Scooter'" {
		t.Errorf("Expected 'The Scooter'.  Actual: %v", res.Nickname)
	}

	if res.LastName != "Rizzuto" {
		t.Errorf("Expected 'Rizzuto'.  Actual: %v", res.LastName)
	}
}

func TestLongPrefixedLastName(t *testing.T) {
	t.Parallel()

	res := Parse("Saleh ibn Tariq ibn Khalid al-Fulan")

	if res.FirstName != "Saleh" {
		t.Errorf("Expected 'Saleh'.  Actual: %v", res.FirstName)
	}

	if res.LastName != "ibn Tariq ibn Khalid al-Fulan" {
		t.Errorf("Expected 'ibn Tariq ibn Khalid al-Fulan'.  Actual: %v", res.LastName)

	}
}

func TestMisplacedApostrophe(t *testing.T) {
	t.Parallel()

	res := Parse("John O' Hurley")

	if res.FirstName != "John" {
		t.Errorf("Expected 'John'.  Actual: %v", res.FirstName)
	}

	if res.LastName != "O'Hurley" {
		t.Errorf("Expected 'O'Hurley'.  Actual: %v", res.LastName)
	}

}

func TestMultipleAKA(t *testing.T) {
	t.Parallel()

	res := Parse("Tony Stark a/k/a Ironman a/k/a Stark, Anthony a/k/a Anthony Edward \"Tony\" Stark")

	if len(res.Aliases) != 3 {
		t.Errorf("Expected 3 aliases.  Actual: %v", len(res.Aliases))
	}

	if res.FirstName != "Tony" {
		t.Errorf("Expected 'Tony'.  Actual: %v", res.FirstName)
	}

	if res.LastName != "Stark" {
		t.Errorf("Expected 'Stark'.  Actual: %v", res.LastName)
	}

}

func TestBuildFullName(t *testing.T) {
	res := Parse("President George Herbert Walker Bush")

	if res.FullName != "President George Herbert Walker Bush" {

		t.Errorf("Expected 'President George Herbert Walker Bush'.  Actual: %v", res.FullName)
	}

}

func TestDottedAka(t *testing.T) {
	res := Parse("James Polera a.k.a James K. Polera")
	if len(res.Aliases) != 1 {
		t.Errorf("Expected 1 alias.  Actual: %v", len(res.Aliases))
	}
}

func TestUnicodeCharsInName(t *testing.T) {
	res := Parse("König Ludwig")

	if res.FirstName != "König" {
		t.Errorf("Expected 'König'.  Actual: %v", res.FirstName)

	}
}

func ExampleParse() {
	res := Parse("Thurston Howell III")
	fmt.Println("FirstName:", res.FirstName)
	fmt.Println("LastName:", res.LastName)
	fmt.Println("Generation:", res.Generation)

	// Output:
	// FirstName: Thurston
	// LastName: Howell
	// Generation: III

}

func ExampleParse_second() {

	res := Parse("President George Herbert Walker Bush")
	fmt.Println("Salutation:", res.Salutation)
	fmt.Println("FirstName:", res.FirstName)
	fmt.Println("MiddleName:", res.MiddleName)
	fmt.Println("LastName:", res.LastName)

	// Output:
	// Salutation: President
	// FirstName: George
	// MiddleName: Herbert Walker
	// LastName: Bush

}
<commit_msg>Added tabs in name test<commit_after>package gonameparts

import (
	"fmt"
	"testing"
)

func TestLooksCorporate(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Sprockets Inc"}

	res := n.looksCorporate()

	if res != true {
		t.Errorf("Expected true.  Actual %v", res)
	}

}

func TestSearchParts(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Mr. James Polera"}

	res := n.searchParts(&salutations)

	if res != 0 {
		t.Errorf("Expected true.  Actual %v", res)
	}

}

func TestClean(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Mr. James Polera"}

	res := n.cleaned()

	if res[0] != "Mr" {
		t.Errorf("Expected 'Mr'.  Actual %v", res[0])
	}

}

func TestLocateSalutation(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Mr. James Polera"}

	res := n.find("salutation")

	if res != 0 {
		t.Errorf("Expected 0.  Actual %v", res)
	}
}

func TestHasComma(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Polera, James"}
	res := n.hasComma()

	if res != true {
		t.Errorf("Expected true.  Actual %v", res)
	}

}

func TestNormalize(t *testing.T) {
	t.Parallel()
	n := nameString{FullName: "Polera, James"}
	res := n.normalize()

	if res[0] != "James" {
		t.Errorf("Expected James.  Actual %v", res[0])
	}

	if res[1] != "Polera" {
		t.Errorf("Expected Polera.  Actual %v", res[1])
	}

}

func TestParseAllFields(t *testing.T) {
	t.Parallel()
	res := Parse("Mr. James J. Polera Jr. Esq.")

	if res.Salutation != "Mr." {
		t.Errorf("Expected 'Mr.'.  Actual %v", res.Salutation)
	}

	if res.FirstName != "James" {
		t.Errorf("Expected 'James'.  Actual %v", res.FirstName)
	}

	if res.MiddleName != "J." {
		t.Errorf("Expected 'J.'.  Actual %v", res.MiddleName)
	}

	if res.LastName != "Polera" {
		t.Errorf("Expected 'Polera'.  Actual %v", res.LastName)
	}

	if res.Generation != "Jr." {
		t.Errorf("Expected 'Jr.'.  Actual %v", res.Generation)
	}

	if res.Suffix != "Esq." {
		t.Errorf("Expected 'Esq.'.  Actual %v", res.Suffix)
	}
}

func TestParseFirstLast(t *testing.T) {
	t.Parallel()

	res := Parse("James Polera")
	if res.FirstName != "James" {
		t.Errorf("Expected 'James'.  Actual %v", res.FirstName)
	}

	if res.LastName != "Polera" {
		t.Errorf("Expected 'Polera'.  Actual %v", res.LastName)
	}
}

func TestLastNamePrefix(t *testing.T) {
	t.Parallel()

	res := Parse("Otto von Bismark")

	if res.FirstName != "Otto" {
		t.Errorf("Expected 'Otto'.  Actual %v", res.FirstName)
	}

	if res.LastName != "von Bismark" {
		t.Errorf("Expected 'von Bismark'.  Actual %v", res.LastName)
	}

}

func TestAliases(t *testing.T) {
	t.Parallel()

	res := Parse("James Polera a/k/a Batman")

	if res.Aliases[0].FirstName != "Batman" {
		t.Errorf("Expected 'Batman'.  Actual: %v", res.Aliases[0].FirstName)
	}

}

func TestNickname(t *testing.T) {
	t.Parallel()

	res := Parse("Philip Francis 'The Scooter' Rizzuto")

	if res.Nickname != "'The Scooter'" {
		t.Errorf("Expected 'The Scooter'.  Actual: %v", res.Nickname)
	}
}

func TestStripSupplemental(t *testing.T) {
	t.Parallel()

	res := Parse("Philip Francis 'The Scooter' Rizzuto, deceased")

	if res.FirstName != "Philip" {
		t.Errorf("Expected 'Philip'.  Actual: %v", res.FirstName)
	}

	if res.MiddleName != "Francis" {
		t.Errorf("Expected 'Francis'.  Actual: %v", res.MiddleName)
	}

	if res.Nickname != "'The Scooter'" {
		t.Errorf("Expected 'The Scooter'.  Actual: %v", res.Nickname)
	}

	if res.LastName != "Rizzuto" {
		t.Errorf("Expected 'Rizzuto'.  Actual: %v", res.LastName)
	}
}

func TestLongPrefixedLastName(t *testing.T) {
	t.Parallel()

	res := Parse("Saleh ibn Tariq ibn Khalid al-Fulan")

	if res.FirstName != "Saleh" {
		t.Errorf("Expected 'Saleh'.  Actual: %v", res.FirstName)
	}

	if res.LastName != "ibn Tariq ibn Khalid al-Fulan" {
		t.Errorf("Expected 'ibn Tariq ibn Khalid al-Fulan'.  Actual: %v", res.LastName)

	}
}

func TestMisplacedApostrophe(t *testing.T) {
	t.Parallel()

	res := Parse("John O' Hurley")

	if res.FirstName != "John" {
		t.Errorf("Expected 'John'.  Actual: %v", res.FirstName)
	}

	if res.LastName != "O'Hurley" {
		t.Errorf("Expected 'O'Hurley'.  Actual: %v", res.LastName)
	}

}

func TestMultipleAKA(t *testing.T) {
	t.Parallel()

	res := Parse("Tony Stark a/k/a Ironman a/k/a Stark, Anthony a/k/a Anthony Edward \"Tony\" Stark")

	if len(res.Aliases) != 3 {
		t.Errorf("Expected 3 aliases.  Actual: %v", len(res.Aliases))
	}

	if res.FirstName != "Tony" {
		t.Errorf("Expected 'Tony'.  Actual: %v", res.FirstName)
	}

	if res.LastName != "Stark" {
		t.Errorf("Expected 'Stark'.  Actual: %v", res.LastName)
	}

}

func TestBuildFullName(t *testing.T) {
	res := Parse("President George Herbert Walker Bush")

	if res.FullName != "President George Herbert Walker Bush" {

		t.Errorf("Expected 'President George Herbert Walker Bush'.  Actual: %v", res.FullName)
	}

}

func TestDottedAka(t *testing.T) {
	res := Parse("James Polera a.k.a James K. Polera")
	if len(res.Aliases) != 1 {
		t.Errorf("Expected 1 alias.  Actual: %v", len(res.Aliases))
	}
}

func TestUnicodeCharsInName(t *testing.T) {
	res := Parse("König Ludwig")

	if res.FirstName != "König" {
		t.Errorf("Expected 'König'.  Actual: %v", res.FirstName)
	}
}

func TestTabsInName(t *testing.T) {
	res := Parse("Dr. James\tPolera\tEsq.")

	if res.Salutation != "Dr." {
		t.Errorf("Expected 'Dr.'.  Actual: %v", res.Salutation)
	}

	if res.FirstName != "James" {
		t.Errorf("Expected 'James'.  Actual: %v", res.FirstName)
	}

	if res.LastName != "Polera" {
		t.Errorf("Expected 'Polera'.  Actual: %v", res.LastName)
	}

	if res.Suffix != "Esq." {
		t.Errorf("Expected 'Esq.'.  Actual: %v", res.Suffix)
	}
}

func ExampleParse() {
	res := Parse("Thurston Howell III")
	fmt.Println("FirstName:", res.FirstName)
	fmt.Println("LastName:", res.LastName)
	fmt.Println("Generation:", res.Generation)

	// Output:
	// FirstName: Thurston
	// LastName: Howell
	// Generation: III

}

func ExampleParse_second() {

	res := Parse("President George Herbert Walker Bush")
	fmt.Println("Salutation:", res.Salutation)
	fmt.Println("FirstName:", res.FirstName)
	fmt.Println("MiddleName:", res.MiddleName)
	fmt.Println("LastName:", res.LastName)

	// Output:
	// Salutation: President
	// FirstName: George
	// MiddleName: Herbert Walker
	// LastName: Bush

}
<|endoftext|>


<commit_before>// +build go1.7

package nethttp

import (
	"context"
	"io"
	"net/http"
	"net/http/httptrace"
	"net/url"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/opentracing/opentracing-go/log"
)

type contextKey int

const (
	keyTracer contextKey = iota
)

const defaultComponentName = "net/http"

// Transport wraps a RoundTripper. If a request is being traced with
// Tracer, Transport will inject the current span into the headers,
// and set HTTP related tags on the span.
type Transport struct {
	// The actual RoundTripper to use for the request. A nil
	// RoundTripper defaults to http.DefaultTransport.
	http.RoundTripper
}

type clientOptions struct {
	operationName            string
	componentName            string
	urlTagFunc               func(u *url.URL) string
	disableClientTrace       bool
	disableInjectSpanContext bool
	spanObserver             func(span opentracing.Span, r *http.Request)
}

// ClientOption contols the behavior of TraceRequest.
type ClientOption func(*clientOptions)

// OperationName returns a ClientOption that sets the operation
// name for the client-side span.
func OperationName(operationName string) ClientOption {
	return func(options *clientOptions) {
		options.operationName = operationName
	}
}

// URLTagFunc returns a ClientOption that uses given function f
// to set the span's http.url tag. Can be used to change the default
// http.url tag, eg to redact sensitive information.
func URLTagFunc(f func(u *url.URL) string) ClientOption {
	return func(options *clientOptions) {
		options.urlTagFunc = f
	}
}

// ComponentName returns a ClientOption that sets the component
// name for the client-side span.
func ComponentName(componentName string) ClientOption {
	return func(options *clientOptions) {
		options.componentName = componentName
	}
}

// ClientTrace returns a ClientOption that turns on or off
// extra instrumentation via httptrace.WithClientTrace.
func ClientTrace(enabled bool) ClientOption {
	return func(options *clientOptions) {
		options.disableClientTrace = !enabled
	}
}

// InjectSpanContext returns a ClientOption that turns on or off
// injection of the Span context in the request HTTP headers.
// If this option is not used, the default behaviour is to
// inject the span context.
func InjectSpanContext(enabled bool) ClientOption {
	return func(options *clientOptions) {
		options.disableInjectSpanContext = !enabled
	}
}

// ClientSpanObserver returns a ClientOption that observes the span
// for the client-side span.
func ClientSpanObserver(f func(span opentracing.Span, r *http.Request)) ClientOption {
	return func(options *clientOptions) {
		options.spanObserver = f
	}
}

// TraceRequest adds a ClientTracer to req, tracing the request and
// all requests caused due to redirects. When tracing requests this
// way you must also use Transport.
//
// Example:
//
// 	func AskGoogle(ctx context.Context) error {
// 		client := &http.Client{Transport: &nethttp.Transport{}}
// 		req, err := http.NewRequest("GET", "http://google.com", nil)
// 		if err != nil {
// 			return err
// 		}
// 		req = req.WithContext(ctx) // extend existing trace, if any
//
// 		req, ht := nethttp.TraceRequest(tracer, req)
// 		defer ht.Finish()
//
// 		res, err := client.Do(req)
// 		if err != nil {
// 			return err
// 		}
// 		res.Body.Close()
// 		return nil
// 	}
func TraceRequest(tr opentracing.Tracer, req *http.Request, options ...ClientOption) (*http.Request, *Tracer) {
	opts := &clientOptions{
		urlTagFunc: func(u *url.URL) string {
			return u.String()
		},
		spanObserver: func(_ opentracing.Span, _ *http.Request) {},
	}
	for _, opt := range options {
		opt(opts)
	}
	ht := &Tracer{tr: tr, opts: opts}
	ctx := req.Context()
	if !opts.disableClientTrace {
		ctx = httptrace.WithClientTrace(ctx, ht.clientTrace())
	}
	req = req.WithContext(context.WithValue(ctx, keyTracer, ht))
	return req, ht
}

type closeTracker struct {
	io.ReadCloser
	sp opentracing.Span
}

func (c closeTracker) Close() error {
	err := c.ReadCloser.Close()
	c.sp.LogFields(log.String("event", "ClosedBody"))
	c.sp.Finish()
	return err
}

type writerCloseTracker struct {
	io.ReadWriteCloser
	sp opentracing.Span
}

func (c writerCloseTracker) Close() error {
	err := c.ReadWriteCloser.Close()
	c.sp.LogFields(log.String("event", "ClosedBody"))
	c.sp.Finish()
	return err
}

// TracerFromRequest retrieves the Tracer from the request. If the request does
// not have a Tracer it will return nil.
func TracerFromRequest(req *http.Request) *Tracer {
	tr, ok := req.Context().Value(keyTracer).(*Tracer)
	if !ok {
		return nil
	}
	return tr
}

// RoundTrip implements the RoundTripper interface.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	rt := t.RoundTripper
	if rt == nil {
		rt = http.DefaultTransport
	}
	tracer := TracerFromRequest(req)
	if tracer == nil {
		return rt.RoundTrip(req)
	}

	tracer.start(req)

	ext.HTTPMethod.Set(tracer.sp, req.Method)
	ext.HTTPUrl.Set(tracer.sp, tracer.opts.urlTagFunc(req.URL))
	ext.PeerAddress.Set(tracer.sp, req.URL.Host)
	tracer.opts.spanObserver(tracer.sp, req)

	if !tracer.opts.disableInjectSpanContext {
		carrier := opentracing.HTTPHeadersCarrier(req.Header)
		tracer.sp.Tracer().Inject(tracer.sp.Context(), opentracing.HTTPHeaders, carrier)
	}

	resp, err := rt.RoundTrip(req)

	if err != nil {
		tracer.sp.Finish()
		return resp, err
	}
	ext.HTTPStatusCode.Set(tracer.sp, uint16(resp.StatusCode))
	if resp.StatusCode >= http.StatusInternalServerError {
		ext.Error.Set(tracer.sp, true)
	}
	if req.Method == "HEAD" {
		tracer.sp.Finish()
	} else {
		readWriteCloser, ok := resp.Body.(io.ReadWriteCloser)
		if ok {
			resp.Body = writerCloseTracker{readWriteCloser, tracer.sp}
		} else {
			resp.Body = closeTracker{resp.Body, tracer.sp}
		}
	}
	return resp, nil
}

// Tracer holds tracing details for one HTTP request.
type Tracer struct {
	tr   opentracing.Tracer
	root opentracing.Span
	sp   opentracing.Span
	opts *clientOptions
}

func (h *Tracer) start(req *http.Request) opentracing.Span {
	if h.root == nil {
		parent := opentracing.SpanFromContext(req.Context())
		var spanctx opentracing.SpanContext
		if parent != nil {
			spanctx = parent.Context()
		}
		operationName := h.opts.operationName
		if operationName == "" {
			operationName = "HTTP Client"
		}
		root := h.tr.StartSpan(operationName, opentracing.ChildOf(spanctx))
		h.root = root
	}

	ctx := h.root.Context()
	h.sp = h.tr.StartSpan("HTTP "+req.Method, opentracing.ChildOf(ctx), ext.SpanKindRPCClient)

	componentName := h.opts.componentName
	if componentName == "" {
		componentName = defaultComponentName
	}
	ext.Component.Set(h.sp, componentName)

	return h.sp
}

// Finish finishes the span of the traced request.
func (h *Tracer) Finish() {
	if h.root != nil {
		h.root.Finish()
	}
}

// Span returns the root span of the traced request. This function
// should only be called after the request has been executed.
func (h *Tracer) Span() opentracing.Span {
	return h.root
}

func (h *Tracer) clientTrace() *httptrace.ClientTrace {
	return &httptrace.ClientTrace{
		GetConn:              h.getConn,
		GotConn:              h.gotConn,
		PutIdleConn:          h.putIdleConn,
		GotFirstResponseByte: h.gotFirstResponseByte,
		Got100Continue:       h.got100Continue,
		DNSStart:             h.dnsStart,
		DNSDone:              h.dnsDone,
		ConnectStart:         h.connectStart,
		ConnectDone:          h.connectDone,
		WroteHeaders:         h.wroteHeaders,
		Wait100Continue:      h.wait100Continue,
		WroteRequest:         h.wroteRequest,
	}
}

func (h *Tracer) getConn(hostPort string) {
	h.sp.LogFields(log.String("event", "GetConn"), log.String("hostPort", hostPort))
}

func (h *Tracer) gotConn(info httptrace.GotConnInfo) {
	h.sp.SetTag("net/http.reused", info.Reused)
	h.sp.SetTag("net/http.was_idle", info.WasIdle)
	h.sp.LogFields(log.String("event", "GotConn"))
}

func (h *Tracer) putIdleConn(error) {
	h.sp.LogFields(log.String("event", "PutIdleConn"))
}

func (h *Tracer) gotFirstResponseByte() {
	h.sp.LogFields(log.String("event", "GotFirstResponseByte"))
}

func (h *Tracer) got100Continue() {
	h.sp.LogFields(log.String("event", "Got100Continue"))
}

func (h *Tracer) dnsStart(info httptrace.DNSStartInfo) {
	h.sp.LogFields(
		log.String("event", "DNSStart"),
		log.String("host", info.Host),
	)
}

func (h *Tracer) dnsDone(info httptrace.DNSDoneInfo) {
	fields := []log.Field{log.String("event", "DNSDone")}
	for _, addr := range info.Addrs {
		fields = append(fields, log.String("addr", addr.String()))
	}
	if info.Err != nil {
		fields = append(fields, log.Error(info.Err))
	}
	h.sp.LogFields(fields...)
}

func (h *Tracer) connectStart(network, addr string) {
	h.sp.LogFields(
		log.String("event", "ConnectStart"),
		log.String("network", network),
		log.String("addr", addr),
	)
}

func (h *Tracer) connectDone(network, addr string, err error) {
	if err != nil {
		h.sp.LogFields(
			log.String("message", "ConnectDone"),
			log.String("network", network),
			log.String("addr", addr),
			log.String("event", "error"),
			log.Error(err),
		)
	} else {
		h.sp.LogFields(
			log.String("event", "ConnectDone"),
			log.String("network", network),
			log.String("addr", addr),
		)
	}
}

func (h *Tracer) wroteHeaders() {
	h.sp.LogFields(log.String("event", "WroteHeaders"))
}

func (h *Tracer) wait100Continue() {
	h.sp.LogFields(log.String("event", "Wait100Continue"))
}

func (h *Tracer) wroteRequest(info httptrace.WroteRequestInfo) {
	if info.Err != nil {
		h.sp.LogFields(
			log.String("message", "WroteRequest"),
			log.String("event", "error"),
			log.Error(info.Err),
		)
		ext.Error.Set(h.sp, true)
	} else {
		h.sp.LogFields(log.String("event", "WroteRequest"))
	}
}
<commit_msg>Use local variable for span (#63)<commit_after>//go:build go1.7
// +build go1.7

package nethttp

import (
	"context"
	"io"
	"net/http"
	"net/http/httptrace"
	"net/url"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/opentracing/opentracing-go/log"
)

type contextKey int

const (
	keyTracer contextKey = iota
)

const defaultComponentName = "net/http"

// Transport wraps a RoundTripper. If a request is being traced with
// Tracer, Transport will inject the current span into the headers,
// and set HTTP related tags on the span.
type Transport struct {
	// The actual RoundTripper to use for the request. A nil
	// RoundTripper defaults to http.DefaultTransport.
	http.RoundTripper
}

type clientOptions struct {
	operationName            string
	componentName            string
	urlTagFunc               func(u *url.URL) string
	disableClientTrace       bool
	disableInjectSpanContext bool
	spanObserver             func(span opentracing.Span, r *http.Request)
}

// ClientOption contols the behavior of TraceRequest.
type ClientOption func(*clientOptions)

// OperationName returns a ClientOption that sets the operation
// name for the client-side span.
func OperationName(operationName string) ClientOption {
	return func(options *clientOptions) {
		options.operationName = operationName
	}
}

// URLTagFunc returns a ClientOption that uses given function f
// to set the span's http.url tag. Can be used to change the default
// http.url tag, eg to redact sensitive information.
func URLTagFunc(f func(u *url.URL) string) ClientOption {
	return func(options *clientOptions) {
		options.urlTagFunc = f
	}
}

// ComponentName returns a ClientOption that sets the component
// name for the client-side span.
func ComponentName(componentName string) ClientOption {
	return func(options *clientOptions) {
		options.componentName = componentName
	}
}

// ClientTrace returns a ClientOption that turns on or off
// extra instrumentation via httptrace.WithClientTrace.
func ClientTrace(enabled bool) ClientOption {
	return func(options *clientOptions) {
		options.disableClientTrace = !enabled
	}
}

// InjectSpanContext returns a ClientOption that turns on or off
// injection of the Span context in the request HTTP headers.
// If this option is not used, the default behaviour is to
// inject the span context.
func InjectSpanContext(enabled bool) ClientOption {
	return func(options *clientOptions) {
		options.disableInjectSpanContext = !enabled
	}
}

// ClientSpanObserver returns a ClientOption that observes the span
// for the client-side span.
func ClientSpanObserver(f func(span opentracing.Span, r *http.Request)) ClientOption {
	return func(options *clientOptions) {
		options.spanObserver = f
	}
}

// TraceRequest adds a ClientTracer to req, tracing the request and
// all requests caused due to redirects. When tracing requests this
// way you must also use Transport.
//
// Example:
//
// 	func AskGoogle(ctx context.Context) error {
// 		client := &http.Client{Transport: &nethttp.Transport{}}
// 		req, err := http.NewRequest("GET", "http://google.com", nil)
// 		if err != nil {
// 			return err
// 		}
// 		req = req.WithContext(ctx) // extend existing trace, if any
//
// 		req, ht := nethttp.TraceRequest(tracer, req)
// 		defer ht.Finish()
//
// 		res, err := client.Do(req)
// 		if err != nil {
// 			return err
// 		}
// 		res.Body.Close()
// 		return nil
// 	}
func TraceRequest(tr opentracing.Tracer, req *http.Request, options ...ClientOption) (*http.Request, *Tracer) {
	opts := &clientOptions{
		urlTagFunc: func(u *url.URL) string {
			return u.String()
		},
		spanObserver: func(_ opentracing.Span, _ *http.Request) {},
	}
	for _, opt := range options {
		opt(opts)
	}
	ht := &Tracer{tr: tr, opts: opts}
	ctx := req.Context()
	if !opts.disableClientTrace {
		ctx = httptrace.WithClientTrace(ctx, ht.clientTrace())
	}
	req = req.WithContext(context.WithValue(ctx, keyTracer, ht))
	return req, ht
}

type closeTracker struct {
	io.ReadCloser
	sp opentracing.Span
}

func (c closeTracker) Close() error {
	err := c.ReadCloser.Close()
	c.sp.LogFields(log.String("event", "ClosedBody"))
	c.sp.Finish()
	return err
}

type writerCloseTracker struct {
	io.ReadWriteCloser
	sp opentracing.Span
}

func (c writerCloseTracker) Close() error {
	err := c.ReadWriteCloser.Close()
	c.sp.LogFields(log.String("event", "ClosedBody"))
	c.sp.Finish()
	return err
}

// TracerFromRequest retrieves the Tracer from the request. If the request does
// not have a Tracer it will return nil.
func TracerFromRequest(req *http.Request) *Tracer {
	tr, ok := req.Context().Value(keyTracer).(*Tracer)
	if !ok {
		return nil
	}
	return tr
}

// RoundTrip implements the RoundTripper interface.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	rt := t.RoundTripper
	if rt == nil {
		rt = http.DefaultTransport
	}
	tracer := TracerFromRequest(req)
	if tracer == nil {
		return rt.RoundTrip(req)
	}

	sp := tracer.start(req)

	ext.HTTPMethod.Set(sp, req.Method)
	ext.HTTPUrl.Set(sp, tracer.opts.urlTagFunc(req.URL))
	ext.PeerAddress.Set(sp, req.URL.Host)
	tracer.opts.spanObserver(sp, req)

	if !tracer.opts.disableInjectSpanContext {
		carrier := opentracing.HTTPHeadersCarrier(req.Header)
		sp.Tracer().Inject(sp.Context(), opentracing.HTTPHeaders, carrier)
	}

	resp, err := rt.RoundTrip(req)

	if err != nil {
		sp.Finish()
		return resp, err
	}
	ext.HTTPStatusCode.Set(sp, uint16(resp.StatusCode))
	if resp.StatusCode >= http.StatusInternalServerError {
		ext.Error.Set(sp, true)
	}
	if req.Method == "HEAD" {
		sp.Finish()
	} else {
		readWriteCloser, ok := resp.Body.(io.ReadWriteCloser)
		if ok {
			resp.Body = writerCloseTracker{readWriteCloser, sp}
		} else {
			resp.Body = closeTracker{resp.Body, sp}
		}
	}
	return resp, nil
}

// Tracer holds tracing details for one HTTP request.
type Tracer struct {
	tr   opentracing.Tracer
	root opentracing.Span
	sp   opentracing.Span
	opts *clientOptions
}

func (h *Tracer) start(req *http.Request) opentracing.Span {
	if h.root == nil {
		parent := opentracing.SpanFromContext(req.Context())
		var spanctx opentracing.SpanContext
		if parent != nil {
			spanctx = parent.Context()
		}
		operationName := h.opts.operationName
		if operationName == "" {
			operationName = "HTTP Client"
		}
		root := h.tr.StartSpan(operationName, opentracing.ChildOf(spanctx))
		h.root = root
	}

	ctx := h.root.Context()
	h.sp = h.tr.StartSpan("HTTP "+req.Method, opentracing.ChildOf(ctx), ext.SpanKindRPCClient)

	componentName := h.opts.componentName
	if componentName == "" {
		componentName = defaultComponentName
	}
	ext.Component.Set(h.sp, componentName)

	return h.sp
}

// Finish finishes the span of the traced request.
func (h *Tracer) Finish() {
	if h.root != nil {
		h.root.Finish()
	}
}

// Span returns the root span of the traced request. This function
// should only be called after the request has been executed.
func (h *Tracer) Span() opentracing.Span {
	return h.root
}

func (h *Tracer) clientTrace() *httptrace.ClientTrace {
	return &httptrace.ClientTrace{
		GetConn:              h.getConn,
		GotConn:              h.gotConn,
		PutIdleConn:          h.putIdleConn,
		GotFirstResponseByte: h.gotFirstResponseByte,
		Got100Continue:       h.got100Continue,
		DNSStart:             h.dnsStart,
		DNSDone:              h.dnsDone,
		ConnectStart:         h.connectStart,
		ConnectDone:          h.connectDone,
		WroteHeaders:         h.wroteHeaders,
		Wait100Continue:      h.wait100Continue,
		WroteRequest:         h.wroteRequest,
	}
}

func (h *Tracer) getConn(hostPort string) {
	h.sp.LogFields(log.String("event", "GetConn"), log.String("hostPort", hostPort))
}

func (h *Tracer) gotConn(info httptrace.GotConnInfo) {
	h.sp.SetTag("net/http.reused", info.Reused)
	h.sp.SetTag("net/http.was_idle", info.WasIdle)
	h.sp.LogFields(log.String("event", "GotConn"))
}

func (h *Tracer) putIdleConn(error) {
	h.sp.LogFields(log.String("event", "PutIdleConn"))
}

func (h *Tracer) gotFirstResponseByte() {
	h.sp.LogFields(log.String("event", "GotFirstResponseByte"))
}

func (h *Tracer) got100Continue() {
	h.sp.LogFields(log.String("event", "Got100Continue"))
}

func (h *Tracer) dnsStart(info httptrace.DNSStartInfo) {
	h.sp.LogFields(
		log.String("event", "DNSStart"),
		log.String("host", info.Host),
	)
}

func (h *Tracer) dnsDone(info httptrace.DNSDoneInfo) {
	fields := []log.Field{log.String("event", "DNSDone")}
	for _, addr := range info.Addrs {
		fields = append(fields, log.String("addr", addr.String()))
	}
	if info.Err != nil {
		fields = append(fields, log.Error(info.Err))
	}
	h.sp.LogFields(fields...)
}

func (h *Tracer) connectStart(network, addr string) {
	h.sp.LogFields(
		log.String("event", "ConnectStart"),
		log.String("network", network),
		log.String("addr", addr),
	)
}

func (h *Tracer) connectDone(network, addr string, err error) {
	if err != nil {
		h.sp.LogFields(
			log.String("message", "ConnectDone"),
			log.String("network", network),
			log.String("addr", addr),
			log.String("event", "error"),
			log.Error(err),
		)
	} else {
		h.sp.LogFields(
			log.String("event", "ConnectDone"),
			log.String("network", network),
			log.String("addr", addr),
		)
	}
}

func (h *Tracer) wroteHeaders() {
	h.sp.LogFields(log.String("event", "WroteHeaders"))
}

func (h *Tracer) wait100Continue() {
	h.sp.LogFields(log.String("event", "Wait100Continue"))
}

func (h *Tracer) wroteRequest(info httptrace.WroteRequestInfo) {
	if info.Err != nil {
		h.sp.LogFields(
			log.String("message", "WroteRequest"),
			log.String("event", "error"),
			log.Error(info.Err),
		)
		ext.Error.Set(h.sp, true)
	} else {
		h.sp.LogFields(log.String("event", "WroteRequest"))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/buildkite/buildkite-metrics/backend"
	"github.com/buildkite/buildkite-metrics/collector"
	"github.com/eawsy/aws-lambda-go/service/lambda/runtime"
	"gopkg.in/buildkite/go-buildkite.v2/buildkite"
)

func handle(evt json.RawMessage, ctx *runtime.Context) (interface{}, error) {
	org := os.Getenv("BUILDKITE_ORG")
	token := os.Getenv("BUILDKITE_TOKEN")
	backendOpt := os.Getenv("BUILDKITE_BACKEND")
	queue := os.Getenv("BUILDKITE_QUEUE")
	quiet := os.Getenv("BUILDKITE_QUIET")

	if quiet == "1" || quiet == "false" {
		log.SetOutput(ioutil.Discard)
	}

	config, err := buildkite.NewTokenConfig(token, false)
	if err != nil {
		return nil, err
	}

	client := buildkite.NewClient(config.Client())
	t := time.Now()

	col := collector.New(client, collector.Opts{
		OrgSlug:    org,
		Historical: time.Hour * 24,
	})

	if queue != "" {
		col.Queue = queue
	}

	var bk backend.Backend
	if backendOpt == "statsd" {
		bk, err = backend.NewStatsDBackend(os.Getenv("STATSD_HOST"), strings.ToLower(os.Getenv("STATSD_TAGS")) == "true")
		if err != nil {
			return nil, err
		}
	} else {
		bk = &backend.CloudWatchBackend{}
	}

	res, err := col.Collect()
	if err != nil {
		return nil, err
	}

	res.Dump()

	err = bk.Collect(res)
	if err != nil {
		return nil, err
	}

	log.Printf("Finished in %s", time.Now().Sub(t))
	return "", nil
}

func init() {
	runtime.HandleFunc(handle)
}
<commit_msg>Add retry for failed bk calls to lambda<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/buildkite/buildkite-metrics/backend"
	"github.com/buildkite/buildkite-metrics/collector"
	"github.com/eawsy/aws-lambda-go/service/lambda/runtime"
	"gopkg.in/buildkite/go-buildkite.v2/buildkite"
)

func handle(evt json.RawMessage, ctx *runtime.Context) (interface{}, error) {
	org := os.Getenv("BUILDKITE_ORG")
	token := os.Getenv("BUILDKITE_TOKEN")
	backendOpt := os.Getenv("BUILDKITE_BACKEND")
	queue := os.Getenv("BUILDKITE_QUEUE")
	quiet := os.Getenv("BUILDKITE_QUIET")

	if quiet == "1" || quiet == "false" {
		log.SetOutput(ioutil.Discard)
	}

	config, err := buildkite.NewTokenConfig(token, false)
	if err != nil {
		return nil, err
	}

	client := buildkite.NewClient(config.Client())
	t := time.Now()

	col := collector.New(client, collector.Opts{
		OrgSlug:    org,
		Historical: time.Hour * 24,
	})

	if queue != "" {
		col.Queue = queue
	}

	var bk backend.Backend
	if backendOpt == "statsd" {
		bk, err = backend.NewStatsDBackend(os.Getenv("STATSD_HOST"), strings.ToLower(os.Getenv("STATSD_TAGS")) == "true")
		if err != nil {
			return nil, err
		}
	} else {
		bk = &backend.CloudWatchBackend{}
	}

	return "", retry(time.Minute, func() error {
		res, err := col.Collect()
		if err != nil {
			return err
		}

		res.Dump()

		err = bk.Collect(res)
		if err != nil {
			return err
		}

		log.Printf("Finished in %s", time.Now().Sub(t))
		return nil
	})
}

func retry(timeout time.Duration, callback func() error) (err error) {
	t0 := time.Now()
	i := 0
	for {
		i++

		err = callback()
		if err == nil {
			return
		}

		delta := time.Now().Sub(t0)
		if delta > timeout {
			return fmt.Errorf("after %d attempts (during %s), last error: %s", i, delta, err)
		}

		time.Sleep(time.Second * 2)
		log.Println("retrying after error:", err)
	}
}

func init() {
	runtime.HandleFunc(handle)
}
<|endoftext|>