<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package embed

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/coreos/etcd/discovery"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/pkg/cors"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/ghodss/yaml"
)

const (
	ClusterStateFlagNew      = "new"
	ClusterStateFlagExisting = "existing"

	DefaultName                     = "default"
	DefaultInitialAdvertisePeerURLs = "http://localhost:2380"
	DefaultAdvertiseClientURLs      = "http://localhost:2379"
	DefaultListenPeerURLs           = "http://localhost:2380"
	DefaultListenClientURLs         = "http://localhost:2379"
	DefaultMaxSnapshots             = 5
	DefaultMaxWALs                  = 5

	// maxElectionMs specifies the maximum value of election timeout.
	// More details are listed in ../Documentation/tuning.md#time-parameters.
	maxElectionMs = 50000
)

var (
	ErrConflictBootstrapFlags = fmt.Errorf("multiple discovery or bootstrap flags are set. " +
		"Choose one of \"initial-cluster\", \"discovery\" or \"discovery-srv\"")
	ErrUnsetAdvertiseClientURLsFlag = fmt.Errorf("--advertise-client-urls is required when --listen-client-urls is set explicitly")
)

// Config holds the arguments for configuring an etcd server.
type Config struct {
	// member

	CorsInfo                *cors.CORSInfo
	LPUrls, LCUrls          []url.URL
	Dir                     string `json:"data-dir"`
	WalDir                  string `json:"wal-dir"`
	MaxSnapFiles            uint   `json:"max-snapshots"`
	MaxWalFiles             uint   `json:"max-wals"`
	Name                    string `json:"name"`
	SnapCount               uint64 `json:"snapshot-count"`
	AutoCompactionRetention int    `json:"auto-compaction-retention"`

	// TickMs is the number of milliseconds between heartbeat ticks.
	// TODO: decouple tickMs and heartbeat tick (current heartbeat tick = 1).
	// make ticks a cluster wide configuration.
	TickMs            uint  `json:"heartbeat-interval"`
	ElectionMs        uint  `json:"election-timeout"`
	QuotaBackendBytes int64 `json:"quota-backend-bytes"`

	// clustering

	APUrls, ACUrls      []url.URL
	ClusterState        string `json:"initial-cluster-state"`
	DNSCluster          string `json:"discovery-srv"`
	Dproxy              string `json:"discovery-proxy"`
	Durl                string `json:"discovery"`
	InitialCluster      string `json:"initial-cluster"`
	InitialClusterToken string `json:"initial-cluster-token"`
	StrictReconfigCheck bool   `json:"strict-reconfig-check"`

	// security

	ClientTLSInfo transport.TLSInfo
	ClientAutoTLS bool
	PeerTLSInfo   transport.TLSInfo
	PeerAutoTLS   bool

	// debug

	Debug        bool   `json:"debug"`
	LogPkgLevels string `json:"log-package-levels"`
	EnablePprof  bool

	// ForceNewCluster starts a new cluster even if previously started; unsafe.
	ForceNewCluster bool `json:"force-new-cluster"`

	// UserHandlers is for registering users handlers and only used for
	// embedding etcd into other applications.
	// The map key is the route path for the handler, and
	// you must ensure it can't be conflicted with etcd's.
	UserHandlers map[string]http.Handler `json:"-"`
}

// configYAML holds the config suitable for yaml parsing
type configYAML struct {
	Config
	configJSON
}

// configJSON has file options that are translated into Config options
type configJSON struct {
	LPUrlsJSON         string         `json:"listen-peer-urls"`
	LCUrlsJSON         string         `json:"listen-client-urls"`
	CorsJSON           string         `json:"cors"`
	APUrlsJSON         string         `json:"initial-advertise-peer-urls"`
	ACUrlsJSON         string         `json:"advertise-client-urls"`
	ClientSecurityJSON securityConfig `json:"client-transport-security"`
	PeerSecurityJSON   securityConfig `json:"peer-transport-security"`
}

type securityConfig struct {
	CAFile        string `json:"ca-file"`
	CertFile      string `json:"cert-file"`
	KeyFile       string `json:"key-file"`
	CertAuth      bool   `json:"client-cert-auth"`
	TrustedCAFile string `json:"trusted-ca-file"`
	AutoTLS       bool   `json:"auto-tls"`
}

// NewConfig creates a new Config populated with default values.
func NewConfig() *Config {
	lpurl, _ := url.Parse(DefaultListenPeerURLs)
	apurl, _ := url.Parse(DefaultInitialAdvertisePeerURLs)
	lcurl, _ := url.Parse(DefaultListenClientURLs)
	acurl, _ := url.Parse(DefaultAdvertiseClientURLs)
	cfg := &Config{
		CorsInfo:            &cors.CORSInfo{},
		MaxSnapFiles:        DefaultMaxSnapshots,
		MaxWalFiles:         DefaultMaxWALs,
		Name:                DefaultName,
		SnapCount:           etcdserver.DefaultSnapCount,
		TickMs:              100,
		ElectionMs:          1000,
		LPUrls:              []url.URL{*lpurl},
		LCUrls:              []url.URL{*lcurl},
		APUrls:              []url.URL{*apurl},
		ACUrls:              []url.URL{*acurl},
		ClusterState:        ClusterStateFlagNew,
		InitialClusterToken: "etcd-cluster",
		StrictReconfigCheck: true,
	}
	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
	return cfg
}

func ConfigFromFile(path string) (*Config, error) {
	cfg := &configYAML{Config: *NewConfig()}
	if err := cfg.configFromFile(path); err != nil {
		return nil, err
	}
	return &cfg.Config, nil
}

func (cfg *configYAML) configFromFile(path string) error {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(b, cfg)
	if err != nil {
		return err
	}

	if cfg.LPUrlsJSON != "" {
		u, err := types.NewURLs(strings.Split(cfg.LPUrlsJSON, ","))
		if err != nil {
			plog.Fatalf("unexpected error setting up listen-peer-urls: %v", err)
		}
		cfg.LPUrls = []url.URL(u)
	}

	if cfg.LCUrlsJSON != "" {
		u, err := types.NewURLs(strings.Split(cfg.LCUrlsJSON, ","))
		if err != nil {
			plog.Fatalf("unexpected error setting up listen-client-urls: %v", err)
		}
		cfg.LCUrls = []url.URL(u)
	}

	if cfg.CorsJSON != "" {
		if err := cfg.CorsInfo.Set(cfg.CorsJSON); err != nil {
			plog.Panicf("unexpected error setting up cors: %v", err)
		}
	}

	if cfg.APUrlsJSON != "" {
		u, err := types.NewURLs(strings.Split(cfg.APUrlsJSON, ","))
		if err != nil {
			plog.Fatalf("unexpected error setting up initial-advertise-peer-urls: %v", err)
		}
		cfg.APUrls = []url.URL(u)
	}

	if cfg.ACUrlsJSON != "" {
		u, err := types.NewURLs(strings.Split(cfg.ACUrlsJSON, ","))
		if err != nil {
			plog.Fatalf("unexpected error setting up advertise-peer-urls: %v", err)
		}
		cfg.ACUrls = []url.URL(u)
	}

	if cfg.ClusterState == "" {
		cfg.ClusterState = ClusterStateFlagNew
	}

	copySecurityDetails := func(tls *transport.TLSInfo, ysc *securityConfig) {
		tls.CAFile = ysc.CAFile
		tls.CertFile = ysc.CertFile
		tls.KeyFile = ysc.KeyFile
		tls.ClientCertAuth = ysc.CertAuth
		tls.TrustedCAFile = ysc.TrustedCAFile
	}
	copySecurityDetails(&cfg.ClientTLSInfo, &cfg.ClientSecurityJSON)
	copySecurityDetails(&cfg.PeerTLSInfo, &cfg.PeerSecurityJSON)
	cfg.ClientAutoTLS = cfg.ClientSecurityJSON.AutoTLS
	cfg.PeerAutoTLS = cfg.PeerSecurityJSON.AutoTLS

	return cfg.Validate()
}

func (cfg *Config) Validate() error {
	// Check if conflicting flags are passed.
	nSet := 0
	for _, v := range []bool{cfg.Durl != "", cfg.InitialCluster != "", cfg.DNSCluster != ""} {
		if v {
			nSet++
		}
	}

	if cfg.ClusterState != ClusterStateFlagNew && cfg.ClusterState != ClusterStateFlagExisting {
		return fmt.Errorf("unexpected clusterState %q", cfg.ClusterState)
	}

	if nSet > 1 {
		return ErrConflictBootstrapFlags
	}

	if 5*cfg.TickMs > cfg.ElectionMs {
		return fmt.Errorf("--election-timeout[%vms] should be at least as 5 times as --heartbeat-interval[%vms]", cfg.ElectionMs, cfg.TickMs)
	}
	if cfg.ElectionMs > maxElectionMs {
		return fmt.Errorf("--election-timeout[%vms] is too long, and should be set less than %vms", cfg.ElectionMs, maxElectionMs)
	}

	// check this last since proxying in etcdmain may make this OK
	if cfg.LCUrls != nil && cfg.ACUrls == nil {
		return ErrUnsetAdvertiseClientURLsFlag
	}

	return nil
}

// PeerURLsMapAndToken sets up an initial peer URLsMap and cluster token for bootstrap or discovery.
func (cfg *Config) PeerURLsMapAndToken(which string) (urlsmap types.URLsMap, token string, err error) {
	switch {
	case cfg.Durl != "":
		urlsmap = types.URLsMap{}
		// If using discovery, generate a temporary cluster based on
		// self's advertised peer URLs
		urlsmap[cfg.Name] = cfg.APUrls
		token = cfg.Durl
	case cfg.DNSCluster != "":
		var clusterStr string
		clusterStr, token, err = discovery.SRVGetCluster(cfg.Name, cfg.DNSCluster, cfg.InitialClusterToken, cfg.APUrls)
		if err != nil {
			return nil, "", err
		}
		if strings.Contains(clusterStr, "https://") && cfg.PeerTLSInfo.CAFile == "" {
			cfg.PeerTLSInfo.ServerName = cfg.DNSCluster
		}
		urlsmap, err = types.NewURLsMap(clusterStr)
		// only etcd member must belong to the discovered cluster.
		// proxy does not need to belong to the discovered cluster.
		if which == "etcd" {
			if _, ok := urlsmap[cfg.Name]; !ok {
				return nil, "", fmt.Errorf("cannot find local etcd member %q in SRV records", cfg.Name)
			}
		}
	default:
		// We're statically configured, and cluster has appropriately been set.
		urlsmap, err = types.NewURLsMap(cfg.InitialCluster)
		token = cfg.InitialClusterToken
	}
	return urlsmap, token, err
}

func (cfg Config) InitialClusterFromName(name string) (ret string) {
	if len(cfg.APUrls) == 0 {
		return ""
	}
	n := name
	if name == "" {
		n = DefaultName
	}
	for i := range cfg.APUrls {
		ret = ret + "," + n + "=" + cfg.APUrls[i].String()
	}
	return ret[1:]
}

func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
<commit_msg>embed: use default route IP for default advertise URL<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package embed

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/coreos/etcd/discovery"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/pkg/cors"
	"github.com/coreos/etcd/pkg/netutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/ghodss/yaml"
)

const (
	ClusterStateFlagNew      = "new"
	ClusterStateFlagExisting = "existing"

	DefaultName         = "default"
	DefaultMaxSnapshots = 5
	DefaultMaxWALs      = 5

	// maxElectionMs specifies the maximum value of election timeout.
	// More details are listed in ../Documentation/tuning.md#time-parameters.
	maxElectionMs = 50000
)

var (
	ErrConflictBootstrapFlags = fmt.Errorf("multiple discovery or bootstrap flags are set. " +
		"Choose one of \"initial-cluster\", \"discovery\" or \"discovery-srv\"")
	ErrUnsetAdvertiseClientURLsFlag = fmt.Errorf("--advertise-client-urls is required when --listen-client-urls is set explicitly")

	DefaultListenPeerURLs           = "http://localhost:2380"
	DefaultListenClientURLs         = "http://localhost:2379"
	DefaultInitialAdvertisePeerURLs = "http://localhost:2380"
	DefaultAdvertiseClientURLs      = "http://localhost:2379"

	defaultHostname   string = "localhost"
	defaultHostStatus error
)

func init() {
	ip, err := netutil.GetDefaultHost()
	if err != nil {
		defaultHostStatus = err
		return
	}
	// found default host, advertise on it
	DefaultInitialAdvertisePeerURLs = "http://" + ip + ":2380"
	DefaultAdvertiseClientURLs = "http://" + ip + ":2379"
	defaultHostname = ip
}

// Config holds the arguments for configuring an etcd server.
type Config struct {
	// member

	CorsInfo                *cors.CORSInfo
	LPUrls, LCUrls          []url.URL
	Dir                     string `json:"data-dir"`
	WalDir                  string `json:"wal-dir"`
	MaxSnapFiles            uint   `json:"max-snapshots"`
	MaxWalFiles             uint   `json:"max-wals"`
	Name                    string `json:"name"`
	SnapCount               uint64 `json:"snapshot-count"`
	AutoCompactionRetention int    `json:"auto-compaction-retention"`

	// TickMs is the number of milliseconds between heartbeat ticks.
	// TODO: decouple tickMs and heartbeat tick (current heartbeat tick = 1).
	// make ticks a cluster wide configuration.
	TickMs            uint  `json:"heartbeat-interval"`
	ElectionMs        uint  `json:"election-timeout"`
	QuotaBackendBytes int64 `json:"quota-backend-bytes"`

	// clustering

	APUrls, ACUrls      []url.URL
	ClusterState        string `json:"initial-cluster-state"`
	DNSCluster          string `json:"discovery-srv"`
	Dproxy              string `json:"discovery-proxy"`
	Durl                string `json:"discovery"`
	InitialCluster      string `json:"initial-cluster"`
	InitialClusterToken string `json:"initial-cluster-token"`
	StrictReconfigCheck bool   `json:"strict-reconfig-check"`

	// security

	ClientTLSInfo transport.TLSInfo
	ClientAutoTLS bool
	PeerTLSInfo   transport.TLSInfo
	PeerAutoTLS   bool

	// debug

	Debug        bool   `json:"debug"`
	LogPkgLevels string `json:"log-package-levels"`
	EnablePprof  bool

	// ForceNewCluster starts a new cluster even if previously started; unsafe.
	ForceNewCluster bool `json:"force-new-cluster"`

	// UserHandlers is for registering users handlers and only used for
	// embedding etcd into other applications.
	// The map key is the route path for the handler, and
	// you must ensure it can't be conflicted with etcd's.
	UserHandlers map[string]http.Handler `json:"-"`
}

// configYAML holds the config suitable for yaml parsing
type configYAML struct {
	Config
	configJSON
}

// configJSON has file options that are translated into Config options
type configJSON struct {
	LPUrlsJSON         string         `json:"listen-peer-urls"`
	LCUrlsJSON         string         `json:"listen-client-urls"`
	CorsJSON           string         `json:"cors"`
	APUrlsJSON         string         `json:"initial-advertise-peer-urls"`
	ACUrlsJSON         string         `json:"advertise-client-urls"`
	ClientSecurityJSON securityConfig `json:"client-transport-security"`
	PeerSecurityJSON   securityConfig `json:"peer-transport-security"`
}

type securityConfig struct {
	CAFile        string `json:"ca-file"`
	CertFile      string `json:"cert-file"`
	KeyFile       string `json:"key-file"`
	CertAuth      bool   `json:"client-cert-auth"`
	TrustedCAFile string `json:"trusted-ca-file"`
	AutoTLS       bool   `json:"auto-tls"`
}

// NewConfig creates a new Config populated with default values.
func NewConfig() *Config {
	lpurl, _ := url.Parse(DefaultListenPeerURLs)
	apurl, _ := url.Parse(DefaultInitialAdvertisePeerURLs)
	lcurl, _ := url.Parse(DefaultListenClientURLs)
	acurl, _ := url.Parse(DefaultAdvertiseClientURLs)
	cfg := &Config{
		CorsInfo:            &cors.CORSInfo{},
		MaxSnapFiles:        DefaultMaxSnapshots,
		MaxWalFiles:         DefaultMaxWALs,
		Name:                DefaultName,
		SnapCount:           etcdserver.DefaultSnapCount,
		TickMs:              100,
		ElectionMs:          1000,
		LPUrls:              []url.URL{*lpurl},
		LCUrls:              []url.URL{*lcurl},
		APUrls:              []url.URL{*apurl},
		ACUrls:              []url.URL{*acurl},
		ClusterState:        ClusterStateFlagNew,
		InitialClusterToken: "etcd-cluster",
		StrictReconfigCheck: true,
	}
	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
	return cfg
}

func ConfigFromFile(path string) (*Config, error) {
	cfg := &configYAML{Config: *NewConfig()}
	if err := cfg.configFromFile(path); err != nil {
		return nil, err
	}
	return &cfg.Config, nil
}

func (cfg *configYAML) configFromFile(path string) error {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(b, cfg)
	if err != nil {
		return err
	}

	if cfg.LPUrlsJSON != "" {
		u, err := types.NewURLs(strings.Split(cfg.LPUrlsJSON, ","))
		if err != nil {
			plog.Fatalf("unexpected error setting up listen-peer-urls: %v", err)
		}
		cfg.LPUrls = []url.URL(u)
	}

	if cfg.LCUrlsJSON != "" {
		u, err := types.NewURLs(strings.Split(cfg.LCUrlsJSON, ","))
		if err != nil {
			plog.Fatalf("unexpected error setting up listen-client-urls: %v", err)
		}
		cfg.LCUrls = []url.URL(u)
	}

	if cfg.CorsJSON != "" {
		if err := cfg.CorsInfo.Set(cfg.CorsJSON); err != nil {
			plog.Panicf("unexpected error setting up cors: %v", err)
		}
	}

	if cfg.APUrlsJSON != "" {
		u, err := types.NewURLs(strings.Split(cfg.APUrlsJSON, ","))
		if err != nil {
			plog.Fatalf("unexpected error setting up initial-advertise-peer-urls: %v", err)
		}
		cfg.APUrls = []url.URL(u)
	}

	if cfg.ACUrlsJSON != "" {
		u, err := types.NewURLs(strings.Split(cfg.ACUrlsJSON, ","))
		if err != nil {
			plog.Fatalf("unexpected error setting up advertise-peer-urls: %v", err)
		}
		cfg.ACUrls = []url.URL(u)
	}

	if cfg.ClusterState == "" {
		cfg.ClusterState = ClusterStateFlagNew
	}

	copySecurityDetails := func(tls *transport.TLSInfo, ysc *securityConfig) {
		tls.CAFile = ysc.CAFile
		tls.CertFile = ysc.CertFile
		tls.KeyFile = ysc.KeyFile
		tls.ClientCertAuth = ysc.CertAuth
		tls.TrustedCAFile = ysc.TrustedCAFile
	}
	copySecurityDetails(&cfg.ClientTLSInfo, &cfg.ClientSecurityJSON)
	copySecurityDetails(&cfg.PeerTLSInfo, &cfg.PeerSecurityJSON)
	cfg.ClientAutoTLS = cfg.ClientSecurityJSON.AutoTLS
	cfg.PeerAutoTLS = cfg.PeerSecurityJSON.AutoTLS

	return cfg.Validate()
}

func (cfg *Config) Validate() error {
	// Check if conflicting flags are passed.
	nSet := 0
	for _, v := range []bool{cfg.Durl != "", cfg.InitialCluster != "", cfg.DNSCluster != ""} {
		if v {
			nSet++
		}
	}

	if cfg.ClusterState != ClusterStateFlagNew && cfg.ClusterState != ClusterStateFlagExisting {
		return fmt.Errorf("unexpected clusterState %q", cfg.ClusterState)
	}

	if nSet > 1 {
		return ErrConflictBootstrapFlags
	}

	if 5*cfg.TickMs > cfg.ElectionMs {
		return fmt.Errorf("--election-timeout[%vms] should be at least as 5 times as --heartbeat-interval[%vms]", cfg.ElectionMs, cfg.TickMs)
	}
	if cfg.ElectionMs > maxElectionMs {
		return fmt.Errorf("--election-timeout[%vms] is too long, and should be set less than %vms", cfg.ElectionMs, maxElectionMs)
	}

	// check this last since proxying in etcdmain may make this OK
	if cfg.LCUrls != nil && cfg.ACUrls == nil {
		return ErrUnsetAdvertiseClientURLsFlag
	}

	return nil
}

// PeerURLsMapAndToken sets up an initial peer URLsMap and cluster token for bootstrap or discovery.
func (cfg *Config) PeerURLsMapAndToken(which string) (urlsmap types.URLsMap, token string, err error) {
	switch {
	case cfg.Durl != "":
		urlsmap = types.URLsMap{}
		// If using discovery, generate a temporary cluster based on
		// self's advertised peer URLs
		urlsmap[cfg.Name] = cfg.APUrls
		token = cfg.Durl
	case cfg.DNSCluster != "":
		var clusterStr string
		clusterStr, token, err = discovery.SRVGetCluster(cfg.Name, cfg.DNSCluster, cfg.InitialClusterToken, cfg.APUrls)
		if err != nil {
			return nil, "", err
		}
		if strings.Contains(clusterStr, "https://") && cfg.PeerTLSInfo.CAFile == "" {
			cfg.PeerTLSInfo.ServerName = cfg.DNSCluster
		}
		urlsmap, err = types.NewURLsMap(clusterStr)
		// only etcd member must belong to the discovered cluster.
		// proxy does not need to belong to the discovered cluster.
		if which == "etcd" {
			if _, ok := urlsmap[cfg.Name]; !ok {
				return nil, "", fmt.Errorf("cannot find local etcd member %q in SRV records", cfg.Name)
			}
		}
	default:
		// We're statically configured, and cluster has appropriately been set.
		urlsmap, err = types.NewURLsMap(cfg.InitialCluster)
		token = cfg.InitialClusterToken
	}
	return urlsmap, token, err
}

func (cfg Config) InitialClusterFromName(name string) (ret string) {
	if len(cfg.APUrls) == 0 {
		return ""
	}
	n := name
	if name == "" {
		n = DefaultName
	}
	for i := range cfg.APUrls {
		ret = ret + "," + n + "=" + cfg.APUrls[i].String()
	}
	return ret[1:]
}

func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }

// IsDefaultHost returns the default hostname, if used, and the error, if any,
// from getting the machine's default host.
func (cfg Config) IsDefaultHost() (string, error) {
	if len(cfg.APUrls) == 1 && cfg.APUrls[0].String() == DefaultInitialAdvertisePeerURLs {
		return defaultHostname, defaultHostStatus
	}
	if len(cfg.ACUrls) == 1 && cfg.ACUrls[0].String() == DefaultAdvertiseClientURLs {
		return defaultHostname, defaultHostStatus
	}
	return "", defaultHostStatus
}
<|endoftext|>


<commit_before><commit_msg>Update hec_writer.go<commit_after><|endoftext|>


<commit_before>package db

// OperationType is a numeric code indentifying the type of an Operation.
type OperationType int64

// Possible values for OperationType
//
// WARNING: The type codes are stored in the database, so this list of
//          definitions should be normally append-only. Any other change
//          requires a database update.
const (
	OperationUnknown OperationType = iota
	OperationClusterBootstrap
	OperationClusterJoin
	OperationBackupCreate
	OperationBackupRename
	OperationBackupRestore
	OperationBackupRemove
	OperationConsoleShow
	OperationInstanceCreate
	OperationInstanceUpdate
	OperationInstanceRename
	OperationInstanceMigrate
	OperationInstanceLiveMigrate
	OperationInstanceFreeze
	OperationInstanceUnfreeze
	OperationInstanceDelete
	OperationInstanceStart
	OperationInstanceStop
	OperationInstanceRestart
	OperationCommandExec
	OperationSnapshotCreate
	OperationSnapshotRename
	OperationSnapshotRestore
	OperationSnapshotTransfer
	OperationSnapshotUpdate
	OperationSnapshotDelete
	OperationImageDownload
	OperationImageDelete
	OperationImageToken
	OperationImageRefresh
	OperationVolumeCopy
	OperationVolumeCreate
	OperationVolumeMigrate
	OperationVolumeMove
	OperationVolumeSnapshotCreate
	OperationVolumeSnapshotDelete
	OperationVolumeSnapshotUpdate
	OperationProjectRename
	OperationImagesExpire
	OperationImagesPruneLeftover
	OperationImagesUpdate
	OperationImagesSynchronize
	OperationLogsExpire
	OperationInstanceTypesUpdate
	OperationBackupsExpire
	OperationSnapshotsExpire
	OperationCustomVolumeSnapshotsExpire
	OperationCustomVolumeBackupCreate
	OperationCustomVolumeBackupRemove
	OperationCustomVolumeBackupRename
	OperationCustomVolumeBackupRestore
	OperationWarningsPruneResolved
)

// Description return a human-readable description of the operation type.
func (t OperationType) Description() string {
	switch t {
	case OperationClusterBootstrap:
		return "Creating bootstrap node"
	case OperationClusterJoin:
		return "Joining cluster"
	case OperationBackupCreate:
		return "Backing up instance"
	case OperationBackupRename:
		return "Renaming instance backup"
	case OperationBackupRestore:
		return "Restoring backup"
	case OperationBackupRemove:
		return "Removing instance backup"
	case OperationConsoleShow:
		return "Showing console"
	case OperationInstanceCreate:
		return "Creating instance"
	case OperationInstanceUpdate:
		return "Updating instance"
	case OperationInstanceRename:
		return "Renaming instance"
	case OperationInstanceMigrate:
		return "Migrating instance"
	case OperationInstanceLiveMigrate:
		return "Live-migrating instance"
	case OperationInstanceFreeze:
		return "Freezing instance"
	case OperationInstanceUnfreeze:
		return "Unfreezing instance"
	case OperationInstanceDelete:
		return "Deleting instance"
	case OperationInstanceStart:
		return "Starting instance"
	case OperationInstanceStop:
		return "Stopping instance"
	case OperationInstanceRestart:
		return "Restarting instance"
	case OperationCommandExec:
		return "Executing command"
	case OperationSnapshotCreate:
		return "Snapshotting instance"
	case OperationSnapshotRename:
		return "Renaming snapshot"
	case OperationSnapshotRestore:
		return "Restoring snapshot"
	case OperationSnapshotTransfer:
		return "Transferring snapshot"
	case OperationSnapshotUpdate:
		return "Updating snapshot"
	case OperationSnapshotDelete:
		return "Deleting snapshot"
	case OperationImageDownload:
		return "Downloading image"
	case OperationImageDelete:
		return "Deleting image"
	case OperationImageToken:
		return "Image download token"
	case OperationImageRefresh:
		return "Refreshing image"
	case OperationVolumeCopy:
		return "Copying storage volume"
	case OperationVolumeCreate:
		return "Creating storage volume"
	case OperationVolumeMigrate:
		return "Migrating storage volume"
	case OperationVolumeMove:
		return "Moving storage volume"
	case OperationVolumeSnapshotCreate:
		return "Creating storage volume snapshot"
	case OperationVolumeSnapshotDelete:
		return "Deleting storage volume snapshot"
	case OperationVolumeSnapshotUpdate:
		return "Updating storage volume snapshot"
	case OperationProjectRename:
		return "Renaming project"
	case OperationImagesExpire:
		return "Cleaning up expired images"
	case OperationImagesPruneLeftover:
		return "Pruning leftover image files"
	case OperationImagesUpdate:
		return "Updating images"
	case OperationImagesSynchronize:
		return "Synchronizing images"
	case OperationLogsExpire:
		return "Expiring log files"
	case OperationInstanceTypesUpdate:
		return "Updating instance types"
	case OperationBackupsExpire:
		return "Cleaning up expired instance backups"
	case OperationSnapshotsExpire:
		return "Cleaning up expired instance snapshots"
	case OperationCustomVolumeSnapshotsExpire:
		return "Cleaning up expired volume snapshots"
	case OperationCustomVolumeBackupCreate:
		return "Creating custom volume backup"
	case OperationCustomVolumeBackupRemove:
		return "Deleting custom volume backup"
	case OperationCustomVolumeBackupRename:
		return "Renaming custom volume backup"
	case OperationCustomVolumeBackupRestore:
		return "Restoring custom volume backup"
	case OperationWarningsPruneResolved:
		return "Pruning resolved warnings"
	default:
		return "Executing operation"
	}
}

// Permission returns the needed RBAC permission to cancel the oepration
func (t OperationType) Permission() string {
	switch t {
	case OperationBackupCreate:
		return "operate-containers"
	case OperationBackupRename:
		return "operate-containers"
	case OperationBackupRestore:
		return "operate-containers"
	case OperationBackupRemove:
		return "operate-containers"
	case OperationConsoleShow:
		return "operate-containers"
	case OperationInstanceFreeze:
		return "operate-containers"
	case OperationInstanceUnfreeze:
		return "operate-containers"
	case OperationInstanceStart:
		return "operate-containers"
	case OperationInstanceStop:
		return "operate-containers"
	case OperationInstanceRestart:
		return "operate-containers"
	case OperationCommandExec:
		return "operate-containers"
	case OperationSnapshotCreate:
		return "operate-containers"
	case OperationSnapshotRename:
		return "operate-containers"
	case OperationSnapshotTransfer:
		return "operate-containers"
	case OperationSnapshotUpdate:
		return "operate-containers"
	case OperationSnapshotDelete:
		return "operate-containers"

	case OperationInstanceCreate:
		return "manage-containers"
	case OperationInstanceUpdate:
		return "manage-containers"
	case OperationInstanceRename:
		return "manage-containers"
	case OperationInstanceMigrate:
		return "manage-containers"
	case OperationInstanceLiveMigrate:
		return "manage-containers"
	case OperationInstanceDelete:
		return "manage-containers"
	case OperationSnapshotRestore:
		return "manage-containers"

	case OperationImageDownload:
		return "manage-images"
	case OperationImageDelete:
		return "manage-images"
	case OperationImageToken:
		return "manage-images"
	case OperationImageRefresh:
		return "manage-images"
	case OperationImagesUpdate:
		return "manage-images"
	case OperationImagesSynchronize:
		return "manage-images"

	case OperationCustomVolumeSnapshotsExpire:
		return "operate-volumes"
	case OperationCustomVolumeBackupCreate:
		return "manage-storage-volumes"
	case OperationCustomVolumeBackupRemove:
		return "manage-storage-volumes"
	case OperationCustomVolumeBackupRename:
		return "manage-storage-volumes"
	case OperationCustomVolumeBackupRestore:
		return "manage-storage-volumes"
	}

	return ""
}
<commit_msg>lxd/db/operations/types: Adds OperationClusterJoinToken type<commit_after>package db

// OperationType is a numeric code indentifying the type of an Operation.
type OperationType int64

// Possible values for OperationType
//
// WARNING: The type codes are stored in the database, so this list of
//          definitions should be normally append-only. Any other change
//          requires a database update.
const (
	OperationUnknown OperationType = iota
	OperationClusterBootstrap
	OperationClusterJoin
	OperationBackupCreate
	OperationBackupRename
	OperationBackupRestore
	OperationBackupRemove
	OperationConsoleShow
	OperationInstanceCreate
	OperationInstanceUpdate
	OperationInstanceRename
	OperationInstanceMigrate
	OperationInstanceLiveMigrate
	OperationInstanceFreeze
	OperationInstanceUnfreeze
	OperationInstanceDelete
	OperationInstanceStart
	OperationInstanceStop
	OperationInstanceRestart
	OperationCommandExec
	OperationSnapshotCreate
	OperationSnapshotRename
	OperationSnapshotRestore
	OperationSnapshotTransfer
	OperationSnapshotUpdate
	OperationSnapshotDelete
	OperationImageDownload
	OperationImageDelete
	OperationImageToken
	OperationImageRefresh
	OperationVolumeCopy
	OperationVolumeCreate
	OperationVolumeMigrate
	OperationVolumeMove
	OperationVolumeSnapshotCreate
	OperationVolumeSnapshotDelete
	OperationVolumeSnapshotUpdate
	OperationProjectRename
	OperationImagesExpire
	OperationImagesPruneLeftover
	OperationImagesUpdate
	OperationImagesSynchronize
	OperationLogsExpire
	OperationInstanceTypesUpdate
	OperationBackupsExpire
	OperationSnapshotsExpire
	OperationCustomVolumeSnapshotsExpire
	OperationCustomVolumeBackupCreate
	OperationCustomVolumeBackupRemove
	OperationCustomVolumeBackupRename
	OperationCustomVolumeBackupRestore
	OperationWarningsPruneResolved
	OperationClusterJoinToken
)

// Description return a human-readable description of the operation type.
func (t OperationType) Description() string {
	switch t {
	case OperationClusterBootstrap:
		return "Creating bootstrap node"
	case OperationClusterJoin:
		return "Joining cluster"
	case OperationBackupCreate:
		return "Backing up instance"
	case OperationBackupRename:
		return "Renaming instance backup"
	case OperationBackupRestore:
		return "Restoring backup"
	case OperationBackupRemove:
		return "Removing instance backup"
	case OperationConsoleShow:
		return "Showing console"
	case OperationInstanceCreate:
		return "Creating instance"
	case OperationInstanceUpdate:
		return "Updating instance"
	case OperationInstanceRename:
		return "Renaming instance"
	case OperationInstanceMigrate:
		return "Migrating instance"
	case OperationInstanceLiveMigrate:
		return "Live-migrating instance"
	case OperationInstanceFreeze:
		return "Freezing instance"
	case OperationInstanceUnfreeze:
		return "Unfreezing instance"
	case OperationInstanceDelete:
		return "Deleting instance"
	case OperationInstanceStart:
		return "Starting instance"
	case OperationInstanceStop:
		return "Stopping instance"
	case OperationInstanceRestart:
		return "Restarting instance"
	case OperationCommandExec:
		return "Executing command"
	case OperationSnapshotCreate:
		return "Snapshotting instance"
	case OperationSnapshotRename:
		return "Renaming snapshot"
	case OperationSnapshotRestore:
		return "Restoring snapshot"
	case OperationSnapshotTransfer:
		return "Transferring snapshot"
	case OperationSnapshotUpdate:
		return "Updating snapshot"
	case OperationSnapshotDelete:
		return "Deleting snapshot"
	case OperationImageDownload:
		return "Downloading image"
	case OperationImageDelete:
		return "Deleting image"
	case OperationImageToken:
		return "Image download token"
	case OperationImageRefresh:
		return "Refreshing image"
	case OperationVolumeCopy:
		return "Copying storage volume"
	case OperationVolumeCreate:
		return "Creating storage volume"
	case OperationVolumeMigrate:
		return "Migrating storage volume"
	case OperationVolumeMove:
		return "Moving storage volume"
	case OperationVolumeSnapshotCreate:
		return "Creating storage volume snapshot"
	case OperationVolumeSnapshotDelete:
		return "Deleting storage volume snapshot"
	case OperationVolumeSnapshotUpdate:
		return "Updating storage volume snapshot"
	case OperationProjectRename:
		return "Renaming project"
	case OperationImagesExpire:
		return "Cleaning up expired images"
	case OperationImagesPruneLeftover:
		return "Pruning leftover image files"
	case OperationImagesUpdate:
		return "Updating images"
	case OperationImagesSynchronize:
		return "Synchronizing images"
	case OperationLogsExpire:
		return "Expiring log files"
	case OperationInstanceTypesUpdate:
		return "Updating instance types"
	case OperationBackupsExpire:
		return "Cleaning up expired instance backups"
	case OperationSnapshotsExpire:
		return "Cleaning up expired instance snapshots"
	case OperationCustomVolumeSnapshotsExpire:
		return "Cleaning up expired volume snapshots"
	case OperationCustomVolumeBackupCreate:
		return "Creating custom volume backup"
	case OperationCustomVolumeBackupRemove:
		return "Deleting custom volume backup"
	case OperationCustomVolumeBackupRename:
		return "Renaming custom volume backup"
	case OperationCustomVolumeBackupRestore:
		return "Restoring custom volume backup"
	case OperationWarningsPruneResolved:
		return "Pruning resolved warnings"
	default:
		return "Executing operation"
	}
}

// Permission returns the needed RBAC permission to cancel the oepration
func (t OperationType) Permission() string {
	switch t {
	case OperationBackupCreate:
		return "operate-containers"
	case OperationBackupRename:
		return "operate-containers"
	case OperationBackupRestore:
		return "operate-containers"
	case OperationBackupRemove:
		return "operate-containers"
	case OperationConsoleShow:
		return "operate-containers"
	case OperationInstanceFreeze:
		return "operate-containers"
	case OperationInstanceUnfreeze:
		return "operate-containers"
	case OperationInstanceStart:
		return "operate-containers"
	case OperationInstanceStop:
		return "operate-containers"
	case OperationInstanceRestart:
		return "operate-containers"
	case OperationCommandExec:
		return "operate-containers"
	case OperationSnapshotCreate:
		return "operate-containers"
	case OperationSnapshotRename:
		return "operate-containers"
	case OperationSnapshotTransfer:
		return "operate-containers"
	case OperationSnapshotUpdate:
		return "operate-containers"
	case OperationSnapshotDelete:
		return "operate-containers"

	case OperationInstanceCreate:
		return "manage-containers"
	case OperationInstanceUpdate:
		return "manage-containers"
	case OperationInstanceRename:
		return "manage-containers"
	case OperationInstanceMigrate:
		return "manage-containers"
	case OperationInstanceLiveMigrate:
		return "manage-containers"
	case OperationInstanceDelete:
		return "manage-containers"
	case OperationSnapshotRestore:
		return "manage-containers"

	case OperationImageDownload:
		return "manage-images"
	case OperationImageDelete:
		return "manage-images"
	case OperationImageToken:
		return "manage-images"
	case OperationImageRefresh:
		return "manage-images"
	case OperationImagesUpdate:
		return "manage-images"
	case OperationImagesSynchronize:
		return "manage-images"

	case OperationCustomVolumeSnapshotsExpire:
		return "operate-volumes"
	case OperationCustomVolumeBackupCreate:
		return "manage-storage-volumes"
	case OperationCustomVolumeBackupRemove:
		return "manage-storage-volumes"
	case OperationCustomVolumeBackupRename:
		return "manage-storage-volumes"
	case OperationCustomVolumeBackupRestore:
		return "manage-storage-volumes"
	}

	return ""
}
<|endoftext|>


<commit_before>package goose

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// Crawler can fetch the target HTML page
type Crawler struct {
	config  Configuration
	url     string
	RawHTML string
	Charset string
}

// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
func NewCrawler(config Configuration, url string, RawHTML string) Crawler {
	return Crawler{
		config:  config,
		url:     url,
		RawHTML: RawHTML,
		Charset: "",
	}
}

func getCharsetFromContentType(cs string) string {
	cs = strings.ToLower(strings.Replace(cs, " ", "", -1))
	if strings.HasPrefix(cs, "text/html;charset=") {
		cs = strings.TrimPrefix(cs, "text/html;charset=")
	}
	if strings.HasPrefix(cs, "application/xhtml+xml;charset=") {
		cs = strings.TrimPrefix(cs, "application/xhtml+xml;charset=")
	}
	return NormaliseCharset(cs)
}

// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
// rather than relying on the detection from the HTML meta tags
func (c *Crawler) SetCharset(cs string) {
	c.Charset = getCharsetFromContentType(cs)
}

// GetContentType returns the Content-Type string extracted from the meta tags
func (c Crawler) GetContentType(document *goquery.Document) string {
	var attr string
	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
		attr, _ = s.Attr("content")
	})
	return attr
}

// GetCharset returns a normalised charset string extracted from the meta tags
func (c Crawler) GetCharset(document *goquery.Document) string {
	// manually-provided charset (from HTTP headers?) takes priority
	if "" != c.Charset {
		return c.Charset
	}

	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	ct := c.GetContentType(document)
	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
		return getCharsetFromContentType(ct)
	}

	// <meta charset="utf-8">
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		_, exists := s.Attr("charset")
		return !exists
	})

	if selection != nil {
		cs, _ := selection.Attr("charset")
		return NormaliseCharset(cs)
	}

	return ""
}

// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
// some text normalisation to guarantee better results when extracting the content
func (c *Crawler) Preprocess() (*goquery.Document, error) {
	if c.RawHTML == "" {
		c.RawHTML = c.fetchHTML(c.url, c.config.timeout)
	}
	if c.RawHTML == "" {
		return nil, nil
	}

	c.RawHTML = c.addSpacesBetweenTags(c.RawHTML)

	reader := strings.NewReader(c.RawHTML)
	document, err := goquery.NewDocumentFromReader(reader)

	if err != nil {
		return nil, err
	}

	cs := c.GetCharset(document)
	//log.Println("-------------------------------------------CHARSET:", cs)
	if "" != cs && "UTF-8" != cs {
		// the net/html parser and goquery require UTF-8 data
		c.RawHTML = UTF8encode(c.RawHTML, cs)
		reader = strings.NewReader(c.RawHTML)
		document, err = goquery.NewDocumentFromReader(reader)

		if nil != err {
			return nil, err
		}
	}

	return document, nil
}

// Crawl fetches the HTML body and returns an Article
func (c Crawler) Crawl() (*Article, error) {
	article := new(Article)

	document, err := c.Preprocess()
	if nil != err {
		return nil, err
	}
	if nil == document {
		return article, nil
	}

	extractor := NewExtractor(c.config)

	startTime := time.Now().UnixNano()

	article.RawHTML, err = document.Html()
	if nil != err {
		return nil, err
	}
	article.FinalURL = c.url
	article.Doc = document

	article.Title = extractor.GetTitle(document)
	article.MetaLang = extractor.GetMetaLanguage(document)
	article.MetaFavicon = extractor.GetFavicon(document)

	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
	article.CanonicalLink = extractor.GetCanonicalLink(document)
	if "" == article.CanonicalLink {
		article.CanonicalLink = article.FinalURL
	}
	article.Domain = extractor.GetDomain(article.CanonicalLink)
	article.Tags = extractor.GetTags(document)

	cleaner := NewCleaner(c.config)
	article.Doc = cleaner.Clean(article.Doc)

	article.TopImage = OpenGraphResolver(document)
	if article.TopImage == "" {
		article.TopImage = WebPageResolver(article)
	}

	article.TopNode = extractor.CalculateBestNode(document)
	if article.TopNode != nil {
		article.TopNode = extractor.PostCleanup(article.TopNode)

		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)

		videoExtractor := NewVideoExtractor()
		article.Movies = videoExtractor.GetVideos(document)
	}

	article.Delta = time.Now().UnixNano() - startTime

	return article, nil
}

// In many cases, like at the end of each
 element or between  tags,
// we need to add spaces, otherwise the text on either side will get joined together into one word.
// This method also adds newlines after each
 tag to preserve paragraphs.
func (c Crawler) addSpacesBetweenTags(text string) string {
	text = strings.Replace(text, "><", "> <", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	text = strings.Replace(text, "<img ", "\n<img ", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	return strings.Replace(text, "
", "
\n", -1)
}

func (c *Crawler) fetchHTML(u string, timeout time.Duration) string {
	cookieJar, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookieJar,
		Timeout: timeout,
	}
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		log.Println(err.Error())
		return ""
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30")
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err.Error())
		return ""
	}
	contents, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		c.RawHTML = string(contents)
	} else {
		log.Println(err.Error())
	}
	err = resp.Body.Close()
	if err != nil {
		log.Println(err.Error())
	}

	return c.RawHTML
}
<commit_msg>Detect charset when Content-Type contains text/xhtml<commit_after>package goose

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// Crawler can fetch the target HTML page
type Crawler struct {
	config  Configuration
	url     string
	RawHTML string
	Charset string
}

// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
func NewCrawler(config Configuration, url string, RawHTML string) Crawler {
	return Crawler{
		config:  config,
		url:     url,
		RawHTML: RawHTML,
		Charset: "",
	}
}

func getCharsetFromContentType(cs string) string {
	cs = strings.ToLower(strings.Replace(cs, " ", "", -1))
	cs = strings.TrimPrefix(cs, "text/html;charset=")
	cs = strings.TrimPrefix(cs, "text/xhtml;charset=")
	cs = strings.TrimPrefix(cs, "application/xhtml+xml;charset=")
	return NormaliseCharset(cs)
}

// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
// rather than relying on the detection from the HTML meta tags
func (c *Crawler) SetCharset(cs string) {
	c.Charset = getCharsetFromContentType(cs)
}

// GetContentType returns the Content-Type string extracted from the meta tags
func (c Crawler) GetContentType(document *goquery.Document) string {
	var attr string
	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
		attr, _ = s.Attr("content")
	})
	return attr
}

// GetCharset returns a normalised charset string extracted from the meta tags
func (c Crawler) GetCharset(document *goquery.Document) string {
	// manually-provided charset (from HTTP headers?) takes priority
	if "" != c.Charset {
		return c.Charset
	}

	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	ct := c.GetContentType(document)
	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
		return getCharsetFromContentType(ct)
	}

	// <meta charset="utf-8">
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		_, exists := s.Attr("charset")
		return !exists
	})

	if selection != nil {
		cs, _ := selection.Attr("charset")
		return NormaliseCharset(cs)
	}

	return ""
}

// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
// some text normalisation to guarantee better results when extracting the content
func (c *Crawler) Preprocess() (*goquery.Document, error) {
	if c.RawHTML == "" {
		c.RawHTML = c.fetchHTML(c.url, c.config.timeout)
	}
	if c.RawHTML == "" {
		return nil, nil
	}

	c.RawHTML = c.addSpacesBetweenTags(c.RawHTML)

	reader := strings.NewReader(c.RawHTML)
	document, err := goquery.NewDocumentFromReader(reader)

	if err != nil {
		return nil, err
	}

	cs := c.GetCharset(document)
	//log.Println("-------------------------------------------CHARSET:", cs)
	if "" != cs && "UTF-8" != cs {
		// the net/html parser and goquery require UTF-8 data
		c.RawHTML = UTF8encode(c.RawHTML, cs)
		reader = strings.NewReader(c.RawHTML)
		document, err = goquery.NewDocumentFromReader(reader)

		if nil != err {
			return nil, err
		}
	}

	return document, nil
}

// Crawl fetches the HTML body and returns an Article
func (c Crawler) Crawl() (*Article, error) {
	article := new(Article)

	document, err := c.Preprocess()
	if nil != err {
		return nil, err
	}
	if nil == document {
		return article, nil
	}

	extractor := NewExtractor(c.config)

	startTime := time.Now().UnixNano()

	article.RawHTML, err = document.Html()
	if nil != err {
		return nil, err
	}
	article.FinalURL = c.url
	article.Doc = document

	article.Title = extractor.GetTitle(document)
	article.MetaLang = extractor.GetMetaLanguage(document)
	article.MetaFavicon = extractor.GetFavicon(document)

	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
	article.CanonicalLink = extractor.GetCanonicalLink(document)
	if "" == article.CanonicalLink {
		article.CanonicalLink = article.FinalURL
	}
	article.Domain = extractor.GetDomain(article.CanonicalLink)
	article.Tags = extractor.GetTags(document)

	cleaner := NewCleaner(c.config)
	article.Doc = cleaner.Clean(article.Doc)

	article.TopImage = OpenGraphResolver(document)
	if article.TopImage == "" {
		article.TopImage = WebPageResolver(article)
	}

	article.TopNode = extractor.CalculateBestNode(document)
	if article.TopNode != nil {
		article.TopNode = extractor.PostCleanup(article.TopNode)

		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)

		videoExtractor := NewVideoExtractor()
		article.Movies = videoExtractor.GetVideos(document)
	}

	article.Delta = time.Now().UnixNano() - startTime

	return article, nil
}

// In many cases, like at the end of each
 element or between  tags,
// we need to add spaces, otherwise the text on either side will get joined together into one word.
// This method also adds newlines after each
 tag to preserve paragraphs.
func (c Crawler) addSpacesBetweenTags(text string) string {
	text = strings.Replace(text, "><", "> <", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	text = strings.Replace(text, "<img ", "\n<img ", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	return strings.Replace(text, "
", "
\n", -1)
}

func (c *Crawler) fetchHTML(u string, timeout time.Duration) string {
	cookieJar, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookieJar,
		Timeout: timeout,
	}
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		log.Println(err.Error())
		return ""
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30")
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err.Error())
		return ""
	}
	contents, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		c.RawHTML = string(contents)
	} else {
		log.Println(err.Error())
	}
	err = resp.Body.Close()
	if err != nil {
		log.Println(err.Error())
	}

	return c.RawHTML
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/hex"
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path"
	"time"

	"github.com/OpenBazaar/jsonpb"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/imdario/mergo"
	ipnspath "github.com/ipfs/go-ipfs/path"
)

func (n *OpenBazaarNode) GetProfile() (pb.Profile, error) {
	var profile pb.Profile
	f, err := os.Open(path.Join(n.RepoPath, "root", "profile"))
	if err != nil {
		return profile, err
	}
	defer f.Close()
	err = jsonpb.Unmarshal(f, &profile)
	if err != nil {
		return profile, err
	}
	return profile, nil
}

func (n *OpenBazaarNode) FetchProfile(peerId string) (pb.Profile, error) {
	profile, err := ipfs.ResolveThenCat(n.Context, ipnspath.FromString(path.Join(peerId, "profile")))
	if err != nil || len(profile) == 0 {
		return pb.Profile{}, err
	}
	var pro pb.Profile
	err = jsonpb.UnmarshalString(string(profile), &pro)
	if err != nil {
		return pb.Profile{}, err
	}
	return pro, nil
}

func (n *OpenBazaarNode) UpdateProfile(profile *pb.Profile) error {
	mPubkey, err := n.Wallet.MasterPublicKey().ECPubKey()
	if err != nil {
		return err
	}
	profile.BitcoinPubkey = hex.EncodeToString(mPubkey.SerializeCompressed())
	m := jsonpb.Marshaler{
		EnumsAsInts:  false,
		EmitDefaults: true,
		Indent:       "    ",
		OrigName:     false,
	}
	out, err := m.MarshalToString(profile)
	if err != nil {
		return err
	}
	profilePath := path.Join(n.RepoPath, "root", "profile")
	f, err := os.Create(profilePath)
	defer f.Close()
	if err != nil {
		return err
	}
	if _, err := f.WriteString(out); err != nil {
		return err
	}
	return nil
}

func (n *OpenBazaarNode) PatchProfile(patch map[string]interface{}) error {
	profilePath := path.Join(n.RepoPath, "root", "profile")

	// Read stored profile data
	profile := make(map[string]interface{})
	profileBytes, err := ioutil.ReadFile(profilePath)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(profileBytes, &profile); err != nil {
		return err
	}

	modInfo, ok := patch["modInfo"]
	if ok {
		fee, ok := modInfo.(map[string]interface{})["fee"]
		if ok {
			fixedFee, ok := fee.(map[string]interface{})["fixedFee"]
			if ok {
				amt := fixedFee.(map[string]interface{})["amount"].(float64)
				fixedFee.(map[string]interface{})["amount"] = uint64(amt)
			}
		}
	}

	patchMod, pok := patch["moderator"]
	storedMod, sok := profile["moderator"]
	if pok && sok {
		patchBool, ok := patchMod.(bool)
		if !ok {
			return errors.New("Invalid moderator type")
		}
		storedBool, ok := storedMod.(bool)
		if !ok {
			return errors.New("Invalid moderator type")
		}
		if patchBool && patchBool != storedBool {
			if err := n.SetSelfAsModerator(nil); err != nil {
				return err
			}
		} else if !patchBool && patchBool != storedBool {
			if err := n.RemoveSelfAsModerator(); err != nil {
				return err
			}
		}
	}

	// Assuming that `profile` map contains complete data, as it is read
	// from storage, and `patch` map is possibly incomplete, merge first
	// into second recursively, preserving new fields and adding missing
	// old ones
	if err := mergo.Map(&patch, &profile); err != nil {
		return err
	}

	// Execute UpdateProfile with new profile
	newProfile, err := json.Marshal(patch)
	p := new(pb.Profile)
	if err := jsonpb.UnmarshalString(string(newProfile), p); err != nil {
		return err
	}
	return n.UpdateProfile(p)
}

func (n *OpenBazaarNode) appendCountsToProfile(profile *pb.Profile) (*pb.Profile, error) {
	profile.ListingCount = uint32(n.GetListingCount())
	profile.FollowerCount = uint32(n.Datastore.Followers().Count())
	profile.FollowingCount = uint32(n.Datastore.Following().Count())

	ts := new(timestamp.Timestamp)
	ts.Seconds = time.Now().Unix()
	ts.Nanos = 0
	profile.LastModified = ts
	return profile, nil
}

func (n *OpenBazaarNode) updateProfileCounts() error {
	profilePath := path.Join(n.RepoPath, "root", "profile")
	profile := new(pb.Profile)
	_, ferr := os.Stat(profilePath)
	if !os.IsNotExist(ferr) {
		// Read existing file
		file, err := ioutil.ReadFile(profilePath)
		if err != nil {
			return err
		}
		err = jsonpb.UnmarshalString(string(file), profile)
		if err != nil {
			return err
		}
	} else {
		return nil
	}
	profile, err := n.appendCountsToProfile(profile)
	if err != nil {
		return err
	}
	return n.UpdateProfile(profile)
}
<commit_msg>Fix bug patching profile<commit_after>package core

import (
	"encoding/hex"
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path"
	"time"

	"github.com/OpenBazaar/jsonpb"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/imdario/mergo"
	ipnspath "github.com/ipfs/go-ipfs/path"
)

func (n *OpenBazaarNode) GetProfile() (pb.Profile, error) {
	var profile pb.Profile
	f, err := os.Open(path.Join(n.RepoPath, "root", "profile"))
	if err != nil {
		return profile, err
	}
	defer f.Close()
	err = jsonpb.Unmarshal(f, &profile)
	if err != nil {
		return profile, err
	}
	return profile, nil
}

func (n *OpenBazaarNode) FetchProfile(peerId string) (pb.Profile, error) {
	profile, err := ipfs.ResolveThenCat(n.Context, ipnspath.FromString(path.Join(peerId, "profile")))
	if err != nil || len(profile) == 0 {
		return pb.Profile{}, err
	}
	var pro pb.Profile
	err = jsonpb.UnmarshalString(string(profile), &pro)
	if err != nil {
		return pb.Profile{}, err
	}
	return pro, nil
}

func (n *OpenBazaarNode) UpdateProfile(profile *pb.Profile) error {
	mPubkey, err := n.Wallet.MasterPublicKey().ECPubKey()
	if err != nil {
		return err
	}
	profile.BitcoinPubkey = hex.EncodeToString(mPubkey.SerializeCompressed())
	m := jsonpb.Marshaler{
		EnumsAsInts:  false,
		EmitDefaults: true,
		Indent:       "    ",
		OrigName:     false,
	}
	out, err := m.MarshalToString(profile)
	if err != nil {
		return err
	}
	profilePath := path.Join(n.RepoPath, "root", "profile")
	f, err := os.Create(profilePath)
	defer f.Close()
	if err != nil {
		return err
	}
	if _, err := f.WriteString(out); err != nil {
		return err
	}
	return nil
}

func (n *OpenBazaarNode) PatchProfile(patch map[string]interface{}) error {
	profilePath := path.Join(n.RepoPath, "root", "profile")

	// Read stored profile data
	profile := make(map[string]interface{})
	profileBytes, err := ioutil.ReadFile(profilePath)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(profileBytes, &profile); err != nil {
		return err
	}

	formatModeratorAmount := func(modInfo interface{}) {
		fee, ok := modInfo.(map[string]interface{})["fee"]
		if ok {
			fixedFee, ok := fee.(map[string]interface{})["fixedFee"]
			if ok {
				amt := fixedFee.(map[string]interface{})["amount"].(float64)
				fixedFee.(map[string]interface{})["amount"] = uint64(amt)
			}
		}
	}
	modInfo, ok := patch["modInfo"]
	if ok {
		formatModeratorAmount(modInfo)
	}
	modInfo, ok = profile["modInfo"]
	if ok {
		formatModeratorAmount(modInfo)
	}

	patchMod, pok := patch["moderator"]
	storedMod, sok := profile["moderator"]
	if pok && sok {
		patchBool, ok := patchMod.(bool)
		if !ok {
			return errors.New("Invalid moderator type")
		}
		storedBool, ok := storedMod.(bool)
		if !ok {
			return errors.New("Invalid moderator type")
		}
		if patchBool && patchBool != storedBool {
			if err := n.SetSelfAsModerator(nil); err != nil {
				return err
			}
		} else if !patchBool && patchBool != storedBool {
			if err := n.RemoveSelfAsModerator(); err != nil {
				return err
			}
		}
	}

	// Assuming that `profile` map contains complete data, as it is read
	// from storage, and `patch` map is possibly incomplete, merge first
	// into second recursively, preserving new fields and adding missing
	// old ones
	if err := mergo.Map(&patch, &profile); err != nil {
		return err
	}

	// Execute UpdateProfile with new profile
	newProfile, err := json.Marshal(patch)
	p := new(pb.Profile)
	if err := jsonpb.UnmarshalString(string(newProfile), p); err != nil {
		return err
	}
	return n.UpdateProfile(p)
}

func (n *OpenBazaarNode) appendCountsToProfile(profile *pb.Profile) (*pb.Profile, error) {
	profile.ListingCount = uint32(n.GetListingCount())
	profile.FollowerCount = uint32(n.Datastore.Followers().Count())
	profile.FollowingCount = uint32(n.Datastore.Following().Count())

	ts := new(timestamp.Timestamp)
	ts.Seconds = time.Now().Unix()
	ts.Nanos = 0
	profile.LastModified = ts
	return profile, nil
}

func (n *OpenBazaarNode) updateProfileCounts() error {
	profilePath := path.Join(n.RepoPath, "root", "profile")
	profile := new(pb.Profile)
	_, ferr := os.Stat(profilePath)
	if !os.IsNotExist(ferr) {
		// Read existing file
		file, err := ioutil.ReadFile(profilePath)
		if err != nil {
			return err
		}
		err = jsonpb.UnmarshalString(string(file), profile)
		if err != nil {
			return err
		}
	} else {
		return nil
	}
	profile, err := n.appendCountsToProfile(profile)
	if err != nil {
		return err
	}
	return n.UpdateProfile(profile)
}
<|endoftext|>


<commit_before>/*
package fileseq is a library for parsing file sequence strings commonly
used in VFX and animation applications.

Frame Range Shorthand

Support for:

    Standard: 1-10
    Comma Delimted: 1-10,10-20
    Chunked: 1-100x5
    Filled: 1-100y5
    Staggered: 1-100:3 (1-100x3, 1-100x2, 1-100)
    Negative frame numbers: -10-100
    Padding: #=4 padded, @=single pad

*/
package fileseq

import (
	"bytes"
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

const Version = "2.2.3"

var (
	rangePatterns []*regexp.Regexp
	splitPattern  *regexp.Regexp
	singleFrame   *regexp.Regexp
)

func init() {
	// Regular expression patterns for matching frame set strings.
	// Examples:
	//     1-100
	//     100
	//     1-100x5
	rangePatterns = []*regexp.Regexp{
		// Frame range:  1-10
		regexp.MustCompile(`^(-?\d+)-(-?\d+)$`),
		// Single frame:  10
		regexp.MustCompile(`^(-?\d+)$`),
		// Complex range:  1-10x2
		regexp.MustCompile(`^(-?\d+)-(-?\d+)([:xy])(\d+)$`),
	}

	// Regular expression for matching a file sequence string.
	// Example:
	//     /film/shot/renders/hero_bty.1-100#.exr
	//     /film/shot/renders/hero_bty.@@.exr
	splitPattern = regexp.MustCompile(`^(.*?)([:xy\d,-]*)?([#@]+)(\.[a-zA-Z0-9]+)?$`)

	// /film/shot/renders/hero_bty.100.exr
	singleFrame = regexp.MustCompile(`^(.*?)(-?\d+)(\.[a-zA-Z0-9]+)?$`)
}

// IsFrameRange returns true if the given string is a valid frame
// range format.  Any padding characters, such as '#' and '@' are ignored.
func IsFrameRange(frange string) bool {
	_, err := frameRangeMatches(frange)
	if err == nil {
		return true
	}
	return false
}

// FramesToFrameRange takes a slice of frame numbers and
// compresses them into a frame range string.
//
// If sorted == true, pre-sort the frames instead of respecting
// their current order in the range.
//
// If zfill > 1, then pad out each number with "0" to the given
// total width.
func FramesToFrameRange(frames []int, sorted bool, zfill int) string {
	count := len(frames)
	if count == 0 {
		return ""
	}

	if count == 1 {
		return zfillInt(frames[0], zfill)
	}

	if sorted {
		sort.Ints(frames)
	}

	var i, frame, step int
	var start, end string
	var buf bytes.Buffer

	// Keep looping until all frames are consumed
	for len(frames) > 0 {
		count = len(frames)
		// If we get to the last element, just write it
		// and end
		if count <= 2 {
			for i, frame = range frames {
				if buf.Len() > 0 {
					buf.WriteString(",")
				}
				buf.WriteString(zfillInt(frame, zfill))
			}
			break
		}
		// At this point, we have 3 or more frames to check.
		// Scan the current window of the slice to see how
		// many frames we can consume into a group
		step = frames[1] - frames[0]
		for i = 0; i < len(frames)-1; i++ {
			// We have scanned as many frames as we can
			// for this group. Now write them and stop
			// looping on this window
			if (frames[i+1] - frames[i]) != step {
				break
			}
		}

		// Subsequent groups are comma-separated
		if buf.Len() > 0 {
			buf.WriteString(",")
		}

		// We only have a single frame to write for this group
		if i == 0 {
			buf.WriteString(zfillInt(frames[0], zfill))
			frames = frames[1:]
			continue
		}

		// First do a check to see if we could have gotten a larger range
		// out of subsequent values with a different step size
		if i == 1 && count > 3 {
			// Check if the next two pairwise frames have the same step.
			// If so, then it is better than our current grouping.
			if (frames[2] - frames[1]) == (frames[3] - frames[2]) {
				// Just consume the first frame, and allow the next
				// loop to scan the new stepping
				buf.WriteString(zfillInt(frames[0], zfill))
				frames = frames[1:]
				continue
			}
		}

		// Otherwise write out this step range
		start = zfillInt(frames[0], zfill)
		end = zfillInt(frames[i], zfill)
		buf.WriteString(fmt.Sprintf("%s-%s", start, end))
		if step > 1 {
			buf.WriteString(fmt.Sprintf("x%d", step))
		}
		frames = frames[i+1:]
	}

	return buf.String()
}

// frameRangeMatches breaks down the string frame range
// into groups of range matches, for further processing.
func frameRangeMatches(frange string) ([][]string, error) {
	for _, k := range defaultPadding.AllChars() {
		frange = strings.Replace(frange, k, "", -1)
	}

	var (
		matched bool
		match   []string
		rx      *regexp.Regexp
	)

	frange = strings.Replace(frange, " ", "", -1)

	// For each comma-sep component, we will parse a frame range
	parts := strings.Split(frange, ",")
	size := len(parts)
	matches := make([][]string, size, size)

	for i, part := range parts {

		matched = false

		// Build up frames for all comma-sep components
		for _, rx = range rangePatterns {
			if match = rx.FindStringSubmatch(part); match == nil {
				continue
			}
			matched = true
			matches[i] = match[1:]
		}

		// If any component of the comma-sep frame range fails to
		// parse, we bail out
		if !matched {
			err := fmt.Errorf("Failed to parse frame range: %s on part %q", frange, part)
			return nil, err
		}
	}

	return matches, nil
}

// Expands a start, end, and stepping value
// into the full range of int values.
func toRange(start, end, step int) []int {
	nums := []int{}
	if step < 1 {
		step = 1
	}
	if start <= end {
		for i := start; i <= end; {
			nums = append(nums, i)
			i += step
		}
	} else {
		for i := start; i >= end; {
			nums = append(nums, i)
			i -= step
		}
	}
	return nums
}

// Parse an int from a specific part of a frame
// range string component
var parseIntErr error = errors.New("Failed to parse int from part of range string")

func parseInt(s string) (int, error) {
	val, err := strconv.Atoi(s)
	if err != nil {
		return 0, parseIntErr
	}
	return int(val), nil
}

// Return whether a string component from a frame
// range string is a valid modifier symbol
func isModifier(s string) bool {
	return len(s) == 1 && strings.ContainsAny(s, "xy:")
}

// Return the min/max frames from an unsorted list
func minMaxFrame(frames []int) (int, int) {
	srcframes := make([]int, len(frames), len(frames))
	copy(srcframes, frames)
	sort.Ints(srcframes)
	min, max := srcframes[0], srcframes[len(srcframes)-1]
	return min, max
}
<commit_msg>bump v2.3.0<commit_after>/*
package fileseq is a library for parsing file sequence strings commonly
used in VFX and animation applications.

Frame Range Shorthand

Support for:

    Standard: 1-10
    Comma Delimted: 1-10,10-20
    Chunked: 1-100x5
    Filled: 1-100y5
    Staggered: 1-100:3 (1-100x3, 1-100x2, 1-100)
    Negative frame numbers: -10-100
    Padding: #=4 padded, @=single pad

*/
package fileseq

import (
	"bytes"
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

const Version = "2.3.0"

var (
	rangePatterns []*regexp.Regexp
	splitPattern  *regexp.Regexp
	singleFrame   *regexp.Regexp
)

func init() {
	// Regular expression patterns for matching frame set strings.
	// Examples:
	//     1-100
	//     100
	//     1-100x5
	rangePatterns = []*regexp.Regexp{
		// Frame range:  1-10
		regexp.MustCompile(`^(-?\d+)-(-?\d+)$`),
		// Single frame:  10
		regexp.MustCompile(`^(-?\d+)$`),
		// Complex range:  1-10x2
		regexp.MustCompile(`^(-?\d+)-(-?\d+)([:xy])(\d+)$`),
	}

	// Regular expression for matching a file sequence string.
	// Example:
	//     /film/shot/renders/hero_bty.1-100#.exr
	//     /film/shot/renders/hero_bty.@@.exr
	splitPattern = regexp.MustCompile(`^(.*?)([:xy\d,-]*)?([#@]+)(\.[a-zA-Z0-9]+)?$`)

	// /film/shot/renders/hero_bty.100.exr
	singleFrame = regexp.MustCompile(`^(.*?)(-?\d+)(\.[a-zA-Z0-9]+)?$`)
}

// IsFrameRange returns true if the given string is a valid frame
// range format.  Any padding characters, such as '#' and '@' are ignored.
func IsFrameRange(frange string) bool {
	_, err := frameRangeMatches(frange)
	if err == nil {
		return true
	}
	return false
}

// FramesToFrameRange takes a slice of frame numbers and
// compresses them into a frame range string.
//
// If sorted == true, pre-sort the frames instead of respecting
// their current order in the range.
//
// If zfill > 1, then pad out each number with "0" to the given
// total width.
func FramesToFrameRange(frames []int, sorted bool, zfill int) string {
	count := len(frames)
	if count == 0 {
		return ""
	}

	if count == 1 {
		return zfillInt(frames[0], zfill)
	}

	if sorted {
		sort.Ints(frames)
	}

	var i, frame, step int
	var start, end string
	var buf bytes.Buffer

	// Keep looping until all frames are consumed
	for len(frames) > 0 {
		count = len(frames)
		// If we get to the last element, just write it
		// and end
		if count <= 2 {
			for i, frame = range frames {
				if buf.Len() > 0 {
					buf.WriteString(",")
				}
				buf.WriteString(zfillInt(frame, zfill))
			}
			break
		}
		// At this point, we have 3 or more frames to check.
		// Scan the current window of the slice to see how
		// many frames we can consume into a group
		step = frames[1] - frames[0]
		for i = 0; i < len(frames)-1; i++ {
			// We have scanned as many frames as we can
			// for this group. Now write them and stop
			// looping on this window
			if (frames[i+1] - frames[i]) != step {
				break
			}
		}

		// Subsequent groups are comma-separated
		if buf.Len() > 0 {
			buf.WriteString(",")
		}

		// We only have a single frame to write for this group
		if i == 0 {
			buf.WriteString(zfillInt(frames[0], zfill))
			frames = frames[1:]
			continue
		}

		// First do a check to see if we could have gotten a larger range
		// out of subsequent values with a different step size
		if i == 1 && count > 3 {
			// Check if the next two pairwise frames have the same step.
			// If so, then it is better than our current grouping.
			if (frames[2] - frames[1]) == (frames[3] - frames[2]) {
				// Just consume the first frame, and allow the next
				// loop to scan the new stepping
				buf.WriteString(zfillInt(frames[0], zfill))
				frames = frames[1:]
				continue
			}
		}

		// Otherwise write out this step range
		start = zfillInt(frames[0], zfill)
		end = zfillInt(frames[i], zfill)
		buf.WriteString(fmt.Sprintf("%s-%s", start, end))
		if step > 1 {
			buf.WriteString(fmt.Sprintf("x%d", step))
		}
		frames = frames[i+1:]
	}

	return buf.String()
}

// frameRangeMatches breaks down the string frame range
// into groups of range matches, for further processing.
func frameRangeMatches(frange string) ([][]string, error) {
	for _, k := range defaultPadding.AllChars() {
		frange = strings.Replace(frange, k, "", -1)
	}

	var (
		matched bool
		match   []string
		rx      *regexp.Regexp
	)

	frange = strings.Replace(frange, " ", "", -1)

	// For each comma-sep component, we will parse a frame range
	parts := strings.Split(frange, ",")
	size := len(parts)
	matches := make([][]string, size, size)

	for i, part := range parts {

		matched = false

		// Build up frames for all comma-sep components
		for _, rx = range rangePatterns {
			if match = rx.FindStringSubmatch(part); match == nil {
				continue
			}
			matched = true
			matches[i] = match[1:]
		}

		// If any component of the comma-sep frame range fails to
		// parse, we bail out
		if !matched {
			err := fmt.Errorf("Failed to parse frame range: %s on part %q", frange, part)
			return nil, err
		}
	}

	return matches, nil
}

// Expands a start, end, and stepping value
// into the full range of int values.
func toRange(start, end, step int) []int {
	nums := []int{}
	if step < 1 {
		step = 1
	}
	if start <= end {
		for i := start; i <= end; {
			nums = append(nums, i)
			i += step
		}
	} else {
		for i := start; i >= end; {
			nums = append(nums, i)
			i -= step
		}
	}
	return nums
}

// Parse an int from a specific part of a frame
// range string component
var parseIntErr error = errors.New("Failed to parse int from part of range string")

func parseInt(s string) (int, error) {
	val, err := strconv.Atoi(s)
	if err != nil {
		return 0, parseIntErr
	}
	return int(val), nil
}

// Return whether a string component from a frame
// range string is a valid modifier symbol
func isModifier(s string) bool {
	return len(s) == 1 && strings.ContainsAny(s, "xy:")
}

// Return the min/max frames from an unsorted list
func minMaxFrame(frames []int) (int, int) {
	srcframes := make([]int, len(frames), len(frames))
	copy(srcframes, frames)
	sort.Ints(srcframes)
	min, max := srcframes[0], srcframes[len(srcframes)-1]
	return min, max
}
<|endoftext|>


<commit_before>package core

// transfer2go data core module, request implementation
// Author: Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"container/heap"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/utils"
)

// AgentStatus data type
type AgentStatus struct {
	Url       string            `json:"url"`      // agent url
	Name      string            `json:"name"`     // agent name or alias
	TimeStamp int64             `json:"ts"`       // time stamp
	Catalog   string            `json:"catalog"`  // underlying TFC catalog
	Protocol  string            `json:"protocol"` // underlying transfer protocol
	Backend   string            `json:"backend"`  // underlying transfer backend
	Tool      string            `json:"tool"`     // underlying transfer tool, e.g. xrdcp
	ToolOpts  string            `json:"toolopts"` // options for backend tool
	Agents    map[string]string `json:"agents"`   // list of known agents
	Addrs     []string          `json:"addrs"`    // list of all IP addresses
	Metrics   map[string]int64  `json:"metrics"`  // agent metrics
}

// Processor is an object who process' given task
// The logic of the Processor should be implemented.
type Processor struct {
}

// Request interface defines a task process
type Request interface {
	Process(*TransferRequest) error
}

// RequestFunc is a function type that implements the Request interface
type RequestFunc func(*TransferRequest) error

// Decorator wraps a request with extra behavior
type Decorator func(Request) Request

// DefaultProcessor is a default processor instance
var DefaultProcessor = &Processor{}

// String provides string representation of given agent status
func (a *AgentStatus) String() string {
	return fmt.Sprintf("<Agent name=%s url=%s catalog=%s protocol=%s backend=%s tool=%s toolOpts=%s agents=%v addrs=%v metrics(%v)>", a.Name, a.Url, a.Catalog, a.Protocol, a.Backend, a.Tool, a.ToolOpts, a.Agents, a.Addrs, a.Metrics)
}

// Process defines execution process for a given task
func (e *Processor) Process(t *TransferRequest) error {
	return nil
}

// Process is a method of TransferRequest
func (f RequestFunc) Process(t *TransferRequest) error {
	return f(t)
}

// filleTransferRequest creates HTTP request to transfer a given file name
// https://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/
func fileTransferRequest(c CatalogEntry, tr *TransferRequest) (*http.Response, error) {
	file, err := os.Open(c.Pfn)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	// Define go pipe
	pr, pw := io.Pipe()
	writer := multipart.NewWriter(pw)
	var resp *http.Response
	// we need to wait for everything to be done
	done := make(chan error)
	go func() {
		url := fmt.Sprintf("%s/upload", tr.DstUrl)
		req, err := http.NewRequest("POST", url, pr)
		if err != nil {
			done <- err
			return
		}
		req.Header.Set("Content-Type", writer.FormDataContentType())
		req.Header.Set("Pfn", c.Pfn)
		req.Header.Set("Lfn", c.Lfn)
		req.Header.Set("Bytes", fmt.Sprintf("%d", c.Bytes))
		req.Header.Set("Hash", c.Hash)
		req.Header.Set("Src", tr.SrcAlias)
		req.Header.Set("Dst", tr.DstAlias)
		client := utils.HttpClient()
		resp, err = client.Do(req)
		if err != nil {
			done <- err
			return
		}
		if resp.StatusCode != 200 {
			done <- errors.New("Status Code is not 200")
			return
		}
		done <- nil
	}()
	part, err := writer.CreateFormFile("data", filepath.Base(c.Pfn))
	if err != nil {
		return nil, err
	}
	// Use copy of writer to avoid deadlock condition
	out := io.MultiWriter(part)
	_, err = io.Copy(out, file)
	if err != nil {
		return nil, err
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}
	err = pw.Close()
	if err != nil {
		return nil, err
	}
	err = <-done
	if err != nil {
		return nil, err
	}
	return resp, nil
}

// helper function to perform transfer via HTTP protocol
func httpTransfer(c CatalogEntry, t *TransferRequest) (string, error) {
	// create file transfer request
	resp, err := fileTransferRequest(c, t)
	if err != nil || resp==nil || resp.StatusCode != 200 {
		return "", err
	}
        defer resp.Body.Close()
	var r CatalogEntry
	err = json.NewDecoder(resp.Body).Decode(&r)
	if err != nil {
		return "", err
	}
	return r.Pfn, nil
}

// Store returns a Decorator that stores request
func Store() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			t.Id = time.Now().Unix()
			item := &Item{
				Value:    *t,
				priority: t.Priority,
			}
			fmt.Println(*t)
			err := TFC.InsertRequest(*t)
			if err != nil {
				return err
			} else {
				heap.Push(&RequestQueue, item)
			}
			log.WithFields(log.Fields{
				"Request": t,
			}).Println("Request Saved")
			return r.Process(t)
		})
	}
}

// Delete returns a Decorator that deletes request from heap
func Delete() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			// Delete request from PriorityQueue. The complexity is O(n) where n = heap.Len()
			index := -1
			var err error

			for _, item := range RequestQueue {
				if item.Value.Id == t.Id {
					index = item.index
					break
				}
			}

			if index < RequestQueue.Len() && index >= 0 {
				err = TFC.UpdateRequest(t.Id, "deleted")
				if err != nil {
					t.Status = "error"
					return err
				} else {
					// TODO: May be we need to add lock over here.
					heap.Remove(&RequestQueue, index)
					t.Status = "deleted"
					log.WithFields(log.Fields{
						"Request": t,
					}).Println("Request Deleted")
				}
			} else {
				t.Status = "error"
				err = errors.New("Can't find request in heap")
				return err
			}

			return r.Process(t)
		})
	}
}

// Transfer returns a Decorator that performs request transfers by pull model
func PullTransfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer")
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}
			// if both are up then send acknowledge message to destination on /pullack url.
			body, err := json.Marshal(t)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/pull", t.DstUrl)
			resp = utils.FetchResponse(url, body)
			// check return status code
			if resp.StatusCode != 200 {
				return fmt.Errorf("Response %s, error=%s", resp.Status, string(resp.Data))
			}
			return r.Process(t)
		})
	}
}

// Transfer returns a Decorator that performs request transfers
func PushTransfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer")
			var records []CatalogEntry
			// Consider those requests which are failed in previous iteration.
			// If it is nil then request must be passing through first iteration.
			if t.FailedRecords != nil {
				records = t.FailedRecords
			} else {
				records = TFC.Records(*t)
			}
			if len(records) == 0 {
				// file does not exists in TFC, nothing to do, return immediately
				log.WithFields(log.Fields{
					"TransferRequest": t,
				}).Warn("Does not match anything in TFC of this agent\n", t)
				return r.Process(t)
			}
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}

			// TODO: I need to implement bulk transfer for all files in found records
			// so far I loop over them individually and transfer one by one
			var trRecords []CatalogEntry // list of successfully transferred records
			var failedRecords []CatalogEntry
			// Overwrite the previous error status
			t.Status = ""
			for _, rec := range records {

				time0 := time.Now().Unix()

				AgentMetrics.Bytes.Inc(rec.Bytes)

				// if protocol is not given use default one: HTTP
				var rpfn string // remote PFN
				if srcAgent.Protocol == "" || srcAgent.Protocol == "http" {
					log.WithFields(log.Fields{
						"dstAgent": dstAgent.String(),
					}).Println("Transfer via HTTP protocol to", dstAgent.String())
					rpfn, err = httpTransfer(rec, t)
					if err != nil {
						log.WithFields(log.Fields{
							"TransferRequest": t.String(),
							"Record":          rec.String(),
							"Err":             err,
						}).Error("Transfer", rec.String(), t.String(), err)
						t.Status = err.Error()
						failedRecords = append(failedRecords, rec)
						continue // if we fail on single record we continue with others
					}
				} else {
					// construct remote PFN by using destination agent backend and record LFN
					rpfn = fmt.Sprintf("%s%s", dstAgent.Backend, rec.Lfn)
					// perform transfer with the help of backend tool
					var cmd *exec.Cmd
					if srcAgent.ToolOpts == "" {
						cmd = exec.Command(srcAgent.Tool, rec.Pfn, rpfn)
					} else {
						cmd = exec.Command(srcAgent.Tool, srcAgent.ToolOpts, rec.Pfn, rpfn)
					}
					log.WithFields(log.Fields{
						"Command": cmd,
					}).Println("Transfer command")
					err = cmd.Run()
					if err != nil {
						log.WithFields(log.Fields{
							"Tool":         srcAgent.Tool,
							"Tool options": srcAgent.ToolOpts,
							"PFN":          rec.Pfn,
							"Remote PFN":   rpfn,
							"Err":          err,
						}).Error("Transfer")
						t.Status = err.Error()
						failedRecords = append(failedRecords, rec)
						continue // if we fail on single record we continue with others
					}
				}
				r := CatalogEntry{Dataset: rec.Dataset, Block: rec.Block, Lfn: rec.Lfn, Pfn: rpfn, Bytes: rec.Bytes, Hash: rec.Hash, TransferTime: (time.Now().Unix() - time0), Timestamp: time.Now().Unix()}
				trRecords = append(trRecords, r)

				// record how much we transferred
				AgentMetrics.TotalBytes.Inc(r.Bytes) // keep growing
				AgentMetrics.Total.Inc(1)            // keep growing
				AgentMetrics.Bytes.Dec(rec.Bytes)    // decrement since we're done

			}
			// Add entry for remote TFC after transfer is completed
			url = fmt.Sprintf("%s/tfc", t.DstUrl)
			d, e := json.Marshal(trRecords)
			if e != nil {
				return e
			}
			resp = utils.FetchResponse(url, d) // POST request
			if resp.Error != nil {
				return resp.Error
			}
			t.FailedRecords = failedRecords
			return r.Process(t)
		})
	}
}

// Logging returns a Decorator that logs client requests
func Logging(l *log.Logger) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			l.Println("TransferRequest", t)
			return r.Process(t)
		})
	}
}

// Pause returns a Decorator that pauses request for a given time interval
func Pause(interval time.Duration) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			if interval > 0 {
				log.WithFields(log.Fields{
					"Request":  t,
					"Interval": interval,
				}).Println("TransferRequest is paused by")
				time.Sleep(interval)
			}
			return r.Process(t)
		})
	}
}

// Tracer returns a Decorator that traces given request
func Tracer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"TransferRequest": t,
			}).Println("Trace")
			return r.Process(t)
		})
	}
}

// Decorate decorates a Request r with all given Decorators
func Decorate(r Request, ds ...Decorator) Request {
	decorated := r
	for _, decorate := range ds {
		decorated = decorate(decorated)
	}
	return decorated
}
<commit_msg>Add separate error catch<commit_after>package core

// transfer2go data core module, request implementation
// Author: Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"container/heap"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/utils"
)

// AgentStatus data type
type AgentStatus struct {
	Url       string            `json:"url"`      // agent url
	Name      string            `json:"name"`     // agent name or alias
	TimeStamp int64             `json:"ts"`       // time stamp
	Catalog   string            `json:"catalog"`  // underlying TFC catalog
	Protocol  string            `json:"protocol"` // underlying transfer protocol
	Backend   string            `json:"backend"`  // underlying transfer backend
	Tool      string            `json:"tool"`     // underlying transfer tool, e.g. xrdcp
	ToolOpts  string            `json:"toolopts"` // options for backend tool
	Agents    map[string]string `json:"agents"`   // list of known agents
	Addrs     []string          `json:"addrs"`    // list of all IP addresses
	Metrics   map[string]int64  `json:"metrics"`  // agent metrics
}

// Processor is an object who process' given task
// The logic of the Processor should be implemented.
type Processor struct {
}

// Request interface defines a task process
type Request interface {
	Process(*TransferRequest) error
}

// RequestFunc is a function type that implements the Request interface
type RequestFunc func(*TransferRequest) error

// Decorator wraps a request with extra behavior
type Decorator func(Request) Request

// DefaultProcessor is a default processor instance
var DefaultProcessor = &Processor{}

// String provides string representation of given agent status
func (a *AgentStatus) String() string {
	return fmt.Sprintf("<Agent name=%s url=%s catalog=%s protocol=%s backend=%s tool=%s toolOpts=%s agents=%v addrs=%v metrics(%v)>", a.Name, a.Url, a.Catalog, a.Protocol, a.Backend, a.Tool, a.ToolOpts, a.Agents, a.Addrs, a.Metrics)
}

// Process defines execution process for a given task
func (e *Processor) Process(t *TransferRequest) error {
	return nil
}

// Process is a method of TransferRequest
func (f RequestFunc) Process(t *TransferRequest) error {
	return f(t)
}

// filleTransferRequest creates HTTP request to transfer a given file name
// https://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/
func fileTransferRequest(c CatalogEntry, tr *TransferRequest) (*http.Response, error) {
	file, err := os.Open(c.Pfn)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	// Define go pipe
	pr, pw := io.Pipe()
	writer := multipart.NewWriter(pw)
	var resp *http.Response
	// we need to wait for everything to be done
	done := make(chan error)
	go func() {
		url := fmt.Sprintf("%s/upload", tr.DstUrl)
		req, err := http.NewRequest("POST", url, pr)
		if err != nil {
			done <- err
			return
		}
		req.Header.Set("Content-Type", writer.FormDataContentType())
		req.Header.Set("Pfn", c.Pfn)
		req.Header.Set("Lfn", c.Lfn)
		req.Header.Set("Bytes", fmt.Sprintf("%d", c.Bytes))
		req.Header.Set("Hash", c.Hash)
		req.Header.Set("Src", tr.SrcAlias)
		req.Header.Set("Dst", tr.DstAlias)
		client := utils.HttpClient()
		resp, err = client.Do(req)
		if err != nil {
			done <- err
			return
		}
		if resp.StatusCode != 200 {
			done <- errors.New("Status Code is not 200")
			return
		}
		done <- nil
	}()
	part, err := writer.CreateFormFile("data", filepath.Base(c.Pfn))
	if err != nil {
		return nil, err
	}
	// Use copy of writer to avoid deadlock condition
	out := io.MultiWriter(part)
	_, err = io.Copy(out, file)
	if err != nil {
		return nil, err
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}
	err = pw.Close()
	if err != nil {
		return nil, err
	}
	err = <-done
	if err != nil {
		return nil, err
	}
	return resp, nil
}

// helper function to perform transfer via HTTP protocol
func httpTransfer(c CatalogEntry, t *TransferRequest) (string, error) {
	// create file transfer request
	resp, err := fileTransferRequest(c, t)
	if err != nil {
		return "", err
	}
	if resp==nil || resp.StatusCode != 200 {
		return "", errors.New("Empty response from destination")
	}
        defer resp.Body.Close()
	var r CatalogEntry
	err = json.NewDecoder(resp.Body).Decode(&r)
	if err != nil {
		return "", err
	}
	return r.Pfn, nil
}

// Store returns a Decorator that stores request
func Store() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			t.Id = time.Now().Unix()
			item := &Item{
				Value:    *t,
				priority: t.Priority,
			}
			fmt.Println(*t)
			err := TFC.InsertRequest(*t)
			if err != nil {
				return err
			} else {
				heap.Push(&RequestQueue, item)
			}
			log.WithFields(log.Fields{
				"Request": t,
			}).Println("Request Saved")
			return r.Process(t)
		})
	}
}

// Delete returns a Decorator that deletes request from heap
func Delete() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			// Delete request from PriorityQueue. The complexity is O(n) where n = heap.Len()
			index := -1
			var err error

			for _, item := range RequestQueue {
				if item.Value.Id == t.Id {
					index = item.index
					break
				}
			}

			if index < RequestQueue.Len() && index >= 0 {
				err = TFC.UpdateRequest(t.Id, "deleted")
				if err != nil {
					t.Status = "error"
					return err
				} else {
					// TODO: May be we need to add lock over here.
					heap.Remove(&RequestQueue, index)
					t.Status = "deleted"
					log.WithFields(log.Fields{
						"Request": t,
					}).Println("Request Deleted")
				}
			} else {
				t.Status = "error"
				err = errors.New("Can't find request in heap")
				return err
			}

			return r.Process(t)
		})
	}
}

// Transfer returns a Decorator that performs request transfers by pull model
func PullTransfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer")
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}
			// if both are up then send acknowledge message to destination on /pullack url.
			body, err := json.Marshal(t)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/pull", t.DstUrl)
			resp = utils.FetchResponse(url, body)
			// check return status code
			if resp.StatusCode != 200 {
				return fmt.Errorf("Response %s, error=%s", resp.Status, string(resp.Data))
			}
			return r.Process(t)
		})
	}
}

// Transfer returns a Decorator that performs request transfers
func PushTransfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer")
			var records []CatalogEntry
			// Consider those requests which are failed in previous iteration.
			// If it is nil then request must be passing through first iteration.
			if t.FailedRecords != nil {
				records = t.FailedRecords
			} else {
				records = TFC.Records(*t)
			}
			if len(records) == 0 {
				// file does not exists in TFC, nothing to do, return immediately
				log.WithFields(log.Fields{
					"TransferRequest": t,
				}).Warn("Does not match anything in TFC of this agent\n", t)
				return r.Process(t)
			}
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}

			// TODO: I need to implement bulk transfer for all files in found records
			// so far I loop over them individually and transfer one by one
			var trRecords []CatalogEntry // list of successfully transferred records
			var failedRecords []CatalogEntry
			// Overwrite the previous error status
			t.Status = ""
			for _, rec := range records {

				time0 := time.Now().Unix()

				AgentMetrics.Bytes.Inc(rec.Bytes)

				// if protocol is not given use default one: HTTP
				var rpfn string // remote PFN
				if srcAgent.Protocol == "" || srcAgent.Protocol == "http" {
					log.WithFields(log.Fields{
						"dstAgent": dstAgent.String(),
					}).Println("Transfer via HTTP protocol to", dstAgent.String())
					rpfn, err = httpTransfer(rec, t)
					if err != nil {
						log.WithFields(log.Fields{
							"TransferRequest": t.String(),
							"Record":          rec.String(),
							"Err":             err,
						}).Error("Transfer", rec.String(), t.String(), err)
						t.Status = err.Error()
						failedRecords = append(failedRecords, rec)
						continue // if we fail on single record we continue with others
					}
				} else {
					// construct remote PFN by using destination agent backend and record LFN
					rpfn = fmt.Sprintf("%s%s", dstAgent.Backend, rec.Lfn)
					// perform transfer with the help of backend tool
					var cmd *exec.Cmd
					if srcAgent.ToolOpts == "" {
						cmd = exec.Command(srcAgent.Tool, rec.Pfn, rpfn)
					} else {
						cmd = exec.Command(srcAgent.Tool, srcAgent.ToolOpts, rec.Pfn, rpfn)
					}
					log.WithFields(log.Fields{
						"Command": cmd,
					}).Println("Transfer command")
					err = cmd.Run()
					if err != nil {
						log.WithFields(log.Fields{
							"Tool":         srcAgent.Tool,
							"Tool options": srcAgent.ToolOpts,
							"PFN":          rec.Pfn,
							"Remote PFN":   rpfn,
							"Err":          err,
						}).Error("Transfer")
						t.Status = err.Error()
						failedRecords = append(failedRecords, rec)
						continue // if we fail on single record we continue with others
					}
				}
				r := CatalogEntry{Dataset: rec.Dataset, Block: rec.Block, Lfn: rec.Lfn, Pfn: rpfn, Bytes: rec.Bytes, Hash: rec.Hash, TransferTime: (time.Now().Unix() - time0), Timestamp: time.Now().Unix()}
				trRecords = append(trRecords, r)

				// record how much we transferred
				AgentMetrics.TotalBytes.Inc(r.Bytes) // keep growing
				AgentMetrics.Total.Inc(1)            // keep growing
				AgentMetrics.Bytes.Dec(rec.Bytes)    // decrement since we're done

			}
			// Add entry for remote TFC after transfer is completed
			url = fmt.Sprintf("%s/tfc", t.DstUrl)
			d, e := json.Marshal(trRecords)
			if e != nil {
				return e
			}
			resp = utils.FetchResponse(url, d) // POST request
			if resp.Error != nil {
				return resp.Error
			}
			t.FailedRecords = failedRecords
			return r.Process(t)
		})
	}
}

// Logging returns a Decorator that logs client requests
func Logging(l *log.Logger) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			l.Println("TransferRequest", t)
			return r.Process(t)
		})
	}
}

// Pause returns a Decorator that pauses request for a given time interval
func Pause(interval time.Duration) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			if interval > 0 {
				log.WithFields(log.Fields{
					"Request":  t,
					"Interval": interval,
				}).Println("TransferRequest is paused by")
				time.Sleep(interval)
			}
			return r.Process(t)
		})
	}
}

// Tracer returns a Decorator that traces given request
func Tracer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"TransferRequest": t,
			}).Println("Trace")
			return r.Process(t)
		})
	}
}

// Decorate decorates a Request r with all given Decorators
func Decorate(r Request, ds ...Decorator) Request {
	decorated := r
	for _, decorate := range ds {
		decorated = decorate(decorated)
	}
	return decorated
}
<|endoftext|>