<commit_before>package config

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/util"
)

var configLogger = logging.GetLogger("config")

// `apibase` and `agentName` are set from build flags
var apibase string

func getApibase() string {
	if apibase != "" {
		return apibase
	}
	return "https://mackerel.io"
}

var agentName string

func getAgentName() string {
	if agentName != "" {
		return agentName
	}
	return "mackerel-agent"
}

// Config represents mackerel-agent's configuration file.
type Config struct {
	Apibase     string
	Apikey      string
	Root        string
	Pidfile     string
	Conffile    string
	Roles       []string
	Verbose     bool
	Silent      bool
	Diagnostic  bool `toml:"diagnostic"`
	Connection  ConnectionConfig
	DisplayName string      `toml:"display_name"`
	HostStatus  HostStatus  `toml:"host_status"`
	Filesystems Filesystems `toml:"filesystems"`
	HTTPProxy   string      `toml:"http_proxy"`

	// Corresponds to the set of [plugin.<kind>.<name>] sections
	// the key of the map is <kind>, which should be one of "metrics" or "checks".
	Plugin map[string]PluginConfigs

	Include string

	// Cannot exist in configuration files
	HostIDStorage HostIDStorage
	MetricPlugins map[string]MetricPlugin
	CheckPlugins  map[string]CheckPlugin
}

// PluginConfigs represents a set of [plugin.<kind>.<name>] sections in the configuration file
// under a specific <kind>. The key of the map is <name>, for example "mysql" of "plugin.metrics.mysql".
type PluginConfigs map[string]*PluginConfig

// PluginConfig represents a section of [plugin.*].
// `MaxCheckAttempts`, `NotificationInterval` and `CheckInterval` options are used with check monitoring plugins. Custom metrics plugins ignore these options.
// `User` option is ignore in windows
type PluginConfig struct {
	CommandRaw           interface{} `toml:"command"`
	Command              string
	CommandArgs          []string
	User                 string
	NotificationInterval *int32  `toml:"notification_interval"`
	CheckInterval        *int32  `toml:"check_interval"`
	MaxCheckAttempts     *int32  `toml:"max_check_attempts"`
	CustomIdentifier     *string `toml:"custom_identifier"`
}

// MetricPlugin represents the configuration of a metric plugin
// The `User` option is ignored in Windows
type MetricPlugin struct {
	Command          string
	CommandArgs      []string
	User             string
	CustomIdentifier *string
}

// CheckPlugin represents the configuration of a check plugin
// The `User` option is ignored in Windows
type CheckPlugin struct {
	Command              string
	CommandArgs          []string
	User                 string
	NotificationInterval *int32
	CheckInterval        *int32
	MaxCheckAttempts     *int32
}

func (pconf *PluginConfig) prepareCommand() error {
	const errFmt = "failed to prepare plugin command. A configuration value of `command` should be string or string slice, but %T"
	v := pconf.CommandRaw
	switch t := v.(type) {
	case string:
		pconf.Command = t
	case []interface{}:
		if len(t) > 0 {
			for _, vv := range t {
				str, ok := vv.(string)
				if !ok {
					return fmt.Errorf(errFmt, v)
				}
				pconf.CommandArgs = append(pconf.CommandArgs, str)
			}
		} else {
			return fmt.Errorf(errFmt, v)
		}
	case []string:
		pconf.CommandArgs = t
	default:
		return fmt.Errorf(errFmt, v)
	}
	return nil
}

// Run the plugin
func (pconf *PluginConfig) Run() (string, string, int, error) {
	if len(pconf.CommandArgs) > 0 {
		return util.RunCommandArgs(pconf.CommandArgs, pconf.User)
	}
	return util.RunCommand(pconf.Command, pconf.User)
}

const postMetricsDequeueDelaySecondsMax = 59   // max delay seconds for dequeuing from buffer queue
const postMetricsRetryDelaySecondsMax = 3 * 60 // max delay seconds for retrying a request that caused errors

// PostMetricsInterval XXX
var PostMetricsInterval = 1 * time.Minute

// ConnectionConfig XXX
type ConnectionConfig struct {
	PostMetricsDequeueDelaySeconds int `toml:"post_metrics_dequeue_delay_seconds"` // delay for dequeuing from buffer queue
	PostMetricsRetryDelaySeconds   int `toml:"post_metrics_retry_delay_seconds"`   // delay for retrying a request that caused errors
	PostMetricsRetryMax            int `toml:"post_metrics_retry_max"`             // max numbers of retries for a request that causes errors
	PostMetricsBufferSize          int `toml:"post_metrics_buffer_size"`           // max numbers of requests stored in buffer queue.
}

// HostStatus configure host status on agent start/stop
type HostStatus struct {
	OnStart string `toml:"on_start"`
	OnStop  string `toml:"on_stop"`
}

// Filesystems configure filesystem related settings
type Filesystems struct {
	Ignore        Regexpwrapper `toml:"ignore"`
	UseMountpoint bool          `toml:"use_mountpoint"`
}

// Regexpwrapper is a wrapper type for marshalling string
type Regexpwrapper struct {
	*regexp.Regexp
}

// UnmarshalText for compiling regexp string while loading toml
func (r *Regexpwrapper) UnmarshalText(text []byte) error {
	var err error
	r.Regexp, err = regexp.Compile(string(text))
	return err
}

// CheckNames return list of plugin.checks._name_
func (conf *Config) CheckNames() []string {
	checks := []string{}
	for name := range conf.Plugin["checks"] {
		checks = append(checks, name)
	}
	return checks
}

// LoadConfig XXX
func LoadConfig(conffile string) (*Config, error) {
	config, err := loadConfigFile(conffile)
	if err != nil {
		return nil, err
	}

	// set default values if config does not have values
	if config.Apibase == "" {
		config.Apibase = DefaultConfig.Apibase
	}
	if config.Root == "" {
		config.Root = DefaultConfig.Root
	}
	if config.Pidfile == "" {
		config.Pidfile = DefaultConfig.Pidfile
	}
	if config.Verbose == false {
		config.Verbose = DefaultConfig.Verbose
	}
	if config.Diagnostic == false {
		config.Diagnostic = DefaultConfig.Diagnostic
	}
	if config.Connection.PostMetricsDequeueDelaySeconds == 0 {
		config.Connection.PostMetricsDequeueDelaySeconds = DefaultConfig.Connection.PostMetricsDequeueDelaySeconds
	}
	if config.Connection.PostMetricsDequeueDelaySeconds > postMetricsDequeueDelaySecondsMax {
		configLogger.Warningf("'post_metrics_dequese_delay_seconds' is set to %d (Maximum Value).", postMetricsDequeueDelaySecondsMax)
		config.Connection.PostMetricsDequeueDelaySeconds = postMetricsDequeueDelaySecondsMax
	}
	if config.Connection.PostMetricsRetryDelaySeconds == 0 {
		config.Connection.PostMetricsRetryDelaySeconds = DefaultConfig.Connection.PostMetricsRetryDelaySeconds
	}
	if config.Connection.PostMetricsRetryDelaySeconds > postMetricsRetryDelaySecondsMax {
		configLogger.Warningf("'post_metrics_retry_delay_seconds' is set to %d (Maximum Value).", postMetricsRetryDelaySecondsMax)
		config.Connection.PostMetricsRetryDelaySeconds = postMetricsRetryDelaySecondsMax
	}
	if config.Connection.PostMetricsRetryMax == 0 {
		config.Connection.PostMetricsRetryMax = DefaultConfig.Connection.PostMetricsRetryMax
	}
	if config.Connection.PostMetricsBufferSize == 0 {
		config.Connection.PostMetricsBufferSize = DefaultConfig.Connection.PostMetricsBufferSize
	}

	return config, err
}

func loadConfigFile(file string) (*Config, error) {
	config := &Config{}
	if _, err := toml.DecodeFile(file, config); err != nil {
		return config, err
	}

	if config.Include != "" {
		if err := includeConfigFile(config, config.Include); err != nil {
			return config, err
		}
	}
	for _, pconfs := range config.Plugin {
		for _, pconf := range pconfs {
			err := pconf.prepareCommand()
			if err != nil {
				return nil, err
			}
		}
	}
	return config, nil
}

func includeConfigFile(config *Config, include string) error {
	files, err := filepath.Glob(include)
	if err != nil {
		return err
	}

	for _, file := range files {
		// Save current "roles" value and reset it
		// because toml.DecodeFile()-ing on a fulfilled struct
		// produces bizarre array values.
		rolesSaved := config.Roles
		config.Roles = nil

		// Also, save plugin values for later merging
		pluginSaved := map[string]PluginConfigs{}
		for kind, plugins := range config.Plugin {
			pluginSaved[kind] = plugins
		}

		meta, err := toml.DecodeFile(file, &config)
		if err != nil {
			return fmt.Errorf("while loading included config file %s: %s", file, err)
		}

		// If included config does not have "roles" key,
		// use the previous roles configuration value.
		if meta.IsDefined("roles") == false {
			config.Roles = rolesSaved
		}

		for kind, plugins := range config.Plugin {
			for key, conf := range plugins {
				if pluginSaved[kind] == nil {
					pluginSaved[kind] = PluginConfigs{}
				}
				pluginSaved[kind][key] = conf
			}
		}

		config.Plugin = pluginSaved
	}

	return nil
}

func (conf *Config) hostIDStorage() HostIDStorage {
	if conf.HostIDStorage == nil {
		conf.HostIDStorage = &FileSystemHostIDStorage{Root: conf.Root}
	}
	return conf.HostIDStorage
}

// LoadHostID loads the previously saved host id.
func (conf *Config) LoadHostID() (string, error) {
	return conf.hostIDStorage().LoadHostID()
}

// SaveHostID saves the host id, which may be restored by LoadHostID.
func (conf *Config) SaveHostID(id string) error {
	return conf.hostIDStorage().SaveHostID(id)
}

// DeleteSavedHostID deletes the host id saved by SaveHostID.
func (conf *Config) DeleteSavedHostID() error {
	return conf.hostIDStorage().DeleteSavedHostID()
}

// HostIDStorage is an interface which maintains persistency
// of the "Host ID" for the current host where the agent is running on.
// The ID is always generated and given by Mackerel (mackerel.io).
type HostIDStorage interface {
	LoadHostID() (string, error)
	SaveHostID(id string) error
	DeleteSavedHostID() error
}

// FileSystemHostIDStorage is the default HostIDStorage
// which saves/loads the host id using an id file on the local filesystem.
// The file will be located at /var/lib/mackerel-agent/id by default on linux.
type FileSystemHostIDStorage struct {
	Root string
}

const idFileName = "id"

// HostIDFile is the location of the host id file.
func (s FileSystemHostIDStorage) HostIDFile() string {
	return filepath.Join(s.Root, idFileName)
}

// LoadHostID loads the current host ID from the mackerel-agent's id file.
func (s FileSystemHostIDStorage) LoadHostID() (string, error) {
	content, err := ioutil.ReadFile(s.HostIDFile())
	if err != nil {
		return "", err
	}
	return strings.TrimRight(string(content), "\r\n"), nil
}

// SaveHostID saves the host ID to the mackerel-agent's id file.
func (s FileSystemHostIDStorage) SaveHostID(id string) error {
	err := os.MkdirAll(s.Root, 0755)
	if err != nil {
		return err
	}

	file, err := os.Create(s.HostIDFile())
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Write([]byte(id))
	if err != nil {
		return err
	}

	return nil
}

// DeleteSavedHostID deletes the mackerel-agent's id file.
func (s FileSystemHostIDStorage) DeleteSavedHostID() error {
	return os.Remove(s.HostIDFile())
}
<commit_msg>create MetricPlugins and CheckPlugins<commit_after>package config

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/util"
)

var configLogger = logging.GetLogger("config")

// `apibase` and `agentName` are set from build flags
var apibase string

func getApibase() string {
	if apibase != "" {
		return apibase
	}
	return "https://mackerel.io"
}

var agentName string

func getAgentName() string {
	if agentName != "" {
		return agentName
	}
	return "mackerel-agent"
}

// Config represents mackerel-agent's configuration file.
type Config struct {
	Apibase     string
	Apikey      string
	Root        string
	Pidfile     string
	Conffile    string
	Roles       []string
	Verbose     bool
	Silent      bool
	Diagnostic  bool `toml:"diagnostic"`
	Connection  ConnectionConfig
	DisplayName string      `toml:"display_name"`
	HostStatus  HostStatus  `toml:"host_status"`
	Filesystems Filesystems `toml:"filesystems"`
	HTTPProxy   string      `toml:"http_proxy"`

	// Corresponds to the set of [plugin.<kind>.<name>] sections
	// the key of the map is <kind>, which should be one of "metrics" or "checks".
	Plugin map[string]PluginConfigs

	Include string

	// Cannot exist in configuration files
	HostIDStorage HostIDStorage
	MetricPlugins map[string]*MetricPlugin
	CheckPlugins  map[string]*CheckPlugin
}

// PluginConfigs represents a set of [plugin.<kind>.<name>] sections in the configuration file
// under a specific <kind>. The key of the map is <name>, for example "mysql" of "plugin.metrics.mysql".
type PluginConfigs map[string]*PluginConfig

// PluginConfig represents a section of [plugin.*].
// `MaxCheckAttempts`, `NotificationInterval` and `CheckInterval` options are used with check monitoring plugins. Custom metrics plugins ignore these options.
// `User` option is ignore in windows
type PluginConfig struct {
	CommandRaw           interface{} `toml:"command"`
	Command              string
	CommandArgs          []string
	User                 string
	NotificationInterval *int32  `toml:"notification_interval"`
	CheckInterval        *int32  `toml:"check_interval"`
	MaxCheckAttempts     *int32  `toml:"max_check_attempts"`
	CustomIdentifier     *string `toml:"custom_identifier"`
}

func makeMetricPlugin(pconf *PluginConfig) (*MetricPlugin, error) {
	err := pconf.prepareCommand()
	if err != nil {
		return nil, err
	}
	return &MetricPlugin{
		Command:          pconf.Command,
		CommandArgs:      pconf.CommandArgs,
		User:             pconf.User,
		CustomIdentifier: pconf.CustomIdentifier,
	}, nil
}

func makeCheckPlugin(pconf *PluginConfig) (*CheckPlugin, error) {
	err := pconf.prepareCommand()
	if err != nil {
		return nil, err
	}
	return &CheckPlugin{
		Command:              pconf.Command,
		CommandArgs:          pconf.CommandArgs,
		User:                 pconf.User,
		NotificationInterval: pconf.NotificationInterval,
		CheckInterval:        pconf.CheckInterval,
		MaxCheckAttempts:     pconf.MaxCheckAttempts,
	}, nil
}

// MetricPlugin represents the configuration of a metric plugin
// The `User` option is ignored in Windows
type MetricPlugin struct {
	Command          string
	CommandArgs      []string
	User             string
	CustomIdentifier *string
}

// CheckPlugin represents the configuration of a check plugin
// The `User` option is ignored in Windows
type CheckPlugin struct {
	Command              string
	CommandArgs          []string
	User                 string
	NotificationInterval *int32
	CheckInterval        *int32
	MaxCheckAttempts     *int32
}

func (pconf *PluginConfig) prepareCommand() error {
	const errFmt = "failed to prepare plugin command. A configuration value of `command` should be string or string slice, but %T"
	v := pconf.CommandRaw
	switch t := v.(type) {
	case string:
		pconf.Command = t
	case []interface{}:
		if len(t) > 0 {
			for _, vv := range t {
				str, ok := vv.(string)
				if !ok {
					return fmt.Errorf(errFmt, v)
				}
				pconf.CommandArgs = append(pconf.CommandArgs, str)
			}
		} else {
			return fmt.Errorf(errFmt, v)
		}
	case []string:
		pconf.CommandArgs = t
	default:
		return fmt.Errorf(errFmt, v)
	}
	return nil
}

// Run the plugin
func (pconf *PluginConfig) Run() (string, string, int, error) {
	if len(pconf.CommandArgs) > 0 {
		return util.RunCommandArgs(pconf.CommandArgs, pconf.User)
	}
	return util.RunCommand(pconf.Command, pconf.User)
}

const postMetricsDequeueDelaySecondsMax = 59   // max delay seconds for dequeuing from buffer queue
const postMetricsRetryDelaySecondsMax = 3 * 60 // max delay seconds for retrying a request that caused errors

// PostMetricsInterval XXX
var PostMetricsInterval = 1 * time.Minute

// ConnectionConfig XXX
type ConnectionConfig struct {
	PostMetricsDequeueDelaySeconds int `toml:"post_metrics_dequeue_delay_seconds"` // delay for dequeuing from buffer queue
	PostMetricsRetryDelaySeconds   int `toml:"post_metrics_retry_delay_seconds"`   // delay for retrying a request that caused errors
	PostMetricsRetryMax            int `toml:"post_metrics_retry_max"`             // max numbers of retries for a request that causes errors
	PostMetricsBufferSize          int `toml:"post_metrics_buffer_size"`           // max numbers of requests stored in buffer queue.
}

// HostStatus configure host status on agent start/stop
type HostStatus struct {
	OnStart string `toml:"on_start"`
	OnStop  string `toml:"on_stop"`
}

// Filesystems configure filesystem related settings
type Filesystems struct {
	Ignore        Regexpwrapper `toml:"ignore"`
	UseMountpoint bool          `toml:"use_mountpoint"`
}

// Regexpwrapper is a wrapper type for marshalling string
type Regexpwrapper struct {
	*regexp.Regexp
}

// UnmarshalText for compiling regexp string while loading toml
func (r *Regexpwrapper) UnmarshalText(text []byte) error {
	var err error
	r.Regexp, err = regexp.Compile(string(text))
	return err
}

// CheckNames return list of plugin.checks._name_
func (conf *Config) CheckNames() []string {
	checks := []string{}
	for name := range conf.Plugin["checks"] {
		checks = append(checks, name)
	}
	return checks
}

// LoadConfig XXX
func LoadConfig(conffile string) (*Config, error) {
	config, err := loadConfigFile(conffile)
	if err != nil {
		return nil, err
	}

	// set default values if config does not have values
	if config.Apibase == "" {
		config.Apibase = DefaultConfig.Apibase
	}
	if config.Root == "" {
		config.Root = DefaultConfig.Root
	}
	if config.Pidfile == "" {
		config.Pidfile = DefaultConfig.Pidfile
	}
	if config.Verbose == false {
		config.Verbose = DefaultConfig.Verbose
	}
	if config.Diagnostic == false {
		config.Diagnostic = DefaultConfig.Diagnostic
	}
	if config.Connection.PostMetricsDequeueDelaySeconds == 0 {
		config.Connection.PostMetricsDequeueDelaySeconds = DefaultConfig.Connection.PostMetricsDequeueDelaySeconds
	}
	if config.Connection.PostMetricsDequeueDelaySeconds > postMetricsDequeueDelaySecondsMax {
		configLogger.Warningf("'post_metrics_dequese_delay_seconds' is set to %d (Maximum Value).", postMetricsDequeueDelaySecondsMax)
		config.Connection.PostMetricsDequeueDelaySeconds = postMetricsDequeueDelaySecondsMax
	}
	if config.Connection.PostMetricsRetryDelaySeconds == 0 {
		config.Connection.PostMetricsRetryDelaySeconds = DefaultConfig.Connection.PostMetricsRetryDelaySeconds
	}
	if config.Connection.PostMetricsRetryDelaySeconds > postMetricsRetryDelaySecondsMax {
		configLogger.Warningf("'post_metrics_retry_delay_seconds' is set to %d (Maximum Value).", postMetricsRetryDelaySecondsMax)
		config.Connection.PostMetricsRetryDelaySeconds = postMetricsRetryDelaySecondsMax
	}
	if config.Connection.PostMetricsRetryMax == 0 {
		config.Connection.PostMetricsRetryMax = DefaultConfig.Connection.PostMetricsRetryMax
	}
	if config.Connection.PostMetricsBufferSize == 0 {
		config.Connection.PostMetricsBufferSize = DefaultConfig.Connection.PostMetricsBufferSize
	}

	return config, err
}

func loadConfigFile(file string) (*Config, error) {
	config := &Config{}
	if _, err := toml.DecodeFile(file, config); err != nil {
		return config, err
	}

	if config.Include != "" {
		if err := includeConfigFile(config, config.Include); err != nil {
			return config, err
		}
	}

	config.MetricPlugins = make(map[string]*MetricPlugin)
	if pconfs, ok := config.Plugin["metrics"]; ok {
		var err error
		for name, pconf := range pconfs {
			config.MetricPlugins[name], err = makeMetricPlugin(pconf)
			if err != nil {
				return nil, err
			}
		}
	}

	config.CheckPlugins = make(map[string]*CheckPlugin)
	if pconfs, ok := config.Plugin["checks"]; ok {
		var err error
		for name, pconf := range pconfs {
			config.CheckPlugins[name], err = makeCheckPlugin(pconf)
			if err != nil {
				return nil, err
			}
		}
	}
	return config, nil
}

func includeConfigFile(config *Config, include string) error {
	files, err := filepath.Glob(include)
	if err != nil {
		return err
	}

	for _, file := range files {
		// Save current "roles" value and reset it
		// because toml.DecodeFile()-ing on a fulfilled struct
		// produces bizarre array values.
		rolesSaved := config.Roles
		config.Roles = nil

		// Also, save plugin values for later merging
		pluginSaved := map[string]PluginConfigs{}
		for kind, plugins := range config.Plugin {
			pluginSaved[kind] = plugins
		}

		meta, err := toml.DecodeFile(file, &config)
		if err != nil {
			return fmt.Errorf("while loading included config file %s: %s", file, err)
		}

		// If included config does not have "roles" key,
		// use the previous roles configuration value.
		if meta.IsDefined("roles") == false {
			config.Roles = rolesSaved
		}

		for kind, plugins := range config.Plugin {
			for key, conf := range plugins {
				if pluginSaved[kind] == nil {
					pluginSaved[kind] = PluginConfigs{}
				}
				pluginSaved[kind][key] = conf
			}
		}

		config.Plugin = pluginSaved
	}

	return nil
}

func (conf *Config) hostIDStorage() HostIDStorage {
	if conf.HostIDStorage == nil {
		conf.HostIDStorage = &FileSystemHostIDStorage{Root: conf.Root}
	}
	return conf.HostIDStorage
}

// LoadHostID loads the previously saved host id.
func (conf *Config) LoadHostID() (string, error) {
	return conf.hostIDStorage().LoadHostID()
}

// SaveHostID saves the host id, which may be restored by LoadHostID.
func (conf *Config) SaveHostID(id string) error {
	return conf.hostIDStorage().SaveHostID(id)
}

// DeleteSavedHostID deletes the host id saved by SaveHostID.
func (conf *Config) DeleteSavedHostID() error {
	return conf.hostIDStorage().DeleteSavedHostID()
}

// HostIDStorage is an interface which maintains persistency
// of the "Host ID" for the current host where the agent is running on.
// The ID is always generated and given by Mackerel (mackerel.io).
type HostIDStorage interface {
	LoadHostID() (string, error)
	SaveHostID(id string) error
	DeleteSavedHostID() error
}

// FileSystemHostIDStorage is the default HostIDStorage
// which saves/loads the host id using an id file on the local filesystem.
// The file will be located at /var/lib/mackerel-agent/id by default on linux.
type FileSystemHostIDStorage struct {
	Root string
}

const idFileName = "id"

// HostIDFile is the location of the host id file.
func (s FileSystemHostIDStorage) HostIDFile() string {
	return filepath.Join(s.Root, idFileName)
}

// LoadHostID loads the current host ID from the mackerel-agent's id file.
func (s FileSystemHostIDStorage) LoadHostID() (string, error) {
	content, err := ioutil.ReadFile(s.HostIDFile())
	if err != nil {
		return "", err
	}
	return strings.TrimRight(string(content), "\r\n"), nil
}

// SaveHostID saves the host ID to the mackerel-agent's id file.
func (s FileSystemHostIDStorage) SaveHostID(id string) error {
	err := os.MkdirAll(s.Root, 0755)
	if err != nil {
		return err
	}

	file, err := os.Create(s.HostIDFile())
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Write([]byte(id))
	if err != nil {
		return err
	}

	return nil
}

// DeleteSavedHostID deletes the mackerel-agent's id file.
func (s FileSystemHostIDStorage) DeleteSavedHostID() error {
	return os.Remove(s.HostIDFile())
}
<|endoftext|>


<commit_before>// Config is put into a different package to prevent cyclic imports in case
// it is needed in several locations

package config

type Config struct {
	Jmxproxybeat JmxproxybeatConfig
}

type JmxproxybeatConfig struct {
	Period         string               `yaml:"period"`
	URLs           []string             `yaml:"urls"`
	Authentication AuthenticationConfig `yaml:"authentication"`
	Beans          []BeanConfig         `yaml:"beans"`
}

type AuthenticationConfig struct {
	Username string
	Password string
}

type BeanConfig struct {
	Name       string   `yaml:"name"`
	Attributes []Attribute `yaml:"attributes"`
	Keys       []string `yaml:"keys"`
}

type Attribute struct {
	Name string `yaml:"name"`
	Keys []string `yaml:"keys"`
}<commit_msg>Cleanup code<commit_after>// Config is put into a different package to prevent cyclic imports in case
// it is needed in several locations

package config

type Config struct {
	Jmxproxybeat JmxproxybeatConfig
}

type JmxproxybeatConfig struct {
	Period         string               `yaml:"period"`
	URLs           []string             `yaml:"urls"`
	Authentication AuthenticationConfig `yaml:"authentication"`
	Beans          []BeanConfig         `yaml:"beans"`
}

type AuthenticationConfig struct {
	Username string
	Password string
}

type BeanConfig struct {
	Name       string      `yaml:"name"`
	Attributes []Attribute `yaml:"attributes"`
	Keys       []string    `yaml:"keys"`
}

type Attribute struct {
	Name string   `yaml:"name"`
	Keys []string `yaml:"keys"`
}
<|endoftext|>


<commit_before>// Defines the structure of our configuration file
package config

import (
	"code.google.com/p/gcfg"
	"log"
	"path"
	"runtime"
)

type Config struct {
	Server struct {
		Host string
		Port int
	}
	Database struct {
		Host    string
		Port    int
		Network string
	}
}

func (self *Config) Load(name string) {
	_, filename, _, _ := runtime.Caller(1)
	configPath := path.Join(path.Dir(filename), name)
	if err := gcfg.ReadFileInto(self, configPath); err != nil {
		log.Fatal("Error loading cfg:", err)
	}
}

type Entities struct {
	Planets struct {
		RingOffset int
		PlanetRadius int
		PlanetCount int
		PlanetHashArgs int
	}
	Suns struct {
		RandomSpawnZoneRadius int
		SolarSystemRadius float64
	}
}

func (self *Entities) Load(name string) {
	_, filename, _, _ := runtime.Caller(1)
	configPath := path.Join(path.Dir(filename), name)
	if err := gcfg.ReadFileInto(self, configPath); err != nil {
		log.Fatal("Error loading cfg:", err)
	}
}
<commit_msg>Remove config.Entities<commit_after>// Defines the structure of our configuration file
package config

import (
	"code.google.com/p/gcfg"
	"log"
	"path"
	"runtime"
)

type Config struct {
	Server struct {
		Host string
		Port int
	}
	Database struct {
		Host    string
		Port    int
		Network string
	}
}

func (self *Config) Load(name string) {
	_, filename, _, _ := runtime.Caller(1)
	configPath := path.Join(path.Dir(filename), name)
	if err := gcfg.ReadFileInto(self, configPath); err != nil {
		log.Fatal("Error loading cfg:", err)
	}
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"

	"github.com/reviewboard/rb-gateway/repositories"
)

const DefaultConfigPath = "config.json"

type repositoryData struct {
	Name string `json:"name"`
	Path string `json:"path"`
	Scm  string `json:"scm"`
}

type Config struct {
	HtpasswdPath   string           `json:"htpasswdPath"`
	Port           uint16           `json:"port"`
	RepositoryData []repositoryData `json:"repositories"`
	SSLCertificate string           `json:"sslCertificate"`
	SSLKey         string           `json:"sslKey"`
	TokenStorePath string           `json:"tokenStorePath"`
	UseTLS         bool             `json:"useTLS"`

	Repositories map[string]repositories.Repository
}

func Load(path string) (*Config, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var config Config
	if err = json.Unmarshal(content, &config); err != nil {
		return nil, err
	}

	var cfgDir string
	if cfgDir, err = filepath.Abs(path); err != nil {
		return nil, err
	} else {
		cfgDir = filepath.Dir(cfgDir)
	}

	if err = validate(cfgDir, &config); err != nil {
		return nil, err
	}

	config.Repositories = make(map[string]repositories.Repository)

	for _, repo := range config.RepositoryData {
		switch repo.Scm {
		case "git":
			config.Repositories[repo.Name] = &repositories.GitRepository{
				repositories.RepositoryInfo{
					Name: repo.Name,
					Path: repo.Path,
				},
			}

		default:
			log.Printf("Unknown SCM '%s' while loading configuration '%s'; ignoring.", repo.Scm, path)
		}
	}

	return &config, nil
}

func validate(cfgDir string, config *Config) (err error) {
	missingFields := []string{}

	if config.Port == 0 {
		log.Printf("WARNING: Port missing, defaulting to 8888.")
		config.Port = 8888
	}

	if len(config.RepositoryData) == 0 {
		missingFields = append(missingFields, "repositories")
	}

	if config.UseTLS {
		if config.SSLCertificate == "" {
			missingFields = append(missingFields, "ssl_certificate")
		} else {
			config.SSLCertificate = resolvePath(cfgDir, config.SSLCertificate)
		}

		if config.SSLKey == "" {
			missingFields = append(missingFields, "ssl_key")
		} else {
			config.SSLKey = resolvePath(cfgDir, config.SSLKey)
		}
	}

	if config.TokenStorePath == "" {
		missingFields = append(missingFields, "tokenStorePath")
	} else if config.TokenStorePath != ":memory:" {
		config.TokenStorePath = resolvePath(cfgDir, config.TokenStorePath)
	}

	if config.HtpasswdPath == "" {
		missingFields = append(missingFields, "htpasswdPath")
	} else {
		config.HtpasswdPath = resolvePath(cfgDir, config.HtpasswdPath)
	}

	if len(missingFields) != 0 {
		err = fmt.Errorf("Some required fields were missing from the configuration: %s.", strings.Join(missingFields, ","))
	}

	return
}

// Resolve a path so that . is treated as cfgDir
func resolvePath(cfgDir string, path string) string {
	if !filepath.IsAbs(path) {
		path = filepath.Join(cfgDir, path)
	}
	return path
}
<commit_msg>Add defaults for Config.HtpasswdPath, Config.TokenStorePath<commit_after>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"

	"github.com/reviewboard/rb-gateway/repositories"
)

const DefaultConfigPath = "config.json"

const (
	defaultPort uint16 = 8888
)

type repositoryData struct {
	Name string `json:"name"`
	Path string `json:"path"`
	Scm  string `json:"scm"`
}

type Config struct {
	HtpasswdPath   string           `json:"htpasswdPath"`
	Port           uint16           `json:"port"`
	RepositoryData []repositoryData `json:"repositories"`
	SSLCertificate string           `json:"sslCertificate"`
	SSLKey         string           `json:"sslKey"`
	TokenStorePath string           `json:"tokenStorePath"`
	UseTLS         bool             `json:"useTLS"`

	Repositories map[string]repositories.Repository
}

func Load(path string) (*Config, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var config Config
	if err = json.Unmarshal(content, &config); err != nil {
		return nil, err
	}

	var cfgDir string
	if cfgDir, err = filepath.Abs(path); err != nil {
		return nil, err
	} else {
		cfgDir = filepath.Dir(cfgDir)
	}

	if err = validate(cfgDir, &config); err != nil {
		return nil, err
	}

	config.Repositories = make(map[string]repositories.Repository)

	for _, repo := range config.RepositoryData {
		switch repo.Scm {
		case "git":
			config.Repositories[repo.Name] = &repositories.GitRepository{
				repositories.RepositoryInfo{
					Name: repo.Name,
					Path: repo.Path,
				},
			}

		default:
			log.Printf("Unknown SCM '%s' while loading configuration '%s'; ignoring.", repo.Scm, path)
		}
	}

	return &config, nil
}

func validate(cfgDir string, config *Config) (err error) {
	missingFields := []string{}

	if config.Port == 0 {
		log.Printf("WARNING: Port missing from config, defaulting to %d.", defaultPort)
		config.Port = defaultPort
	}

	if len(config.RepositoryData) == 0 {
		missingFields = append(missingFields, "repositories")
	}

	if config.UseTLS {
		if config.SSLCertificate == "" {
			missingFields = append(missingFields, "ssl_certificate")
		} else {
			config.SSLCertificate = resolvePath(cfgDir, config.SSLCertificate)
		}

		if config.SSLKey == "" {
			missingFields = append(missingFields, "ssl_key")
		} else {
			config.SSLKey = resolvePath(cfgDir, config.SSLKey)
		}
	}

	optionalPathFields := []struct {
		field        *string
		name         string
		defaultValue string
	}{
		{&config.TokenStorePath, "tokenStorePath", "tokens.dat"},
		{&config.HtpasswdPath, "htpasswdPath", "htpasswd"},
	}

	for _, fieldInfo := range optionalPathFields {
		if *fieldInfo.field == "" {
			log.Printf(`Warning: %s missing from config, defaulting to "%v".`, fieldInfo.name, fieldInfo.defaultValue)
			*fieldInfo.field = fieldInfo.defaultValue
		}
	}

	if config.TokenStorePath != ":memory:" {
		config.TokenStorePath = resolvePath(cfgDir, config.TokenStorePath)
	}

	config.HtpasswdPath = resolvePath(cfgDir, config.HtpasswdPath)

	if len(missingFields) != 0 {
		err = fmt.Errorf("Some required fields were missing from the configuration: %s.", strings.Join(missingFields, ","))
	}

	return
}

// Resolve a path so that . is treated as cfgDir
func resolvePath(cfgDir string, path string) string {
	if !filepath.IsAbs(path) {
		path = filepath.Join(cfgDir, path)
	}
	return path
}
<|endoftext|>


<commit_before>package config

import (
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/user"
	"path"
)

const DefaultConfigFileName = ".graven.yaml"

type Config struct {
	configFileName string
	data map[string]interface{}
}

func NewConfig() Config {
	config := Config{}
	config.data = map[string]interface{}{}
	config.configFileName = DefaultConfigFileName
	return config
}

func (c Config) Set(name string, value interface{}) {
	c.data[name] = value
}

func (c Config) Get(name string) interface{} {
	return c.data[name]
}

func (c Config) GetString(name string) string {
	if v, ok := c.data[name]; ok {
		return v.(string)
	}
	return ""
}

func (c Config) GetMap(name string) map[string]interface{} {
	if v, ok := c.data[name]; ok {
		return v.(map[string]interface{})
	}
	return map[string]interface{}{}
}

func (c Config) GetMaps(name string) []map[string]interface{} {
	if v, ok := c.data[name]; ok {
		return v.([]map[string]interface{})
	}
	return []map[string]interface{}{}
}

func (c Config) Read() error {
	usr, err := user.Current()
	if err != nil {
		return err
	}
	file, err := os.Open(path.Join(usr.HomeDir, c.configFileName))
	if err != nil {
		return err
	}
	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}
	err = yaml.Unmarshal(bytes, c)
	if err != nil {
		return err
	}
	return nil
}

func (c Config) Write() error {
	usr, err := user.Current()
	if err != nil {
		return err
	}
	bytes, err := yaml.Marshal(c)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(path.Join(usr.HomeDir, c.configFileName), bytes, 0600)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>fixing login<commit_after>package config

import (
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/user"
	"path"
)

const DefaultConfigFileName = ".graven.yaml"

type Config struct {
	configFileName string
	data map[string]interface{}
}

func NewConfig() Config {
	config := Config{}
	config.data = map[string]interface{}{}
	config.configFileName = DefaultConfigFileName
	return config
}

func (c Config) Set(name string, value interface{}) {
	c.data[name] = value
}

func (c Config) Get(name string) interface{} {
	return c.data[name]
}

func (c Config) GetString(name string) string {
	if v, ok := c.data[name]; ok {
		return v.(string)
	}
	return ""
}

func (c Config) GetMap(name string) map[string]interface{} {
	if v, ok := c.data[name]; ok {
		return v.(map[string]interface{})
	}
	return map[string]interface{}{}
}

func (c Config) GetMaps(name string) []map[string]interface{} {
	if v, ok := c.data[name]; ok {
		return v.([]map[string]interface{})
	}
	return []map[string]interface{}{}
}

func (c Config) Read() error {
	usr, err := user.Current()
	if err != nil {
		return err
	}
	file, err := os.Open(path.Join(usr.HomeDir, c.configFileName))
	if err != nil {
		return err
	}
	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}
	err = yaml.Unmarshal(bytes, c)
	if err != nil {
		return err
	}
	return nil
}

func (c Config) Write() error {
	usr, err := user.Current()
	if err != nil {
		return err
	}
	bytes, err := yaml.Marshal(c.data)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(path.Join(usr.HomeDir, c.configFileName), bytes, 0600)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package release

import (
	"os"
	"testing"

	"github.com/goreleaser/goreleaser/context"
	"github.com/stretchr/testify/assert"
)

func TestDescribeBody(t *testing.T) {
	var assert = assert.New(t)
	var changelog = "\nfeature1: description\nfeature2: other description"
	var ctx = &context.Context{
		ReleaseNotes: changelog,
	}
	out, err := describeBody(ctx)
	assert.NoError(err)
	assert.Contains(out.String(), changelog)
	assert.Contains(out.String(), "Automated with @goreleaser")
	assert.Contains(out.String(), "Built with go version go1.8")
}

func TestGoVersionFails(t *testing.T) {
	var assert = assert.New(t)
	var path = os.Getenv("PATH")
	defer func() {
		assert.NoError(os.Setenv("PATH", path))
	}()
	assert.NoError(os.Setenv("PATH", ""))
	var ctx = &context.Context{
		ReleaseNotes: "changelog",
	}
	_, err := describeBody(ctx)
	assert.Error(err)
}
<commit_msg>fixed unit test<commit_after>package release

import (
	"os"
	"testing"

	"github.com/goreleaser/goreleaser/context"
	"github.com/stretchr/testify/assert"
)

func TestDescribeBody(t *testing.T) {
	var assert = assert.New(t)
	var changelog = "\nfeature1: description\nfeature2: other description"
	var ctx = &context.Context{
		ReleaseNotes: changelog,
	}
	out, err := describeBody(ctx)
	assert.NoError(err)
	assert.Contains(out.String(), changelog)
	assert.Contains(out.String(), "Automated with [GoReleaser]")
	assert.Contains(out.String(), "Built with go version go1.8")
}

func TestGoVersionFails(t *testing.T) {
	var assert = assert.New(t)
	var path = os.Getenv("PATH")
	defer func() {
		assert.NoError(os.Setenv("PATH", path))
	}()
	assert.NoError(os.Setenv("PATH", ""))
	var ctx = &context.Context{
		ReleaseNotes: "changelog",
	}
	_, err := describeBody(ctx)
	assert.Error(err)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gps

import (
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/dep/internal/fs"
	"github.com/pkg/errors"
)

// PruneOptions represents the pruning options used to write the dependecy tree.
type PruneOptions uint8

const (
	// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
	PruneNestedVendorDirs PruneOptions = 1 << iota
	// PruneUnusedPackages indicates if unused Go packages should be pruned.
	PruneUnusedPackages
	// PruneNonGoFiles indicates if non-Go files should be pruned.
	// Files matching licenseFilePrefixes and legalFileSubstrings are kept in
	// an attempt to comply with legal requirements.
	PruneNonGoFiles
	// PruneGoTestFiles indicates if Go test files should be pruned.
	PruneGoTestFiles
)

// PruneOptionSet represents trinary distinctions for each of the types of
// prune rules (as expressed via PruneOptions): nested vendor directories,
// unused packages, non-go files, and go test files.
//
// The three-way distinction is between "none", "true", and "false", represented
// by uint8 values of 0, 1, and 2, respectively.
//
// This trinary distinction is necessary in order to record, with full fidelity,
// a cascading tree of pruning values, as expressed in CascadingPruneOptions; a
// simple boolean cannot delineate between "false" and "none".
type PruneOptionSet struct {
	NestedVendor   uint8
	UnusedPackages uint8
	NonGoFiles     uint8
	GoTests        uint8
}

// CascadingPruneOptions is a set of rules for pruning a dependency tree.
//
// The DefaultOptions are the global default pruning rules, expressed as a
// single PruneOptions bitfield. These global rules will cascade down to
// individual project rules, unless superseded.
type CascadingPruneOptions struct {
	DefaultOptions    PruneOptions
	PerProjectOptions map[ProjectRoot]PruneOptionSet
}

// PruneOptionsFor returns the PruneOptions bits for the given project,
// indicating which pruning rules should be applied to the project's code.
//
// It computes the cascade from default to project-specific options (if any) on
// the fly.
func (o CascadingPruneOptions) PruneOptionsFor(pr ProjectRoot) PruneOptions {
	po, has := o.PerProjectOptions[pr]
	if !has {
		return o.DefaultOptions
	}

	ops := o.DefaultOptions
	if po.NestedVendor != 0 {
		if po.NestedVendor == 1 {
			ops |= PruneNestedVendorDirs
		} else {
			ops &^= PruneNestedVendorDirs
		}
	}

	if po.UnusedPackages != 0 {
		if po.UnusedPackages == 1 {
			ops |= PruneUnusedPackages
		} else {
			ops &^= PruneUnusedPackages
		}
	}

	if po.NonGoFiles != 0 {
		if po.NonGoFiles == 1 {
			ops |= PruneNonGoFiles
		} else {
			ops &^= PruneNonGoFiles
		}
	}

	if po.GoTests != 0 {
		if po.GoTests == 1 {
			ops |= PruneGoTestFiles
		} else {
			ops &^= PruneGoTestFiles
		}
	}

	return ops
}

func defaultCascadingPruneOptions() CascadingPruneOptions {
	return CascadingPruneOptions{
		DefaultOptions:    PruneNestedVendorDirs,
		PerProjectOptions: map[ProjectRoot]PruneOptionSet{},
	}
}

var (
	// licenseFilePrefixes is a list of name prefixes for license files.
	licenseFilePrefixes = []string{
		"license",
		"licence",
		"copying",
		"unlicense",
		"copyright",
		"copyleft",
	}
	// legalFileSubstrings contains substrings that are likey part of a legal
	// declaration file.
	legalFileSubstrings = []string{
		"authors",
		"contributors",
		"legal",
		"notice",
		"disclaimer",
		"patent",
		"third-party",
		"thirdparty",
	}
)

// PruneProject remove excess files according to the options passed, from
// the lp directory in baseDir.
func PruneProject(baseDir string, lp LockedProject, options PruneOptions, logger *log.Logger) error {
	fsState, err := deriveFilesystemState(baseDir)

	if err != nil {
		return errors.Wrap(err, "could not derive filesystem state")
	}

	if (options & PruneNestedVendorDirs) != 0 {
		if err := pruneVendorDirs(fsState); err != nil {
			return errors.Wrapf(err, "failed to prune nested vendor directories")
		}
	}

	if (options & PruneUnusedPackages) != 0 {
		if _, err := pruneUnusedPackages(lp, fsState); err != nil {
			return errors.Wrap(err, "failed to prune unused packages")
		}
	}

	if (options & PruneNonGoFiles) != 0 {
		if err := pruneNonGoFiles(fsState); err != nil {
			return errors.Wrap(err, "failed to prune non-Go files")
		}
	}

	if (options & PruneGoTestFiles) != 0 {
		if err := pruneGoTestFiles(fsState); err != nil {
			return errors.Wrap(err, "failed to prune Go test files")
		}
	}

	if err := deleteEmptyDirs(fsState); err != nil {
		return errors.Wrap(err, "could not delete empty dirs")
	}

	return nil
}

// pruneVendorDirs deletes all nested vendor directories within baseDir.
func pruneVendorDirs(fsState filesystemState) error {
	for _, dir := range fsState.dirs {
		if filepath.Base(dir) == "vendor" {
			err := os.RemoveAll(filepath.Join(fsState.root, dir))
			if err != nil && !os.IsNotExist(err) {
				return err
			}
		}
	}

	for _, link := range fsState.links {
		if filepath.Base(link.path) == "vendor" {
			err := os.Remove(filepath.Join(fsState.root, link.path))
			if err != nil && !os.IsNotExist(err) {
				return err
			}
		}
	}

	return nil
}

// pruneUnusedPackages deletes unimported packages found in fsState.
// Determining whether packages are imported or not is based on the passed LockedProject.
func pruneUnusedPackages(lp LockedProject, fsState filesystemState) (map[string]interface{}, error) {
	unusedPackages := calculateUnusedPackages(lp, fsState)
	toDelete := collectUnusedPackagesFiles(fsState, unusedPackages)

	for _, path := range toDelete {
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			return nil, err
		}
	}

	return unusedPackages, nil
}

// calculateUnusedPackages generates a list of unused packages in lp.
func calculateUnusedPackages(lp LockedProject, fsState filesystemState) map[string]interface{} {
	unused := make(map[string]interface{})
	imported := make(map[string]interface{})

	for _, pkg := range lp.Packages() {
		imported[pkg] = nil
	}

	// Add the root package if it's not imported.
	if _, ok := imported["."]; !ok {
		unused["."] = nil
	}

	for _, dirPath := range fsState.dirs {
		pkg := filepath.ToSlash(dirPath)

		if _, ok := imported[pkg]; !ok {
			unused[pkg] = nil
		}
	}

	return unused
}

// collectUnusedPackagesFiles returns a slice of all files in the unused
// packages based on fsState.
func collectUnusedPackagesFiles(fsState filesystemState, unusedPackages map[string]interface{}) []string {
	// TODO(ibrasho): is this useful?
	files := make([]string, 0, len(unusedPackages))

	for _, path := range fsState.files {
		// Keep perserved files.
		if isPreservedFile(filepath.Base(path)) {
			continue
		}

		pkg := filepath.ToSlash(filepath.Dir(path))

		if _, ok := unusedPackages[pkg]; ok {
			files = append(files, filepath.Join(fsState.root, path))
		}
	}

	return files
}

func isSourceFile(path string) bool {
	ext := fileExt(path)

	// Refer to: https://github.com/golang/go/blob/release-branch.go1.9/src/go/build/build.go#L750
	switch ext {
	case ".go":
		return true
	case ".c":
		return true
	case ".cc", ".cpp", ".cxx":
		return true
	case ".m":
		return true
	case ".h", ".hh", ".hpp", ".hxx":
		return true
	case ".f", ".F", ".for", ".f90":
		return true
	case ".s":
		return true
	case ".S":
		return true
	case ".swig":
		return true
	case ".swigcxx":
		return true
	case ".syso":
		return true
	}
	return false
}

// pruneNonGoFiles delete all non-Go files existing in fsState.
//
// Files matching licenseFilePrefixes and legalFileSubstrings are not pruned.
func pruneNonGoFiles(fsState filesystemState) error {
	toDelete := make([]string, 0, len(fsState.files)/4)

	for _, path := range fsState.files {
		if isSourceFile(path) {
			continue
		}

		// Ignore perserved files.
		if isPreservedFile(filepath.Base(path)) {
			continue
		}

		toDelete = append(toDelete, filepath.Join(fsState.root, path))
	}

	for _, path := range toDelete {
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	return nil
}

// isPreservedFile checks if the file name indicates that the file should be
// preserved based on licenseFilePrefixes or legalFileSubstrings.
func isPreservedFile(name string) bool {
	name = strings.ToLower(name)

	for _, prefix := range licenseFilePrefixes {
		if strings.HasPrefix(name, prefix) {
			return true
		}
	}

	for _, substring := range legalFileSubstrings {
		if strings.Contains(name, substring) {
			return true
		}
	}

	return false
}

// pruneGoTestFiles deletes all Go test files (*_test.go) in fsState.
func pruneGoTestFiles(fsState filesystemState) error {
	toDelete := make([]string, 0, len(fsState.files)/2)

	for _, path := range fsState.files {
		if strings.HasSuffix(path, "_test.go") {
			toDelete = append(toDelete, filepath.Join(fsState.root, path))
		}
	}

	for _, path := range toDelete {
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	return nil
}

func deleteEmptyDirs(fsState filesystemState) error {
	sort.Sort(sort.Reverse(sort.StringSlice(fsState.dirs)))

	for _, dir := range fsState.dirs {
		path := filepath.Join(fsState.root, dir)

		notEmpty, err := fs.IsNonEmptyDir(path)
		if err != nil {
			return err
		}

		if !notEmpty {
			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
				return err
			}
		}
	}

	return nil
}

func fileExt(name string) string {
	i := strings.LastIndex(name, ".")
	if i < 0 {
		return ""
	}
	return name[i:]
}
<commit_msg>prune: Fix handling of "legal" source files<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gps

import (
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/dep/internal/fs"
	"github.com/pkg/errors"
)

// PruneOptions represents the pruning options used to write the dependecy tree.
type PruneOptions uint8

const (
	// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
	PruneNestedVendorDirs PruneOptions = 1 << iota
	// PruneUnusedPackages indicates if unused Go packages should be pruned.
	PruneUnusedPackages
	// PruneNonGoFiles indicates if non-Go files should be pruned.
	// Files matching licenseFilePrefixes and legalFileSubstrings are kept in
	// an attempt to comply with legal requirements.
	PruneNonGoFiles
	// PruneGoTestFiles indicates if Go test files should be pruned.
	PruneGoTestFiles
)

// PruneOptionSet represents trinary distinctions for each of the types of
// prune rules (as expressed via PruneOptions): nested vendor directories,
// unused packages, non-go files, and go test files.
//
// The three-way distinction is between "none", "true", and "false", represented
// by uint8 values of 0, 1, and 2, respectively.
//
// This trinary distinction is necessary in order to record, with full fidelity,
// a cascading tree of pruning values, as expressed in CascadingPruneOptions; a
// simple boolean cannot delineate between "false" and "none".
type PruneOptionSet struct {
	NestedVendor   uint8
	UnusedPackages uint8
	NonGoFiles     uint8
	GoTests        uint8
}

// CascadingPruneOptions is a set of rules for pruning a dependency tree.
//
// The DefaultOptions are the global default pruning rules, expressed as a
// single PruneOptions bitfield. These global rules will cascade down to
// individual project rules, unless superseded.
type CascadingPruneOptions struct {
	DefaultOptions    PruneOptions
	PerProjectOptions map[ProjectRoot]PruneOptionSet
}

// PruneOptionsFor returns the PruneOptions bits for the given project,
// indicating which pruning rules should be applied to the project's code.
//
// It computes the cascade from default to project-specific options (if any) on
// the fly.
func (o CascadingPruneOptions) PruneOptionsFor(pr ProjectRoot) PruneOptions {
	po, has := o.PerProjectOptions[pr]
	if !has {
		return o.DefaultOptions
	}

	ops := o.DefaultOptions
	if po.NestedVendor != 0 {
		if po.NestedVendor == 1 {
			ops |= PruneNestedVendorDirs
		} else {
			ops &^= PruneNestedVendorDirs
		}
	}

	if po.UnusedPackages != 0 {
		if po.UnusedPackages == 1 {
			ops |= PruneUnusedPackages
		} else {
			ops &^= PruneUnusedPackages
		}
	}

	if po.NonGoFiles != 0 {
		if po.NonGoFiles == 1 {
			ops |= PruneNonGoFiles
		} else {
			ops &^= PruneNonGoFiles
		}
	}

	if po.GoTests != 0 {
		if po.GoTests == 1 {
			ops |= PruneGoTestFiles
		} else {
			ops &^= PruneGoTestFiles
		}
	}

	return ops
}

func defaultCascadingPruneOptions() CascadingPruneOptions {
	return CascadingPruneOptions{
		DefaultOptions:    PruneNestedVendorDirs,
		PerProjectOptions: map[ProjectRoot]PruneOptionSet{},
	}
}

var (
	// licenseFilePrefixes is a list of name prefixes for license files.
	licenseFilePrefixes = []string{
		"license",
		"licence",
		"copying",
		"unlicense",
		"copyright",
		"copyleft",
	}
	// legalFileSubstrings contains substrings that are likey part of a legal
	// declaration file.
	legalFileSubstrings = []string{
		"authors",
		"contributors",
		"legal",
		"notice",
		"disclaimer",
		"patent",
		"third-party",
		"thirdparty",
	}
)

// PruneProject remove excess files according to the options passed, from
// the lp directory in baseDir.
func PruneProject(baseDir string, lp LockedProject, options PruneOptions, logger *log.Logger) error {
	fsState, err := deriveFilesystemState(baseDir)

	if err != nil {
		return errors.Wrap(err, "could not derive filesystem state")
	}

	if (options & PruneNestedVendorDirs) != 0 {
		if err := pruneVendorDirs(fsState); err != nil {
			return errors.Wrapf(err, "failed to prune nested vendor directories")
		}
	}

	if (options & PruneUnusedPackages) != 0 {
		if _, err := pruneUnusedPackages(lp, fsState); err != nil {
			return errors.Wrap(err, "failed to prune unused packages")
		}
	}

	if (options & PruneNonGoFiles) != 0 {
		if err := pruneNonGoFiles(fsState); err != nil {
			return errors.Wrap(err, "failed to prune non-Go files")
		}
	}

	if (options & PruneGoTestFiles) != 0 {
		if err := pruneGoTestFiles(fsState); err != nil {
			return errors.Wrap(err, "failed to prune Go test files")
		}
	}

	if err := deleteEmptyDirs(fsState); err != nil {
		return errors.Wrap(err, "could not delete empty dirs")
	}

	return nil
}

// pruneVendorDirs deletes all nested vendor directories within baseDir.
func pruneVendorDirs(fsState filesystemState) error {
	for _, dir := range fsState.dirs {
		if filepath.Base(dir) == "vendor" {
			err := os.RemoveAll(filepath.Join(fsState.root, dir))
			if err != nil && !os.IsNotExist(err) {
				return err
			}
		}
	}

	for _, link := range fsState.links {
		if filepath.Base(link.path) == "vendor" {
			err := os.Remove(filepath.Join(fsState.root, link.path))
			if err != nil && !os.IsNotExist(err) {
				return err
			}
		}
	}

	return nil
}

// pruneUnusedPackages deletes unimported packages found in fsState.
// Determining whether packages are imported or not is based on the passed LockedProject.
func pruneUnusedPackages(lp LockedProject, fsState filesystemState) (map[string]interface{}, error) {
	unusedPackages := calculateUnusedPackages(lp, fsState)
	toDelete := collectUnusedPackagesFiles(fsState, unusedPackages)

	for _, path := range toDelete {
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			return nil, err
		}
	}

	return unusedPackages, nil
}

// calculateUnusedPackages generates a list of unused packages in lp.
func calculateUnusedPackages(lp LockedProject, fsState filesystemState) map[string]interface{} {
	unused := make(map[string]interface{})
	imported := make(map[string]interface{})

	for _, pkg := range lp.Packages() {
		imported[pkg] = nil
	}

	// Add the root package if it's not imported.
	if _, ok := imported["."]; !ok {
		unused["."] = nil
	}

	for _, dirPath := range fsState.dirs {
		pkg := filepath.ToSlash(dirPath)

		if _, ok := imported[pkg]; !ok {
			unused[pkg] = nil
		}
	}

	return unused
}

// collectUnusedPackagesFiles returns a slice of all files in the unused
// packages based on fsState.
func collectUnusedPackagesFiles(fsState filesystemState, unusedPackages map[string]interface{}) []string {
	// TODO(ibrasho): is this useful?
	files := make([]string, 0, len(unusedPackages))

	for _, path := range fsState.files {
		// Keep perserved files.
		if isPreservedFile(filepath.Base(path)) {
			continue
		}

		pkg := filepath.ToSlash(filepath.Dir(path))

		if _, ok := unusedPackages[pkg]; ok {
			files = append(files, filepath.Join(fsState.root, path))
		}
	}

	return files
}

func isSourceFile(path string) bool {
	ext := fileExt(path)

	// Refer to: https://github.com/golang/go/blob/release-branch.go1.9/src/go/build/build.go#L750
	switch ext {
	case ".go":
		return true
	case ".c":
		return true
	case ".cc", ".cpp", ".cxx":
		return true
	case ".m":
		return true
	case ".h", ".hh", ".hpp", ".hxx":
		return true
	case ".f", ".F", ".for", ".f90":
		return true
	case ".s":
		return true
	case ".S":
		return true
	case ".swig":
		return true
	case ".swigcxx":
		return true
	case ".syso":
		return true
	}
	return false
}

// pruneNonGoFiles delete all non-Go files existing in fsState.
//
// Files matching licenseFilePrefixes and legalFileSubstrings are not pruned.
func pruneNonGoFiles(fsState filesystemState) error {
	toDelete := make([]string, 0, len(fsState.files)/4)

	for _, path := range fsState.files {
		if isSourceFile(path) {
			continue
		}

		// Ignore perserved files.
		if isPreservedFile(filepath.Base(path)) {
			continue
		}

		toDelete = append(toDelete, filepath.Join(fsState.root, path))
	}

	for _, path := range toDelete {
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	return nil
}

// isPreservedFile checks if the file name indicates that the file should be
// preserved based on licenseFilePrefixes or legalFileSubstrings.
// This applies only to non-source files.
func isPreservedFile(name string) bool {
	if isSourceFile(name) {
		return false
	}

	name = strings.ToLower(name)

	for _, prefix := range licenseFilePrefixes {
		if strings.HasPrefix(name, prefix) {
			return true
		}
	}

	for _, substring := range legalFileSubstrings {
		if strings.Contains(name, substring) {
			return true
		}
	}

	return false
}

// pruneGoTestFiles deletes all Go test files (*_test.go) in fsState.
func pruneGoTestFiles(fsState filesystemState) error {
	toDelete := make([]string, 0, len(fsState.files)/2)

	for _, path := range fsState.files {
		if strings.HasSuffix(path, "_test.go") {
			toDelete = append(toDelete, filepath.Join(fsState.root, path))
		}
	}

	for _, path := range toDelete {
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	return nil
}

func deleteEmptyDirs(fsState filesystemState) error {
	sort.Sort(sort.Reverse(sort.StringSlice(fsState.dirs)))

	for _, dir := range fsState.dirs {
		path := filepath.Join(fsState.root, dir)

		notEmpty, err := fs.IsNonEmptyDir(path)
		if err != nil {
			return err
		}

		if !notEmpty {
			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
				return err
			}
		}
	}

	return nil
}

func fileExt(name string) string {
	i := strings.LastIndex(name, ".")
	if i < 0 {
		return ""
	}
	return name[i:]
}
<|endoftext|>


<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package annotations makes it possible to track use of resource annotations within a procress
// in order to generate documentation for these uses.
package annotations

import (
	"sort"
	"sync"

	"istio.io/istio/pkg/log"
)

// Annotation describes a single resource annotation
type Annotation struct {
	// The name of the annotation.
	Name string

	// Description of the annotation.
	Description string

	// Hide the existence of this annotation when outputting usage information.
	Hidden bool

	// Mark this annotation as deprecated when generating usage information.
	Deprecated bool
}

var allAnnotations = make(map[string]Annotation)
var mutex sync.Mutex

// Returns a description of this process' annotations, sorted by name.
func Descriptions() []Annotation {
	mutex.Lock()
	sorted := make([]Annotation, 0, len(allAnnotations))
	for _, v := range allAnnotations {
		sorted = append(sorted, v)
	}
	mutex.Unlock()

	sort.Slice(sorted, func(i, j int) bool {
		return sorted[i].Name < sorted[j].Name
	})

	return sorted
}

// Register registers a new annotation.
func Register(name string, description string) Annotation {
	a := Annotation{Name: name, Description: description}
	RegisterFull(a)

	// get what's actually been registered
	mutex.Lock()
	result := allAnnotations[name]
	mutex.Unlock()

	return result
}

// RegisterFull registers a new annotation.
func RegisterFull(a Annotation) {
	mutex.Lock()

	if old, ok := allAnnotations[a.Name]; ok {
		if a.Description != "" {
			allAnnotations[a.Name] = a // last one with a description wins if the same annotation name is registered multiple times
		}

		if old.Description != a.Description || old.Deprecated != a.Deprecated || old.Hidden != a.Hidden {
			log.Warnf("The annotation %s was registered multiple times using different metadata: %v, %v", a.Name, old, a)
		}
	} else {
		allAnnotations[a.Name] = a
	}

	mutex.Unlock()
}
<commit_msg>Update import path of log package.<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package annotations makes it possible to track use of resource annotations within a procress
// in order to generate documentation for these uses.
package annotations

import (
	"sort"
	"sync"

	"istio.io/common/pkg/log"
)

// Annotation describes a single resource annotation
type Annotation struct {
	// The name of the annotation.
	Name string

	// Description of the annotation.
	Description string

	// Hide the existence of this annotation when outputting usage information.
	Hidden bool

	// Mark this annotation as deprecated when generating usage information.
	Deprecated bool
}

var allAnnotations = make(map[string]Annotation)
var mutex sync.Mutex

// Returns a description of this process' annotations, sorted by name.
func Descriptions() []Annotation {
	mutex.Lock()
	sorted := make([]Annotation, 0, len(allAnnotations))
	for _, v := range allAnnotations {
		sorted = append(sorted, v)
	}
	mutex.Unlock()

	sort.Slice(sorted, func(i, j int) bool {
		return sorted[i].Name < sorted[j].Name
	})

	return sorted
}

// Register registers a new annotation.
func Register(name string, description string) Annotation {
	a := Annotation{Name: name, Description: description}
	RegisterFull(a)

	// get what's actually been registered
	mutex.Lock()
	result := allAnnotations[name]
	mutex.Unlock()

	return result
}

// RegisterFull registers a new annotation.
func RegisterFull(a Annotation) {
	mutex.Lock()

	if old, ok := allAnnotations[a.Name]; ok {
		if a.Description != "" {
			allAnnotations[a.Name] = a // last one with a description wins if the same annotation name is registered multiple times
		}

		if old.Description != a.Description || old.Deprecated != a.Deprecated || old.Hidden != a.Hidden {
			log.Warnf("The annotation %s was registered multiple times using different metadata: %v, %v", a.Name, old, a)
		}
	} else {
		allAnnotations[a.Name] = a
	}

	mutex.Unlock()
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/validation"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/fielderrors"
)

func TestCompatibility_v1_Pod(t *testing.T) {
	// Test "spec.host" -> "spec.nodeName"
	expectedHost := "my-host"
	// Test "spec.serviceAccount" -> "spec.serviceAccountName"
	expectedServiceAccount := "my-service-account"
	// Test "tcp" protocol gets converted to "TCP" and validated
	originalProtocol := "tcp"
	expectedProtocol := "TCP"

	input := []byte(fmt.Sprintf(`
{
	"kind":"Pod",
	"apiVersion":"v1",
	"metadata":{"name":"my-pod-name", "namespace":"my-pod-namespace"},
	"spec": {
		"host":"%s",
		"serviceAccount":"%s",
		"containers":[{
			"name":"my-container-name",
			"image":"my-container-image",
			"ports":[{"containerPort":1,"protocol":"%s"}]
		}]
	}
}
`, expectedHost, expectedServiceAccount, originalProtocol))

	t.Log("Testing 1.0.0 v1 migration added in PR #3592")
	testCompatibility(
		t, "v1", input,
		func(obj runtime.Object) fielderrors.ValidationErrorList {
			return validation.ValidatePod(obj.(*api.Pod))
		},
		map[string]string{
			"spec.host":     expectedHost,
			"spec.nodeName": expectedHost,

			"spec.serviceAccount":     expectedServiceAccount,
			"spec.serviceAccountName": expectedServiceAccount,

			"spec.containers[0].ports[0].protocol": expectedProtocol,
		},
	)
}

func TestCompatibility_v1_Service(t *testing.T) {
	// Test "spec.portalIP" -> "spec.clusterIP"
	expectedIP := "1.2.3.4"
	// Test "tcp" protocol gets converted to "TCP" and validated
	originalProtocol := "tcp"
	expectedProtocol := "TCP"

	input := []byte(fmt.Sprintf(`
{
	"kind":"Service",
	"apiVersion":"v1",
	"metadata":{"name":"my-service-name", "namespace":"my-service-namespace"},
	"spec": {
		"portalIP":"%s",
		"ports":[{"port":1,"protocol":"%s"}]
	}
}
`, expectedIP, originalProtocol))

	t.Log("Testing 1.0.0 v1 migration added in PR #3592")
	testCompatibility(
		t, "v1", input,
		func(obj runtime.Object) fielderrors.ValidationErrorList {
			return validation.ValidateService(obj.(*api.Service))
		},
		map[string]string{
			"spec.portalIP":          expectedIP,
			"spec.clusterIP":         expectedIP,
			"spec.ports[0].protocol": expectedProtocol,
		},
	)
}

func TestCompatibility_v1_Endpoints(t *testing.T) {
	// Test "tcp" protocol gets converted to "TCP" and validated
	originalProtocol := "tcp"
	expectedProtocol := "TCP"

	input := []byte(fmt.Sprintf(`
{
	"kind":"Endpoints",
	"apiVersion":"v1",
	"metadata":{"name":"my-endpoints-name", "namespace":"my-endpoints-namespace"},
	"subsets": [{
		"addresses":[{"ip":"1.2.3.4"}],
		"ports": [{"port":1,"targetPort":1,"protocol":"%s"}]
	}]
}
`, originalProtocol))

	t.Log("Testing 1.0.0 v1 migration added in PR #3592")
	testCompatibility(
		t, "v1", input,
		func(obj runtime.Object) fielderrors.ValidationErrorList {
			return validation.ValidateEndpoints(obj.(*api.Endpoints))
		},
		map[string]string{
			"subsets[0].ports[0].protocol": expectedProtocol,
		},
	)
}

func testCompatibility(
	t *testing.T,
	version string,
	input []byte,
	validator func(obj runtime.Object) fielderrors.ValidationErrorList,
	serialized map[string]string,
) {

	// Decode
	codec := runtime.CodecFor(api.Scheme, version)
	obj, err := codec.Decode(input)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	// Validate
	errs := validator(obj)
	if len(errs) != 0 {
		t.Fatalf("Unexpected errors: %v", errs)
	}

	// Encode
	output, err := codec.Encode(obj)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	// Validate old and new fields are encoded
	generic := map[string]interface{}{}
	if err := json.Unmarshal(output, &generic); err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	for k, expectedValue := range serialized {
		keys := strings.Split(k, ".")
		if actualValue, ok, err := getJSONValue(generic, keys...); err != nil || !ok {
			t.Errorf("Unexpected error for %s: %v", k, err)
		} else if !reflect.DeepEqual(expectedValue, actualValue) {
			t.Errorf("Expected %v, got %v", expectedValue, actualValue)
		}
	}
}

func getJSONValue(data map[string]interface{}, keys ...string) (interface{}, bool, error) {
	// No keys, current value is it
	if len(keys) == 0 {
		return data, true, nil
	}

	// Get the key (and optional index)
	key := keys[0]
	index := -1
	if matches := regexp.MustCompile(`^(.*)\[(\d+)\]$`).FindStringSubmatch(key); len(matches) > 0 {
		key = matches[1]
		index, _ = strconv.Atoi(matches[2])
	}

	// Look up the value
	value, ok := data[key]
	if !ok {
		return nil, false, fmt.Errorf("No key %s found", key)
	}

	// Get the indexed value if an index is specified
	if index >= 0 {
		valueSlice, ok := value.([]interface{})
		if !ok {
			return nil, false, fmt.Errorf("Key %s did not hold a slice", key)
		}
		if index >= len(valueSlice) {
			return nil, false, fmt.Errorf("Index %d out of bounds for slice at key: %v", index, key)
		}
		value = valueSlice[index]
	}

	if len(keys) == 1 {
		return value, true, nil
	}

	childData, ok := value.(map[string]interface{})
	if !ok {
		return nil, false, fmt.Errorf("Key %s did not hold a map", keys[0])
	}
	return getJSONValue(childData, keys[1:]...)
}
<commit_msg>Compatibility test for Volume Source<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/validation"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/fielderrors"
)

func TestCompatibility_v1_Pod(t *testing.T) {
	// Test "spec.host" -> "spec.nodeName"
	expectedHost := "my-host"
	// Test "spec.serviceAccount" -> "spec.serviceAccountName"
	expectedServiceAccount := "my-service-account"
	// Test "tcp" protocol gets converted to "TCP" and validated
	originalProtocol := "tcp"
	expectedProtocol := "TCP"

	input := []byte(fmt.Sprintf(`
{
	"kind":"Pod",
	"apiVersion":"v1",
	"metadata":{"name":"my-pod-name", "namespace":"my-pod-namespace"},
	"spec": {
		"host":"%s",
		"serviceAccount":"%s",
		"containers":[{
			"name":"my-container-name",
			"image":"my-container-image",
			"ports":[{"containerPort":1,"protocol":"%s"}]
		}]
	}
}
`, expectedHost, expectedServiceAccount, originalProtocol))

	t.Log("Testing 1.0.0 v1 migration added in PR #3592")
	testCompatibility(
		t, "v1", input,
		func(obj runtime.Object) fielderrors.ValidationErrorList {
			return validation.ValidatePod(obj.(*api.Pod))
		},
		map[string]string{
			"spec.host":     expectedHost,
			"spec.nodeName": expectedHost,

			"spec.serviceAccount":     expectedServiceAccount,
			"spec.serviceAccountName": expectedServiceAccount,

			"spec.containers[0].ports[0].protocol": expectedProtocol,
		},
	)
}

// TestCompatibility_v1_VolumeSource tests that the metadata field in volume
// sources gets properly round-tripped
func TestCompatibility_v1_VolumeSource(t *testing.T) {
	// Test volume source compatibility
	path := "test/volume/source/compat"

	metadata := []byte(fmt.Sprintf(`
{
	"kind":"Pod",
	"apiVersion":"v1",
	"metadata":{"name":"my-pod-name", "namespace":"my-pod-namespace"},
	"spec": {
		"containers":[{
			"name":"my-container-name",
			"image":"my-container-image",
			"ports":[{"containerPort":1,"protocol":"TCP"}]
		}],
		"volumes":[{
			"name":"a-metadata-volume",
			"metadata":{"items":[{"name":"%s","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.name"}}]}
		}]
	}
}
`, path))

	downward := []byte(fmt.Sprintf(`
{
	"kind":"Pod",
	"apiVersion":"v1",
	"metadata":{"name":"my-pod-name", "namespace":"my-pod-namespace"},
	"spec": {
		"containers":[{
			"name":"my-container-name",
			"image":"my-container-image",
			"ports":[{"containerPort":1,"protocol":"TCP"}]
		}],
		"volumes":[{
			"name":"a-downwardapi-volume",
			"downwardAPI":{"items":[{"path":"%s","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.name"}}]}
		}]
	}
}
`, path))

	t.Log("Testing 1.0.6 v1 migration added in PR #4663")
	testCompatibility(
		t, "v1", metadata,
		func(obj runtime.Object) fielderrors.ValidationErrorList {
			return validation.ValidatePod(obj.(*api.Pod))
		},
		map[string]string{
			"spec.volumes[0].metadata.items[0].name":    path,
			"spec.volumes[0].downwardAPI.items[0].path": path,
		},
	)
	testCompatibility(
		t, "v1", downward,
		func(obj runtime.Object) fielderrors.ValidationErrorList {
			return validation.ValidatePod(obj.(*api.Pod))
		},
		map[string]string{
			"spec.volumes[0].metadata.items[0].name":    path,
			"spec.volumes[0].downwardAPI.items[0].path": path,
		},
	)
}

func TestCompatibility_v1_Service(t *testing.T) {
	// Test "spec.portalIP" -> "spec.clusterIP"
	expectedIP := "1.2.3.4"
	// Test "tcp" protocol gets converted to "TCP" and validated
	originalProtocol := "tcp"
	expectedProtocol := "TCP"

	input := []byte(fmt.Sprintf(`
{
	"kind":"Service",
	"apiVersion":"v1",
	"metadata":{"name":"my-service-name", "namespace":"my-service-namespace"},
	"spec": {
		"portalIP":"%s",
		"ports":[{"port":1,"protocol":"%s"}]
	}
}
`, expectedIP, originalProtocol))

	t.Log("Testing 1.0.0 v1 migration added in PR #3592")
	testCompatibility(
		t, "v1", input,
		func(obj runtime.Object) fielderrors.ValidationErrorList {
			return validation.ValidateService(obj.(*api.Service))
		},
		map[string]string{
			"spec.portalIP":          expectedIP,
			"spec.clusterIP":         expectedIP,
			"spec.ports[0].protocol": expectedProtocol,
		},
	)
}

func TestCompatibility_v1_Endpoints(t *testing.T) {
	// Test "tcp" protocol gets converted to "TCP" and validated
	originalProtocol := "tcp"
	expectedProtocol := "TCP"

	input := []byte(fmt.Sprintf(`
{
	"kind":"Endpoints",
	"apiVersion":"v1",
	"metadata":{"name":"my-endpoints-name", "namespace":"my-endpoints-namespace"},
	"subsets": [{
		"addresses":[{"ip":"1.2.3.4"}],
		"ports": [{"port":1,"targetPort":1,"protocol":"%s"}]
	}]
}
`, originalProtocol))

	t.Log("Testing 1.0.0 v1 migration added in PR #3592")
	testCompatibility(
		t, "v1", input,
		func(obj runtime.Object) fielderrors.ValidationErrorList {
			return validation.ValidateEndpoints(obj.(*api.Endpoints))
		},
		map[string]string{
			"subsets[0].ports[0].protocol": expectedProtocol,
		},
	)
}

func testCompatibility(
	t *testing.T,
	version string,
	input []byte,
	validator func(obj runtime.Object) fielderrors.ValidationErrorList,
	serialized map[string]string,
) {

	// Decode
	codec := runtime.CodecFor(api.Scheme, version)
	obj, err := codec.Decode(input)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	// Validate
	errs := validator(obj)
	if len(errs) != 0 {
		t.Fatalf("Unexpected errors: %v", errs)
	}

	// Encode
	output, err := codec.Encode(obj)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	// Validate old and new fields are encoded
	generic := map[string]interface{}{}
	if err := json.Unmarshal(output, &generic); err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	for k, expectedValue := range serialized {
		keys := strings.Split(k, ".")
		if actualValue, ok, err := getJSONValue(generic, keys...); err != nil || !ok {
			t.Errorf("Unexpected error for %s: %v", k, err)
		} else if !reflect.DeepEqual(expectedValue, actualValue) {
			t.Errorf("Expected %v, got %v", expectedValue, actualValue)
		}
	}
}

func getJSONValue(data map[string]interface{}, keys ...string) (interface{}, bool, error) {
	// No keys, current value is it
	if len(keys) == 0 {
		return data, true, nil
	}

	// Get the key (and optional index)
	key := keys[0]
	index := -1
	if matches := regexp.MustCompile(`^(.*)\[(\d+)\]$`).FindStringSubmatch(key); len(matches) > 0 {
		key = matches[1]
		index, _ = strconv.Atoi(matches[2])
	}

	// Look up the value
	value, ok := data[key]
	if !ok {
		return nil, false, fmt.Errorf("No key %s found", key)
	}

	// Get the indexed value if an index is specified
	if index >= 0 {
		valueSlice, ok := value.([]interface{})
		if !ok {
			return nil, false, fmt.Errorf("Key %s did not hold a slice", key)
		}
		if index >= len(valueSlice) {
			return nil, false, fmt.Errorf("Index %d out of bounds for slice at key: %v", index, key)
		}
		value = valueSlice[index]
	}

	if len(keys) == 1 {
		return value, true, nil
	}

	childData, ok := value.(map[string]interface{})
	if !ok {
		return nil, false, fmt.Errorf("Key %s did not hold a map", keys[0])
	}
	return getJSONValue(childData, keys[1:]...)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 the Velero contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package restore

import (
	"fmt"
	"sort"
	"time"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"

	api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
	"github.com/vmware-tanzu/velero/pkg/client"
	"github.com/vmware-tanzu/velero/pkg/cmd"
	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
	"github.com/vmware-tanzu/velero/pkg/cmd/util/output"
	veleroclient "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
	v1 "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1"
	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
)

func NewCreateCommand(f client.Factory, use string) *cobra.Command {
	o := NewCreateOptions()

	c := &cobra.Command{
		Use:   use + " [RESTORE_NAME] [--from-backup BACKUP_NAME | --from-schedule SCHEDULE_NAME]",
		Short: "Create a restore",
		Example: `  # create a restore named "restore-1" from backup "backup-1"
  velero restore create restore-1 --from-backup backup-1

  # create a restore with a default name ("backup-1-<timestamp>") from backup "backup-1"
  velero restore create --from-backup backup-1

  # create a restore from the latest successful backup triggered by schedule "schedule-1"
  velero restore create --from-schedule schedule-1

  # create a restore from the latest successful OR partially-failed backup triggered by schedule "schedule-1"
  velero restore create --from-schedule schedule-1 --allow-partially-failed

  # create a restore for only persistentvolumeclaims and persistentvolumes within a backup
  velero restore create --from-backup backup-2 --include-resources persistentvolumeclaims,persistentvolumes
  `,
		Args: cobra.MaximumNArgs(1),
		Run: func(c *cobra.Command, args []string) {
			cmd.CheckError(o.Complete(args, f))
			cmd.CheckError(o.Validate(c, args, f))
			cmd.CheckError(o.Run(c, f))
		},
	}

	o.BindFlags(c.Flags())
	output.BindFlags(c.Flags())
	output.ClearOutputFlagDefault(c)

	return c
}

type CreateOptions struct {
	BackupName              string
	ScheduleName            string
	RestoreName             string
	RestoreVolumes          flag.OptionalBool
	Labels                  flag.Map
	IncludeNamespaces       flag.StringArray
	ExcludeNamespaces       flag.StringArray
	IncludeResources        flag.StringArray
	ExcludeResources        flag.StringArray
	NamespaceMappings       flag.Map
	Selector                flag.LabelSelector
	IncludeClusterResources flag.OptionalBool
	Wait                    bool
	AllowPartiallyFailed    flag.OptionalBool

	client veleroclient.Interface
}

func NewCreateOptions() *CreateOptions {
	return &CreateOptions{
		Labels:                  flag.NewMap(),
		IncludeNamespaces:       flag.NewStringArray("*"),
		NamespaceMappings:       flag.NewMap().WithEntryDelimiter(",").WithKeyValueDelimiter(":"),
		RestoreVolumes:          flag.NewOptionalBool(nil),
		IncludeClusterResources: flag.NewOptionalBool(nil),
	}
}

func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
	flags.StringVar(&o.BackupName, "from-backup", "", "backup to restore from")
	flags.StringVar(&o.ScheduleName, "from-schedule", "", "schedule to restore from")
	flags.Var(&o.IncludeNamespaces, "include-namespaces", "namespaces to include in the restore (use '*' for all namespaces)")
	flags.Var(&o.ExcludeNamespaces, "exclude-namespaces", "namespaces to exclude from the restore")
	flags.Var(&o.NamespaceMappings, "namespace-mappings", "namespace mappings from name in the backup to desired restored name in the form src1:dst1,src2:dst2,...")
	flags.Var(&o.Labels, "labels", "labels to apply to the restore")
	flags.Var(&o.IncludeResources, "include-resources", "resources to include in the restore, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources)")
	flags.Var(&o.ExcludeResources, "exclude-resources", "resources to exclude from the restore, formatted as resource.group, such as storageclasses.storage.k8s.io")
	flags.VarP(&o.Selector, "selector", "l", "only restore resources matching this label selector")
	f := flags.VarPF(&o.RestoreVolumes, "restore-volumes", "", "whether to restore volumes from snapshots")
	// this allows the user to just specify "--restore-volumes" as shorthand for "--restore-volumes=true"
	// like a normal bool flag
	f.NoOptDefVal = "true"

	f = flags.VarPF(&o.IncludeClusterResources, "include-cluster-resources", "", "include cluster-scoped resources in the restore")
	f.NoOptDefVal = "true"

	f = flags.VarPF(&o.AllowPartiallyFailed, "allow-partially-failed", "", "if using --from-schedule, whether to consider PartiallyFailed backups when looking for the most recent one. This flag has no effect if not using --from-schedule.")
	f.NoOptDefVal = "true"

	flags.BoolVarP(&o.Wait, "wait", "w", o.Wait, "wait for the operation to complete")
}

func (o *CreateOptions) Complete(args []string, f client.Factory) error {
	if len(args) == 1 {
		o.RestoreName = args[0]
	} else {
		sourceName := o.BackupName
		if o.ScheduleName != "" {
			sourceName = o.ScheduleName
		}

		o.RestoreName = fmt.Sprintf("%s-%s", sourceName, time.Now().Format("20060102150405"))
	}

	client, err := f.Client()
	if err != nil {
		return err
	}
	o.client = client

	return nil
}

func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error {
	if o.BackupName != "" && o.ScheduleName != "" {
		return errors.New("either a backup or schedule must be specified, but not both")
	}

	if o.BackupName == "" && o.ScheduleName == "" {
		return errors.New("either a backup or schedule must be specified, but not both")
	}

	if err := output.ValidateFlags(c); err != nil {
		return err
	}

	if o.client == nil {
		// This should never happen
		return errors.New("Velero client is not set; unable to proceed")
	}

	switch {
	case o.BackupName != "":
		if _, err := o.client.VeleroV1().Backups(f.Namespace()).Get(o.BackupName, metav1.GetOptions{}); err != nil {
			return err
		}
	case o.ScheduleName != "":
		if _, err := o.client.VeleroV1().Schedules(f.Namespace()).Get(o.ScheduleName, metav1.GetOptions{}); err != nil {
			return err
		}
	}

	return nil
}

// mostRecentBackup returns the backup with the most recent start timestamp that has a phase that's
// in the provided list of allowed phases.
func mostRecentBackup(backups []api.Backup, allowedPhases ...api.BackupPhase) *api.Backup {
	// sort the backups in descending order of start time (i.e. most recent to least recent)
	sort.Slice(backups, func(i, j int) bool {
		// Use .After() because we want descending sort.

		var iStartTime, jStartTime time.Time
		if backups[i].Status.StartTimestamp != nil {
			iStartTime = backups[i].Status.StartTimestamp.Time
		}
		if backups[j].Status.StartTimestamp != nil {
			jStartTime = backups[j].Status.StartTimestamp.Time
		}
		return iStartTime.After(jStartTime)
	})

	// create a map of the allowed phases for easy lookup below
	phases := map[api.BackupPhase]struct{}{}
	for _, phase := range allowedPhases {
		phases[phase] = struct{}{}
	}

	var res *api.Backup
	for i, backup := range backups {
		// if the backup's phase is one of the allowable ones, record
		// the backup and break the loop so we can return it
		if _, ok := phases[backup.Status.Phase]; ok {
			res = &backups[i]
			break
		}
	}

	return res
}

func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
	if o.client == nil {
		// This should never happen
		return errors.New("Velero client is not set; unable to proceed")
	}

	// if --allow-partially-failed was specified, look up the most recent Completed or
	// PartiallyFailed backup for the provided schedule, and use that specific backup
	// to restore from.
	if o.ScheduleName != "" && boolptr.IsSetToTrue(o.AllowPartiallyFailed.Value) {
		backups, err := o.client.VeleroV1().Backups(f.Namespace()).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", api.ScheduleNameLabel, o.ScheduleName)})
		if err != nil {
			return err
		}

		// if we find a Completed or PartiallyFailed backup for the schedule, restore specifically from that backup. If we don't
		// find one, proceed as-is -- the Velero server will handle validation.
		if backup := mostRecentBackup(backups.Items, api.BackupPhaseCompleted, api.BackupPhasePartiallyFailed); backup != nil {
			// TODO(sk): this is kind of a hack -- we should revisit this and probably
			// move this logic to the server side or otherwise solve this problem.
			o.BackupName = backup.Name
			o.ScheduleName = ""
		}
	}

	restore := &api.Restore{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace(),
			Name:      o.RestoreName,
			Labels:    o.Labels.Data(),
		},
		Spec: api.RestoreSpec{
			BackupName:              o.BackupName,
			ScheduleName:            o.ScheduleName,
			IncludedNamespaces:      o.IncludeNamespaces,
			ExcludedNamespaces:      o.ExcludeNamespaces,
			IncludedResources:       o.IncludeResources,
			ExcludedResources:       o.ExcludeResources,
			NamespaceMapping:        o.NamespaceMappings.Data(),
			LabelSelector:           o.Selector.LabelSelector,
			RestorePVs:              o.RestoreVolumes.Value,
			IncludeClusterResources: o.IncludeClusterResources.Value,
		},
	}

	if printed, err := output.PrintWithFormat(c, restore); printed || err != nil {
		return err
	}

	var restoreInformer cache.SharedIndexInformer
	var updates chan *api.Restore
	if o.Wait {
		stop := make(chan struct{})
		defer close(stop)

		updates = make(chan *api.Restore)

		restoreInformer = v1.NewRestoreInformer(o.client, f.Namespace(), 0, nil)

		restoreInformer.AddEventHandler(
			cache.FilteringResourceEventHandler{
				FilterFunc: func(obj interface{}) bool {
					restore, ok := obj.(*api.Restore)
					if !ok {
						return false
					}
					return restore.Name == o.RestoreName
				},
				Handler: cache.ResourceEventHandlerFuncs{
					UpdateFunc: func(_, obj interface{}) {
						restore, ok := obj.(*api.Restore)
						if !ok {
							return
						}
						updates <- restore
					},
					DeleteFunc: func(obj interface{}) {
						restore, ok := obj.(*api.Restore)
						if !ok {
							return
						}
						updates <- restore
					},
				},
			},
		)
		go restoreInformer.Run(stop)
	}

	restore, err := o.client.VeleroV1().Restores(restore.Namespace).Create(restore)
	if err != nil {
		return err
	}

	fmt.Printf("Restore request %q submitted successfully.\n", restore.Name)
	if o.Wait {
		fmt.Println("Waiting for restore to complete. You may safely press ctrl-c to stop waiting - your restore will continue in the background.")
		ticker := time.NewTicker(time.Second)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				fmt.Print(".")
			case restore, ok := <-updates:
				if !ok {
					fmt.Println("\nError waiting: unable to watch restores.")
					return nil
				}

				if restore.Status.Phase != api.RestorePhaseNew && restore.Status.Phase != api.RestorePhaseInProgress {
					fmt.Printf("\nRestore completed with status: %s. You may check for more information using the commands `velero restore describe %s` and `velero restore logs %s`.\n", restore.Status.Phase, restore.Name, restore.Name)
					return nil
				}
			}
		}
	}

	// Not waiting

	fmt.Printf("Run `velero restore describe %s` or `velero restore logs %s` for more details.\n", restore.Name, restore.Name)

	return nil
}
<commit_msg>remove schedule validation<commit_after>/*
Copyright 2017 the Velero contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package restore

import (
	"fmt"
	"sort"
	"time"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"

	api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
	"github.com/vmware-tanzu/velero/pkg/client"
	"github.com/vmware-tanzu/velero/pkg/cmd"
	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
	"github.com/vmware-tanzu/velero/pkg/cmd/util/output"
	veleroclient "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
	v1 "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1"
	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
)

func NewCreateCommand(f client.Factory, use string) *cobra.Command {
	o := NewCreateOptions()

	c := &cobra.Command{
		Use:   use + " [RESTORE_NAME] [--from-backup BACKUP_NAME | --from-schedule SCHEDULE_NAME]",
		Short: "Create a restore",
		Example: `  # create a restore named "restore-1" from backup "backup-1"
  velero restore create restore-1 --from-backup backup-1

  # create a restore with a default name ("backup-1-<timestamp>") from backup "backup-1"
  velero restore create --from-backup backup-1

  # create a restore from the latest successful backup triggered by schedule "schedule-1"
  velero restore create --from-schedule schedule-1

  # create a restore from the latest successful OR partially-failed backup triggered by schedule "schedule-1"
  velero restore create --from-schedule schedule-1 --allow-partially-failed

  # create a restore for only persistentvolumeclaims and persistentvolumes within a backup
  velero restore create --from-backup backup-2 --include-resources persistentvolumeclaims,persistentvolumes
  `,
		Args: cobra.MaximumNArgs(1),
		Run: func(c *cobra.Command, args []string) {
			cmd.CheckError(o.Complete(args, f))
			cmd.CheckError(o.Validate(c, args, f))
			cmd.CheckError(o.Run(c, f))
		},
	}

	o.BindFlags(c.Flags())
	output.BindFlags(c.Flags())
	output.ClearOutputFlagDefault(c)

	return c
}

type CreateOptions struct {
	BackupName              string
	ScheduleName            string
	RestoreName             string
	RestoreVolumes          flag.OptionalBool
	Labels                  flag.Map
	IncludeNamespaces       flag.StringArray
	ExcludeNamespaces       flag.StringArray
	IncludeResources        flag.StringArray
	ExcludeResources        flag.StringArray
	NamespaceMappings       flag.Map
	Selector                flag.LabelSelector
	IncludeClusterResources flag.OptionalBool
	Wait                    bool
	AllowPartiallyFailed    flag.OptionalBool

	client veleroclient.Interface
}

func NewCreateOptions() *CreateOptions {
	return &CreateOptions{
		Labels:                  flag.NewMap(),
		IncludeNamespaces:       flag.NewStringArray("*"),
		NamespaceMappings:       flag.NewMap().WithEntryDelimiter(",").WithKeyValueDelimiter(":"),
		RestoreVolumes:          flag.NewOptionalBool(nil),
		IncludeClusterResources: flag.NewOptionalBool(nil),
	}
}

func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
	flags.StringVar(&o.BackupName, "from-backup", "", "backup to restore from")
	flags.StringVar(&o.ScheduleName, "from-schedule", "", "schedule to restore from")
	flags.Var(&o.IncludeNamespaces, "include-namespaces", "namespaces to include in the restore (use '*' for all namespaces)")
	flags.Var(&o.ExcludeNamespaces, "exclude-namespaces", "namespaces to exclude from the restore")
	flags.Var(&o.NamespaceMappings, "namespace-mappings", "namespace mappings from name in the backup to desired restored name in the form src1:dst1,src2:dst2,...")
	flags.Var(&o.Labels, "labels", "labels to apply to the restore")
	flags.Var(&o.IncludeResources, "include-resources", "resources to include in the restore, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources)")
	flags.Var(&o.ExcludeResources, "exclude-resources", "resources to exclude from the restore, formatted as resource.group, such as storageclasses.storage.k8s.io")
	flags.VarP(&o.Selector, "selector", "l", "only restore resources matching this label selector")
	f := flags.VarPF(&o.RestoreVolumes, "restore-volumes", "", "whether to restore volumes from snapshots")
	// this allows the user to just specify "--restore-volumes" as shorthand for "--restore-volumes=true"
	// like a normal bool flag
	f.NoOptDefVal = "true"

	f = flags.VarPF(&o.IncludeClusterResources, "include-cluster-resources", "", "include cluster-scoped resources in the restore")
	f.NoOptDefVal = "true"

	f = flags.VarPF(&o.AllowPartiallyFailed, "allow-partially-failed", "", "if using --from-schedule, whether to consider PartiallyFailed backups when looking for the most recent one. This flag has no effect if not using --from-schedule.")
	f.NoOptDefVal = "true"

	flags.BoolVarP(&o.Wait, "wait", "w", o.Wait, "wait for the operation to complete")
}

func (o *CreateOptions) Complete(args []string, f client.Factory) error {
	if len(args) == 1 {
		o.RestoreName = args[0]
	} else {
		sourceName := o.BackupName
		if o.ScheduleName != "" {
			sourceName = o.ScheduleName
		}

		o.RestoreName = fmt.Sprintf("%s-%s", sourceName, time.Now().Format("20060102150405"))
	}

	client, err := f.Client()
	if err != nil {
		return err
	}
	o.client = client

	return nil
}

func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error {
	if o.BackupName != "" && o.ScheduleName != "" {
		return errors.New("either a backup or schedule must be specified, but not both")
	}

	if o.BackupName == "" && o.ScheduleName == "" {
		return errors.New("either a backup or schedule must be specified, but not both")
	}

	if err := output.ValidateFlags(c); err != nil {
		return err
	}

	if o.client == nil {
		// This should never happen
		return errors.New("Velero client is not set; unable to proceed")
	}

	if o.BackupName != "" {
		if _, err := o.client.VeleroV1().Backups(f.Namespace()).Get(o.BackupName, metav1.GetOptions{}); err != nil {
			return err
		}
	}

	return nil
}

// mostRecentBackup returns the backup with the most recent start timestamp that has a phase that's
// in the provided list of allowed phases.
func mostRecentBackup(backups []api.Backup, allowedPhases ...api.BackupPhase) *api.Backup {
	// sort the backups in descending order of start time (i.e. most recent to least recent)
	sort.Slice(backups, func(i, j int) bool {
		// Use .After() because we want descending sort.

		var iStartTime, jStartTime time.Time
		if backups[i].Status.StartTimestamp != nil {
			iStartTime = backups[i].Status.StartTimestamp.Time
		}
		if backups[j].Status.StartTimestamp != nil {
			jStartTime = backups[j].Status.StartTimestamp.Time
		}
		return iStartTime.After(jStartTime)
	})

	// create a map of the allowed phases for easy lookup below
	phases := map[api.BackupPhase]struct{}{}
	for _, phase := range allowedPhases {
		phases[phase] = struct{}{}
	}

	var res *api.Backup
	for i, backup := range backups {
		// if the backup's phase is one of the allowable ones, record
		// the backup and break the loop so we can return it
		if _, ok := phases[backup.Status.Phase]; ok {
			res = &backups[i]
			break
		}
	}

	return res
}

func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
	if o.client == nil {
		// This should never happen
		return errors.New("Velero client is not set; unable to proceed")
	}

	// if --allow-partially-failed was specified, look up the most recent Completed or
	// PartiallyFailed backup for the provided schedule, and use that specific backup
	// to restore from.
	if o.ScheduleName != "" && boolptr.IsSetToTrue(o.AllowPartiallyFailed.Value) {
		backups, err := o.client.VeleroV1().Backups(f.Namespace()).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", api.ScheduleNameLabel, o.ScheduleName)})
		if err != nil {
			return err
		}

		// if we find a Completed or PartiallyFailed backup for the schedule, restore specifically from that backup. If we don't
		// find one, proceed as-is -- the Velero server will handle validation.
		if backup := mostRecentBackup(backups.Items, api.BackupPhaseCompleted, api.BackupPhasePartiallyFailed); backup != nil {
			// TODO(sk): this is kind of a hack -- we should revisit this and probably
			// move this logic to the server side or otherwise solve this problem.
			o.BackupName = backup.Name
			o.ScheduleName = ""
		}
	}

	restore := &api.Restore{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace(),
			Name:      o.RestoreName,
			Labels:    o.Labels.Data(),
		},
		Spec: api.RestoreSpec{
			BackupName:              o.BackupName,
			ScheduleName:            o.ScheduleName,
			IncludedNamespaces:      o.IncludeNamespaces,
			ExcludedNamespaces:      o.ExcludeNamespaces,
			IncludedResources:       o.IncludeResources,
			ExcludedResources:       o.ExcludeResources,
			NamespaceMapping:        o.NamespaceMappings.Data(),
			LabelSelector:           o.Selector.LabelSelector,
			RestorePVs:              o.RestoreVolumes.Value,
			IncludeClusterResources: o.IncludeClusterResources.Value,
		},
	}

	if printed, err := output.PrintWithFormat(c, restore); printed || err != nil {
		return err
	}

	var restoreInformer cache.SharedIndexInformer
	var updates chan *api.Restore
	if o.Wait {
		stop := make(chan struct{})
		defer close(stop)

		updates = make(chan *api.Restore)

		restoreInformer = v1.NewRestoreInformer(o.client, f.Namespace(), 0, nil)

		restoreInformer.AddEventHandler(
			cache.FilteringResourceEventHandler{
				FilterFunc: func(obj interface{}) bool {
					restore, ok := obj.(*api.Restore)
					if !ok {
						return false
					}
					return restore.Name == o.RestoreName
				},
				Handler: cache.ResourceEventHandlerFuncs{
					UpdateFunc: func(_, obj interface{}) {
						restore, ok := obj.(*api.Restore)
						if !ok {
							return
						}
						updates <- restore
					},
					DeleteFunc: func(obj interface{}) {
						restore, ok := obj.(*api.Restore)
						if !ok {
							return
						}
						updates <- restore
					},
				},
			},
		)
		go restoreInformer.Run(stop)
	}

	restore, err := o.client.VeleroV1().Restores(restore.Namespace).Create(restore)
	if err != nil {
		return err
	}

	fmt.Printf("Restore request %q submitted successfully.\n", restore.Name)
	if o.Wait {
		fmt.Println("Waiting for restore to complete. You may safely press ctrl-c to stop waiting - your restore will continue in the background.")
		ticker := time.NewTicker(time.Second)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				fmt.Print(".")
			case restore, ok := <-updates:
				if !ok {
					fmt.Println("\nError waiting: unable to watch restores.")
					return nil
				}

				if restore.Status.Phase != api.RestorePhaseNew && restore.Status.Phase != api.RestorePhaseInProgress {
					fmt.Printf("\nRestore completed with status: %s. You may check for more information using the commands `velero restore describe %s` and `velero restore logs %s`.\n", restore.Status.Phase, restore.Name, restore.Name)
					return nil
				}
			}
		}
	}

	// Not waiting

	fmt.Printf("Run `velero restore describe %s` or `velero restore logs %s` for more details.\n", restore.Name, restore.Name)

	return nil
}
<|endoftext|>