<commit_before>/*
Copyright 2016 Skippbox, Ltd.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"gopkg.in/yaml.v2"
)

var ConfigFileName = ".kubewatch.yaml"

type Handler struct {
	Slack Slack `json:"slack"`
	Hipchat Hipchat `json:"hipchat"`
}

// Resource contains resource configuration
type Resource struct {
	Deployment            bool `json:"deployment"`
	ReplicationController bool `json:"rc"`
	ReplicaSet            bool `json:"rs"`
	DaemonSet             bool `json:"ds"`
	Services              bool `json:"svc"`
	Pod                   bool `json:"po"`
	Job                   bool `json:"job"`
	PersistentVolume      bool `json:"pv"`
}

// Config struct contains kubewatch configuration
type Config struct {
	Handler Handler `json:"handler"`
	//Reason   []string `json:"reason"`
	Resource Resource `json:"resource"`
}

// Slack contains slack configuration
type Slack struct {
	Token   string `json:"token"`
	Channel string `json:"channel"`
}

// Hipchat contains slack configuration
type Hipchat struct {
	Token   string `json:"token"`
	Room string `json:"room"`
	Url string `json:"url"`
}

// New creates new config object
func New() (*Config, error) {
	c := &Config{}
	if err := c.Load(); err != nil {
		return c, err
	}

	return c, nil
}

func createIfNotExist() error {
	// create file if not exist
	configFile := filepath.Join(configDir(), ConfigFileName)
	_, err := os.Stat(configFile)
	if err != nil {
		if os.IsNotExist(err) {
			file, err := os.Create(configFile)
			if err != nil {
				return err
			}
			file.Close()
		} else {
			return err
		}
	}
	return nil
}

// Load loads configuration from config file
func (c *Config) Load() error {
	err := createIfNotExist()
	if err != nil {
		return err
	}

	file, err := os.Open(getConfigFile())
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}

	if len(b) != 0 {
		return yaml.Unmarshal(b, c)
	}

	return nil
}

func (c *Config) CheckMissingResourceEnvvars() {
	if !c.Resource.DaemonSet && os.Getenv("KW_DAEMONSET") == "true" {
		c.Resource.DaemonSet = true
	}
	if !c.Resource.ReplicaSet && os.Getenv("KW_REPLICASET") == "true" {
		c.Resource.ReplicaSet = true
	}
	if !c.Resource.Deployment && os.Getenv("KW_DEPLOYMENT") == "true" {
		c.Resource.Deployment = true
	}
	if !c.Resource.Pod && os.Getenv("KW_POD") == "true" {
		c.Resource.Pod = true
	}
	if !c.Resource.ReplicationController && os.Getenv("KW_REPLICATION_CONTROLLER") == "true" {
		c.Resource.ReplicationController = true
	}
	if !c.Resource.Services && os.Getenv("KW_SERVICE") == "true" {
		c.Resource.Services = true
	}
	if !c.Resource.Job && os.Getenv("KW_JOB") == "true" {
		c.Resource.Job = true
	}
	if !c.Resource.PersistentVolume && os.Getenv("KW_PERSISTENT_VOLUME") == "true" {
		c.Resource.PersistentVolume = true
	}
	if (c.Handler.Slack.Channel == "") && (os.Getenv("SLACK_CHANNEL") != "") {
		c.Handler.Slack.Channel = os.Getenv("SLACK_CHANNEL")
	}
	if (c.Handler.Slack.Token == "") && (os.Getenv("SLACK_TOKEN") != "") {
		c.Handler.Slack.Token = os.Getenv("SLACK_TOKEN")
	}
}

func (c *Config) Write() error {
	b, err := yaml.Marshal(c)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(getConfigFile(), b, 0644)
	if err != nil {
		return err
	}

	return nil
}

func getConfigFile() string {
	configFile := filepath.Join(configDir(), ConfigFileName)
	if _, err := os.Stat(configFile); err == nil {
		return configFile
	}

	return ""
}

func configDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("USERPROFILE")
		return home
	}
	return os.Getenv("HOME")
	//path := "/etc/kubewatch"
	//if _, err := os.Stat(path); os.IsNotExist(err) {
	//	os.Mkdir(path, 755)
	//}
	//return path
}
<commit_msg>config/config.go: add namespace to resource struct and environment variables<commit_after>/*
Copyright 2016 Skippbox, Ltd.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"gopkg.in/yaml.v2"
)

var ConfigFileName = ".kubewatch.yaml"

type Handler struct {
	Slack Slack `json:"slack"`
	Hipchat Hipchat `json:"hipchat"`
}

// Resource contains resource configuration
type Resource struct {
	Deployment            bool `json:"deployment"`
	ReplicationController bool `json:"rc"`
	ReplicaSet            bool `json:"rs"`
	DaemonSet             bool `json:"ds"`
	Services              bool `json:"svc"`
	Pod                   bool `json:"po"`
	Job                   bool `json:"job"`
	PersistentVolume      bool `json:"pv"`
	Namespace	      bool `json:"ns"`
}

// Config struct contains kubewatch configuration
type Config struct {
	Handler Handler `json:"handler"`
	//Reason   []string `json:"reason"`
	Resource Resource `json:"resource"`
}

// Slack contains slack configuration
type Slack struct {
	Token   string `json:"token"`
	Channel string `json:"channel"`
}

// Hipchat contains slack configuration
type Hipchat struct {
	Token   string `json:"token"`
	Room string `json:"room"`
	Url string `json:"url"`
}

// New creates new config object
func New() (*Config, error) {
	c := &Config{}
	if err := c.Load(); err != nil {
		return c, err
	}

	return c, nil
}

func createIfNotExist() error {
	// create file if not exist
	configFile := filepath.Join(configDir(), ConfigFileName)
	_, err := os.Stat(configFile)
	if err != nil {
		if os.IsNotExist(err) {
			file, err := os.Create(configFile)
			if err != nil {
				return err
			}
			file.Close()
		} else {
			return err
		}
	}
	return nil
}

// Load loads configuration from config file
func (c *Config) Load() error {
	err := createIfNotExist()
	if err != nil {
		return err
	}

	file, err := os.Open(getConfigFile())
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}

	if len(b) != 0 {
		return yaml.Unmarshal(b, c)
	}

	return nil
}

func (c *Config) CheckMissingResourceEnvvars() {
	if !c.Resource.DaemonSet && os.Getenv("KW_DAEMONSET") == "true" {
		c.Resource.DaemonSet = true
	}
	if !c.Resource.ReplicaSet && os.Getenv("KW_REPLICASET") == "true" {
		c.Resource.ReplicaSet = true
	}
	if !c.Resource.Namespace && os.Getenv("KW_NAMESPACE") == "true" {
		c.Resource.Namespace = true
	}
	if !c.Resource.Deployment && os.Getenv("KW_DEPLOYMENT") == "true" {
		c.Resource.Deployment = true
	}
	if !c.Resource.Pod && os.Getenv("KW_POD") == "true" {
		c.Resource.Pod = true
	}
	if !c.Resource.ReplicationController && os.Getenv("KW_REPLICATION_CONTROLLER") == "true" {
		c.Resource.ReplicationController = true
	}
	if !c.Resource.Services && os.Getenv("KW_SERVICE") == "true" {
		c.Resource.Services = true
	}
	if !c.Resource.Job && os.Getenv("KW_JOB") == "true" {
		c.Resource.Job = true
	}
	if !c.Resource.PersistentVolume && os.Getenv("KW_PERSISTENT_VOLUME") == "true" {
		c.Resource.PersistentVolume = true
	}
	if (c.Handler.Slack.Channel == "") && (os.Getenv("SLACK_CHANNEL") != "") {
		c.Handler.Slack.Channel = os.Getenv("SLACK_CHANNEL")
	}
	if (c.Handler.Slack.Token == "") && (os.Getenv("SLACK_TOKEN") != "") {
		c.Handler.Slack.Token = os.Getenv("SLACK_TOKEN")
	}
}

func (c *Config) Write() error {
	b, err := yaml.Marshal(c)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(getConfigFile(), b, 0644)
	if err != nil {
		return err
	}

	return nil
}

func getConfigFile() string {
	configFile := filepath.Join(configDir(), ConfigFileName)
	if _, err := os.Stat(configFile); err == nil {
		return configFile
	}

	return ""
}

func configDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("USERPROFILE")
		return home
	}
	return os.Getenv("HOME")
	//path := "/etc/kubewatch"
	//if _, err := os.Stat(path); os.IsNotExist(err) {
	//	os.Mkdir(path, 755)
	//}
	//return path
}
<|endoftext|>


<commit_before>package config

import (
	"io/ioutil"
	"log"

	"gopkg.in/yaml.v2"
)

// config struct
type config struct {
	Port  int `yaml:"app"`
	Mongo struct {
		Host string
		Name string
		Mode string
	}
}

// Config is globle configuration
var Config *config

// Init read the configuration file
func Init(conf string) {
	data, err := ioutil.ReadFile(conf)
	if err != nil {
		log.Panic(err.Error())
	}

	Config = &config{}

	err = yaml.Unmarshal([]byte(data), Config)
	if err != nil {
		log.Panic(err.Error())
	}
}
<commit_msg>Fix config.Config struct comment<commit_after>package config

import (
	"io/ioutil"
	"log"

	"gopkg.in/yaml.v2"
)

// config struct
type config struct {
	Port  int `yaml:"port"`
	Mongo struct {
		Host string `yaml:"host"`
		Name string `yaml:"name"`
		Mode string `yaml:"mode"`
	}
}

// Config is globle configuration
var Config *config

// Init read the configuration file
func Init(conf string) {
	data, err := ioutil.ReadFile(conf)
	if err != nil {
		log.Panic(err.Error())
	}

	Config = &config{}

	err = yaml.Unmarshal([]byte(data), Config)
	if err != nil {
		log.Panic(err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 NetApp, Inc. All Rights Reserved.

package config

import (
	"fmt"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/netapp/trident/utils"
)

type Protocol string
type AccessMode string
type VolumeMode string
type VolumeType string
type DriverContext string
type Platform string

type Telemetry struct {
	TridentVersion  string `json:"version"`
	Platform        string `json:"platform"`
	PlatformVersion string `json:"platformVersion"`
}

type PersistentStateVersion struct {
	PersistentStoreVersion string `json:"store_version"`
	OrchestratorAPIVersion string `json:"orchestrator_api_version"`
}

const (
	/* Misc. orchestrator constants */
	OrchestratorName                 = "trident"
	OrchestratorClientName           = OrchestratorName + "ctl"
	orchestratorVersion              = "20.04.0"
	OrchestratorAPIVersion           = "1"
	PersistentStoreBootstrapAttempts = 30
	PersistentStoreBootstrapTimeout  = PersistentStoreBootstrapAttempts * time.Second
	PersistentStoreTimeout           = 10 * time.Second
	DockerCreateTimeout              = 115 * time.Second
	DockerDefaultTimeout             = 55 * time.Second

	/* REST/HTTP constants */
	HTTPTimeout = 90 * time.Second

	CACertName     = "trident-ca"
	ServerCertName = "trident-csi" // Must match CSI service name
	ClientCertName = "trident-node"

	CAKeyFile      = "caKey"
	CACertFile     = "caCert"
	ServerKeyFile  = "serverKey"
	ServerCertFile = "serverCert"
	ClientKeyFile  = "clientKey"
	ClientCertFile = "clientCert"

	certsPath = "/certs/"

	CAKeyPath      = certsPath + CAKeyFile
	CACertPath     = certsPath + CACertFile
	ServerKeyPath  = certsPath + ServerKeyFile
	ServerCertPath = certsPath + ServerCertFile
	ClientKeyPath  = certsPath + ClientKeyFile
	ClientCertPath = certsPath + ClientCertFile

	/* Protocol constants. This value denotes a volume's backing storage protocol. For example,
	a Trident volume with  'file' protocol is most likely NFS, while a 'block' protocol volume is probably iSCSI. */
	File        Protocol = "file"
	Block       Protocol = "block"
	ProtocolAny Protocol = ""

	/* Access mode constants */
	ReadWriteOnce AccessMode = "ReadWriteOnce"
	ReadOnlyMany  AccessMode = "ReadOnlyMany"
	ReadWriteMany AccessMode = "ReadWriteMany"
	ModeAny       AccessMode = ""

	/* Volume mode constants. This value describes how a volume will be consumed by application containers.
	Most Trident volumes (regardless of protocol) probably use the 'Filesystem' mode, where the volume contains
	a filesystem and is mounted into a container. By contrast, volumes with 'Block' mode always use 'block' protocol
	and are attached to a container as raw block devices. */
	RawBlock   VolumeMode = "Block"
	Filesystem VolumeMode = "Filesystem"

	/* Volume type constants */
	OntapNFS          VolumeType = "ONTAP_NFS"
	OntapISCSI        VolumeType = "ONTAP_iSCSI"
	SolidFireISCSI    VolumeType = "SolidFire_iSCSI"
	ESeriesISCSI      VolumeType = "Eseries_iSCSI"
	UnknownVolumeType VolumeType = ""

	/* Driver-related constants */
	DefaultSolidFireVAG      = OrchestratorName
	UnknownDriver            = "UnknownDriver"
	StorageAPITimeoutSeconds = 90
	SANResizeDelta           = 50000000 // 50mb

	/* REST frontend constants */
	MaxRESTRequestSize = 10240

	/* Kubernetes deployment constants */
	ContainerTrident = "trident-main"
	ContainerEtcd    = "etcd"

	ContextDocker     DriverContext = "docker"
	ContextKubernetes DriverContext = "kubernetes"
	ContextCSI        DriverContext = "csi"
	ContextCRD        DriverContext = "crd"

	PlatformDocker     Platform = "docker"
	PlatformKubernetes Platform = "kubernetes"
	PlatformCSI        Platform = "csi" // plain CSI, no other CO present

	// Minimum and maximum supported Kubernetes versions
	KubernetesVersionMin = "v1.11.0"
	KubernetesVersionMax = "v1.17.0"

	// Minimum Kubernetes version for CSI Trident (non-CSI is the default)
	KubernetesCSIVersionMinOptional = "v1.13.0"

	// Minimum Kubernetes version for CSI Trident default (non-CSI not supported)
	KubernetesCSIVersionMinForced = "v1.14.0"

	TridentNamespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
)

var (
	validProtocols = map[Protocol]bool{
		File:        true,
		Block:       true,
		ProtocolAny: true,
	}

	// BuildHash is the git hash the binary was built from
	BuildHash = "unknown"

	// BuildType is the type of build: custom, beta or stable
	BuildType = "custom"

	// BuildTypeRev is the revision of the build
	BuildTypeRev = "0"

	// BuildTime is the time the binary was built
	BuildTime = "unknown"

	// BuildImage is the Trident image that was built
	BuildImage = "netapp/trident:" + orchestratorVersion + "-custom.0"

	// BuildEtcdVersion is the etcd version that Trident should be deployed with
	BuildEtcdVersion = "v3.3.18"

	// BuildEtcdImage is the etcd image that Trident should be deployed with
	BuildEtcdImage = "quay.io/coreos/etcd:" + BuildEtcdVersion

	OrchestratorVersion = utils.MustParseDate(version())

	/* API Server and persistent store variables */
	BaseURL         = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion
	VersionURL      = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/version"
	BackendURL      = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/backend"
	BackendUUIDURL  = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/backendUUID"
	VolumeURL       = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/volume"
	TransactionURL  = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/txn"
	StorageClassURL = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/storageclass"
	NodeURL         = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/node"
	SnapshotURL     = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/snapshot"
	StoreURL        = "/" + OrchestratorName + "/store"

	UsingPassthroughStore bool
	CurrentDriverContext  DriverContext
	OrchestratorTelemetry = Telemetry{TridentVersion: OrchestratorVersion.String()}
)

func IsValidProtocol(p Protocol) bool {
	_, ok := validProtocols[p]
	return ok
}

func GetValidProtocolNames() []string {
	ret := make([]string, len(validProtocols))
	for key := range validProtocols {
		ret = append(ret, string(key))
	}
	return ret
}

func PlatformAtLeast(platformName string, version string) bool {
	if OrchestratorTelemetry.Platform == platformName {
		platformVersion := utils.MustParseSemantic(OrchestratorTelemetry.PlatformVersion)
		requiredVersion, err := utils.ParseSemantic(version)
		if err != nil {
			log.WithFields(log.Fields{
				"platform": platformName,
				"version":  version,
			}).Errorf("Platform version check failed. %+v", err)
			return false
		}
		if platformVersion.AtLeast(requiredVersion) {
			return true
		}
	}
	return false
}

func version() string {

	var version string

	if BuildType != "stable" {
		if BuildType == "custom" {
			version = fmt.Sprintf("%v-%v+%v", orchestratorVersion, BuildType, BuildHash)
		} else {
			version = fmt.Sprintf("%v-%v.%v+%v", orchestratorVersion, BuildType, BuildTypeRev, BuildHash)
		}
	} else {
		version = orchestratorVersion
	}

	return version
}
<commit_msg>Bump up the K8S version for Trident installation / Run Trident on K8S 1.18<commit_after>// Copyright 2020 NetApp, Inc. All Rights Reserved.

package config

import (
	"fmt"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/netapp/trident/utils"
)

type Protocol string
type AccessMode string
type VolumeMode string
type VolumeType string
type DriverContext string
type Platform string

type Telemetry struct {
	TridentVersion  string `json:"version"`
	Platform        string `json:"platform"`
	PlatformVersion string `json:"platformVersion"`
}

type PersistentStateVersion struct {
	PersistentStoreVersion string `json:"store_version"`
	OrchestratorAPIVersion string `json:"orchestrator_api_version"`
}

const (
	/* Misc. orchestrator constants */
	OrchestratorName                 = "trident"
	OrchestratorClientName           = OrchestratorName + "ctl"
	orchestratorVersion              = "20.04.0"
	OrchestratorAPIVersion           = "1"
	PersistentStoreBootstrapAttempts = 30
	PersistentStoreBootstrapTimeout  = PersistentStoreBootstrapAttempts * time.Second
	PersistentStoreTimeout           = 10 * time.Second
	DockerCreateTimeout              = 115 * time.Second
	DockerDefaultTimeout             = 55 * time.Second

	/* REST/HTTP constants */
	HTTPTimeout = 90 * time.Second

	CACertName     = "trident-ca"
	ServerCertName = "trident-csi" // Must match CSI service name
	ClientCertName = "trident-node"

	CAKeyFile      = "caKey"
	CACertFile     = "caCert"
	ServerKeyFile  = "serverKey"
	ServerCertFile = "serverCert"
	ClientKeyFile  = "clientKey"
	ClientCertFile = "clientCert"

	certsPath = "/certs/"

	CAKeyPath      = certsPath + CAKeyFile
	CACertPath     = certsPath + CACertFile
	ServerKeyPath  = certsPath + ServerKeyFile
	ServerCertPath = certsPath + ServerCertFile
	ClientKeyPath  = certsPath + ClientKeyFile
	ClientCertPath = certsPath + ClientCertFile

	/* Protocol constants. This value denotes a volume's backing storage protocol. For example,
	a Trident volume with  'file' protocol is most likely NFS, while a 'block' protocol volume is probably iSCSI. */
	File        Protocol = "file"
	Block       Protocol = "block"
	ProtocolAny Protocol = ""

	/* Access mode constants */
	ReadWriteOnce AccessMode = "ReadWriteOnce"
	ReadOnlyMany  AccessMode = "ReadOnlyMany"
	ReadWriteMany AccessMode = "ReadWriteMany"
	ModeAny       AccessMode = ""

	/* Volume mode constants. This value describes how a volume will be consumed by application containers.
	Most Trident volumes (regardless of protocol) probably use the 'Filesystem' mode, where the volume contains
	a filesystem and is mounted into a container. By contrast, volumes with 'Block' mode always use 'block' protocol
	and are attached to a container as raw block devices. */
	RawBlock   VolumeMode = "Block"
	Filesystem VolumeMode = "Filesystem"

	/* Volume type constants */
	OntapNFS          VolumeType = "ONTAP_NFS"
	OntapISCSI        VolumeType = "ONTAP_iSCSI"
	SolidFireISCSI    VolumeType = "SolidFire_iSCSI"
	ESeriesISCSI      VolumeType = "Eseries_iSCSI"
	UnknownVolumeType VolumeType = ""

	/* Driver-related constants */
	DefaultSolidFireVAG      = OrchestratorName
	UnknownDriver            = "UnknownDriver"
	StorageAPITimeoutSeconds = 90
	SANResizeDelta           = 50000000 // 50mb

	/* REST frontend constants */
	MaxRESTRequestSize = 10240

	/* Kubernetes deployment constants */
	ContainerTrident = "trident-main"
	ContainerEtcd    = "etcd"

	ContextDocker     DriverContext = "docker"
	ContextKubernetes DriverContext = "kubernetes"
	ContextCSI        DriverContext = "csi"
	ContextCRD        DriverContext = "crd"

	PlatformDocker     Platform = "docker"
	PlatformKubernetes Platform = "kubernetes"
	PlatformCSI        Platform = "csi" // plain CSI, no other CO present

	// Minimum and maximum supported Kubernetes versions
	KubernetesVersionMin = "v1.11.0"
	KubernetesVersionMax = "v1.18.0"

	// Minimum Kubernetes version for CSI Trident (non-CSI is the default)
	KubernetesCSIVersionMinOptional = "v1.13.0"

	// Minimum Kubernetes version for CSI Trident default (non-CSI not supported)
	KubernetesCSIVersionMinForced = "v1.14.0"

	TridentNamespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
)

var (
	validProtocols = map[Protocol]bool{
		File:        true,
		Block:       true,
		ProtocolAny: true,
	}

	// BuildHash is the git hash the binary was built from
	BuildHash = "unknown"

	// BuildType is the type of build: custom, beta or stable
	BuildType = "custom"

	// BuildTypeRev is the revision of the build
	BuildTypeRev = "0"

	// BuildTime is the time the binary was built
	BuildTime = "unknown"

	// BuildImage is the Trident image that was built
	BuildImage = "netapp/trident:" + orchestratorVersion + "-custom.0"

	// BuildEtcdVersion is the etcd version that Trident should be deployed with
	BuildEtcdVersion = "v3.3.18"

	// BuildEtcdImage is the etcd image that Trident should be deployed with
	BuildEtcdImage = "quay.io/coreos/etcd:" + BuildEtcdVersion

	OrchestratorVersion = utils.MustParseDate(version())

	/* API Server and persistent store variables */
	BaseURL         = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion
	VersionURL      = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/version"
	BackendURL      = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/backend"
	BackendUUIDURL  = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/backendUUID"
	VolumeURL       = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/volume"
	TransactionURL  = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/txn"
	StorageClassURL = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/storageclass"
	NodeURL         = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/node"
	SnapshotURL     = "/" + OrchestratorName + "/v" + OrchestratorAPIVersion + "/snapshot"
	StoreURL        = "/" + OrchestratorName + "/store"

	UsingPassthroughStore bool
	CurrentDriverContext  DriverContext
	OrchestratorTelemetry = Telemetry{TridentVersion: OrchestratorVersion.String()}
)

func IsValidProtocol(p Protocol) bool {
	_, ok := validProtocols[p]
	return ok
}

func GetValidProtocolNames() []string {
	ret := make([]string, len(validProtocols))
	for key := range validProtocols {
		ret = append(ret, string(key))
	}
	return ret
}

func PlatformAtLeast(platformName string, version string) bool {
	if OrchestratorTelemetry.Platform == platformName {
		platformVersion := utils.MustParseSemantic(OrchestratorTelemetry.PlatformVersion)
		requiredVersion, err := utils.ParseSemantic(version)
		if err != nil {
			log.WithFields(log.Fields{
				"platform": platformName,
				"version":  version,
			}).Errorf("Platform version check failed. %+v", err)
			return false
		}
		if platformVersion.AtLeast(requiredVersion) {
			return true
		}
	}
	return false
}

func version() string {

	var version string

	if BuildType != "stable" {
		if BuildType == "custom" {
			version = fmt.Sprintf("%v-%v+%v", orchestratorVersion, BuildType, BuildHash)
		} else {
			version = fmt.Sprintf("%v-%v.%v+%v", orchestratorVersion, BuildType, BuildTypeRev, BuildHash)
		}
	} else {
		version = orchestratorVersion
	}

	return version
}
<|endoftext|>


<commit_before>package cloudmutex

import (
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"sync"
	"testing"
	"time"
)

const (
	project = "marc-general"
	bucket  = "cloudmutex"
	object  = "lock"
)

var (
	limit = 10

	lockHolderMu sync.Mutex
	lockHolder   = -1
)

func TestLock(t *testing.T) {
	// google cloud storage stub
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			t.Errorf("r.Method = %q; want POST", r.Method)
		}
		path := "/b/cloudmutex/o"
		if r.URL.Path != path {
			t.Errorf("r.URL.Path = %q; want %q", r.URL.Path, path)
		}
		vals := url.Values{
			"ifGenerationMatch": []string{"0"},
			"name":              []string{"lock"},
			"uploadType":        []string{"media"},
		}
		if !reflect.DeepEqual(r.URL.Query(), vals) {
			t.Errorf("query params = %q; want %q", r.URL.Query(), vals)
		}
	}))
	defer storage.Close()
	storageLockURL = storage.URL

	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{})
	go func() {
		m.Lock()
		close(done)
	}()
	select {
	case <-time.After(time.Second):
		t.Errorf("m.Lock() took too long to lock")
	case <-done:
		// pass
	}
}

func TestLockRetry(t *testing.T) {
	var (
		retryCount int
		retryLimit = 2
	)
	// google cloud storage stub
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if retryCount < retryLimit {
			w.WriteHeader(http.StatusInternalServerError)
			retryCount++
		}
	}))
	defer storage.Close()
	storageLockURL = storage.URL

	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{})
	go func() {
		m.Lock()
		close(done)
	}()
	select {
	case <-time.After(time.Second):
		t.Errorf("m.Lock() took too long to lock")
	case <-done:
		// pass
	}
	if retryCount < retryLimit {
		t.Errorf("retryCount = %d; want %d", retryCount, retryLimit)
	}
}

func TestUnlock(t *testing.T) {
	// google cloud storage stub
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "DELETE" {
			t.Errorf("r.Method = %q; want DELETE", r.Method)
		}
		path := "/b/cloudmutex/o/lock"
		if r.URL.Path != path {
			t.Errorf("r.URL.Path = %q; want %q", r.URL.Path, path)
		}
		w.WriteHeader(http.StatusNoContent)
	}))
	defer storage.Close()
	storageUnlockURL = storage.URL

	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{})
	go func() {
		m.Unlock()
		close(done)
	}()
	select {
	case <-time.After(time.Second):
		t.Errorf("m.Unlock() took too long to lock")
	case <-done:
		// pass
	}
}

func TestUnlockRetry(t *testing.T) {
	var (
		retryCount int
		retryLimit = 2
	)
	// google cloud storage stub
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if retryCount < retryLimit {
			w.WriteHeader(http.StatusInternalServerError)
			retryCount++
		} else {
			w.WriteHeader(http.StatusNoContent)
		}
	}))
	defer storage.Close()
	storageUnlockURL = storage.URL

	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{})
	go func() {
		m.Unlock()
		close(done)
	}()
	select {
	case <-time.After(time.Second):
		t.Errorf("m.Unlock() took too long to lock")
	case <-done:
		// pass
	}
	if retryCount < retryLimit {
		t.Errorf("retryCount = %d; want %d", retryCount, retryLimit)
	}
}

func locker(done chan struct{}, t *testing.T, i int, m sync.Locker) {
	m.Lock()
	lockHolderMu.Lock()
	if lockHolder != -1 {
		t.Errorf("%d trying to lock, but already held by %d",
			i, lockHolder)
	}
	lockHolder = i
	lockHolderMu.Unlock()
	t.Logf("locked by %d", i)
	time.Sleep(5 * time.Millisecond)
	lockHolderMu.Lock()
	lockHolder = -1
	lockHolderMu.Unlock()
	m.Unlock()
	done <- struct{}{}
}

func TestParallel(t *testing.T) {
	storageLockURL = defaultStorageLockURL
	storageUnlockURL = defaultStorageUnlockURL
	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{}, 1)
	total := 0
	for i := 0; i < limit; i++ {
		total++
		go locker(done, t, i, m)
	}
	for ; total > 0; total-- {
		<-done
	}
}

const timeoutSeconds = 1

// This type is used to mock the sync.Lock interface provided by cloudmutex.
type mockLocker struct {
	wait time.Duration
}

func (l *mockLocker) Lock() {
	time.Sleep(l.wait)
}
func (l *mockLocker) Unlock() {
	time.Sleep(l.wait)
}

func TestLockTimeout(t *testing.T) {
	m := &mockLocker{10 * time.Millisecond}
	if err := Lock(m, time.Millisecond); err == nil {
		t.Errorf("want lock error for Lock(m, 1ms)")
	}
	if err := Lock(m, 100*time.Millisecond); err != nil {
		t.Errorf("Lock(m, 100ms): %v", err)
	}
}

func TestUnlockTimeout(t *testing.T) {
	m := &mockLocker{10 * time.Millisecond}
	if err := Unlock(m, time.Millisecond); err == nil {
		t.Errorf("want unlock error for Unlock(m, 1ms)")
	}
	if err := Unlock(m, 100*time.Millisecond); err != nil {
		t.Errorf("Unlock(m, 100ms): %v", err)
	}
}
<commit_msg>remove unused const<commit_after>package cloudmutex

import (
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"sync"
	"testing"
	"time"
)

const (
	project = "marc-general"
	bucket  = "cloudmutex"
	object  = "lock"
)

var (
	limit = 10

	lockHolderMu sync.Mutex
	lockHolder   = -1
)

func TestLock(t *testing.T) {
	// google cloud storage stub
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			t.Errorf("r.Method = %q; want POST", r.Method)
		}
		path := "/b/cloudmutex/o"
		if r.URL.Path != path {
			t.Errorf("r.URL.Path = %q; want %q", r.URL.Path, path)
		}
		vals := url.Values{
			"ifGenerationMatch": []string{"0"},
			"name":              []string{"lock"},
			"uploadType":        []string{"media"},
		}
		if !reflect.DeepEqual(r.URL.Query(), vals) {
			t.Errorf("query params = %q; want %q", r.URL.Query(), vals)
		}
	}))
	defer storage.Close()
	storageLockURL = storage.URL

	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{})
	go func() {
		m.Lock()
		close(done)
	}()
	select {
	case <-time.After(time.Second):
		t.Errorf("m.Lock() took too long to lock")
	case <-done:
		// pass
	}
}

func TestLockRetry(t *testing.T) {
	var (
		retryCount int
		retryLimit = 2
	)
	// google cloud storage stub
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if retryCount < retryLimit {
			w.WriteHeader(http.StatusInternalServerError)
			retryCount++
		}
	}))
	defer storage.Close()
	storageLockURL = storage.URL

	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{})
	go func() {
		m.Lock()
		close(done)
	}()
	select {
	case <-time.After(time.Second):
		t.Errorf("m.Lock() took too long to lock")
	case <-done:
		// pass
	}
	if retryCount < retryLimit {
		t.Errorf("retryCount = %d; want %d", retryCount, retryLimit)
	}
}

func TestUnlock(t *testing.T) {
	// google cloud storage stub
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "DELETE" {
			t.Errorf("r.Method = %q; want DELETE", r.Method)
		}
		path := "/b/cloudmutex/o/lock"
		if r.URL.Path != path {
			t.Errorf("r.URL.Path = %q; want %q", r.URL.Path, path)
		}
		w.WriteHeader(http.StatusNoContent)
	}))
	defer storage.Close()
	storageUnlockURL = storage.URL

	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{})
	go func() {
		m.Unlock()
		close(done)
	}()
	select {
	case <-time.After(time.Second):
		t.Errorf("m.Unlock() took too long to lock")
	case <-done:
		// pass
	}
}

func TestUnlockRetry(t *testing.T) {
	var (
		retryCount int
		retryLimit = 2
	)
	// google cloud storage stub
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if retryCount < retryLimit {
			w.WriteHeader(http.StatusInternalServerError)
			retryCount++
		} else {
			w.WriteHeader(http.StatusNoContent)
		}
	}))
	defer storage.Close()
	storageUnlockURL = storage.URL

	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{})
	go func() {
		m.Unlock()
		close(done)
	}()
	select {
	case <-time.After(time.Second):
		t.Errorf("m.Unlock() took too long to lock")
	case <-done:
		// pass
	}
	if retryCount < retryLimit {
		t.Errorf("retryCount = %d; want %d", retryCount, retryLimit)
	}
}

func locker(done chan struct{}, t *testing.T, i int, m sync.Locker) {
	m.Lock()
	lockHolderMu.Lock()
	if lockHolder != -1 {
		t.Errorf("%d trying to lock, but already held by %d",
			i, lockHolder)
	}
	lockHolder = i
	lockHolderMu.Unlock()
	t.Logf("locked by %d", i)
	time.Sleep(5 * time.Millisecond)
	lockHolderMu.Lock()
	lockHolder = -1
	lockHolderMu.Unlock()
	m.Unlock()
	done <- struct{}{}
}

func TestParallel(t *testing.T) {
	storageLockURL = defaultStorageLockURL
	storageUnlockURL = defaultStorageUnlockURL
	m, err := New(nil, project, bucket, object)
	if err != nil {
		t.Fatal("unable to allocate a cloudmutex global object")
	}
	done := make(chan struct{}, 1)
	total := 0
	for i := 0; i < limit; i++ {
		total++
		go locker(done, t, i, m)
	}
	for ; total > 0; total-- {
		<-done
	}
}

// This type is used to mock the sync.Lock interface provided by cloudmutex.
type mockLocker struct {
	wait time.Duration
}

func (l *mockLocker) Lock() {
	time.Sleep(l.wait)
}
func (l *mockLocker) Unlock() {
	time.Sleep(l.wait)
}

func TestLockTimeout(t *testing.T) {
	m := &mockLocker{10 * time.Millisecond}
	if err := Lock(m, time.Millisecond); err == nil {
		t.Errorf("want lock error for Lock(m, 1ms)")
	}
	if err := Lock(m, 100*time.Millisecond); err != nil {
		t.Errorf("Lock(m, 100ms): %v", err)
	}
}

func TestUnlockTimeout(t *testing.T) {
	m := &mockLocker{10 * time.Millisecond}
	if err := Unlock(m, time.Millisecond); err == nil {
		t.Errorf("want unlock error for Unlock(m, 1ms)")
	}
	if err := Unlock(m, 100*time.Millisecond); err != nil {
		t.Errorf("Unlock(m, 100ms): %v", err)
	}
}
<|endoftext|>


<commit_before>package cluster

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"h12.me/kpax/log"
	"h12.me/kpax/model"
	"h12.me/kpax/proto"
)

var (
	ErrLeaderNotFound = errors.New("leader not found")
	ErrCoordNotFound  = errors.New("coordinator not found")
	ErrNoBrokerFound  = errors.New("no broker found")
)

type (
	C struct {
		topics *topicPartitions
		pool   *brokerPool
		mu     sync.Mutex
	}
	NewBrokerFunc func(addr string) model.Broker
)

func New(newBroker NewBrokerFunc, brokers []string) model.Cluster {
	c := &C{
		topics: newTopicPartitions(),
		pool:   newBrokerPool(newBroker),
	}
	for _, addr := range brokers {
		c.pool.AddAddr(addr)
	}
	return c
}

func (c *C) Coordinator(group string) (model.Broker, error) {
	if coord, err := c.pool.GetCoordinator(group); err == nil {
		return coord, nil
	}
	if err := c.updateCoordinator(group); err != nil {
		return nil, err
	}
	return c.pool.GetCoordinator(group)
}

func (c *C) CoordinatorIsDown(group string) {
	log.Warnf("coordinator (%s) is down", group)
	c.pool.DeleteCoordinator(group)
}

func (c *C) Leader(topic string, partition int32) (model.Broker, error) {
	if leader, err := c.pool.GetLeader(topic, partition); err == nil {
		return leader, nil
	}
	if err := c.updateFromTopicMetadata(topic); err != nil {
		return nil, err
	}
	return c.pool.GetLeader(topic, partition)
}

func (c *C) LeaderIsDown(topic string, partition int32) {
	log.Warnf("leader (%s,%d) is down", topic, partition)
	c.pool.DeleteLeader(topic, partition)
}

func (c *C) Partitions(topic string) ([]int32, error) {
	partitions := c.topics.getPartitions(topic)
	if len(partitions) > 0 {
		return partitions, nil
	}
	if err := c.updateFromTopicMetadata(topic); err != nil {
		return nil, err
	}
	partitions = c.topics.getPartitions(topic)
	if len(partitions) > 0 {
		return partitions, nil
	}
	return nil, fmt.Errorf("topic %s not found", topic)
}

func (c *C) updateCoordinator(group string) error {
	brokers, err := c.pool.Brokers()
	if err != nil {
		return err
	}
	var merr MultiError
	for _, broker := range brokers {
		coord, err := proto.GroupCoordinator(group).Fetch(broker)
		if err != nil {
			merr.Add(err)
			continue
		}
		c.pool.SetCoordinator(group, coord.NodeID, coord.Addr())
		return nil
	}
	return merr
}

func (c *C) updateFromTopicMetadata(topic string) error {
	brokers, err := c.pool.Brokers()
	if err != nil {
		return err
	}
	var merr MultiError
	for _, broker := range brokers {
		var err error
		var m *proto.TopicMetadataResponse
		for i := 0; i < 3; i++ {
			// retry for automatic topic creation
			m, err = proto.Metadata(topic).Fetch(broker)
			if err == proto.ErrLeaderNotAvailable {
				time.Sleep(time.Second)
				continue
			}
			break
		}
		if err != nil {
			merr.Add(err)
			continue
		}
		for i := range m.Brokers {
			b := &m.Brokers[i]
			c.pool.Add(b.NodeID, b.Addr())
		}
		for i := range m.TopicMetadatas {
			t := &m.TopicMetadatas[i]
			if t.TopicName == topic {
				partitions := make([]int32, len(t.PartitionMetadatas))
				for i := range t.PartitionMetadatas {
					partition := &t.PartitionMetadatas[i]
					partitions[i] = partition.PartitionID
					if err := c.pool.SetLeader(topic, partition.PartitionID, partition.Leader); err != nil {
						merr.Add(err)
						continue
					}
				}
				c.topics.addPartitions(topic, partitions)
				return nil
			}
		}
	}
	return merr
}
<commit_msg>fail fast to fetch metadata<commit_after>package cluster

import (
	"errors"
	"fmt"
	"sync"

	"h12.me/kpax/log"
	"h12.me/kpax/model"
	"h12.me/kpax/proto"
)

var (
	ErrLeaderNotFound = errors.New("leader not found")
	ErrCoordNotFound  = errors.New("coordinator not found")
	ErrNoBrokerFound  = errors.New("no broker found")
)

type (
	C struct {
		topics *topicPartitions
		pool   *brokerPool
		mu     sync.Mutex
	}
	NewBrokerFunc func(addr string) model.Broker
)

func New(newBroker NewBrokerFunc, brokers []string) model.Cluster {
	c := &C{
		topics: newTopicPartitions(),
		pool:   newBrokerPool(newBroker),
	}
	for _, addr := range brokers {
		c.pool.AddAddr(addr)
	}
	return c
}

func (c *C) Coordinator(group string) (model.Broker, error) {
	if coord, err := c.pool.GetCoordinator(group); err == nil {
		return coord, nil
	}
	if err := c.updateCoordinator(group); err != nil {
		return nil, err
	}
	return c.pool.GetCoordinator(group)
}

func (c *C) CoordinatorIsDown(group string) {
	log.Warnf("coordinator (%s) is down", group)
	c.pool.DeleteCoordinator(group)
}

func (c *C) Leader(topic string, partition int32) (model.Broker, error) {
	if leader, err := c.pool.GetLeader(topic, partition); err == nil {
		return leader, nil
	}
	if err := c.updateFromTopicMetadata(topic); err != nil {
		return nil, err
	}
	return c.pool.GetLeader(topic, partition)
}

func (c *C) LeaderIsDown(topic string, partition int32) {
	log.Warnf("leader (%s,%d) is down", topic, partition)
	c.pool.DeleteLeader(topic, partition)
}

func (c *C) Partitions(topic string) ([]int32, error) {
	partitions := c.topics.getPartitions(topic)
	if len(partitions) > 0 {
		return partitions, nil
	}
	if err := c.updateFromTopicMetadata(topic); err != nil {
		return nil, err
	}
	partitions = c.topics.getPartitions(topic)
	if len(partitions) > 0 {
		return partitions, nil
	}
	return nil, fmt.Errorf("topic %s not found", topic)
}

func (c *C) updateCoordinator(group string) error {
	brokers, err := c.pool.Brokers()
	if err != nil {
		return err
	}
	var merr MultiError
	for _, broker := range brokers {
		coord, err := proto.GroupCoordinator(group).Fetch(broker)
		if err != nil {
			merr.Add(err)
			continue
		}
		c.pool.SetCoordinator(group, coord.NodeID, coord.Addr())
		return nil
	}
	return merr
}

func (c *C) updateFromTopicMetadata(topic string) error {
	brokers, err := c.pool.Brokers()
	if err != nil {
		return err
	}
	var merr MultiError
	for _, broker := range brokers {
		// no retry, fail fast
		m, err := proto.Metadata(topic).Fetch(broker)
		if err != nil {
			merr.Add(err)
			continue
		}
		for i := range m.Brokers {
			b := &m.Brokers[i]
			c.pool.Add(b.NodeID, b.Addr())
		}
		for i := range m.TopicMetadatas {
			t := &m.TopicMetadatas[i]
			if t.TopicName == topic {
				partitions := make([]int32, len(t.PartitionMetadatas))
				for i := range t.PartitionMetadatas {
					partition := &t.PartitionMetadatas[i]
					partitions[i] = partition.PartitionID
					if err := c.pool.SetLeader(topic, partition.PartitionID, partition.Leader); err != nil {
						merr.Add(err)
						continue
					}
				}
				c.topics.addPartitions(topic, partitions)
				return nil
			}
		}
	}
	return merr
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager_test

import (
	"errors"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
)

type mockObjectConvertor struct{}

func (c *mockObjectConvertor) Convert(in, out, context interface{}) error {
	out = in
	return nil
}

func (c *mockObjectConvertor) ConvertToVersion(in runtime.Object, _ runtime.GroupVersioner) (runtime.Object, error) {
	return in, nil
}

func (c *mockObjectConvertor) ConvertFieldLabel(_ schema.GroupVersionKind, _, _ string) (string, string, error) {
	return "", "", errors.New("not implemented")
}

type mockObjectDefaulter struct{}

func (d *mockObjectDefaulter) Default(in runtime.Object) {}

func NewTestFieldManager(t *testing.T) *fieldmanager.FieldManager {
	gv := schema.GroupVersion{
		Group:   "apps",
		Version: "v1",
	}

	return fieldmanager.NewCRDFieldManager(
		&mockObjectConvertor{},
		&mockObjectDefaulter{},
		gv,
		gv,
	)
}

func TestFieldManagerCreation(t *testing.T) {
	_ = NewTestFieldManager(t)
}

func TestApplyStripsFields(t *testing.T) {
	f := NewTestFieldManager(t)

	obj := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:              "a",
			Namespace:         "a",
			CreationTimestamp: metav1.Time{Time: time.Now().UTC()},
			SelfLink:          "a",
		},
	}

	newObj, err := f.Apply(obj, []byte(`{
		"apiVersion": "v1",
		"kind": "Deployment",
		"metadata": {
			"name": "b",
			"namespace": "b",
			"creationTimestamp": "`+time.Now().UTC().Format(time.RFC3339)+`",
			"selfLink": "b",
			"uid": "b",
			"resourceVersion": "b"
		}
	}`), false)
	if err != nil {
		t.Fatalf("failed to apply object: %v", err)
	}

	accessor, err := meta.Accessor(newObj)
	if err != nil {
		t.Fatalf("couldn't get accessor: %v", err)
	}

	if m := accessor.GetManagedFields(); len(m) != 0 {
		t.Fatalf("fields did not get stripped on apply: %v", m)
	}
}
func TestApplyDoesNotStripLabels(t *testing.T) {
	f := NewTestFieldManager(t)

	obj := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:              "a",
			Namespace:         "a",
			CreationTimestamp: metav1.Time{Time: time.Now().UTC()},
			SelfLink:          "a",
		},
	}

	newObj, err := f.Apply(obj, []byte(`{
		"apiVersion": "v1",
		"kind": "Pod",
		"metadata": {
			"labels": {
				"a": "b"
			},
		}
	}`), false)
	if err != nil {
		t.Fatalf("failed to apply object: %v", err)
	}

	accessor, err := meta.Accessor(newObj)
	if err != nil {
		t.Fatalf("couldn't get accessor: %v", err)
	}

	if m := accessor.GetManagedFields(); len(m) != 1 {
		t.Fatalf("labels shouldn't get stripped on apply: %v", m)
	}
}

<commit_msg>rename mocks to fakes and add nil check<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager_test

import (
	"errors"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
)

type fakeObjectConvertor struct{}

func (c *fakeObjectConvertor) Convert(in, out, context interface{}) error {
	out = in
	return nil
}

func (c *fakeObjectConvertor) ConvertToVersion(in runtime.Object, _ runtime.GroupVersioner) (runtime.Object, error) {
	return in, nil
}

func (c *fakeObjectConvertor) ConvertFieldLabel(_ schema.GroupVersionKind, _, _ string) (string, string, error) {
	return "", "", errors.New("not implemented")
}

type fakeObjectDefaulter struct{}

func (d *fakeObjectDefaulter) Default(in runtime.Object) {}

func NewTestFieldManager(t *testing.T) *fieldmanager.FieldManager {
	gv := schema.GroupVersion{
		Group:   "apps",
		Version: "v1",
	}

	return fieldmanager.NewCRDFieldManager(
		&fakeObjectConvertor{},
		&fakeObjectDefaulter{},
		gv,
		gv,
	)
}

func TestFieldManagerCreation(t *testing.T) {
	if NewTestFieldManager(t) == nil {
		t.Fatal("failed to create FieldManager")
	}
}

func TestApplyStripsFields(t *testing.T) {
	f := NewTestFieldManager(t)
	ti, _ := time.Parse(time.RFC3339, "2016-05-19T09:59:00Z")

	obj := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:              "a",
			Namespace:         "a",
			CreationTimestamp: metav1.Time{Time: ti},
			SelfLink:          "a",
		},
	}

	newObj, err := f.Apply(obj, []byte(`{
		"apiVersion": "v1",
		"kind": "Deployment",
		"metadata": {
			"name": "b",
			"namespace": "b",
			"creationTimestamp": "`+ti.Add(1*time.Minute).Format(time.RFC3339)+`",
			"selfLink": "b",
			"uid": "b",
			"resourceVersion": "b"
		}
	}`), false)
	if err != nil {
		t.Fatalf("failed to apply object: %v", err)
	}

	accessor, err := meta.Accessor(newObj)
	if err != nil {
		t.Fatalf("couldn't get accessor: %v", err)
	}

	if m := accessor.GetManagedFields(); len(m) != 0 {
		t.Fatalf("fields did not get stripped on apply: %v", m)
	}
}
func TestApplyDoesNotStripLabels(t *testing.T) {
	f := NewTestFieldManager(t)

	obj := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:              "a",
			Namespace:         "a",
			CreationTimestamp: metav1.Time{Time: time.Now().UTC()},
			SelfLink:          "a",
		},
	}

	newObj, err := f.Apply(obj, []byte(`{
		"apiVersion": "v1",
		"kind": "Pod",
		"metadata": {
			"labels": {
				"a": "b"
			},
		}
	}`), false)
	if err != nil {
		t.Fatalf("failed to apply object: %v", err)
	}

	accessor, err := meta.Accessor(newObj)
	if err != nil {
		t.Fatalf("couldn't get accessor: %v", err)
	}

	if m := accessor.GetManagedFields(); len(m) != 1 {
		t.Fatalf("labels shouldn't get stripped on apply: %v", m)
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/retry"
)

type AgentWorker struct {
	// The API Client used when this agent is communicating with the API
	APIClient *api.Client

	// The endpoint that should be used when communicating with the API
	Endpoint string

	// The registred agent API record
	Agent *api.Agent

	// The configuration of the agent from the CLI
	AgentConfiguration *AgentConfiguration

	// Used by the Start call to control the looping of the pings
	ticker *time.Ticker
	stop   chan struct{}

	// When this worker runs a job, we'll store an instance of the
	// JobRunner here
	jobRunner *JobRunner
}

// Creates the agent worker and initializes it's API Client
func (a AgentWorker) Create() AgentWorker {
	var endpoint string
	if a.Agent.Endpoint != "" {
		endpoint = a.Agent.Endpoint
	} else {
		endpoint = a.Endpoint
	}

	a.APIClient = APIClient{Endpoint: endpoint, Token: a.Agent.AccessToken}.Create()

	return a
}

// Starts the agent worker
func (a *AgentWorker) Start() error {
	// Create the ticker and stop channels
	a.ticker = time.NewTicker(5 * time.Second)
	a.stop = make(chan struct{})

	// Continue this loop until the the ticker is stopped, and we received
	// a message on the stop channel.
	for {
		a.Ping()

		select {
		case <-a.ticker.C:
			continue
		case <-a.stop:
			a.ticker.Stop()
			return nil
		}
	}

	return nil
}

// Stops the agent from accepting new work and cancels any current work it's
// running
func (a *AgentWorker) Stop() {
	logger.Debug("Stopping the agent...")

	// If ther'es a running job, kill it.
	if a.jobRunner != nil {
		a.jobRunner.Kill()
	}

	// If we have a ticker, stop it, and send a signal to the stop channel,
	// which will cause the agent worker to stop looping immediatly.
	if a.ticker != nil {
		close(a.stop)
	}
}

// Connects the agent to the Buildkite Agent API, retrying up to 30 times if it
// fails.
func (a *AgentWorker) Connect() error {
	return retry.Do(func(s *retry.Stats) error {
		_, err := a.APIClient.Agents.Connect()
		if err != nil {
			logger.Warn("%s (%s)", err, s)
		}

		return err
	}, &retry.Config{Maximum: 10, Interval: 1 * time.Second})
}

// Performs a ping, which returns what action the agent should take next.
func (a *AgentWorker) Ping() {
	ping, _, err := a.APIClient.Pings.Get()
	if err != nil {
		// If a ping fails, we don't really care, because it'll
		// ping again after the interval.
		logger.Warn("Failed to ping: %s", err)
		return
	}

	// Should we switch endpoints?
	if ping.Endpoint != "" && ping.Endpoint != a.Agent.Endpoint {
		// Before switching to the new one, do a ping test to make sure it's
		// valid. If it is, switch and carry on, otherwise ignore the switch
		// for now.
		newAPIClient := APIClient{Endpoint: ping.Endpoint, Token: a.Agent.AccessToken}.Create()
		newPing, _, err := newAPIClient.Pings.Get()
		if err != nil {
			logger.Warn("Failed to ping the new endpoint %s - ignoring switch for now (%s)", ping.Endpoint, err)
		} else {
			// Replace the APIClient and process the new ping
			a.APIClient = newAPIClient
			a.Agent.Endpoint = ping.Endpoint
			ping = newPing
		}
	}

	// Is there a message that should be shown in the logs?
	if ping.Message != "" {
		logger.Info(ping.Message)
	}

	// Should the agent disconnect?
	if ping.Action == "disconnect" {
		a.Stop()
		return
	}

	// If we don't have a job, there's nothing to do!
	if ping.Job == nil {
		return
	}

	logger.Info("Assigned job %s. Accepting...", ping.Job.ID)

	// Accept the job. We'll retry on connection related issues, but if
	// Buildkite returns a 422 or 500 for example, we'll just bail out,
	// re-ping, and try the whole process again.
	var accepted *api.Job
	retry.Do(func(s *retry.Stats) error {
		accepted, _, err = a.APIClient.Jobs.Accept(ping.Job)

		if err != nil {
			if api.IsRetryableError(err) {
				logger.Warn("%s (%s)", err, s)
			} else {
				logger.Warn("Buildkite rejected the call to accept the job (%s)", err)
				s.Break()
			}
		}

		return err
	}, &retry.Config{Maximum: 30, Interval: 1 * time.Second})

	// Now that the job has been accepted, we can start it.
	a.jobRunner, err = JobRunner{
		Endpoint:           accepted.Endpoint,
		Agent:              a.Agent,
		AgentConfiguration: a.AgentConfiguration,
		Job:                accepted,
	}.Create()

	// Was there an error creating the job runner?
	if err != nil {
		logger.Error("Failed to initialize job: %s", err)
		return
	}

	// Start running the job
	if err = a.jobRunner.Run(); err != nil {
		logger.Error("Failed to run job: %s", err)
	}

	// No more job, no more runner.
	a.jobRunner = nil
}

// Disconnects the agent from the Buildkite Agent API, doesn't bother retrying
// because we want to disconnect as fast as possible.
func (a *AgentWorker) Disconnect() error {
	_, err := a.APIClient.Agents.Disconnect()
	if err != nil {
		logger.Warn("There was an error sending the disconnect API call to Buildkite. If this agent still appears online, you may have to manually stop it (%s)", err)
	}

	return err
}
<commit_msg>Fix a bug where stopping the agent at the same time multiple times causes a panic.<commit_after>package agent

import (
	"sync"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/retry"
)

type AgentWorker struct {
	// The API Client used when this agent is communicating with the API
	APIClient *api.Client

	// The endpoint that should be used when communicating with the API
	Endpoint string

	// The registred agent API record
	Agent *api.Agent

	// The configuration of the agent from the CLI
	AgentConfiguration *AgentConfiguration

	// Used by the Start call to control the looping of the pings
	ticker *time.Ticker

	// Stop controls
	stop      chan struct{}
	stopping  bool
	stopMutex sync.Mutex

	// When this worker runs a job, we'll store an instance of the
	// JobRunner here
	jobRunner *JobRunner
}

// Creates the agent worker and initializes it's API Client
func (a AgentWorker) Create() AgentWorker {
	var endpoint string
	if a.Agent.Endpoint != "" {
		endpoint = a.Agent.Endpoint
	} else {
		endpoint = a.Endpoint
	}

	a.APIClient = APIClient{Endpoint: endpoint, Token: a.Agent.AccessToken}.Create()

	return a
}

// Starts the agent worker
func (a *AgentWorker) Start() error {
	// Create the ticker and stop channels
	a.ticker = time.NewTicker(5 * time.Second)
	a.stop = make(chan struct{})

	// Continue this loop until the the ticker is stopped, and we received
	// a message on the stop channel.
	for {
		a.Ping()

		select {
		case <-a.ticker.C:
			continue
		case <-a.stop:
			a.ticker.Stop()
			return nil
		}
	}

	return nil
}

// Stops the agent from accepting new work and cancels any current work it's
// running
func (a *AgentWorker) Stop() {
	// Only allow one stop to run at a time (because we're playing with channels)
	a.stopMutex.Lock()

	if a.stopping {
		logger.Debug("Agent is already stopping...")
		return
	} else {
		logger.Debug("Stopping the agent...")
	}

	// If ther'es a running job, kill it.
	if a.jobRunner != nil {
		a.jobRunner.Kill()
	}

	// If we have a ticker, stop it, and send a signal to the stop channel,
	// which will cause the agent worker to stop looping immediatly.
	if a.ticker != nil {
		close(a.stop)
	}

	// Mark the agent as stopping
	a.stopping = true

	// Unlock the stop mutex
	a.stopMutex.Unlock()
}

// Connects the agent to the Buildkite Agent API, retrying up to 30 times if it
// fails.
func (a *AgentWorker) Connect() error {
	return retry.Do(func(s *retry.Stats) error {
		_, err := a.APIClient.Agents.Connect()
		if err != nil {
			logger.Warn("%s (%s)", err, s)
		}

		return err
	}, &retry.Config{Maximum: 10, Interval: 1 * time.Second})
}

// Performs a ping, which returns what action the agent should take next.
func (a *AgentWorker) Ping() {
	ping, _, err := a.APIClient.Pings.Get()
	if err != nil {
		// If a ping fails, we don't really care, because it'll
		// ping again after the interval.
		logger.Warn("Failed to ping: %s", err)
		return
	}

	// Should we switch endpoints?
	if ping.Endpoint != "" && ping.Endpoint != a.Agent.Endpoint {
		// Before switching to the new one, do a ping test to make sure it's
		// valid. If it is, switch and carry on, otherwise ignore the switch
		// for now.
		newAPIClient := APIClient{Endpoint: ping.Endpoint, Token: a.Agent.AccessToken}.Create()
		newPing, _, err := newAPIClient.Pings.Get()
		if err != nil {
			logger.Warn("Failed to ping the new endpoint %s - ignoring switch for now (%s)", ping.Endpoint, err)
		} else {
			// Replace the APIClient and process the new ping
			a.APIClient = newAPIClient
			a.Agent.Endpoint = ping.Endpoint
			ping = newPing
		}
	}

	// Is there a message that should be shown in the logs?
	if ping.Message != "" {
		logger.Info(ping.Message)
	}

	// Should the agent disconnect?
	if ping.Action == "disconnect" {
		a.Stop()
		return
	}

	// If we don't have a job, there's nothing to do!
	if ping.Job == nil {
		return
	}

	logger.Info("Assigned job %s. Accepting...", ping.Job.ID)

	// Accept the job. We'll retry on connection related issues, but if
	// Buildkite returns a 422 or 500 for example, we'll just bail out,
	// re-ping, and try the whole process again.
	var accepted *api.Job
	retry.Do(func(s *retry.Stats) error {
		accepted, _, err = a.APIClient.Jobs.Accept(ping.Job)

		if err != nil {
			if api.IsRetryableError(err) {
				logger.Warn("%s (%s)", err, s)
			} else {
				logger.Warn("Buildkite rejected the call to accept the job (%s)", err)
				s.Break()
			}
		}

		return err
	}, &retry.Config{Maximum: 30, Interval: 1 * time.Second})

	// Now that the job has been accepted, we can start it.
	a.jobRunner, err = JobRunner{
		Endpoint:           accepted.Endpoint,
		Agent:              a.Agent,
		AgentConfiguration: a.AgentConfiguration,
		Job:                accepted,
	}.Create()

	// Was there an error creating the job runner?
	if err != nil {
		logger.Error("Failed to initialize job: %s", err)
		return
	}

	// Start running the job
	if err = a.jobRunner.Run(); err != nil {
		logger.Error("Failed to run job: %s", err)
	}

	// No more job, no more runner.
	a.jobRunner = nil
}

// Disconnects the agent from the Buildkite Agent API, doesn't bother retrying
// because we want to disconnect as fast as possible.
func (a *AgentWorker) Disconnect() error {
	_, err := a.APIClient.Agents.Disconnect()
	if err != nil {
		logger.Warn("There was an error sending the disconnect API call to Buildkite. If this agent still appears online, you may have to manually stop it (%s)", err)
	}

	return err
}
<|endoftext|>


<commit_before>package tasks

import (
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/headmade/backuper/backuper"
	"github.com/headmade/backuper/config"
	"github.com/nightlyone/lockfile"
)

type Runner struct {
	agentConfig  *backuper.AgentConfig
	secretConfig *config.Providers
	timestamp    string
	lockfile     lockfile.Lockfile
}

func NewRunner(config *backuper.AgentConfig, secretConfig *config.Providers) *Runner {
	return &Runner{
		agentConfig:  config,
		secretConfig: secretConfig,
		timestamp:    time.Now().Format("20060102_1504"),
	}
}

func (runner *Runner) tmpDirPath() string {
	return runner.agentConfig.TmpDir + "/backuper/"
}

func (runner *Runner) tmpFilePath(tmpFileName string) string {
	return filepath.Join(runner.tmpDirPath(), tmpFileName)
}

func (runner *Runner) prepareTmpDir() error {
	log.Println("prepareTmpDir():", runner.tmpDirPath())
	return os.MkdirAll(runner.tmpDirPath(), 0700)
}

func (runner *Runner) pidFilePath() string {
	return runner.tmpFilePath("backuper.pid")
}

func (runner *Runner) lockPidFile(pidFilePath string) (err error) {
	log.Println("lockPidFile():", pidFilePath)
	runner.lockfile, err = lockfile.New(pidFilePath)
	if err == nil {
		err = runner.lockfile.TryLock()
	}
	return
}

func (runner *Runner) unlockPidFile() error {
	return runner.lockfile.Unlock()
}

func (runner *Runner) CleanupTmpDir() error {
	log.Println("cleanupTmpLocal():", runner.tmpDirPath())
	return nil
	return os.RemoveAll(runner.tmpDirPath())
}

func (runner *Runner) backupFileName() string {
	return runner.appendTimestamp("backup")
}

func (runner *Runner) backupFilePath() string {
	return runner.tmpFilePath(runner.backupFileName())
}

func (runner *Runner) appendTimestamp(str string) string {
	return strings.Join([]string{
		str,
		runner.timestamp,
	}, "_")
}

func (runner *Runner) formatDstPath(path string) string {
	hostname, _ := os.Hostname()
	return ReplaceVars(
		path,
		map[string]string{
			"$server_name": hostname,
			"$timestamp":   runner.timestamp,
		},
	)
}

func (runner *Runner) encryptTmpFiles(backupFilePath string, tmpFiles []string) (output []byte, err error) {

	if len(tmpFiles) == 0 {
		return []byte{}, errors.New("No files to encrypt")
	}

	cmd := fmt.Sprintf(
		"tar -cf - -C %s %s | %s > %s",
		runner.tmpDirPath(),
		strings.Join(tmpFiles, " "),
		EncryptCmd("PASS"),
		backupFilePath,
	)
	log.Println(cmd)

	return System(cmd)
}

func (runner *Runner) uploadBackupFile(backupFilePath, bucket, dstPath string) (output []byte, err error) {

	cmd := fmt.Sprintf(
		"gof3r put -p %s -b %s -k %s",
		backupFilePath,
		bucket,
		runner.formatDstPath(dstPath),
	)
	log.Println(cmd)

	return System(cmd)
}

func (runner *Runner) runTasks(configs *[]backuper.TaskConfig) (results []backuper.BackupTaskResult) {

	results = make([]backuper.BackupTaskResult, 0, len(*configs))

	for _, config := range *configs {
		task, err := GetTask(&config)
		if err == nil {
			taskTmpFileName := task.TmpFileName()
			tmpFilePath := runner.tmpFilePath(taskTmpFileName)

			out, err := task.GenerateTmpFile(tmpFilePath)

			results = append(results, backuper.BackupTaskResult{
				PathResult: backuper.NewPathResult(err, taskTmpFileName, string(out)),
			})
		} else {
			log.Printf("task type: %s, no registered handler found", config.Type)
		}
	}
	return
}

func (runner *Runner) Run() (err error, backupResult *backuper.BackupResult) {
	configs := &runner.agentConfig.Tasks

	backupResult = &backuper.BackupResult{}

	err = runner.prepareTmpDir()

	tmpDirPath := runner.tmpDirPath()
	backupResult.Prepare = backuper.NewPathResult(err, tmpDirPath, "")

	if err != nil {
		log.Println("ERR: prepare:", err.Error())
		return
	}

	pidFilePath := runner.pidFilePath()
	err = runner.lockPidFile(pidFilePath)
	backupResult.Lock = backuper.NewPathResult(err, pidFilePath, "")

	if err != nil {
		log.Println("ERR: lock:", err.Error())
		return
	}

	backupResult.Backup = runner.runTasks(configs)

	tmpFiles := make([]string, 0, len(*configs))
	for _, result := range backupResult.Backup {
		if len(result.Err) == 0 {
			tmpFiles = append(tmpFiles, result.Path)
		}
	}

	backupFilePath := runner.backupFilePath()

	output, err := runner.encryptTmpFiles(backupFilePath, tmpFiles)
	backupResult.Encrypt = backuper.NewPathResult(err, backupFilePath, string(output))

	if err != nil {
		log.Println("ERR: encrypt:", err.Error())
		return
	}

	output, err = runner.uploadBackupFile(backupFilePath, "headmade", "backup/$hostname/$timestamp")
	backupResult.Upload = backuper.NewPathResult(err, backupFilePath, string(output))

	if err != nil {
		log.Println("ERR: upload:", err.Error())
		return
	}

	// TODO: remember to uncomment the unlock()!
	//err = runner.unlockPidFile()
	backupResult.Unlock = backuper.NewPathResult(err, pidFilePath, "")

	err = runner.CleanupTmpDir()
	backupResult.Cleanup = backuper.NewPathResult(err, tmpDirPath, "")

	return
}

// TODO: move to some utils
func System(cmd string) ([]byte, error) {
	return exec.Command("sh", "-c", cmd).CombinedOutput()
}

func EncryptCmd(pass string) string {
	return fmt.Sprintf(
		"openssl aes-128-cbc -pass pass:%s",
		pass,
	)
}
func ReplaceVars(str string, replacements map[string]string) string {
	for from, to := range replacements {
		str = strings.Replace(str, from, to, -1)
	}
	return str
}
<commit_msg>Add AWS credentials<commit_after>package tasks

import (
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/headmade/backuper/backuper"
	"github.com/headmade/backuper/config"
	"github.com/nightlyone/lockfile"
)

type Runner struct {
	agentConfig  *backuper.AgentConfig
	secretConfig *config.Providers
	timestamp    string
	lockfile     lockfile.Lockfile
}

func NewRunner(config *backuper.AgentConfig, secretConfig *config.Providers) *Runner {
	return &Runner{
		agentConfig:  config,
		secretConfig: secretConfig,
		timestamp:    time.Now().Format("20060102_1504"),
	}
}

func (runner *Runner) tmpDirPath() string {
	return runner.agentConfig.TmpDir + "/backuper/"
}

func (runner *Runner) tmpFilePath(tmpFileName string) string {
	return filepath.Join(runner.tmpDirPath(), tmpFileName)
}

func (runner *Runner) prepareTmpDir() error {
	log.Println("prepareTmpDir():", runner.tmpDirPath())
	return os.MkdirAll(runner.tmpDirPath(), 0700)
}

func (runner *Runner) pidFilePath() string {
	return runner.tmpFilePath("backuper.pid")
}

func (runner *Runner) lockPidFile(pidFilePath string) (err error) {
	log.Println("lockPidFile():", pidFilePath)
	runner.lockfile, err = lockfile.New(pidFilePath)
	if err == nil {
		err = runner.lockfile.TryLock()
	}
	return
}

func (runner *Runner) unlockPidFile() error {
	return runner.lockfile.Unlock()
}

func (runner *Runner) CleanupTmpDir() error {
	log.Println("cleanupTmpLocal():", runner.tmpDirPath())
	return nil
	return os.RemoveAll(runner.tmpDirPath())
}

func (runner *Runner) backupFileName() string {
	return runner.appendTimestamp("backup")
}

func (runner *Runner) backupFilePath() string {
	return runner.tmpFilePath(runner.backupFileName())
}

func (runner *Runner) appendTimestamp(str string) string {
	return strings.Join([]string{
		str,
		runner.timestamp,
	}, "_")
}

func (runner *Runner) formatDstPath(path string) string {
	hostname, _ := os.Hostname()
	return ReplaceVars(
		path,
		map[string]string{
			"$hostname": hostname,
			"$timestamp": runner.timestamp,
		},
	)
}

func (runner *Runner) encryptTmpFiles(backupFilePath string, tmpFiles []string) (output []byte, err error) {

	if len(tmpFiles) == 0 {
		return []byte{}, errors.New("No files to encrypt")
	}

	cmd := fmt.Sprintf(
		"tar -cf - -C %s %s | %s > %s",
		runner.tmpDirPath(),
		strings.Join(tmpFiles, " "),
		EncryptCmd("PASS"),
		backupFilePath,
	)
	log.Println(cmd)

	return System(cmd)
}

func (runner *Runner) uploadBackupFile(backupFilePath, bucket, dstPath string) (output []byte, err error) {

	awsProvider := (*runner.secretConfig)["AWS"]
	envPath := os.Getenv("PATH")
	envGopath := os.Getenv("GOPATH")

	cmd := fmt.Sprintf(
		"AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s PATH=%s:%s gof3r put -p %s -b %s -k %s",
		awsProvider["AWS_ACCESS_KEY_ID"],
		awsProvider["AWS_SECRET_ACCESS_KEY"],
		envPath,
		envGopath,
		backupFilePath,
		bucket,
		runner.formatDstPath(dstPath),
	)
	log.Println(cmd)

	return System(cmd)
}

func (runner *Runner) runTasks(configs *[]backuper.TaskConfig) (results []backuper.BackupTaskResult) {

	results = make([]backuper.BackupTaskResult, 0, len(*configs))

	for _, config := range *configs {
		task, err := GetTask(&config)
		if err == nil {
			taskTmpFileName := task.TmpFileName()
			tmpFilePath := runner.tmpFilePath(taskTmpFileName)

			out, err := task.GenerateTmpFile(tmpFilePath)

			results = append(results, backuper.BackupTaskResult{
				PathResult: backuper.NewPathResult(err, taskTmpFileName, string(out)),
			})
		} else {
			log.Printf("task type: %s, no registered handler found", config.Type)
		}
	}
	return
}

func (runner *Runner) Run() (err error, backupResult *backuper.BackupResult) {
	configs := &runner.agentConfig.Tasks

	backupResult = &backuper.BackupResult{}

	err = runner.prepareTmpDir()

	tmpDirPath := runner.tmpDirPath()
	backupResult.Prepare = backuper.NewPathResult(err, tmpDirPath, "")

	if err != nil {
		log.Println("ERR: prepare:", err.Error())
		return
	}

	pidFilePath := runner.pidFilePath()
	err = runner.lockPidFile(pidFilePath)
	backupResult.Lock = backuper.NewPathResult(err, pidFilePath, "")

	if err != nil {
		log.Println("ERR: lock:", err.Error())
		return
	}

	backupResult.Backup = runner.runTasks(configs)

	tmpFiles := make([]string, 0, len(*configs))
	for _, result := range backupResult.Backup {
		if len(result.Err) == 0 {
			tmpFiles = append(tmpFiles, result.Path)
		}
	}

	backupFilePath := runner.backupFilePath()

	output, err := runner.encryptTmpFiles(backupFilePath, tmpFiles)
	backupResult.Encrypt = backuper.NewPathResult(err, backupFilePath, string(output))

	if err != nil {
		log.Println("ERR: encrypt:", err.Error())
		return
	}

	output, err = runner.uploadBackupFile(backupFilePath, "headmade", "backup/$hostname/$timestamp")
	backupResult.Upload = backuper.NewPathResult(err, backupFilePath, string(output))

	if err != nil {
		log.Println("ERR: upload:", err.Error())
		return
	}

	// TODO: remember to uncomment the unlock()!
	//err = runner.unlockPidFile()
	backupResult.Unlock = backuper.NewPathResult(err, pidFilePath, "")

	err = runner.CleanupTmpDir()
	backupResult.Cleanup = backuper.NewPathResult(err, tmpDirPath, "")

	return
}

// TODO: move to some utils
func System(cmd string) ([]byte, error) {
	return exec.Command("sh", "-c", cmd).CombinedOutput()
}

func EncryptCmd(pass string) string {
	return fmt.Sprintf(
		"openssl aes-128-cbc -pass pass:%s",
		pass,
	)
}
func ReplaceVars(str string, replacements map[string]string) string {
	for from, to := range replacements {
		str = strings.Replace(str, from, to, -1)
	}
	return str
}
<|endoftext|>


<commit_before>// Package multitick broadcasts a time.Ticker to multiple receivers,
// all of which receive the same values, aligned to an offset.
package multitick

import (
	"sync"
	"time"
)

// Ticker is a broadcaster for time.Time tick events.
type Ticker struct {
	mux   sync.Mutex
	chans []chan time.Time
	stop  chan bool
}

// NewTicker creates and starts a new Ticker, whose ticks are sent to
// subscribers at the specified interval, delayed until the specified offset
// after the interval (there will be some imprecision). The first parameter is
// equivalent to "d" in time.NewTicker(d). If the offset is negative, ticking
// is not aligned to an offset, and begins immediately.
func NewTicker(interval, offset time.Duration) *Ticker {
	t := &Ticker{stop: make(chan bool)}
	if offset >= 0 { // Sleep till the specified offset past the interval.
		var (
			now   = time.Now().UnixNano()
			intNs = interval.Nanoseconds()
			offNs = offset.Nanoseconds()
			sleep time.Duration
		)
		elapsed := now % intNs // since the last interval boundary
		if elapsed <= offNs {  // we haven't passed the offset, sleep till it
			sleep = time.Duration(offNs - elapsed)
		} else { // we passed the offset, sleep till next interval+offset
			sleep = time.Duration((intNs - elapsed) + offNs)
		}
		time.Sleep(sleep)
	}
	go t.tick(interval)
	return t
}

// Subscribe returns a channel to which ticks will be delivered. Ticks that
// can't be delivered to the channel, because it is not ready to receive, are
// discarded.
func (t *Ticker) Subscribe() <-chan time.Time {
	t.mux.Lock()
	defer t.mux.Unlock()
	c := make(chan time.Time)
	t.chans = append(t.chans, c)
	return c
}

// Stop stops the ticker. As in time.Ticker, it does not close channels.
func (t *Ticker) Stop() {
	t.stop <- true
}

// This could be inlined as an anonymous function, but I think it's easier to
// read stacktraces with real function names in them.
func (t *Ticker) tick(interval time.Duration) {
	ticker := time.NewTicker(interval)
	for {
		select {
		case <-t.stop:
			ticker.Stop()
			return
		case tick := <-ticker.C:
			t.mux.Lock()
			for i := range t.chans {
				select {
				case t.chans[i] <- tick:
				default:
				}
			}
			t.mux.Unlock()
		}
	}
}
<commit_msg>Clean up the interface slightly<commit_after>// Package multitick broadcasts a time.Ticker to multiple receivers,
// all of which receive the same values, aligned to an offset.
package multitick

import (
	"sync"
	"time"
)

// Ticker is a broadcaster for time.Time tick events.
type Ticker struct {
	mux    sync.Mutex
	chans  []chan time.Time
	ticker *time.Ticker
}

// NewTicker creates and starts a new Ticker, whose ticks are sent to
// subscribers at the specified interval, delayed until the specified offset
// after the interval (there will be some imprecision). The first parameter is
// equivalent to "d" in time.NewTicker(d). If the offset is negative, ticking
// is not aligned to an offset, and begins immediately.
func NewTicker(interval, offset time.Duration) *Ticker {
	t := &Ticker{}
	if offset >= 0 { // Sleep till the specified offset past the interval.
		var (
			now   = time.Now().UnixNano()
			intNs = interval.Nanoseconds()
			offNs = offset.Nanoseconds()
			sleep time.Duration
		)
		elapsed := now % intNs // since the last interval boundary
		if elapsed <= offNs {  // we haven't passed the offset, sleep till it
			sleep = time.Duration(offNs - elapsed)
		} else { // we passed the offset, sleep till next interval+offset
			sleep = time.Duration((intNs - elapsed) + offNs)
		}
		time.Sleep(sleep)
	}
	t.ticker = time.NewTicker(interval)
	go t.tick()
	return t
}

// Subscribe returns a channel to which ticks will be delivered. Ticks that
// can't be delivered to the channel, because it is not ready to receive, are
// discarded.
func (t *Ticker) Subscribe() <-chan time.Time {
	t.mux.Lock()
	defer t.mux.Unlock()
	c := make(chan time.Time)
	t.chans = append(t.chans, c)
	return c
}

// Stop stops the ticker. As in time.Ticker, it does not close channels.
func (t *Ticker) Stop() {
	t.ticker.Stop()
}

// This could be inlined as an anonymous function, but I think it's easier to
// read stacktraces with real function names in them.
func (t *Ticker) tick() {
	for tick := range t.ticker.C {
		t.mux.Lock()
		for i := range t.chans {
			select {
			case t.chans[i] <- tick:
			default:
			}
		}
		t.mux.Unlock()
	}
}
<|endoftext|>


<commit_before>package erratum

import (
	"errors"
	"fmt"
)

type opener func() (Resource, error)

func Use(open opener, s string) (err error) {
	resource, err := retry(5, open)
	if err != nil {
		return err
	}
	defer func(resource Resource) {
		resource.Close()
	}(resource)
	defer func(resource Resource) {
		if r := recover(); r != nil {
			fmt.Printf("recovered from %v\n", r)
			if errors.Is(r.(error), FrobError{}) {
				resource.Defrob(fmt.Sprintf("%v", r.(FrobError).defrobTag))
			}
			err = fmt.Errorf("%v", r)
		}
	}(resource)
	resource.Frob(s)
	return nil
}

func retry(attempts int, f func() (Resource, error)) (Resource, error) {
	for i := 0; ; i++ {
		r, err := f()
		if err == nil {
			return r, nil
		}
		if i >= (attempts - 1) {
			return nil, err
		}
	}
}
<commit_msg>Solve error handling<commit_after>package erratum

import (
	"fmt"
)

type opener func() (Resource, error)

func Use(open opener, s string) (err error) {
	resource, err := retry(5, open)
	if err != nil {
		return err
	}
	defer func(resource Resource) {
		resource.Close()
	}(resource)
	defer func(resource Resource) {
		if r := recover(); r != nil {
			fmt.Printf("recovered from %v\n", r)
			switch e := r.(type) {
			case FrobError:
				resource.Defrob(e.defrobTag)
				err = e
			case error:
				err = e
			}
			err = fmt.Errorf("%v", r)
		}
	}(resource)
	resource.Frob(s)
	return nil
}

func retry(attempts int, f func() (Resource, error)) (Resource, error) {
	for i := 0; ; i++ {
		r, err := f()
		if err == nil {
			return r, nil
		}
		if i >= (attempts - 1) {
			return nil, err
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package planbuilder

import (
	"strconv"

	"github.com/youtube/vitess/go/vt/sqlparser"
)

// ListVarName is the bind var name used for plans
// that require VTGate to compute custom list values,
// like for IN clauses.
const ListVarName = "__vals"

// jointab is used for managing join dependencies.
type jointab struct {
	refs map[colref]string
	vars map[string]struct{}
}

// newJointab creates a new jointab for the current plan
// being built. It also needs the current list of bind vars
// used in the original query to make sure that the names
// it generates don't collide with those already in use.
func newJointab(bindvars map[string]struct{}) *jointab {
	return &jointab{
		refs: make(map[colref]string),
		vars: bindvars,
	}
}

// Procure requests for the specified column from the plan
// and returns the join var name for it.
func (jt *jointab) Procure(plan planBuilder, col *sqlparser.ColName, to int) string {
	from, joinVar := jt.Lookup(col)
	// If joinVar is empty, jterate a unique name.
	if joinVar == "" {
		suffix := ""
		i := 0
		for {
			if col.Qualifier != "" {
				joinVar = string(col.Qualifier) + "_" + string(col.Name) + suffix
			} else {
				joinVar = string(col.Name) + suffix
			}
			if _, ok := jt.vars[joinVar]; !ok {
				break
			}
			i++
			suffix = strconv.Itoa(i)
		}
		jt.vars[joinVar] = struct{}{}
		jt.refs[newColref(col)] = joinVar
	}
	plan.SupplyVar(from, to, col, joinVar)
	return joinVar
}

// Lookup returns the order of the route that supplies the column and
// the join var name if one has already been assigned for it.
func (jt *jointab) Lookup(col *sqlparser.ColName) (order int, joinVar string) {
	ref := newColref(col)
	switch meta := col.Metadata.(type) {
	case *colsym:
		return meta.Route().Order(), jt.refs[ref]
	case *tableAlias:
		return meta.Route().Order(), jt.refs[ref]
	}
	panic("unreachable")
}
<commit_msg>v3: code review comments<commit_after>// Copyright 2016, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package planbuilder

import (
	"strconv"

	"github.com/youtube/vitess/go/vt/sqlparser"
)

// ListVarName is the bind var name used for plans
// that require VTGate to compute custom list values,
// like for IN clauses.
const ListVarName = "__vals"

// jointab manages procurement and naming of join
// variables across primitives.
type jointab struct {
	refs map[colref]string
	vars map[string]struct{}
}

// newJointab creates a new jointab for the current plan
// being built. It also needs the current list of bind vars
// used in the original query to make sure that the names
// it generates don't collide with those already in use.
func newJointab(bindvars map[string]struct{}) *jointab {
	return &jointab{
		refs: make(map[colref]string),
		vars: bindvars,
	}
}

// Procure requests for the specified column from the plan
// and returns the join var name for it.
func (jt *jointab) Procure(plan planBuilder, col *sqlparser.ColName, to int) string {
	from, joinVar := jt.Lookup(col)
	// If joinVar is empty, jterate a unique name.
	if joinVar == "" {
		suffix := ""
		i := 0
		for {
			if col.Qualifier != "" {
				joinVar = string(col.Qualifier) + "_" + string(col.Name) + suffix
			} else {
				joinVar = string(col.Name) + suffix
			}
			if _, ok := jt.vars[joinVar]; !ok {
				break
			}
			i++
			suffix = strconv.Itoa(i)
		}
		jt.vars[joinVar] = struct{}{}
		jt.refs[newColref(col)] = joinVar
	}
	plan.SupplyVar(from, to, col, joinVar)
	return joinVar
}

// Lookup returns the order of the route that supplies the column and
// the join var name if one has already been assigned for it.
func (jt *jointab) Lookup(col *sqlparser.ColName) (order int, joinVar string) {
	ref := newColref(col)
	switch meta := col.Metadata.(type) {
	case *colsym:
		return meta.Route().Order(), jt.refs[ref]
	case *tableAlias:
		return meta.Route().Order(), jt.refs[ref]
	}
	panic("unreachable")
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package yarpc_test

import (
	"context"
	"fmt"
	"log"

	"go.uber.org/yarpc"
	"go.uber.org/yarpc/encoding/json"
	"go.uber.org/yarpc/encoding/raw"
)

func ExampleDispatcher_minimal() {
	dispatcher := yarpc.NewDispatcher(yarpc.Config{Name: "myFancyService"})
	if err := dispatcher.Start(); err != nil {
		log.Fatal(err)
	}
	defer dispatcher.Stop()
}

// global dispatcher used in the registration examples
var dispatcher = yarpc.NewDispatcher(yarpc.Config{Name: "service"})

func ExampleDispatcher_Register_raw() {
	handler := func(ctx context.Context, data []byte) ([]byte, error) {
		return data, nil
	}

	dispatcher.Register(raw.Procedure("echo", handler))
}

// Excuse the weird naming of this function. This lets is show as "JSON"
// rather than "Json"

func ExampleDispatcher_Register_jSON() {
	handler := func(ctx context.Context, key string) (string, error) {
		fmt.Println("key", key)
		return "value", nil
	}

	dispatcher.Register(json.Procedure("get", handler))
}
<commit_msg>Fix `Dispatcher.Register` example tests (#1916)<commit_after>// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package yarpc_test

import (
	"context"
	"fmt"
	"log"

	"go.uber.org/yarpc"
	"go.uber.org/yarpc/encoding/json"
	"go.uber.org/yarpc/encoding/raw"
)

func ExampleDispatcher_minimal() {
	dispatcher := yarpc.NewDispatcher(yarpc.Config{Name: "my-fancy-service"})
	if err := dispatcher.Start(); err != nil {
		log.Fatal(err)
	}
	defer dispatcher.Stop()
	// Output:
}

// global dispatcher used in the registration examples
var dispatcher = yarpc.NewDispatcher(yarpc.Config{Name: "service"})

func ExampleDispatcher_Register_raw() {
	handler := func(ctx context.Context, data []byte) ([]byte, error) {
		return data, nil
	}

	dispatcher.Register(raw.Procedure("echo", handler))
	// Output:
}

type Request struct {
	key string
}

type Response struct {
	val string
}

// Excuse the weird naming of this function. This lets is show as "JSON"
// rather than "Json"

func ExampleDispatcher_Register_jSON() {
	handler := func(ctx context.Context, req *Request) (*Response, error) {
		fmt.Println("key", req.key)
		return &Response{val: "value"}, nil
	}

	dispatcher.Register(json.Procedure("get", handler))
	// Output:
}
<|endoftext|>


<commit_before>//Author: Gary Connelly
//date 02/10/2017

package main

import (
	"fmt"
)

func main() {
	word := "HelloWorld" // Word to reverse

	fmt.Printf("The reverse of the String '%s' is %s", word, reverseWord(word)) // Print string in reverse
}<commit_msg>reverse the string method<commit_after>//Author: Gary Connelly
//date 02/10/2017

package main

import (
	"fmt"
)

func main() {
	word := "HelloWorld" // Word to reverse

	fmt.Printf("The reverse of the String '%s' is %s", word, reverseWord(word)) // Print string in reverse
}

func reverseWord(s string) string {
	var reverse string

    for i := len(s)-1; i >= 0; i-- {
        reverse += string(s[i])
    }
    return reverse
}<|endoftext|>