<commit_before>package config

import (
	"github.com/freeformz/shh/utils"
	"time"
)

const (
	VERSION              = "0.2.1"
	DEFAULT_INTERVAL     = "10s"                                                              // Default tick interval for pollers
	DEFAULT_OUTPUTTER    = "stdoutl2metder"                                                   // Default outputter
	DEFAULT_POLLERS      = "conntrack,cpu,df,disk,listen,load,mem,nif,ntpdate,processes,self" // Default pollers
	DEFAULT_PROFILE_PORT = "0"                                                                // Default profile port, 0 disables
	DEFAULT_DF_TYPES     = "btrfs,ext3,ext4,tmpfs,xfs"                                        // Default fs types to report df for
	DEFAULT_NIF_DEVICES  = "eth0,lo"                                                          // Default interfaces to report stats for
)

var (
	Interval            = utils.GetEnvWithDefaultDuration("SHH_INTERVAL", DEFAULT_INTERVAL)                      // Polling Interval
	Outputter           = utils.GetEnvWithDefault("SHH_OUTPUTTER", DEFAULT_OUTPUTTER)                            // Outputter
	Pollers             = utils.GetEnvWithDefaultStrings("SHH_POLLERS", DEFAULT_POLLERS)                         // Pollers to poll
	Source              = utils.GetEnvWithDefault("SHH_SOURCE", "")                                              // Source to emit
	Prefix              = utils.GetEnvWithDefault("SHH_PREFIX", "")                                              // Metric prefix to use
	ProfilePort         = utils.GetEnvWithDefault("SHH_PROFILE_PORT", DEFAULT_PROFILE_PORT)                      // Profile Port
	DfTypes             = utils.GetEnvWithDefaultStrings("SHH_DF_TYPES", DEFAULT_DF_TYPES)                       // Default DF types
	Listen              = utils.GetEnvWithDefault("SHH_LISTEN", "unix,#shh")                                     // Default network socket info for listen
	NifDevices          = utils.GetEnvWithDefaultStrings("SHH_NIF_DEVICES", DEFAULT_NIF_DEVICES)                 // Devices to poll
	NtpdateServers      = utils.GetEnvWithDefaultStrings("SHH_NTPDATE_SERVERS", "0.pool.ntp.org,1.pool.ntp.org") // NTP Servers
	LibratoUser         = utils.GetEnvWithDefault("SHH_LIBRATO_USER", "")                                        // The Librato API User
	LibratoToken        = utils.GetEnvWithDefault("SHH_LIBRATO_TOKEN", "")                                       // The Librato API TOken
	LibratoBatchSize    = utils.GetEnvWithDefaultInt("SHH_LIBRATO_BATCH_SIZE", 50)                               // The max number of metrics to submit in a single request
	LibratoBatchTimeout = utils.GetEnvWithDefaultDuration("SHH_LIBRATO_BATCH_TIMEOUT", "500ms")                  // The max time metrics will sit un-delivered
	CarbonHost          = utils.GetEnvWithDefault("SHH_CARBON_HOST", "")                                         // Where the Carbon Outputter sends it's data

	Start = time.Now() // Start time
)
<commit_msg>Yank conntrack for now<commit_after>package config

import (
	"github.com/freeformz/shh/utils"
	"time"
)

const (
	VERSION              = "0.2.1"
	DEFAULT_INTERVAL     = "10s"                                                              // Default tick interval for pollers
	DEFAULT_OUTPUTTER    = "stdoutl2metder"                                                   // Default outputter
	DEFAULT_POLLERS      = "cpu,df,disk,listen,load,mem,nif,ntpdate,processes,self"           // Default pollers
	DEFAULT_PROFILE_PORT = "0"                                                                // Default profile port, 0 disables
	DEFAULT_DF_TYPES     = "btrfs,ext3,ext4,tmpfs,xfs"                                        // Default fs types to report df for
	DEFAULT_NIF_DEVICES  = "eth0,lo"                                                          // Default interfaces to report stats for
)

var (
	Interval            = utils.GetEnvWithDefaultDuration("SHH_INTERVAL", DEFAULT_INTERVAL)                      // Polling Interval
	Outputter           = utils.GetEnvWithDefault("SHH_OUTPUTTER", DEFAULT_OUTPUTTER)                            // Outputter
	Pollers             = utils.GetEnvWithDefaultStrings("SHH_POLLERS", DEFAULT_POLLERS)                         // Pollers to poll
	Source              = utils.GetEnvWithDefault("SHH_SOURCE", "")                                              // Source to emit
	Prefix              = utils.GetEnvWithDefault("SHH_PREFIX", "")                                              // Metric prefix to use
	ProfilePort         = utils.GetEnvWithDefault("SHH_PROFILE_PORT", DEFAULT_PROFILE_PORT)                      // Profile Port
	DfTypes             = utils.GetEnvWithDefaultStrings("SHH_DF_TYPES", DEFAULT_DF_TYPES)                       // Default DF types
	Listen              = utils.GetEnvWithDefault("SHH_LISTEN", "unix,#shh")                                     // Default network socket info for listen
	NifDevices          = utils.GetEnvWithDefaultStrings("SHH_NIF_DEVICES", DEFAULT_NIF_DEVICES)                 // Devices to poll
	NtpdateServers      = utils.GetEnvWithDefaultStrings("SHH_NTPDATE_SERVERS", "0.pool.ntp.org,1.pool.ntp.org") // NTP Servers
	LibratoUser         = utils.GetEnvWithDefault("SHH_LIBRATO_USER", "")                                        // The Librato API User
	LibratoToken        = utils.GetEnvWithDefault("SHH_LIBRATO_TOKEN", "")                                       // The Librato API TOken
	LibratoBatchSize    = utils.GetEnvWithDefaultInt("SHH_LIBRATO_BATCH_SIZE", 50)                               // The max number of metrics to submit in a single request
	LibratoBatchTimeout = utils.GetEnvWithDefaultDuration("SHH_LIBRATO_BATCH_TIMEOUT", "500ms")                  // The max time metrics will sit un-delivered
	CarbonHost          = utils.GetEnvWithDefault("SHH_CARBON_HOST", "")                                         // Where the Carbon Outputter sends it's data

	Start = time.Now() // Start time
)
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"net/url"
	"os"

	"github.com/koding/kite/kitekey"
)

// Options is passed to kite.New when creating new instance.
type Config struct {
	// Options for Kite
	Username              string
	Environment           string
	Region                string
	KiteKey               string
	DisableAuthentication bool

	// Options for Server
	IP                 string
	Port               int
	DisableConcurrency bool

	KontrolURL  *url.URL
	KontrolKey  string
	KontrolUser string
}

var defaultConfig = &Config{
	Username:    "unknown",
	Environment: "unknown",
	Region:      "unknown",
	IP:          "0.0.0.0",
	Port:        0,
	// KontrolURL:  &url.URL{Scheme: "ws", Host: "localhost:4000"},
}

// New returns a new Config initialized with defaults.
func New() *Config {
	c := new(Config)
	*c = *defaultConfig
	return c
}

func (c *Config) ReadEnvironmentVariables() {
	if environment := os.Getenv("KITE_ENVIRONMENT"); environment != "" {
		c.Environment = environment
	}

	if region := os.Getenv("KITE_REGION"); region != "" {
		c.Region = region
	}
}

// ReadKiteKey parsed the user's kite key and returns a new Config.
func (c *Config) ReadKiteKey() error {
	key, err := kitekey.Parse()
	if err != nil {
		return err
	}

	c.KiteKey = key.Raw

	if username, ok := key.Claims["sub"].(string); ok {
		c.Username = username
	}

	if kontrolUser, ok := key.Claims["iss"].(string); ok {
		c.KontrolUser = kontrolUser
	}

	if kontrolURL, ok := key.Claims["kontrolURL"].(string); ok {
		c.KontrolURL, err = url.Parse(kontrolURL)
		if err != nil {
			return err
		}
	}

	if kontrolKey, ok := key.Claims["kontrolKey"].(string); ok {
		c.KontrolKey = kontrolKey
	}

	return nil
}

// Clone returns a new copy of the config object.
func (c *Config) Clone() *Config {
	cloned := new(Config)
	*cloned = *c
	*cloned.KontrolURL = *c.KontrolURL
	return cloned
}

func Get() (*Config, error) {
	c := New()
	err := c.ReadKiteKey()
	if err != nil {
		return nil, err
	}
	c.ReadEnvironmentVariables()
	return c, nil
}

func MustGet() *Config {
	c, err := Get()
	if err != nil {
		fmt.Printf("Cannot read kite.key: %s\n", err.Error())
		os.Exit(1)
	}
	return c
}

// validate fields of the options struct. It exits if an error is occured.
func (o *Config) validate() {
	// if o.PublicIP == "" {
	// 	o.PublicIP = "0.0.0.0"
	// }

	// if o.Port == "" {
	// 	o.Port = "0" // OS binds to an automatic port
	// }

	// if o.Path == "" {
	// 	o.Path = "/kite/" + o.Kitename
	// }

	// if o.Path[0] != '/' {
	// 	o.Path = "/" + o.Path
	// }
}

// 		// 	k.TrustKontrolKey(k.kiteKey.Claims["iss"].(string), k.kiteKey.Claims["kontrolKey"].(string))
// 		// }
// 	}
<commit_msg>remove dead code<commit_after>package config

import (
	"fmt"
	"net/url"
	"os"

	"github.com/koding/kite/kitekey"
)

// Options is passed to kite.New when creating new instance.
type Config struct {
	// Options for Kite
	Username              string
	Environment           string
	Region                string
	KiteKey               string
	DisableAuthentication bool

	// Options for Server
	IP                 string
	Port               int
	DisableConcurrency bool

	KontrolURL  *url.URL
	KontrolKey  string
	KontrolUser string
}

var defaultConfig = &Config{
	Username:    "unknown",
	Environment: "unknown",
	Region:      "unknown",
	IP:          "0.0.0.0",
	Port:        0,
}

// New returns a new Config initialized with defaults.
func New() *Config {
	c := new(Config)
	*c = *defaultConfig
	return c
}

func (c *Config) ReadEnvironmentVariables() {
	if environment := os.Getenv("KITE_ENVIRONMENT"); environment != "" {
		c.Environment = environment
	}

	if region := os.Getenv("KITE_REGION"); region != "" {
		c.Region = region
	}
}

// ReadKiteKey parsed the user's kite key and returns a new Config.
func (c *Config) ReadKiteKey() error {
	key, err := kitekey.Parse()
	if err != nil {
		return err
	}

	c.KiteKey = key.Raw

	if username, ok := key.Claims["sub"].(string); ok {
		c.Username = username
	}

	if kontrolUser, ok := key.Claims["iss"].(string); ok {
		c.KontrolUser = kontrolUser
	}

	if kontrolURL, ok := key.Claims["kontrolURL"].(string); ok {
		c.KontrolURL, err = url.Parse(kontrolURL)
		if err != nil {
			return err
		}
	}

	if kontrolKey, ok := key.Claims["kontrolKey"].(string); ok {
		c.KontrolKey = kontrolKey
	}

	return nil
}

// Clone returns a new copy of the config object.
func (c *Config) Clone() *Config {
	cloned := new(Config)
	*cloned = *c
	*cloned.KontrolURL = *c.KontrolURL
	return cloned
}

func Get() (*Config, error) {
	c := New()
	err := c.ReadKiteKey()
	if err != nil {
		return nil, err
	}
	c.ReadEnvironmentVariables()
	return c, nil
}

func MustGet() *Config {
	c, err := Get()
	if err != nil {
		fmt.Printf("Cannot read kite.key: %s\n", err.Error())
		os.Exit(1)
	}
	return c
}

// validate fields of the options struct. It exits if an error is occured.
func (o *Config) validate() {
	// if o.PublicIP == "" {
	// 	o.PublicIP = "0.0.0.0"
	// }

	// if o.Port == "" {
	// 	o.Port = "0" // OS binds to an automatic port
	// }

	// if o.Path == "" {
	// 	o.Path = "/kite/" + o.Kitename
	// }

	// if o.Path[0] != '/' {
	// 	o.Path = "/" + o.Path
	// }
}

// 		// 	k.TrustKontrolKey(k.kiteKey.Claims["iss"].(string), k.kiteKey.Claims["kontrolKey"].(string))
// 		// }
// 	}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"strings"
)

// Config describes the config of the system.
type Config struct {
	// Net should be tcp4 or tcp6.
	Net string
	// AddrStr is the local address string.
	AddrStr string
	// Peers is peer list.
	Peers []string
	// LocalTCPAddr is TCP address parsed from
	// Net and AddrStr.
	LocalTCPAddr *net.TCPAddr
	// AViewMinSize is the minimum size of the active view.
	AViewMinSize int
	// AViewMaxSize is the maximum size of the active view.
	AViewMaxSize int
	// PViewSize is the size of the passive view.
	PViewSize int
	// Ka is the number of nodes to choose from active view
	// when shuffling views.
	Ka int
	// Kp is the number of nodes to choose from passive view
	// when shuffling views.
	Kp int
	// Active Random Walk Length.
	ARWL int
	// Passive Random Walk Length.
	PRWL int
	// Shuffle Random Walk Length.
	SRWL int
	// Message life.
	MLife int
	// Shuffle Duration in seconds.
	ShuffleDuration int
	// Heal Duration in seconds.
	HealDuration int
	// The REST server address.
	RESTAddrStr string
	// The path to user message handler(script).
	UserMsgHandler string
}

func ParseConfig() (*Config, error) {
	var peerStr string
	var peerFile string

	cfg := new(Config)

	flag.StringVar(&cfg.Net, "net", "tcp", "The network protocol")
	flag.StringVar(&cfg.AddrStr, "addr", "localhost:8424", "The address the agent listens on")

	flag.StringVar(&peerFile, "peer-file", "", "Peer list file")
	flag.StringVar(&peerStr, "peers", "", "Comma-separated list of peers")

	flag.IntVar(&cfg.AViewMinSize, "min-aview-size", 3, "The minimum size of the active view")
	flag.IntVar(&cfg.AViewMaxSize, "max-aview-size", 5, "The maximum size of the active view")
	flag.IntVar(&cfg.PViewSize, "pview-size", 5, "The size of the passive view")

	flag.IntVar(&cfg.Ka, "ka", 1, "The number of active nodes to shuffle")
	flag.IntVar(&cfg.Kp, "kp", 3, "The number of passive nodes to shuffle")

	flag.IntVar(&cfg.ARWL, "arwl", 5, "The active random walk length")
	flag.IntVar(&cfg.PRWL, "prwl", 5, "The passive random walk length")
	flag.IntVar(&cfg.SRWL, "srwl", 3, "The shuffle random walk length")

	flag.IntVar(&cfg.MLife, "msg-life", 5000, "The default message life (milliconds)")
	flag.IntVar(&cfg.ShuffleDuration, "shuffle-duration", 5, "The default shuffle duration (seconds)")
	flag.IntVar(&cfg.HealDuration, "heal", 1, "The default heal duration (seconds)")
	flag.StringVar(&cfg.RESTAddrStr, "rest-addr", "localhost:8425", "The address of the REST server")
	flag.StringVar(&cfg.UserMsgHandler, "user-message-handler", "", "The path to the user message handler script")

	flag.Parse()

	// Check configuration.
	if peerStr != "" {
		cfg.Peers = strings.Split(peerStr, ",")
	}
	if peerFile != "" {
		peers, err := parsePeerFile(peerFile)
		if err != nil {
			return nil, err
		}
		cfg.Peers = peers
	}

	// Check agent server address.
	tcpAddr, err := net.ResolveTCPAddr(cfg.Net, cfg.AddrStr)
	if err != nil {
		return nil, err
	}
	cfg.LocalTCPAddr = tcpAddr

	// Check REST API address.
	_, err = net.ResolveTCPAddr(cfg.Net, cfg.RESTAddrStr)
	if err != nil {
		return nil, err
	}

	// Check User Message Handler.
	if cfg.UserMsgHandler != "" {
		_, err = exec.LookPath(cfg.UserMsgHandler)
		if err != nil {
			return nil, err
		}
	}
	return cfg, nil
}

func parsePeerFile(path string) ([]string, error) {
	var peers []string
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(b, &peers); err != nil {
		return nil, err
	}
	return peers, nil
}

func (cfg *Config) ShufflePeers() []string {
	shuffledPeers := make([]string, len(cfg.Peers))
	copy(shuffledPeers, cfg.Peers)
	for i := range shuffledPeers {
		if i == 0 {
			continue
		}
		swapIndex := rand.Intn(i)
		shuffledPeers[i], shuffledPeers[swapIndex] = shuffledPeers[swapIndex], shuffledPeers[i]
	}
	return shuffledPeers
}
<commit_msg>Modified default rest-addr<commit_after>package config

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"strings"
)

// Config describes the config of the system.
type Config struct {
	// Net should be tcp4 or tcp6.
	Net string
	// AddrStr is the local address string.
	AddrStr string
	// Peers is peer list.
	Peers []string
	// LocalTCPAddr is TCP address parsed from
	// Net and AddrStr.
	LocalTCPAddr *net.TCPAddr
	// AViewMinSize is the minimum size of the active view.
	AViewMinSize int
	// AViewMaxSize is the maximum size of the active view.
	AViewMaxSize int
	// PViewSize is the size of the passive view.
	PViewSize int
	// Ka is the number of nodes to choose from active view
	// when shuffling views.
	Ka int
	// Kp is the number of nodes to choose from passive view
	// when shuffling views.
	Kp int
	// Active Random Walk Length.
	ARWL int
	// Passive Random Walk Length.
	PRWL int
	// Shuffle Random Walk Length.
	SRWL int
	// Message life.
	MLife int
	// Shuffle Duration in seconds.
	ShuffleDuration int
	// Heal Duration in seconds.
	HealDuration int
	// The REST server address.
	RESTAddrStr string
	// The path to user message handler(script).
	UserMsgHandler string
}

func ParseConfig() (*Config, error) {
	var peerStr string
	var peerFile string

	cfg := new(Config)

	flag.StringVar(&cfg.Net, "net", "tcp", "The network protocol")
	flag.StringVar(&cfg.AddrStr, "addr", "localhost:8424", "The address the agent listens on")

	flag.StringVar(&peerFile, "peer-file", "", "Peer list file")
	flag.StringVar(&peerStr, "peers", "", "Comma-separated list of peers")

	flag.IntVar(&cfg.AViewMinSize, "min-aview-size", 3, "The minimum size of the active view")
	flag.IntVar(&cfg.AViewMaxSize, "max-aview-size", 5, "The maximum size of the active view")
	flag.IntVar(&cfg.PViewSize, "pview-size", 5, "The size of the passive view")

	flag.IntVar(&cfg.Ka, "ka", 1, "The number of active nodes to shuffle")
	flag.IntVar(&cfg.Kp, "kp", 3, "The number of passive nodes to shuffle")

	flag.IntVar(&cfg.ARWL, "arwl", 5, "The active random walk length")
	flag.IntVar(&cfg.PRWL, "prwl", 5, "The passive random walk length")
	flag.IntVar(&cfg.SRWL, "srwl", 3, "The shuffle random walk length")

	flag.IntVar(&cfg.MLife, "msg-life", 5000, "The default message life (milliconds)")
	flag.IntVar(&cfg.ShuffleDuration, "shuffle-duration", 5, "The default shuffle duration (seconds)")
	flag.IntVar(&cfg.HealDuration, "heal", 1, "The default heal duration (seconds)")
	flag.StringVar(&cfg.RESTAddrStr, "rest-addr", ":8425", "The address of the REST server")
	flag.StringVar(&cfg.UserMsgHandler, "user-message-handler", "", "The path to the user message handler script")

	flag.Parse()

	// Check configuration.
	if peerStr != "" {
		cfg.Peers = strings.Split(peerStr, ",")
	}
	if peerFile != "" {
		peers, err := parsePeerFile(peerFile)
		if err != nil {
			return nil, err
		}
		cfg.Peers = peers
	}

	// Check agent server address.
	tcpAddr, err := net.ResolveTCPAddr(cfg.Net, cfg.AddrStr)
	if err != nil {
		return nil, err
	}
	cfg.LocalTCPAddr = tcpAddr

	// Check REST API address.
	_, err = net.ResolveTCPAddr(cfg.Net, cfg.RESTAddrStr)
	if err != nil {
		return nil, err
	}

	// Check User Message Handler.
	if cfg.UserMsgHandler != "" {
		_, err = exec.LookPath(cfg.UserMsgHandler)
		if err != nil {
			return nil, err
		}
	}
	return cfg, nil
}

func parsePeerFile(path string) ([]string, error) {
	var peers []string
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(b, &peers); err != nil {
		return nil, err
	}
	return peers, nil
}

func (cfg *Config) ShufflePeers() []string {
	shuffledPeers := make([]string, len(cfg.Peers))
	copy(shuffledPeers, cfg.Peers)
	for i := range shuffledPeers {
		if i == 0 {
			continue
		}
		swapIndex := rand.Intn(i)
		shuffledPeers[i], shuffledPeers[swapIndex] = shuffledPeers[swapIndex], shuffledPeers[i]
	}
	return shuffledPeers
}
<|endoftext|>


<commit_before>package config

import (
	"github.com/elastic/libbeat/cfgfile"
	"log"
	"math"
	"os"
	"path/filepath"
	"time"
)

// Defaults for config variables which are not set
const (
	DefaultRegistryFile                      = ".filebeat"
	DefaultIgnoreOlderDuration time.Duration = math.MaxInt64
	DefaultScanFrequency       time.Duration = 10 * time.Second // 10 seconds
	DefaultSpoolSize           uint64        = 1024
	DefaultIdleTimeout         time.Duration = time.Second * 5
	DefaultHarvesterBufferSize int           = 16 << 10 // 16384
	DefaultTailOnRotate                      = false
)

type Config struct {
	Filebeat FilebeatConfig
}

type FilebeatConfig struct {
	Prospectors         []ProspectorConfig
	SpoolSize           uint64 `yaml:"spoolSize"`
	CpuProfileFile      string // TODO: Still needed?
	IdleTimeout         string `yaml:"idleTimeout"`
	IdleTimeoutDuration time.Duration
	Quiet               bool
	RegistryFile        string
}

type ProspectorConfig struct {
	Paths                 []string
	Fields                map[string]string
	Input                 string
	IgnoreOlder           string `yaml:"ignoreOlder"`
	IgnoreOlderDuration   time.Duration
	ScanFrequency         string `yaml:"scanFrequency"`
	ScanFrequencyDuration time.Duration
	HarvesterBufferSize   int  `yaml:"harvesterBufferSize"`
	TailOnRotate          bool `yaml:"tailOnRotate"`
}

// getConfigFiles returns list of config files.
// In case path is a file, it will be directly returned.
// In case it is a directory, it will fetch all .yml files inside this directory
func getConfigFiles(path string) (configFiles []string, err error) {

	// Check if path is valid file or dir
	stat, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	// Create empty slice for config file list
	configFiles = make([]string, 0)

	if stat.IsDir() {
		files, err := filepath.Glob(path + "/*.yml")

		if err != nil {
			return nil, err
		}

		configFiles = append(configFiles, files...)

	} else {
		// Only 1 config file
		configFiles = append(configFiles, path)
	}

	return configFiles, nil
}

// mergeConfigFiles reads in all config files given by list configFiles and merges them into config
func mergeConfigFiles(configFiles []string, config *Config) error {

	for _, file := range configFiles {
		tmpConfig := &Config{}
		cfgfile.Read(tmpConfig, file)

		config.Filebeat.Prospectors = append(config.Filebeat.Prospectors, tmpConfig.Filebeat.Prospectors...)
	}

	return nil
}

// Fetches and merges all config files given by Options.configArgs. All are put into one config object
func (config *Config) FetchConfigs(path string) {

	configFiles, err := getConfigFiles(path)

	if err != nil {
		log.Fatal("Could not use -configDir of ", path, err)
	}

	err = mergeConfigFiles(configFiles, config)

	if err != nil {
		log.Fatal("Error merging config files: ", err)
	}

	if len(config.Filebeat.Prospectors) == 0 {
		log.Fatalf("No paths given. What files do you want me to watch?")
	}

}
<commit_msg>Remove Quite option again which sneaked in during merge<commit_after>package config

import (
	"github.com/elastic/libbeat/cfgfile"
	"log"
	"math"
	"os"
	"path/filepath"
	"time"
)

// Defaults for config variables which are not set
const (
	DefaultRegistryFile                      = ".filebeat"
	DefaultIgnoreOlderDuration time.Duration = math.MaxInt64
	DefaultScanFrequency       time.Duration = 10 * time.Second // 10 seconds
	DefaultSpoolSize           uint64        = 1024
	DefaultIdleTimeout         time.Duration = time.Second * 5
	DefaultHarvesterBufferSize int           = 16 << 10 // 16384
	DefaultTailOnRotate                      = false
)

type Config struct {
	Filebeat FilebeatConfig
}

type FilebeatConfig struct {
	Prospectors         []ProspectorConfig
	SpoolSize           uint64 `yaml:"spoolSize"`
	CpuProfileFile      string // TODO: Still needed?
	IdleTimeout         string `yaml:"idleTimeout"`
	IdleTimeoutDuration time.Duration
	RegistryFile        string
}

type ProspectorConfig struct {
	Paths                 []string
	Fields                map[string]string
	Input                 string
	IgnoreOlder           string `yaml:"ignoreOlder"`
	IgnoreOlderDuration   time.Duration
	ScanFrequency         string `yaml:"scanFrequency"`
	ScanFrequencyDuration time.Duration
	HarvesterBufferSize   int  `yaml:"harvesterBufferSize"`
	TailOnRotate          bool `yaml:"tailOnRotate"`
}

// getConfigFiles returns list of config files.
// In case path is a file, it will be directly returned.
// In case it is a directory, it will fetch all .yml files inside this directory
func getConfigFiles(path string) (configFiles []string, err error) {

	// Check if path is valid file or dir
	stat, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	// Create empty slice for config file list
	configFiles = make([]string, 0)

	if stat.IsDir() {
		files, err := filepath.Glob(path + "/*.yml")

		if err != nil {
			return nil, err
		}

		configFiles = append(configFiles, files...)

	} else {
		// Only 1 config file
		configFiles = append(configFiles, path)
	}

	return configFiles, nil
}

// mergeConfigFiles reads in all config files given by list configFiles and merges them into config
func mergeConfigFiles(configFiles []string, config *Config) error {

	for _, file := range configFiles {
		tmpConfig := &Config{}
		cfgfile.Read(tmpConfig, file)

		config.Filebeat.Prospectors = append(config.Filebeat.Prospectors, tmpConfig.Filebeat.Prospectors...)
	}

	return nil
}

// Fetches and merges all config files given by Options.configArgs. All are put into one config object
func (config *Config) FetchConfigs(path string) {

	configFiles, err := getConfigFiles(path)

	if err != nil {
		log.Fatal("Could not use -configDir of ", path, err)
	}

	err = mergeConfigFiles(configFiles, config)

	if err != nil {
		log.Fatal("Error merging config files: ", err)
	}

	if len(config.Filebeat.Prospectors) == 0 {
		log.Fatalf("No paths given. What files do you want me to watch?")
	}

}
<|endoftext|>


<commit_before>package neuralnetwork

import (
	"fmt"
	"math"
	"math/rand"
)

func randomFloat32(a, b float32) float32 {
	return (b-a)*rand.Float32() + a
}

func matrix(I, J int) [][]float32 {
	m := make([][]float32, I)
	for i := 0; i < I; i++ {
		m[i] = make([]float32, J)
	}
	return m
}

func vector(I int, fill float32) []float32 {
	v := make([]float32, I)
	for i := 0; i < I; i++ {
		v[i] = fill
	}
	return v
}

func sigmoid(x float32) float32 {
	return float32(math.Tanh(float64(x)))
}

func dsigmoid(y float32) float32 {
	return 1.0 - y*y
}

type NeuralNetwork struct {
	// Number of input, hidden and output nodes
	ni, nh, no int
	// Whether it is regression or not
	regression bool
	// Activations for nodes
	ai, ah, ao []float32
	// Weights
	wi, wo [][]float32
	// Last change in weights for momentum
	ci, co [][]float32
}

func New(ni, nh, no int, regression bool) *NeuralNetwork {
	nn := &NeuralNetwork{ni: ni + 1, nh: nh + 1, no: no, regression: regression}

	nn.ai = vector(nn.ni, 1.0)
	nn.ah = vector(nn.nh, 1.0)
	nn.ao = vector(nn.no, 1.0)

	nn.wi = matrix(nn.ni, nn.nh)
	nn.wo = matrix(nn.nh, nn.no)

	for i := 0; i < nn.ni; i++ {
		for j := 0; j < nn.nh; j++ {
			nn.wi[i][j] = randomFloat32(-1, 1)
			fmt.Println(nn.wi[i][j])
		}
	}

	for i := 0; i < nn.nh; i++ {
		for j := 0; j < nn.no; j++ {
			nn.wo[i][j] = randomFloat32(-1, 1)
		}
	}

	nn.ci = matrix(nn.ni, nn.nh)
	nn.co = matrix(nn.nh, nn.no)

	return nn
}

func (nn *NeuralNetwork) Update(inputs []float32) []float32 {
	if len(inputs) != nn.ni-1 {
		fmt.Println("Error: wrong number of inputs")
		return []float32{} // should return error
	}

	for i := 0; i < nn.ni-1; i++ {
		nn.ai[i] = inputs[i]
	}

	for i := 0; i < nn.nh-1; i++ {
		var sum float32 = 0.0
		for j := 0; j < nn.ni; j++ {
			sum += nn.ai[j] * nn.wi[j][i]
		}
		nn.ah[i] = sigmoid(sum)
	}

	for i := 0; i < nn.no; i++ {
		var sum float32 = 0.0
		for j := 0; j < nn.nh; j++ {
			sum += nn.ah[j] * nn.wo[j][i]
		}
		if nn.regression {
			nn.ao[i] = sum
		} else {
			nn.ao[i] = sigmoid(sum)
		}
	}

	return nn.ao
}

func (nn *NeuralNetwork) BackPropagate(targets []float32, lRate, mFactor float32) float32 {
	if len(targets) != nn.no {
		fmt.Println("Error: wrong number of target values")
		return float32(0.0)
	}

	output_deltas := vector(nn.no, 0.0)
	for i := 0; i < nn.no; i++ {
		output_deltas[i] = targets[i] - nn.ao[i]

		if !nn.regression {
			output_deltas[i] = dsigmoid(nn.ao[i]) * output_deltas[i]
		}
	}

	hidden_deltas := vector(nn.nh, 0.0)
	for i := 0; i < nn.nh; i++ {
		var e float32 = 0.0

		for j := 0; j < nn.no; j++ {
			e += output_deltas[j] * nn.wo[i][j]
		}
		hidden_deltas[i] = dsigmoid(nn.ah[i]) * e
	}

	for i := 0; i < nn.nh; i++ {
		for j := 0; j < nn.no; j++ {
			change := output_deltas[j] * nn.ah[i]
			nn.wo[i][j] = nn.wo[i][j] + lRate*change + mFactor*nn.co[i][j]
			nn.co[i][j] = change
		}
	}

	for i := 0; i < nn.ni; i++ {
		for j := 0; j < nn.nh; j++ {
			change := hidden_deltas[j] * nn.ai[i]
			nn.wi[i][j] = nn.wi[i][j] + lRate*change + mFactor*nn.ci[i][j]
			nn.ci[i][j] = change
		}
	}

	var e float32 = 0.0

	for i := 0; i < len(targets); i++ {
		e += 0.5 * (float32(math.Pow(float64(targets[i]-nn.ao[i]), 2)))
	}

	return e
}

func (nn *NeuralNetwork) Train(patterns [][][]float32, iterations int, lRate, mFactor float32) {
	for i := 0; i < iterations; i++ {
		var e float32 = 0.0
		for _, p := range patterns {
			nn.Update(p[0])

			tmp := nn.BackPropagate(p[1], lRate, mFactor)
			e += tmp
		}

		if i%100 == 0 {
			fmt.Printf("Error %e\n", e)
		}
	}
}

func (nn *NeuralNetwork) Test(patterns [][][]float32) {
	for _, p := range patterns {
		fmt.Println(p[0], "->", nn.Update(p[0]), " : ", p[1])
	}
}
<commit_msg>removed print<commit_after>package neuralnetwork

import (
	"fmt"
	"math"
	"math/rand"
)

func randomFloat32(a, b float32) float32 {
	return (b-a)*rand.Float32() + a
}

func matrix(I, J int) [][]float32 {
	m := make([][]float32, I)
	for i := 0; i < I; i++ {
		m[i] = make([]float32, J)
	}
	return m
}

func vector(I int, fill float32) []float32 {
	v := make([]float32, I)
	for i := 0; i < I; i++ {
		v[i] = fill
	}
	return v
}

func sigmoid(x float32) float32 {
	return float32(math.Tanh(float64(x)))
}

func dsigmoid(y float32) float32 {
	return 1.0 - y*y
}

type NeuralNetwork struct {
	// Number of input, hidden and output nodes
	ni, nh, no int
	// Whether it is regression or not
	regression bool
	// Activations for nodes
	ai, ah, ao []float32
	// Weights
	wi, wo [][]float32
	// Last change in weights for momentum
	ci, co [][]float32
}

func New(ni, nh, no int, regression bool) *NeuralNetwork {
	nn := &NeuralNetwork{ni: ni + 1, nh: nh + 1, no: no, regression: regression}

	nn.ai = vector(nn.ni, 1.0)
	nn.ah = vector(nn.nh, 1.0)
	nn.ao = vector(nn.no, 1.0)

	nn.wi = matrix(nn.ni, nn.nh)
	nn.wo = matrix(nn.nh, nn.no)

	for i := 0; i < nn.ni; i++ {
		for j := 0; j < nn.nh; j++ {
			nn.wi[i][j] = randomFloat32(-1, 1)
		}
	}

	for i := 0; i < nn.nh; i++ {
		for j := 0; j < nn.no; j++ {
			nn.wo[i][j] = randomFloat32(-1, 1)
		}
	}

	nn.ci = matrix(nn.ni, nn.nh)
	nn.co = matrix(nn.nh, nn.no)

	return nn
}

func (nn *NeuralNetwork) Update(inputs []float32) []float32 {
	if len(inputs) != nn.ni-1 {
		fmt.Println("Error: wrong number of inputs")
		return []float32{} // should return error
	}

	for i := 0; i < nn.ni-1; i++ {
		nn.ai[i] = inputs[i]
	}

	for i := 0; i < nn.nh-1; i++ {
		var sum float32 = 0.0
		for j := 0; j < nn.ni; j++ {
			sum += nn.ai[j] * nn.wi[j][i]
		}
		nn.ah[i] = sigmoid(sum)
	}

	for i := 0; i < nn.no; i++ {
		var sum float32 = 0.0
		for j := 0; j < nn.nh; j++ {
			sum += nn.ah[j] * nn.wo[j][i]
		}
		if nn.regression {
			nn.ao[i] = sum
		} else {
			nn.ao[i] = sigmoid(sum)
		}
	}

	return nn.ao
}

func (nn *NeuralNetwork) BackPropagate(targets []float32, lRate, mFactor float32) float32 {
	if len(targets) != nn.no {
		fmt.Println("Error: wrong number of target values")
		return float32(0.0)
	}

	output_deltas := vector(nn.no, 0.0)
	for i := 0; i < nn.no; i++ {
		output_deltas[i] = targets[i] - nn.ao[i]

		if !nn.regression {
			output_deltas[i] = dsigmoid(nn.ao[i]) * output_deltas[i]
		}
	}

	hidden_deltas := vector(nn.nh, 0.0)
	for i := 0; i < nn.nh; i++ {
		var e float32 = 0.0

		for j := 0; j < nn.no; j++ {
			e += output_deltas[j] * nn.wo[i][j]
		}
		hidden_deltas[i] = dsigmoid(nn.ah[i]) * e
	}

	for i := 0; i < nn.nh; i++ {
		for j := 0; j < nn.no; j++ {
			change := output_deltas[j] * nn.ah[i]
			nn.wo[i][j] = nn.wo[i][j] + lRate*change + mFactor*nn.co[i][j]
			nn.co[i][j] = change
		}
	}

	for i := 0; i < nn.ni; i++ {
		for j := 0; j < nn.nh; j++ {
			change := hidden_deltas[j] * nn.ai[i]
			nn.wi[i][j] = nn.wi[i][j] + lRate*change + mFactor*nn.ci[i][j]
			nn.ci[i][j] = change
		}
	}

	var e float32 = 0.0

	for i := 0; i < len(targets); i++ {
		e += 0.5 * (float32(math.Pow(float64(targets[i]-nn.ao[i]), 2)))
	}

	return e
}

func (nn *NeuralNetwork) Train(patterns [][][]float32, iterations int, lRate, mFactor float32) {
	for i := 0; i < iterations; i++ {
		var e float32 = 0.0
		for _, p := range patterns {
			nn.Update(p[0])

			tmp := nn.BackPropagate(p[1], lRate, mFactor)
			e += tmp
		}

		if i%100 == 0 {
			fmt.Printf("Error %e\n", e)
		}
	}
}

func (nn *NeuralNetwork) Test(patterns [][][]float32) {
	for _, p := range patterns {
		fmt.Println(p[0], "->", nn.Update(p[0]), " : ", p[1])
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"os"
	"time"

	ui "github.com/gizak/termui"
	"github.com/john-pettigrew/irc"
	"github.com/john-pettigrew/irc/message"
)

var ircConn irc.Client

func listenForInput() {
	input := make([]byte, 100)
	for {
		input = make([]byte, 100)
		_, err := os.Stdin.Read(input)
		if err != nil {
			break
		}
		handleInput(string(input))
		time.Sleep(time.Second / 60)
	}
}

func handleInput(input string) {

	//Check for direction input

	//left
	if bytes.Equal([]byte(input)[:3], []byte{27, 79, 68}) {
		buffer.CursorMove(-1)
		return
	}
	//right
	if bytes.Equal([]byte(input)[:3], []byte{27, 79, 67}) {
		buffer.CursorMove(1)
		return
	}

	//up
	if bytes.Equal([]byte(input)[:3], []byte{27, 79, 65}) {
		messagesList.ListMove(-1)
		return
	}
	//down
	if bytes.Equal([]byte(input)[:3], []byte{27, 79, 66}) {
		messagesList.ListMove(1)
		return
	}

	for _, r := range input {
		if r == rune(0) {
			continue
		}
		if r == '\r' {
			//handle buffer
			if buffer.Contents == "/quit" {
				ui.StopLoop()
				return
			} else if len(buffer.Contents) >= 8 && buffer.Contents[:8] == "/connect" {

				messagesList.AddMessage("IRC", "Connecting to host...")
				ircConn, err = irc.NewClient(buffer.Contents[9:])
				if err != nil {
					messagesList.AddMessage("Error", "Error connecting to host: "+err.Error())
					buffer.Clear()
				}

				messagesList.AddMessage("IRC", "Connected to server")
				ircConn.SubscribeForMessages(&msgCh)
				buffer.Clear()
			} else if ircConn == (irc.Client{}) {
				messagesList.AddMessage("Error", "You must connect to a server first")
				buffer.Clear()
			} else {
				msg, err := message.ParseCommand(buffer.Contents)
				if err != nil {
					panic(err)
				}
				err = ircConn.SendMessage(msg)
				if err != nil {
					panic(err)
				}
				msgCh <- msg

				buffer.Clear()
			}
		} else {
			buffer.Type(r)
		}

		textInput.Text = inputPrefix + buffer.Contents
		ui.Render(ui.Body)
	}
}
<commit_msg>subscribe for messages in new goroutine<commit_after>package main

import (
	"bytes"
	"os"
	"time"

	ui "github.com/gizak/termui"
	"github.com/john-pettigrew/irc"
	"github.com/john-pettigrew/irc/message"
)

var ircConn irc.Client

func listenForInput() {
	input := make([]byte, 100)
	for {
		input = make([]byte, 100)
		_, err := os.Stdin.Read(input)
		if err != nil {
			break
		}
		handleInput(string(input))
		time.Sleep(time.Second / 60)
	}
}

func handleInput(input string) {

	//Check for direction input

	//left
	if bytes.Equal([]byte(input)[:3], []byte{27, 79, 68}) {
		buffer.CursorMove(-1)
		return
	}
	//right
	if bytes.Equal([]byte(input)[:3], []byte{27, 79, 67}) {
		buffer.CursorMove(1)
		return
	}

	//up
	if bytes.Equal([]byte(input)[:3], []byte{27, 79, 65}) {
		messagesList.ListMove(-1)
		return
	}
	//down
	if bytes.Equal([]byte(input)[:3], []byte{27, 79, 66}) {
		messagesList.ListMove(1)
		return
	}

	for _, r := range input {
		if r == rune(0) {
			continue
		}
		if r == '\r' {
			//handle buffer
			if buffer.Contents == "/quit" {
				ui.StopLoop()
				return
			} else if len(buffer.Contents) >= 8 && buffer.Contents[:8] == "/connect" {

				messagesList.AddMessage("IRC", "Connecting to host...")
				ircConn, err = irc.NewClient(buffer.Contents[9:])
				if err != nil {
					messagesList.AddMessage("Error", "Error connecting to host: "+err.Error())
					buffer.Clear()
				}

				messagesList.AddMessage("IRC", "Connected to server")
				go ircConn.SubscribeForMessages(&msgCh)
				buffer.Clear()
			} else if ircConn == (irc.Client{}) {
				messagesList.AddMessage("Error", "You must connect to a server first")
				buffer.Clear()
			} else {
				msg, err := message.ParseCommand(buffer.Contents)
				if err != nil {
					panic(err)
				}
				err = ircConn.SendMessage(msg)
				if err != nil {
					panic(err)
				}
				msgCh <- msg

				buffer.Clear()
			}
		} else {
			buffer.Type(r)
		}

		textInput.Text = inputPrefix + buffer.Contents
		ui.Render(ui.Body)
	}
}
<|endoftext|>


<commit_before>
package main

import (
  "fmt"
  "strconv"
  "unicode"
  "utf8"
)

const (
  kSplit = iota         // proceed down out & out1
  kAltBegin             // begin of alt section, i.e. '('
  kAltEnd               // end of alt section, i.e. ')'
  kRune                 // if match rune, proceed down out
  kCall                 // if matcher passes, proceed down out
  kMatch                // success state!
)

/**
 * Single instruction in regexp.
 */
type instr struct {
  idx int               // index of this instr
  mode byte             // mode (as above)
  out *instr            // next instr to process
  out1 *instr           // alt next instr (for kSplit)
  rune int              // rune to match (kRune)
  matcher func(rune int) bool   // matcher method (for kCall)
  alt int               // identifier of alt branch (for kAlt{Begin,End})
}

/**
 * String-representation of an individual instruction.
 */
func (i *instr) str() string {
  str := fmt.Sprintf("{%d", i.idx)
  out := ""
  if i.out != nil {
    out += fmt.Sprintf(" out=%d", i.out.idx)
  }
  switch i.mode {
  case kSplit:
    str += " kSplit"
    if i.out1 != nil {
      out += fmt.Sprintf(" out1=%d", i.out1.idx)
    }
  case kAltBegin:
    str += fmt.Sprintf(" kAltBegin alt=%d", i.alt)
  case kAltEnd:
    str += fmt.Sprintf(" kAltEnd alt=%d", i.alt)
  case kRune:
    str += fmt.Sprintf(" kRune rune=%c", i.rune)
  case kCall:
    str += " kCall meth=?"
  case kMatch:
    str += " kMatch"
  }
  return str + out + "}"
}

type parser struct {
  src string
  ch int
  pos int
  prog []*instr
  inst int
  altpos int
}

/*
 * Generic matcher for consuming instr instances (i.e. kRune/kCall). Does not
 * match anything else.
 */
func (s *instr) match(rune int) bool {
  if s.mode == kRune {
    return s.rune == rune || s.rune == -1
  } else if s.mode == kCall {
    return s.matcher(rune)
  }
  return false
}

/**
 * Generate a new pre-indexed instr.
 */
func (p *parser) instr() *instr {
  if p.inst == len(p.prog) {
    panic("overflow instr buffer")
  }
  i := &instr{p.inst, kSplit, nil, nil, -1, nil, -1}
  p.prog[p.inst] = i
  p.inst += 1
  return i
}

/**
 * Store/return the next character in parser. -1 indicates EOF.
 */
func (p *parser) nextc() int {
  if p.pos >= len(p.src) {
    p.ch = -1
  } else {
		c, w := utf8.DecodeRuneInString(p.src[p.pos:])
		p.ch = c
		p.pos += w
  }
  return p.ch
}

/**
 * Connect from -> to.
 */
func (p *parser) out(from *instr, to *instr) {
  if from.out == nil {
    from.out = to
  } else if from.mode == kSplit && from.out1 == nil {
    from.out1 = to
  } else {
    panic("can't out")
  }
}

func (p *parser) alt() (start *instr, end *instr) {
  altpos := p.altpos
  p.altpos += 1

  if p.ch != '(' {
    panic("alt must start with '('")
  }

  end = p.instr() // shared end state for alt
  end.mode = kAltEnd
  end.alt = altpos

  p.nextc()
  b_start, b_end := p.regexp()
  start = b_start
  p.out(b_end, end)

  for p.ch == '|' {
    start = p.instr()
    p.out(start, b_start)

    p.nextc()
    b_start, b_end = p.regexp()
    p.out(start, b_start)
    p.out(b_end, end)
    b_start = start
  }

  if p.ch != ')' {
    panic("alt must end with ')'")
  }

  alt_begin := p.instr()
  alt_begin.mode = kAltBegin
  alt_begin.alt = altpos
  p.out(alt_begin, start)
  return alt_begin, end
}

func (p *parser) term() (start *instr, end *instr) {
  start = p.instr()
  end = start

  switch p.ch {
  case -1:
    panic("EOF in term")
  case '*', '+', '{', '?':
    panic("unexpected expansion char")
  case ')', '}', ']':
    panic("unexpected close element")
  case '(':
    start, end = p.alt()
  case '[':
    panic("not yet supported: [")
  case '$':
    panic("not yet supported: end of string")
  case '^':
    panic("not yet supported: start of string")
  case '.':
    start.mode = kRune
  case '\\':
    next := p.nextc()
    start.mode = kRune
    switch next {
    case 'n':
      start.rune = '\n'
    case 't':
      start.rune = '\t'
    default:
      // TODO: limit this to punctuation
      start.rune = next
    }
  default:
    start.mode = kRune
    start.rune = p.ch
  }
  p.nextc()
  return start, end
}

func (p *parser) closure() (start *instr, end *instr) {
  start, end = p.term()

  switch p.ch {
  case '?', '*':
    p_start := start
    p_end := end
    start = p.instr()
    end = p.instr()
    p.out(start, p_start)
    p.out(start, end)
    if p.ch == '?' {
      p.out(p_end, end)
    } else {
      p.out(p_end, start)
    }
    p.nextc()
  case '+':
    p_end := end
    end = p.instr()
    p.out(p_end, end)
    p.out(end, start)
    p.nextc()
  case '{':
    count_str := ""
    p.nextc()
    for unicode.IsDigit(p.ch) {
      count_str += fmt.Sprintf("%c", p.ch)
      p.nextc()
    }
    if len(count_str) == 0 {
      panic("{ must be followed by digit")
    }
    if p.ch == '}' {
      // fixed expansion
      count, _ := strconv.Atoi(count_str)
      panic(fmt.Sprintf("can't yet expand to: %d", count))
    } else if p.ch == ',' {
      panic("can't handle anything but {n}")
    } else {
      panic("unexpected char in {}")
    }
  }
  return start, end
}

/**
 * Match a regexp (defined as [closure]*) from parser, until either: EOF, |, or
 * ) is encountered.
 */
func (p *parser) regexp() (start *instr, end *instr) {
  start = p.instr()
  curr := start

  for {
    if p.ch == -1 || p.ch == '|' || p.ch == ')' {
      break
    }
    s, e := p.closure()
    p.out(curr, s)
    curr = e
  }

  end = p.instr()
  p.out(curr, end)
  return start, end
}

/**
 * Cleanup the given program. Assumes the given input is a flat slice containing
 * no nil instructions. Will not clean up the first instruction, as it is always
 * the canonical entry point for the regexp.
 *
 * Returns a similarly flat slice containing no nil instructions, however the
 * slice may potentially be smaller.
 */
func cleanup(prog []*instr) []*instr {
  // Detect kSplit recursion. We can remove this and convert it to a single path.
  states := NewStateSet(len(prog), len(prog))
  for i := 1; i < len(prog); i++ {
    states.Clear()
    pi := prog[i]
    var fn func(ci *instr) bool
    fn = func(ci *instr) bool {
      if ci != nil && ci.mode == kSplit {
        if states.Put(ci.idx) {
          // NOTE: I'm not sure if this will ever happen. Panic for now. If we're
          // confident this won't happen, we could move the panic to runtime.
          panic("regexp should never loop")
          return true
        }
        if fn(ci.out) {
          ci.out = nil
        }
        if fn(ci.out1) {
          ci.out1 = nil
        }
        return true
      }
      return false
    }
    fn(pi)
  }

  // Iterate through the program, and remove single-instr kSplits.
  // NB: Don't parse the first instr, it will always be single.
  for i := 1; i < len(prog); i++ {
    pi := prog[i]
    if pi.mode == kSplit && (pi.out1 == nil || pi.out == pi.out1) {
      for j := 0; j < len(prog); j++ {
        if prog[j] == nil {
          continue
        }
        pj := prog[j]
        if pj.out == pi {
          pj.out = pi.out
        }
        if pj.out1 == pi {
          pj.out1 = pi.out
        }
      }
      prog[i] = nil
    }
  }

  // We may now have nil gaps: shift everything up.
  last := 0
  for i := 0; i < len(prog); i++ {
    if prog[i] != nil {
      last = i
    } else {
      // find next non-nil, move here
      var found int
      for found = i; found < len(prog); found++ {
        if prog[found] != nil {
          break
        }
      }
      if found == len(prog) {
        break // no more entries
      }

      // move found to i
      prog[i] = prog[found]
      prog[i].idx = i
      prog[found] = nil
      last = i
    }
  }

  return prog[0:last+1]
}

/**
 * Generates a simple straight-forward NFA.
 */
func Parse(src string) (r *sregexp) {
  p := parser{src, -1, 0, make([]*instr, 128), 0, 0}
  begin := p.instr()
  match := p.instr()
  match.mode = kMatch

  p.nextc()
  start, end := p.regexp()

  if p.nextc() != -1 {
    panic("could not consume all of regexp!")
  }

  p.out(begin, start)
  p.out(end, match)

  result := p.prog[0:end.idx+1]
  result = cleanup(result)

  return &sregexp{result, p.altpos}
}
<commit_msg>tiny fix to sanity-checker - oops<commit_after>
package main

import (
  "fmt"
  "strconv"
  "unicode"
  "utf8"
)

const (
  kSplit = iota         // proceed down out & out1
  kAltBegin             // begin of alt section, i.e. '('
  kAltEnd               // end of alt section, i.e. ')'
  kRune                 // if match rune, proceed down out
  kCall                 // if matcher passes, proceed down out
  kMatch                // success state!
)

/**
 * Single instruction in regexp.
 */
type instr struct {
  idx int               // index of this instr
  mode byte             // mode (as above)
  out *instr            // next instr to process
  out1 *instr           // alt next instr (for kSplit)
  rune int              // rune to match (kRune)
  matcher func(rune int) bool   // matcher method (for kCall)
  alt int               // identifier of alt branch (for kAlt{Begin,End})
}

/**
 * String-representation of an individual instruction.
 */
func (i *instr) str() string {
  str := fmt.Sprintf("{%d", i.idx)
  out := ""
  if i.out != nil {
    out += fmt.Sprintf(" out=%d", i.out.idx)
  }
  switch i.mode {
  case kSplit:
    str += " kSplit"
    if i.out1 != nil {
      out += fmt.Sprintf(" out1=%d", i.out1.idx)
    }
  case kAltBegin:
    str += fmt.Sprintf(" kAltBegin alt=%d", i.alt)
  case kAltEnd:
    str += fmt.Sprintf(" kAltEnd alt=%d", i.alt)
  case kRune:
    str += fmt.Sprintf(" kRune rune=%c", i.rune)
  case kCall:
    str += " kCall meth=?"
  case kMatch:
    str += " kMatch"
  }
  return str + out + "}"
}

type parser struct {
  src string
  ch int
  pos int
  prog []*instr
  inst int
  altpos int
}

/*
 * Generic matcher for consuming instr instances (i.e. kRune/kCall). Does not
 * match anything else.
 */
func (s *instr) match(rune int) bool {
  if s.mode == kRune {
    return s.rune == rune || s.rune == -1
  } else if s.mode == kCall {
    return s.matcher(rune)
  }
  return false
}

/**
 * Generate a new pre-indexed instr.
 */
func (p *parser) instr() *instr {
  if p.inst == len(p.prog) {
    panic("overflow instr buffer")
  }
  i := &instr{p.inst, kSplit, nil, nil, -1, nil, -1}
  p.prog[p.inst] = i
  p.inst += 1
  return i
}

/**
 * Store/return the next character in parser. -1 indicates EOF.
 */
func (p *parser) nextc() int {
  if p.pos >= len(p.src) {
    p.ch = -1
  } else {
		c, w := utf8.DecodeRuneInString(p.src[p.pos:])
		p.ch = c
		p.pos += w
  }
  return p.ch
}

/**
 * Connect from -> to.
 */
func (p *parser) out(from *instr, to *instr) {
  if from.out == nil {
    from.out = to
  } else if from.mode == kSplit && from.out1 == nil {
    from.out1 = to
  } else {
    panic("can't out")
  }
}

func (p *parser) alt() (start *instr, end *instr) {
  altpos := p.altpos
  p.altpos += 1

  if p.ch != '(' {
    panic("alt must start with '('")
  }

  end = p.instr() // shared end state for alt
  end.mode = kAltEnd
  end.alt = altpos

  p.nextc()
  b_start, b_end := p.regexp()
  start = b_start
  p.out(b_end, end)

  for p.ch == '|' {
    start = p.instr()
    p.out(start, b_start)

    p.nextc()
    b_start, b_end = p.regexp()
    p.out(start, b_start)
    p.out(b_end, end)
    b_start = start
  }

  if p.ch != ')' {
    panic("alt must end with ')'")
  }

  alt_begin := p.instr()
  alt_begin.mode = kAltBegin
  alt_begin.alt = altpos
  p.out(alt_begin, start)
  return alt_begin, end
}

func (p *parser) term() (start *instr, end *instr) {
  start = p.instr()
  end = start

  switch p.ch {
  case -1:
    panic("EOF in term")
  case '*', '+', '{', '?':
    panic("unexpected expansion char")
  case ')', '}', ']':
    panic("unexpected close element")
  case '(':
    start, end = p.alt()
  case '[':
    panic("not yet supported: [")
  case '$':
    panic("not yet supported: end of string")
  case '^':
    panic("not yet supported: start of string")
  case '.':
    start.mode = kRune
  case '\\':
    next := p.nextc()
    start.mode = kRune
    switch next {
    case 'n':
      start.rune = '\n'
    case 't':
      start.rune = '\t'
    default:
      // TODO: limit this to punctuation
      start.rune = next
    }
  default:
    start.mode = kRune
    start.rune = p.ch
  }
  p.nextc()
  return start, end
}

func (p *parser) closure() (start *instr, end *instr) {
  start, end = p.term()

  switch p.ch {
  case '?', '*':
    p_start := start
    p_end := end
    start = p.instr()
    end = p.instr()
    p.out(start, p_start)
    p.out(start, end)
    if p.ch == '?' {
      p.out(p_end, end)
    } else {
      p.out(p_end, start)
    }
    p.nextc()
  case '+':
    p_end := end
    end = p.instr()
    p.out(p_end, end)
    p.out(end, start)
    p.nextc()
  case '{':
    count_str := ""
    p.nextc()
    for unicode.IsDigit(p.ch) {
      count_str += fmt.Sprintf("%c", p.ch)
      p.nextc()
    }
    if len(count_str) == 0 {
      panic("{ must be followed by digit")
    }
    if p.ch == '}' {
      // fixed expansion
      count, _ := strconv.Atoi(count_str)
      panic(fmt.Sprintf("can't yet expand to: %d", count))
    } else if p.ch == ',' {
      panic("can't handle anything but {n}")
    } else {
      panic("unexpected char in {}")
    }
  }
  return start, end
}

/**
 * Match a regexp (defined as [closure]*) from parser, until either: EOF, |, or
 * ) is encountered.
 */
func (p *parser) regexp() (start *instr, end *instr) {
  start = p.instr()
  curr := start

  for {
    if p.ch == -1 || p.ch == '|' || p.ch == ')' {
      break
    }
    s, e := p.closure()
    p.out(curr, s)
    curr = e
  }

  end = p.instr()
  p.out(curr, end)
  return start, end
}

/**
 * Cleanup the given program. Assumes the given input is a flat slice containing
 * no nil instructions. Will not clean up the first instruction, as it is always
 * the canonical entry point for the regexp.
 *
 * Returns a similarly flat slice containing no nil instructions, however the
 * slice may potentially be smaller.
 */
func cleanup(prog []*instr) []*instr {
  // Detect kSplit recursion. We can remove this and convert it to a single path.
  states := NewStateSet(len(prog), len(prog))
  for i := 1; i < len(prog); i++ {
    states.Clear()
    pi := prog[i]
    var fn func(ci *instr) bool
    fn = func(ci *instr) bool {
      if ci != nil && ci.mode == kSplit {
        if states.Put(ci.idx) {
          // NOTE: I'm not sure if this will ever happen. Panic for now. If we're
          // confident this won't happen, we could move the panic to runtime.
          panic("regexp should never loop")
          return true
        }
        if fn(ci.out) {
          ci.out = nil
        }
        if fn(ci.out1) {
          ci.out1 = nil
        }
        return false
      }
      return false
    }
    fn(pi)
  }

  // Iterate through the program, and remove single-instr kSplits.
  // NB: Don't parse the first instr, it will always be single.
  for i := 1; i < len(prog); i++ {
    pi := prog[i]
    if pi.mode == kSplit && (pi.out1 == nil || pi.out == pi.out1) {
      for j := 0; j < len(prog); j++ {
        if prog[j] == nil {
          continue
        }
        pj := prog[j]
        if pj.out == pi {
          pj.out = pi.out
        }
        if pj.out1 == pi {
          pj.out1 = pi.out
        }
      }
      prog[i] = nil
    }
  }

  // We may now have nil gaps: shift everything up.
  last := 0
  for i := 0; i < len(prog); i++ {
    if prog[i] != nil {
      last = i
    } else {
      // find next non-nil, move here
      var found int
      for found = i; found < len(prog); found++ {
        if prog[found] != nil {
          break
        }
      }
      if found == len(prog) {
        break // no more entries
      }

      // move found to i
      prog[i] = prog[found]
      prog[i].idx = i
      prog[found] = nil
      last = i
    }
  }

  return prog[0:last+1]
}

/**
 * Generates a simple straight-forward NFA.
 */
func Parse(src string) (r *sregexp) {
  p := parser{src, -1, 0, make([]*instr, 128), 0, 0}
  begin := p.instr()
  match := p.instr()
  match.mode = kMatch

  p.nextc()
  start, end := p.regexp()

  if p.nextc() != -1 {
    panic("could not consume all of regexp!")
  }

  p.out(begin, start)
  p.out(end, match)

  result := p.prog[0:end.idx+1]
  result = cleanup(result)

  return &sregexp{result, p.altpos}
}
<|endoftext|>


<commit_before>package cluster

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"path/filepath"

	"github.com/rancher/k3s/pkg/bootstrap"
	"github.com/rancher/k3s/pkg/clientaccess"
	"github.com/rancher/k3s/pkg/version"
	"github.com/sirupsen/logrus"
)

func (c *Cluster) Bootstrap(ctx context.Context) error {
	if err := c.assignManagedDriver(ctx); err != nil {
		return err
	}

	runBootstrap, err := c.shouldBootstrapLoad()
	if err != nil {
		return err
	}
	c.shouldBootstrap = runBootstrap

	if runBootstrap {
		if err := c.bootstrap(ctx); err != nil {
			return err
		}
	}

	return nil
}

func (c *Cluster) shouldBootstrapLoad() (bool, error) {
	if c.managedDB != nil {
		c.runtime.HTTPBootstrap = true
		if c.config.JoinURL == "" {
			return false, nil
		}

		token, err := clientaccess.NormalizeAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
		if err != nil {
			return false, err
		}

		info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token)
		if err != nil {
			return false, err
		}
		c.clientAccessInfo = info
	}

	stamp := c.bootstrapStamp()
	if _, err := os.Stat(stamp); err == nil {
		logrus.Info("Cluster bootstrap already complete")
		return false, nil
	}

	if c.managedDB != nil && c.config.Token == "" {
		return false, fmt.Errorf("K3S_TOKEN is required to join a cluster")
	}

	return true, nil
}

func (c *Cluster) bootstrapped() error {
	if err := os.MkdirAll(filepath.Dir(c.bootstrapStamp()), 0700); err != nil {
		return err
	}

	if _, err := os.Stat(c.bootstrapStamp()); err == nil {
		return nil
	}

	f, err := os.Create(c.bootstrapStamp())
	if err != nil {
		return err
	}

	return f.Close()
}

func (c *Cluster) httpBootstrap() error {
	content, err := clientaccess.Get("/v1-"+version.Program+"/server-bootstrap", c.clientAccessInfo)
	if err != nil {
		return err
	}

	return bootstrap.Read(bytes.NewBuffer(content), &c.runtime.ControlRuntimeBootstrap)
}

func (c *Cluster) bootstrap(ctx context.Context) error {
	c.joining = true

	if c.runtime.HTTPBootstrap {
		return c.httpBootstrap()
	}

	if err := c.storageBootstrap(ctx); err != nil {
		return err
	}

	return nil
}

func (c *Cluster) bootstrapStamp() string {
	return filepath.Join(c.config.DataDir, "db/joined-"+keyHash(c.config.Token))
}
<commit_msg>Fix managed etcd cold startup deadlock issue #2249<commit_after>package cluster

import (
	"bytes"
	"context"
	"errors"
	"os"
	"path/filepath"

	"github.com/rancher/k3s/pkg/bootstrap"
	"github.com/rancher/k3s/pkg/clientaccess"
	"github.com/rancher/k3s/pkg/version"
	"github.com/sirupsen/logrus"
)

func (c *Cluster) Bootstrap(ctx context.Context) error {
	if err := c.assignManagedDriver(ctx); err != nil {
		return err
	}

	runBootstrap, err := c.shouldBootstrapLoad(ctx)
	if err != nil {
		return err
	}
	c.shouldBootstrap = runBootstrap

	if runBootstrap {
		if err := c.bootstrap(ctx); err != nil {
			return err
		}
	}

	return nil
}

// shouldBootstrapLoad returns true if we need to load ControlRuntimeBootstrap data again.
// This is controlled by a stamp file on disk that records successful bootstrap using a hash of the join token.
func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, error) {
	// Non-nil managedDB indicates that the database is either initialized, initializing, or joining
	if c.managedDB != nil {
		c.runtime.HTTPBootstrap = true

		isInitialized, err := c.managedDB.IsInitialized(ctx, c.config)
		if err != nil {
			return false, err
		}

		if isInitialized {
			// If the database is initialized we skip bootstrapping; if the user wants to rejoin a
			// cluster they need to delete the database.
			logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName())
			return false, nil
		} else if c.config.JoinURL == "" {
			// Not initialized, not joining - must be initializing (cluster-init)
			logrus.Infof("Managed %s cluster initializing", c.managedDB.EndpointName())
			return false, nil
		} else {
			// Not initialized, but have a Join URL - fail if there's no token; if there is then validate it.
			if c.config.Token == "" {
				return false, errors.New(version.ProgramUpper + "_TOKEN is required to join a cluster")
			}

			token, err := clientaccess.NormalizeAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
			if err != nil {
				return false, err
			}

			info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token)
			if err != nil {
				return false, err
			}

			logrus.Infof("Managed %s cluster not yet initialized", c.managedDB.EndpointName())
			c.clientAccessInfo = info
		}
	}

	// Check the stamp file to see if we have successfully bootstrapped using this token.
	// NOTE: The fact that we use a hash of the token to generate the stamp
	//       means that it is unsafe to use the same token for multiple clusters.
	stamp := c.bootstrapStamp()
	if _, err := os.Stat(stamp); err == nil {
		logrus.Info("Cluster bootstrap already complete")
		return false, nil
	}

	// No errors and no bootstrap stamp, need to bootstrap.
	return true, nil
}

func (c *Cluster) bootstrapped() error {
	if err := os.MkdirAll(filepath.Dir(c.bootstrapStamp()), 0700); err != nil {
		return err
	}

	if _, err := os.Stat(c.bootstrapStamp()); err == nil {
		return nil
	}

	f, err := os.Create(c.bootstrapStamp())
	if err != nil {
		return err
	}

	return f.Close()
}

func (c *Cluster) httpBootstrap() error {
	content, err := clientaccess.Get("/v1-"+version.Program+"/server-bootstrap", c.clientAccessInfo)
	if err != nil {
		return err
	}

	return bootstrap.Read(bytes.NewBuffer(content), &c.runtime.ControlRuntimeBootstrap)
}

func (c *Cluster) bootstrap(ctx context.Context) error {
	c.joining = true

	if c.runtime.HTTPBootstrap {
		return c.httpBootstrap()
	}

	// Bootstrap directly from datastore
	return c.storageBootstrap(ctx)
}

func (c *Cluster) bootstrapStamp() string {
	return filepath.Join(c.config.DataDir, "db/joined-"+keyHash(c.config.Token))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package collectors

import (
	"testing"
	"time"

	"k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type mockDaemonSetStore struct {
	f func() ([]v1beta1.DaemonSet, error)
}

func (ds mockDaemonSetStore) List() (daemonsets []v1beta1.DaemonSet, err error) {
	return ds.f()
}

func TestDaemonSetCollector(t *testing.T) {
	// Fixed metadata on type and help text. We prepend this to every expected
	// output so we only have to modify a single place when doing adjustments.
	const metadata = `
		# HELP kube_daemonset_created Unix creation timestamp
		# TYPE kube_daemonset_created gauge
		# HELP kube_daemonset_metadata_generation Sequence number representing a specific generation of the desired state.
		# TYPE kube_daemonset_metadata_generation gauge
		# HELP kube_daemonset_status_current_number_scheduled The number of nodes running at least one daemon pod and are supposed to.
		# TYPE kube_daemonset_status_current_number_scheduled gauge
		# HELP kube_daemonset_status_number_misscheduled The number of nodes running a daemon pod but are not supposed to.
		# TYPE kube_daemonset_status_number_misscheduled gauge
		# HELP kube_daemonset_status_desired_number_scheduled The number of nodes that should be running the daemon pod.
		# TYPE kube_daemonset_status_desired_number_scheduled gauge
		# HELP kube_daemonset_status_number_available The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available
		# TYPE kube_daemonset_status_number_available gauge
		# HELP kube_daemonset_status_number_ready The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready.
		# TYPE kube_daemonset_status_number_ready gauge
		# HELP kube_daemonset_status_number_unavailable The number of nodes that should be running the daemon pod and have none of the daemon pod running and available
		# TYPE kube_daemonset_status_number_unavailable gauge
		# HELP kube_daemonset_updated_number_scheduled The total number of nodes that are running updated daemon pod
		# TYPE kube_daemonset_updated_number_scheduled gauge
		# HELP kube_daemonset_labels Kubernetes labels converted to Prometheus labels.
		# TYPE kube_daemonset_labels gauge
`
	cases := []struct {
		dss  []v1beta1.DaemonSet
		want string
	}{
		{
			dss: []v1beta1.DaemonSet{
				{
					ObjectMeta: metav1.ObjectMeta{
						Name:       "ds1",
						Namespace:  "ns1",
						Labels: map[string]string{
							"app": "example1",
						},
						Generation: 21,
					},
					Status: v1beta1.DaemonSetStatus{
						CurrentNumberScheduled: 15,
						NumberMisscheduled:     10,
						DesiredNumberScheduled: 5,
						NumberReady:            5,
					},
				}, {
					ObjectMeta: metav1.ObjectMeta{
						Name:              "ds2",
						CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
						Namespace:         "ns2",
						Labels: map[string]string{
							"app": "example2",
						},
						Generation:        14,
					},
					Status: v1beta1.DaemonSetStatus{
						CurrentNumberScheduled: 10,
						NumberMisscheduled:     5,
						DesiredNumberScheduled: 0,
						NumberReady:            0,
					},
				}, {
					ObjectMeta: metav1.ObjectMeta{
						Name:              "ds3",
						CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
						Namespace:         "ns3",
						Labels: map[string]string{
							"app": "example3",
						},
						Generation:        15,
					},
					Status: v1beta1.DaemonSetStatus{
						CurrentNumberScheduled: 10,
						NumberMisscheduled:     5,
						DesiredNumberScheduled: 15,
						NumberReady:            5,
						NumberAvailable:        5,
						NumberUnavailable:      5,
						UpdatedNumberScheduled: 5,
					},
				},
			},
			want: metadata + `
				kube_daemonset_created{daemonset="ds2",namespace="ns2"} 1.5e+09
				kube_daemonset_created{daemonset="ds3",namespace="ns3"} 1.5e+09
				kube_daemonset_metadata_generation{namespace="ns1",daemonset="ds1"} 21
				kube_daemonset_metadata_generation{namespace="ns2",daemonset="ds2"} 14
				kube_daemonset_metadata_generation{namespace="ns3",daemonset="ds3"} 15
				kube_daemonset_status_current_number_scheduled{namespace="ns1",daemonset="ds1"} 15
				kube_daemonset_status_current_number_scheduled{namespace="ns2",daemonset="ds2"} 10
				kube_daemonset_status_current_number_scheduled{namespace="ns3",daemonset="ds3"} 10
				kube_daemonset_status_desired_number_scheduled{namespace="ns1",daemonset="ds1"} 5
				kube_daemonset_status_desired_number_scheduled{namespace="ns2",daemonset="ds2"} 0
				kube_daemonset_status_desired_number_scheduled{namespace="ns3",daemonset="ds3"} 15
				kube_daemonset_status_number_available{daemonset="ds1",namespace="ns1"} 0
				kube_daemonset_status_number_available{daemonset="ds2",namespace="ns2"} 0
				kube_daemonset_status_number_available{daemonset="ds3",namespace="ns3"} 5
				kube_daemonset_status_number_misscheduled{namespace="ns1",daemonset="ds1"} 10
				kube_daemonset_status_number_misscheduled{namespace="ns2",daemonset="ds2"} 5
				kube_daemonset_status_number_misscheduled{namespace="ns3",daemonset="ds3"} 5
				kube_daemonset_status_number_ready{namespace="ns1",daemonset="ds1"} 5
				kube_daemonset_status_number_ready{namespace="ns2",daemonset="ds2"} 0
				kube_daemonset_status_number_ready{namespace="ns3",daemonset="ds3"} 5
				kube_daemonset_status_number_unavailable{daemonset="ds1",namespace="ns1"} 0
				kube_daemonset_status_number_unavailable{daemonset="ds2",namespace="ns2"} 0
				kube_daemonset_status_number_unavailable{daemonset="ds3",namespace="ns3"} 5
				kube_daemonset_updated_number_scheduled{daemonset="ds1",namespace="ns1"} 0
				kube_daemonset_updated_number_scheduled{daemonset="ds2",namespace="ns2"} 0
				kube_daemonset_updated_number_scheduled{daemonset="ds3",namespace="ns3"} 5
				kube_daemonset_labels{label_app="example1",namespace="ns1",daemonset="ds1"} 1
				kube_daemonset_labels{label_app="example2",namespace="ns2",daemonset="ds2"} 1
				kube_daemonset_labels{label_app="example3",namespace="ns3",daemonset="ds3"} 1
			`,
		},
	}
	for _, c := range cases {
		dc := &daemonsetCollector{
			store: mockDaemonSetStore{
				f: func() ([]v1beta1.DaemonSet, error) { return c.dss, nil },
			},
		}
		if err := gatherAndCompare(dc, c.want, nil); err != nil {
			t.Errorf("unexpected collecting result:\n%s", err)
		}
	}
}
<commit_msg>formatting<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package collectors

import (
	"testing"
	"time"

	"k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type mockDaemonSetStore struct {
	f func() ([]v1beta1.DaemonSet, error)
}

func (ds mockDaemonSetStore) List() (daemonsets []v1beta1.DaemonSet, err error) {
	return ds.f()
}

func TestDaemonSetCollector(t *testing.T) {
	// Fixed metadata on type and help text. We prepend this to every expected
	// output so we only have to modify a single place when doing adjustments.
	const metadata = `
		# HELP kube_daemonset_created Unix creation timestamp
		# TYPE kube_daemonset_created gauge
		# HELP kube_daemonset_metadata_generation Sequence number representing a specific generation of the desired state.
		# TYPE kube_daemonset_metadata_generation gauge
		# HELP kube_daemonset_status_current_number_scheduled The number of nodes running at least one daemon pod and are supposed to.
		# TYPE kube_daemonset_status_current_number_scheduled gauge
		# HELP kube_daemonset_status_number_misscheduled The number of nodes running a daemon pod but are not supposed to.
		# TYPE kube_daemonset_status_number_misscheduled gauge
		# HELP kube_daemonset_status_desired_number_scheduled The number of nodes that should be running the daemon pod.
		# TYPE kube_daemonset_status_desired_number_scheduled gauge
		# HELP kube_daemonset_status_number_available The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available
		# TYPE kube_daemonset_status_number_available gauge
		# HELP kube_daemonset_status_number_ready The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready.
		# TYPE kube_daemonset_status_number_ready gauge
		# HELP kube_daemonset_status_number_unavailable The number of nodes that should be running the daemon pod and have none of the daemon pod running and available
		# TYPE kube_daemonset_status_number_unavailable gauge
		# HELP kube_daemonset_updated_number_scheduled The total number of nodes that are running updated daemon pod
		# TYPE kube_daemonset_updated_number_scheduled gauge
		# HELP kube_daemonset_labels Kubernetes labels converted to Prometheus labels.
		# TYPE kube_daemonset_labels gauge
`
	cases := []struct {
		dss  []v1beta1.DaemonSet
		want string
	}{
		{
			dss: []v1beta1.DaemonSet{
				{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "ds1",
						Namespace: "ns1",
						Labels: map[string]string{
							"app": "example1",
						},
						Generation: 21,
					},
					Status: v1beta1.DaemonSetStatus{
						CurrentNumberScheduled: 15,
						NumberMisscheduled:     10,
						DesiredNumberScheduled: 5,
						NumberReady:            5,
					},
				}, {
					ObjectMeta: metav1.ObjectMeta{
						Name:              "ds2",
						CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
						Namespace:         "ns2",
						Labels: map[string]string{
							"app": "example2",
						},
						Generation: 14,
					},
					Status: v1beta1.DaemonSetStatus{
						CurrentNumberScheduled: 10,
						NumberMisscheduled:     5,
						DesiredNumberScheduled: 0,
						NumberReady:            0,
					},
				}, {
					ObjectMeta: metav1.ObjectMeta{
						Name:              "ds3",
						CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
						Namespace:         "ns3",
						Labels: map[string]string{
							"app": "example3",
						},
						Generation: 15,
					},
					Status: v1beta1.DaemonSetStatus{
						CurrentNumberScheduled: 10,
						NumberMisscheduled:     5,
						DesiredNumberScheduled: 15,
						NumberReady:            5,
						NumberAvailable:        5,
						NumberUnavailable:      5,
						UpdatedNumberScheduled: 5,
					},
				},
			},
			want: metadata + `
				kube_daemonset_created{daemonset="ds2",namespace="ns2"} 1.5e+09
				kube_daemonset_created{daemonset="ds3",namespace="ns3"} 1.5e+09
				kube_daemonset_metadata_generation{namespace="ns1",daemonset="ds1"} 21
				kube_daemonset_metadata_generation{namespace="ns2",daemonset="ds2"} 14
				kube_daemonset_metadata_generation{namespace="ns3",daemonset="ds3"} 15
				kube_daemonset_status_current_number_scheduled{namespace="ns1",daemonset="ds1"} 15
				kube_daemonset_status_current_number_scheduled{namespace="ns2",daemonset="ds2"} 10
				kube_daemonset_status_current_number_scheduled{namespace="ns3",daemonset="ds3"} 10
				kube_daemonset_status_desired_number_scheduled{namespace="ns1",daemonset="ds1"} 5
				kube_daemonset_status_desired_number_scheduled{namespace="ns2",daemonset="ds2"} 0
				kube_daemonset_status_desired_number_scheduled{namespace="ns3",daemonset="ds3"} 15
				kube_daemonset_status_number_available{daemonset="ds1",namespace="ns1"} 0
				kube_daemonset_status_number_available{daemonset="ds2",namespace="ns2"} 0
				kube_daemonset_status_number_available{daemonset="ds3",namespace="ns3"} 5
				kube_daemonset_status_number_misscheduled{namespace="ns1",daemonset="ds1"} 10
				kube_daemonset_status_number_misscheduled{namespace="ns2",daemonset="ds2"} 5
				kube_daemonset_status_number_misscheduled{namespace="ns3",daemonset="ds3"} 5
				kube_daemonset_status_number_ready{namespace="ns1",daemonset="ds1"} 5
				kube_daemonset_status_number_ready{namespace="ns2",daemonset="ds2"} 0
				kube_daemonset_status_number_ready{namespace="ns3",daemonset="ds3"} 5
				kube_daemonset_status_number_unavailable{daemonset="ds1",namespace="ns1"} 0
				kube_daemonset_status_number_unavailable{daemonset="ds2",namespace="ns2"} 0
				kube_daemonset_status_number_unavailable{daemonset="ds3",namespace="ns3"} 5
				kube_daemonset_updated_number_scheduled{daemonset="ds1",namespace="ns1"} 0
				kube_daemonset_updated_number_scheduled{daemonset="ds2",namespace="ns2"} 0
				kube_daemonset_updated_number_scheduled{daemonset="ds3",namespace="ns3"} 5
				kube_daemonset_labels{label_app="example1",namespace="ns1",daemonset="ds1"} 1
				kube_daemonset_labels{label_app="example2",namespace="ns2",daemonset="ds2"} 1
				kube_daemonset_labels{label_app="example3",namespace="ns3",daemonset="ds3"} 1
			`,
		},
	}
	for _, c := range cases {
		dc := &daemonsetCollector{
			store: mockDaemonSetStore{
				f: func() ([]v1beta1.DaemonSet, error) { return c.dss, nil },
			},
		}
		if err := gatherAndCompare(dc, c.want, nil); err != nil {
			t.Errorf("unexpected collecting result:\n%s", err)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package syncers

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	"google.golang.org/api/compute/v0.beta"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	negtypes "k8s.io/ingress-gce/pkg/neg/types"
	"k8s.io/ingress-gce/pkg/utils"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
)

const (
	MAX_NETWORK_ENDPOINTS_PER_BATCH = 500
	// For each NEG, only retries 15 times to process it.
	// This is a convention in kube-controller-manager.
	maxRetries    = 15
	minRetryDelay = 5 * time.Second
	maxRetryDelay = 600 * time.Second
	separator     = "||"
)

// NegSyncerKey includes information to uniquely identify a NEG
type NegSyncerKey struct {
	Namespace  string
	Name       string
	Port       int32
	TargetPort string
}

func (key NegSyncerKey) String() string {
	return fmt.Sprintf("%s/%s-%v/%s", key.Namespace, key.Name, key.Port, key.TargetPort)
}

// encodeEndpoint encodes ip and instance into a single string
func encodeEndpoint(ip, instance, port string) string {
	return strings.Join([]string{ip, instance, port}, separator)
}

// decodeEndpoint decodes ip and instance from an encoded string
func decodeEndpoint(str string) (string, string, string) {
	strs := strings.Split(str, separator)
	return strs[0], strs[1], strs[2]
}

// calculateDifference determines what endpoints needs to be added and removed in order to move current state to target state.
func calculateDifference(targetMap, currentMap map[string]sets.String) (map[string]sets.String, map[string]sets.String) {
	addSet := map[string]sets.String{}
	removeSet := map[string]sets.String{}
	for zone, endpointSet := range targetMap {
		diff := endpointSet.Difference(currentMap[zone])
		if len(diff) > 0 {
			addSet[zone] = diff
		}
	}

	for zone, endpointSet := range currentMap {
		diff := endpointSet.Difference(targetMap[zone])
		if len(diff) > 0 {
			removeSet[zone] = diff
		}
	}
	return addSet, removeSet
}

// getService retrieves service object from serviceLister based on the input Namespace and Name
func getService(serviceLister cache.Indexer, namespace, name string) *apiv1.Service {
	if serviceLister == nil {
		return nil
	}
	service, exists, err := serviceLister.GetByKey(utils.ServiceKeyFunc(namespace, name))
	if exists && err == nil {
		return service.(*apiv1.Service)
	}
	if err != nil {
		glog.Errorf("Failed to retrieve service %s/%s from store: %v", namespace, name, err)
	}
	return nil
}

// ensureNetworkEndpointGroup ensures corresponding NEG is configured correctly in the specified zone.
func ensureNetworkEndpointGroup(svcNamespace, svcName, negName, zone, negServicePortName string, cloud negtypes.NetworkEndpointGroupCloud, serviceLister cache.Indexer, recorder record.EventRecorder) error {
	neg, err := cloud.GetNetworkEndpointGroup(negName, zone)
	if err != nil {
		// Most likely to be caused by non-existed NEG
		glog.V(4).Infof("Error while retriving %q in zone %q: %v", negName, zone, err)
	}

	needToCreate := false
	if neg == nil {
		needToCreate = true
	} else if !utils.EqualResourceIDs(neg.LoadBalancer.Network, cloud.NetworkURL()) ||
		!utils.EqualResourceIDs(neg.LoadBalancer.Subnetwork, cloud.SubnetworkURL()) {
		needToCreate = true
		glog.V(2).Infof("NEG %q in %q does not match network and subnetwork of the cluster. Deleting NEG.", negName, zone)
		err = cloud.DeleteNetworkEndpointGroup(negName, zone)
		if err != nil {
			return err
		} else {
			if recorder != nil && serviceLister != nil {
				if svc := getService(serviceLister, svcNamespace, svcName); svc != nil {
					recorder.Eventf(svc, apiv1.EventTypeNormal, "Delete", "Deleted NEG %q for %s in %q.", negName, negServicePortName, zone)
				}
			}
		}
	}

	if needToCreate {
		glog.V(2).Infof("Creating NEG %q for %s in %q.", negName, negServicePortName, zone)
		err = cloud.CreateNetworkEndpointGroup(&compute.NetworkEndpointGroup{
			Name:                negName,
			NetworkEndpointType: gce.NEGIPPortNetworkEndpointType,
			LoadBalancer: &compute.NetworkEndpointGroupLbNetworkEndpointGroup{
				Network:    cloud.NetworkURL(),
				Subnetwork: cloud.SubnetworkURL(),
			},
		}, zone)
		if err != nil {
			return err
		} else {
			if recorder != nil && serviceLister != nil {
				if svc := getService(serviceLister, svcNamespace, svcName); svc != nil {
					recorder.Eventf(svc, apiv1.EventTypeNormal, "Create", "Created NEG %q for %s in %q.", negName, negServicePortName, zone)
				}
			}
		}
	}
	return nil
}

// toZoneNetworkEndpointMap translates addresses in endpoints object into zone and endpoints map
func toZoneNetworkEndpointMap(endpoints *apiv1.Endpoints, zoneGetter negtypes.ZoneGetter, targetPort string) (map[string]sets.String, error) {
	zoneNetworkEndpointMap := map[string]sets.String{}
	if endpoints == nil {
		glog.Errorf("Endpoint object is nil")
		return zoneNetworkEndpointMap, nil
	}
	targetPortNum, _ := strconv.Atoi(targetPort)
	for _, subset := range endpoints.Subsets {
		matchPort := ""
		// service spec allows target Port to be a named Port.
		// support both explicit Port and named Port.
		for _, port := range subset.Ports {
			if targetPortNum != 0 {
				// TargetPort is int
				if int(port.Port) == targetPortNum {
					matchPort = targetPort
				}
			} else {
				// TargetPort is string
				if port.Name == targetPort {
					matchPort = strconv.Itoa(int(port.Port))
				}
			}
			if len(matchPort) > 0 {
				break
			}
		}

		// subset does not contain target Port
		if len(matchPort) == 0 {
			continue
		}
		for _, address := range subset.Addresses {
			if address.NodeName == nil {
				glog.V(2).Infof("Endpoint %q in Endpoints %s/%s does not have an associated node. Skipping", address.IP, endpoints.Namespace, endpoints.Name)
				continue
			}
			zone, err := zoneGetter.GetZoneForNode(*address.NodeName)
			if err != nil {
				return nil, err
			}
			if zoneNetworkEndpointMap[zone] == nil {
				zoneNetworkEndpointMap[zone] = sets.String{}
			}
			zoneNetworkEndpointMap[zone].Insert(encodeEndpoint(address.IP, *address.NodeName, matchPort))
		}
	}
	return zoneNetworkEndpointMap, nil
}

// retrieveExistingZoneNetworkEndpointMap lists existing network endpoints in the neg and return the zone and endpoints map
func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter negtypes.ZoneGetter, cloud negtypes.NetworkEndpointGroupCloud) (map[string]sets.String, error) {
	zones, err := zoneGetter.ListZones()
	if err != nil {
		return nil, err
	}

	zoneNetworkEndpointMap := map[string]sets.String{}
	for _, zone := range zones {
		zoneNetworkEndpointMap[zone] = sets.String{}
		networkEndpointsWithHealthStatus, err := cloud.ListNetworkEndpoints(negName, zone, false)
		if err != nil {
			return nil, err
		}
		for _, ne := range networkEndpointsWithHealthStatus {
			zoneNetworkEndpointMap[zone].Insert(encodeEndpoint(ne.NetworkEndpoint.IpAddress, ne.NetworkEndpoint.Instance, strconv.FormatInt(ne.NetworkEndpoint.Port, 10)))
		}
	}
	return zoneNetworkEndpointMap, nil
}

// makeEndpointBatch return a batch of endpoint from the input and remove the endpoints from input set
// The return map has the encoded endpoint as key and GCE network endpoint object as value
func makeEndpointBatch(endpoints sets.String) (map[string]*compute.NetworkEndpoint, error) {
	endpointBatch := map[string]*compute.NetworkEndpoint{}

	for i := 0; i < MAX_NETWORK_ENDPOINTS_PER_BATCH; i++ {
		encodedEndpoint, ok := endpoints.PopAny()
		if !ok {
			break
		}

		ip, instance, port := decodeEndpoint(encodedEndpoint)
		portNum, err := strconv.Atoi(port)
		if err != nil {
			return nil, fmt.Errorf("Failed to decode endpoint %q: %v", encodedEndpoint, err)
		}

		endpointBatch[encodedEndpoint] = &compute.NetworkEndpoint{
			Instance:  instance,
			IpAddress: ip,
			Port:      int64(portNum),
		}
	}
	return endpointBatch, nil
}
<commit_msg>switch to use the beta+ NEG API<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package syncers

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	"google.golang.org/api/compute/v0.beta"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	negtypes "k8s.io/ingress-gce/pkg/neg/types"
	"k8s.io/ingress-gce/pkg/utils"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
)

const (
	MAX_NETWORK_ENDPOINTS_PER_BATCH = 500
	// For each NEG, only retries 15 times to process it.
	// This is a convention in kube-controller-manager.
	maxRetries    = 15
	minRetryDelay = 5 * time.Second
	maxRetryDelay = 600 * time.Second
	separator     = "||"
)

// NegSyncerKey includes information to uniquely identify a NEG
type NegSyncerKey struct {
	Namespace  string
	Name       string
	Port       int32
	TargetPort string
}

func (key NegSyncerKey) String() string {
	return fmt.Sprintf("%s/%s-%v/%s", key.Namespace, key.Name, key.Port, key.TargetPort)
}

// encodeEndpoint encodes ip and instance into a single string
func encodeEndpoint(ip, instance, port string) string {
	return strings.Join([]string{ip, instance, port}, separator)
}

// decodeEndpoint decodes ip and instance from an encoded string
func decodeEndpoint(str string) (string, string, string) {
	strs := strings.Split(str, separator)
	return strs[0], strs[1], strs[2]
}

// calculateDifference determines what endpoints needs to be added and removed in order to move current state to target state.
func calculateDifference(targetMap, currentMap map[string]sets.String) (map[string]sets.String, map[string]sets.String) {
	addSet := map[string]sets.String{}
	removeSet := map[string]sets.String{}
	for zone, endpointSet := range targetMap {
		diff := endpointSet.Difference(currentMap[zone])
		if len(diff) > 0 {
			addSet[zone] = diff
		}
	}

	for zone, endpointSet := range currentMap {
		diff := endpointSet.Difference(targetMap[zone])
		if len(diff) > 0 {
			removeSet[zone] = diff
		}
	}
	return addSet, removeSet
}

// getService retrieves service object from serviceLister based on the input Namespace and Name
func getService(serviceLister cache.Indexer, namespace, name string) *apiv1.Service {
	if serviceLister == nil {
		return nil
	}
	service, exists, err := serviceLister.GetByKey(utils.ServiceKeyFunc(namespace, name))
	if exists && err == nil {
		return service.(*apiv1.Service)
	}
	if err != nil {
		glog.Errorf("Failed to retrieve service %s/%s from store: %v", namespace, name, err)
	}
	return nil
}

// ensureNetworkEndpointGroup ensures corresponding NEG is configured correctly in the specified zone.
func ensureNetworkEndpointGroup(svcNamespace, svcName, negName, zone, negServicePortName string, cloud negtypes.NetworkEndpointGroupCloud, serviceLister cache.Indexer, recorder record.EventRecorder) error {
	neg, err := cloud.GetNetworkEndpointGroup(negName, zone)
	if err != nil {
		// Most likely to be caused by non-existed NEG
		glog.V(4).Infof("Error while retriving %q in zone %q: %v", negName, zone, err)
	}

	needToCreate := false
	if neg == nil {
		needToCreate = true
	} else if !utils.EqualResourceIDs(neg.Network, cloud.NetworkURL()) ||
		!utils.EqualResourceIDs(neg.Subnetwork, cloud.SubnetworkURL()) {
		needToCreate = true
		glog.V(2).Infof("NEG %q in %q does not match network and subnetwork of the cluster. Deleting NEG.", negName, zone)
		err = cloud.DeleteNetworkEndpointGroup(negName, zone)
		if err != nil {
			return err
		} else {
			if recorder != nil && serviceLister != nil {
				if svc := getService(serviceLister, svcNamespace, svcName); svc != nil {
					recorder.Eventf(svc, apiv1.EventTypeNormal, "Delete", "Deleted NEG %q for %s in %q.", negName, negServicePortName, zone)
				}
			}
		}
	}

	if needToCreate {
		glog.V(2).Infof("Creating NEG %q for %s in %q.", negName, negServicePortName, zone)
		err = cloud.CreateNetworkEndpointGroup(&compute.NetworkEndpointGroup{
			Name:                negName,
			NetworkEndpointType: gce.NEGIPPortNetworkEndpointType,
			Network:             cloud.NetworkURL(),
			Subnetwork:          cloud.SubnetworkURL(),
		}, zone)
		if err != nil {
			return err
		} else {
			if recorder != nil && serviceLister != nil {
				if svc := getService(serviceLister, svcNamespace, svcName); svc != nil {
					recorder.Eventf(svc, apiv1.EventTypeNormal, "Create", "Created NEG %q for %s in %q.", negName, negServicePortName, zone)
				}
			}
		}
	}
	return nil
}

// toZoneNetworkEndpointMap translates addresses in endpoints object into zone and endpoints map
func toZoneNetworkEndpointMap(endpoints *apiv1.Endpoints, zoneGetter negtypes.ZoneGetter, targetPort string) (map[string]sets.String, error) {
	zoneNetworkEndpointMap := map[string]sets.String{}
	if endpoints == nil {
		glog.Errorf("Endpoint object is nil")
		return zoneNetworkEndpointMap, nil
	}
	targetPortNum, _ := strconv.Atoi(targetPort)
	for _, subset := range endpoints.Subsets {
		matchPort := ""
		// service spec allows target Port to be a named Port.
		// support both explicit Port and named Port.
		for _, port := range subset.Ports {
			if targetPortNum != 0 {
				// TargetPort is int
				if int(port.Port) == targetPortNum {
					matchPort = targetPort
				}
			} else {
				// TargetPort is string
				if port.Name == targetPort {
					matchPort = strconv.Itoa(int(port.Port))
				}
			}
			if len(matchPort) > 0 {
				break
			}
		}

		// subset does not contain target Port
		if len(matchPort) == 0 {
			continue
		}
		for _, address := range subset.Addresses {
			if address.NodeName == nil {
				glog.V(2).Infof("Endpoint %q in Endpoints %s/%s does not have an associated node. Skipping", address.IP, endpoints.Namespace, endpoints.Name)
				continue
			}
			zone, err := zoneGetter.GetZoneForNode(*address.NodeName)
			if err != nil {
				return nil, err
			}
			if zoneNetworkEndpointMap[zone] == nil {
				zoneNetworkEndpointMap[zone] = sets.String{}
			}
			zoneNetworkEndpointMap[zone].Insert(encodeEndpoint(address.IP, *address.NodeName, matchPort))
		}
	}
	return zoneNetworkEndpointMap, nil
}

// retrieveExistingZoneNetworkEndpointMap lists existing network endpoints in the neg and return the zone and endpoints map
func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter negtypes.ZoneGetter, cloud negtypes.NetworkEndpointGroupCloud) (map[string]sets.String, error) {
	zones, err := zoneGetter.ListZones()
	if err != nil {
		return nil, err
	}

	zoneNetworkEndpointMap := map[string]sets.String{}
	for _, zone := range zones {
		zoneNetworkEndpointMap[zone] = sets.String{}
		networkEndpointsWithHealthStatus, err := cloud.ListNetworkEndpoints(negName, zone, false)
		if err != nil {
			return nil, err
		}
		for _, ne := range networkEndpointsWithHealthStatus {
			zoneNetworkEndpointMap[zone].Insert(encodeEndpoint(ne.NetworkEndpoint.IpAddress, ne.NetworkEndpoint.Instance, strconv.FormatInt(ne.NetworkEndpoint.Port, 10)))
		}
	}
	return zoneNetworkEndpointMap, nil
}

// makeEndpointBatch return a batch of endpoint from the input and remove the endpoints from input set
// The return map has the encoded endpoint as key and GCE network endpoint object as value
func makeEndpointBatch(endpoints sets.String) (map[string]*compute.NetworkEndpoint, error) {
	endpointBatch := map[string]*compute.NetworkEndpoint{}

	for i := 0; i < MAX_NETWORK_ENDPOINTS_PER_BATCH; i++ {
		encodedEndpoint, ok := endpoints.PopAny()
		if !ok {
			break
		}

		ip, instance, port := decodeEndpoint(encodedEndpoint)
		portNum, err := strconv.Atoi(port)
		if err != nil {
			return nil, fmt.Errorf("Failed to decode endpoint %q: %v", encodedEndpoint, err)
		}

		endpointBatch[encodedEndpoint] = &compute.NetworkEndpoint{
			Instance:  instance,
			IpAddress: ip,
			Port:      int64(portNum),
		}
	}
	return endpointBatch, nil
}
<|endoftext|>


<commit_before>package testutil

import (
	"bytes"
	"code.google.com/p/go.net/context"
	"github.com/fritzpay/paymentd/pkg/config"
	"github.com/fritzpay/paymentd/pkg/service"
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/inconshreveable/log15.v2"
	"net/http"
)

const (
	logChanBufferSize = 32
)

// WithContext is a decorator for GoConvey based tests
//
// It will inject a service context and a log channel, where log messages can be read from
func WithContext(f func(*service.Context, <-chan *log15.Record)) func() {
	return func() {
		logChan := make(chan *log15.Record, logChanBufferSize)
		log := log15.New()
		log.SetHandler(log15.ChannelHandler(logChan))

		ctx, err := service.NewContext(context.Background(), config.DefaultConfig(), log)
		So(err, ShouldBeNil)

		ctx.Keychain().AddBinKey([]byte("test"))
		So(ctx.Keychain().KeyCount(), ShouldEqual, 1)

		f(ctx, logChan)

		Reset(func() {
			close(logChan)
		})
	}
}

// ResponseWriter is a mock http.ResponseWriter which can be used to inspect
// HTTP handler responses
type ResponseWriter struct {
	Buf           bytes.Buffer
	H             http.Header
	HeaderWritten bool
	StatusCode    int
}

// NewResponseWriter creates a response writer to capture http handler responses
func NewResponseWriter() *ResponseWriter {
	return &ResponseWriter{
		H: http.Header(make(map[string][]string)),
	}
}

// Header implementing the http.ResponseWriter
func (r *ResponseWriter) Header() http.Header {
	return r.H
}

// Write implementing the http.ResponseWriter, io.Writer
func (r *ResponseWriter) Write(p []byte) (int, error) {
	return (&(r.Buf)).Write(p)
}

// WriteHeader implementing the http.ResponseWriter
func (r *ResponseWriter) WriteHeader(statusCode int) {
	r.HeaderWritten = true
	r.StatusCode = statusCode
}
<commit_msg>test response writer, set http status ok when write is called with no WriteHeader<commit_after>package testutil

import (
	"bytes"
	"code.google.com/p/go.net/context"
	"github.com/fritzpay/paymentd/pkg/config"
	"github.com/fritzpay/paymentd/pkg/service"
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/inconshreveable/log15.v2"
	"net/http"
)

const (
	logChanBufferSize = 32
)

// WithContext is a decorator for GoConvey based tests
//
// It will inject a service context and a log channel, where log messages can be read from
func WithContext(f func(*service.Context, <-chan *log15.Record)) func() {
	return func() {
		logChan := make(chan *log15.Record, logChanBufferSize)
		log := log15.New()
		log.SetHandler(log15.ChannelHandler(logChan))

		ctx, err := service.NewContext(context.Background(), config.DefaultConfig(), log)
		So(err, ShouldBeNil)

		ctx.Keychain().AddBinKey([]byte("test"))
		So(ctx.Keychain().KeyCount(), ShouldEqual, 1)

		f(ctx, logChan)

		Reset(func() {
			close(logChan)
		})
	}
}

// ResponseWriter is a mock http.ResponseWriter which can be used to inspect
// HTTP handler responses
type ResponseWriter struct {
	Buf           bytes.Buffer
	H             http.Header
	HeaderWritten bool
	StatusCode    int
}

// NewResponseWriter creates a response writer to capture http handler responses
func NewResponseWriter() *ResponseWriter {
	return &ResponseWriter{
		H: http.Header(make(map[string][]string)),
	}
}

// Header implementing the http.ResponseWriter
func (r *ResponseWriter) Header() http.Header {
	return r.H
}

// Write implementing the http.ResponseWriter, io.Writer
func (r *ResponseWriter) Write(p []byte) (int, error) {
	if !r.HeaderWritten {
		r.HeaderWritten = true
	}
	if r.StatusCode == 0 {
		r.StatusCode = http.StatusOK
	}
	return (&(r.Buf)).Write(p)
}

// WriteHeader implementing the http.ResponseWriter
func (r *ResponseWriter) WriteHeader(statusCode int) {
	r.HeaderWritten = true
	r.StatusCode = statusCode
}
<|endoftext|>