<commit_before>package config

import (
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"time"

	"github.com/getlantern/appdir"
	"github.com/getlantern/fronted"
	"github.com/getlantern/geolookup"
	"github.com/getlantern/golog"
	"github.com/getlantern/proxiedsites"
	"github.com/getlantern/waitforserver"
	"github.com/getlantern/yaml"
	"github.com/getlantern/yamlconf"

	"github.com/getlantern/flashlight/client"
	"github.com/getlantern/flashlight/globals"
	"github.com/getlantern/flashlight/server"
	"github.com/getlantern/flashlight/statreporter"
	"github.com/getlantern/flashlight/util"
)

const (
	CloudConfigPollInterval = 1 * time.Minute

	cloudflare  = "cloudflare"
	etag        = "ETag"
	ifNoneMatch = "If-None-Match"
)

var (
	log                 = golog.LoggerFor("flashlight.config")
	m                   *yamlconf.Manager
	lastCloudConfigETag = ""
)

type Config struct {
	Version       int
	CloudConfig   string
	CloudConfigCA string
	Addr          string
	Role          string
	InstanceId    string
	Country       string
	CpuProfile    string
	MemProfile    string
	UIAddr        string // UI HTTP server address
	AutoReport    *bool  // Report anonymous usage to GA
	Stats         *statreporter.Config
	Server        *server.ServerConfig
	Client        *client.ClientConfig
	ProxiedSites  *proxiedsites.Config // List of proxied site domains that get routed through Lantern rather than accessed directly
	TrustedCAs    []*CA
}

// CA represents a certificate authority
type CA struct {
	CommonName string
	Cert       string // PEM-encoded
}

// Init initializes the configuration system.
func Init() (*Config, error) {
	configPath, err := InConfigDir("lantern.yaml")
	if err != nil {
		return nil, err
	}
	m = &yamlconf.Manager{
		FilePath:         configPath,
		FilePollInterval: 1 * time.Second,
		ConfigServerAddr: *configaddr,
		EmptyConfig: func() yamlconf.Config {
			return &Config{}
		},
		OneTimeSetup: func(ycfg yamlconf.Config) error {
			cfg := ycfg.(*Config)
			return cfg.applyFlags()
		},
		CustomPoll: func(currentCfg yamlconf.Config) (mutate func(yamlconf.Config) error, waitTime time.Duration, err error) {
			// By default, do nothing
			mutate = func(ycfg yamlconf.Config) error {
				// do nothing
				return nil
			}
			cfg := currentCfg.(*Config)
			waitTime = cfg.cloudPollSleepTime()
			if cfg.CloudConfig == "" {
				// Config doesn't have a CloudConfig, just ignore
				return
			}

			var bytes []byte
			bytes, err = cfg.fetchCloudConfig()
			if err == nil && bytes != nil {
				mutate = func(ycfg yamlconf.Config) error {
					log.Debugf("Merging cloud configuration")
					cfg := ycfg.(*Config)
					return cfg.updateFrom(bytes)
				}
			}
			return
		},
	}
	initial, err := m.Start()
	var cfg *Config
	if err == nil {
		cfg = initial.(*Config)
		err = updateGlobals(cfg)
		if err != nil {
			return nil, err
		}
	}
	return cfg, err
}

// Run runs the configuration system.
func Run(updateHandler func(updated *Config)) error {
	for {
		next := m.Next()
		nextCfg := next.(*Config)
		err := updateGlobals(nextCfg)
		if err != nil {
			return err
		}
		updateHandler(nextCfg)
	}
}

func updateGlobals(cfg *Config) error {
	globals.InstanceId = cfg.InstanceId
	loc := &geolookup.City{}
	loc.Country.IsoCode = cfg.Country
	globals.SetLocation(loc)
	err := globals.SetTrustedCAs(cfg.TrustedCACerts())
	if err != nil {
		return fmt.Errorf("Unable to configure trusted CAs: %s", err)
	}
	return nil
}

// Update updates the configuration using the given mutator function.
func Update(mutate func(cfg *Config) error) error {
	return m.Update(func(ycfg yamlconf.Config) error {
		return mutate(ycfg.(*Config))
	})
}

// InConfigDir returns the path to the given filename inside of the configdir.
func InConfigDir(filename string) (string, error) {
	cdir := *configdir
	if cdir == "" {
		cdir = appdir.General("Lantern")
	}
	log.Debugf("Placing configuration in %v", cdir)
	if _, err := os.Stat(cdir); err != nil {
		if os.IsNotExist(err) {
			// Create config dir
			if err := os.MkdirAll(cdir, 0755); err != nil {
				return "", fmt.Errorf("Unable to create configdir at %s: %s", cdir, err)
			}
		}
	}
	return filepath.Join(cdir, filename), nil
}

// TrustedCACerts returns a slice of PEM-encoded certs for the trusted CAs
func (cfg *Config) TrustedCACerts() []string {
	certs := make([]string, 0, len(cfg.TrustedCAs))
	for _, ca := range cfg.TrustedCAs {
		certs = append(certs, ca.Cert)
	}
	return certs
}

// GetVersion implements the method from interface yamlconf.Config
func (cfg *Config) GetVersion() int {
	return cfg.Version
}

// SetVersion implements the method from interface yamlconf.Config
func (cfg *Config) SetVersion(version int) {
	cfg.Version = version
}

// ApplyDefaults implements the method from interface yamlconf.Config
//
// ApplyDefaults populates default values on a Config to make sure that we have
// a minimum viable config for running.  As new settings are added to
// flashlight, this function should be updated to provide sensible defaults for
// those settings.
func (cfg *Config) ApplyDefaults() {
	if cfg.Role == "" {
		cfg.Role = "client"
	}

	if cfg.Addr == "" {
		cfg.Addr = "localhost:8787"
	}

	if cfg.UIAddr == "" {
		cfg.UIAddr = "localhost:16823"
	}

	if cfg.CloudConfig == "" {
		cfg.CloudConfig = "https://s3.amazonaws.com/lantern_config/cloud.2.0.0-nl.yaml.gz"
	}

	// Default country
	if cfg.Country == "" {
		cfg.Country = *country
	}

	// Make sure we always have a stats config
	if cfg.Stats == nil {
		cfg.Stats = &statreporter.Config{}
	}

	if cfg.Stats.StatshubAddr == "" {
		cfg.Stats.StatshubAddr = *statshubAddr
	}

	if cfg.Client != nil && cfg.Role == "client" {
		cfg.applyClientDefaults()
	}

	if cfg.ProxiedSites == nil {
		log.Debugf("Adding empty proxiedsites")
		cfg.ProxiedSites = &proxiedsites.Config{
			Delta: &proxiedsites.Delta{
				Additions: []string{},
				Deletions: []string{},
			},
			Cloud: []string{},
		}
	}

	if cfg.ProxiedSites.Cloud == nil || len(cfg.ProxiedSites.Cloud) == 0 {
		log.Debugf("Loading default cloud proxiedsites")
		cfg.ProxiedSites.Cloud = defaultProxiedSites
	}

	if cfg.TrustedCAs == nil || len(cfg.TrustedCAs) == 0 {
		cfg.TrustedCAs = defaultTrustedCAs
	}
}

func (cfg *Config) applyClientDefaults() {
	// Make sure we always have at least one masquerade set
	if cfg.Client.MasqueradeSets == nil {
		cfg.Client.MasqueradeSets = make(map[string][]*fronted.Masquerade)
	}
	if len(cfg.Client.MasqueradeSets) == 0 {
		cfg.Client.MasqueradeSets[cloudflare] = cloudflareMasquerades
	}

	// Make sure we always have at least one server
	if cfg.Client.FrontedServers == nil {
		cfg.Client.FrontedServers = make([]*client.FrontedServerInfo, 0)
	}
	if len(cfg.Client.FrontedServers) == 0 && len(cfg.Client.ChainedServers) == 0 {
		cfg.Client.FrontedServers = []*client.FrontedServerInfo{
			&client.FrontedServerInfo{
				Host:           "nl.fallbacks.getiantem.org",
				Port:           443,
				PoolSize:       30,
				MasqueradeSet:  cloudflare,
				MaxMasquerades: 20,
				QOS:            10,
				Weight:         4000,
			},
		}

		cfg.Client.ChainedServers = make(map[string]*client.ChainedServerInfo, len(fallbacks))
		for key, fb := range fallbacks {
			cfg.Client.ChainedServers[key] = fb
		}
	}

	if cfg.AutoReport == nil {
		cfg.AutoReport = new(bool)
		*cfg.AutoReport = true
	}

	// Make sure all servers have a QOS and Weight configured
	for _, server := range cfg.Client.FrontedServers {
		if server.QOS == 0 {
			server.QOS = 5
		}
		if server.Weight == 0 {
			server.Weight = 100
		}
		if server.RedialAttempts == 0 {
			server.RedialAttempts = 2
		}
	}

	// Always make sure we have a map of ChainedServers
	if cfg.Client.ChainedServers == nil {
		cfg.Client.ChainedServers = make(map[string]*client.ChainedServerInfo)
	}

	// Sort servers so that they're always in a predictable order
	cfg.Client.SortServers()
}

func (cfg *Config) IsDownstream() bool {
	return cfg.Role == "client"
}

func (cfg *Config) IsUpstream() bool {
	return !cfg.IsDownstream()
}

func (cfg Config) cloudPollSleepTime() time.Duration {
	return time.Duration((CloudConfigPollInterval.Nanoseconds() / 2) + rand.Int63n(CloudConfigPollInterval.Nanoseconds()))
}

func (cfg Config) fetchCloudConfig() (bytes []byte, err error) {
	log.Debugf("Fetching cloud config from: %s", cfg.CloudConfig)

	if cfg.IsDownstream() {
		// Clients must always proxy the request
		if cfg.Addr == "" {
			err = fmt.Errorf("No proxyAddr")
		} else {
			bytes, err = cfg.doFetchCloudConfig(cfg.Addr)
		}
	} else {
		bytes, err = cfg.doFetchCloudConfig("")
	}
	if err != nil {
		bytes = nil
		err = fmt.Errorf("Unable to read yaml from %s: %s", cfg.CloudConfig, err)
	}
	return
}

func (cfg Config) doFetchCloudConfig(proxyAddr string) ([]byte, error) {
	log.Tracef("doFetchCloudConfig via '%s'", proxyAddr)

	if proxyAddr != "" {
		// Wait for proxy to become available
		err := waitforserver.WaitForServer("tcp", proxyAddr, 30*time.Second)
		if err != nil {
			return nil, fmt.Errorf("Proxy never came up at %v: %v", proxyAddr, err)
		}
	}

	client, err := util.HTTPClient(cfg.CloudConfigCA, proxyAddr)
	if err != nil {
		return nil, fmt.Errorf("Unable to initialize HTTP client: %s", err)
	}

	log.Debugf("Checking for cloud configuration at: %s", cfg.CloudConfig)
	req, err := http.NewRequest("GET", cfg.CloudConfig, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to construct request for cloud config at %s: %s", cfg.CloudConfig, err)
	}
	if lastCloudConfigETag != "" {
		// Don't bother fetching if unchanged
		req.Header.Set(ifNoneMatch, lastCloudConfigETag)
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Unable to fetch cloud config at %s: %s", cfg.CloudConfig, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 304 {
		log.Debugf("Config unchanged in cloud")
		return nil, nil
	} else if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Unexpected response status: %d", resp.StatusCode)
	}

	lastCloudConfigETag = resp.Header.Get(etag)
	gzReader, err := gzip.NewReader(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("Unable to open gzip reader: %s", err)
	}
	return ioutil.ReadAll(gzReader)
}

// updateFrom creates a new Config by merging the given yaml into this Config.
// Any servers in the updated yaml replace ones in the original Config and any
// masquerade sets in the updated yaml replace ones in the original Config.
func (updated *Config) updateFrom(updateBytes []byte) error {
	err := yaml.Unmarshal(updateBytes, updated)
	if err != nil {
		return fmt.Errorf("Unable to unmarshal YAML for update: %s", err)
	}

	// Need to de-duplicate servers, since yaml appends them
	servers := make(map[string]*client.FrontedServerInfo)
	for _, server := range updated.Client.FrontedServers {
		servers[server.Host] = server
	}
	updated.Client.FrontedServers = make([]*client.FrontedServerInfo, 0, len(servers))
	for _, server := range servers {
		updated.Client.FrontedServers = append(updated.Client.FrontedServers, server)
	}

	// Same with global proxiedsites
	if len(updated.ProxiedSites.Cloud) > 0 {
		wlDomains := make(map[string]bool)
		for _, domain := range updated.ProxiedSites.Cloud {
			wlDomains[domain] = true
		}
		updated.ProxiedSites.Cloud = make([]string, 0, len(wlDomains))
		for domain, _ := range wlDomains {
			updated.ProxiedSites.Cloud = append(updated.ProxiedSites.Cloud, domain)
		}
		sort.Strings(updated.ProxiedSites.Cloud)
	}
	return nil
}
<commit_msg>make cloud config completely overwrite servers and masquerade sets<commit_after>package config

import (
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"time"

	"github.com/getlantern/appdir"
	"github.com/getlantern/fronted"
	"github.com/getlantern/geolookup"
	"github.com/getlantern/golog"
	"github.com/getlantern/proxiedsites"
	"github.com/getlantern/waitforserver"
	"github.com/getlantern/yaml"
	"github.com/getlantern/yamlconf"

	"github.com/getlantern/flashlight/client"
	"github.com/getlantern/flashlight/globals"
	"github.com/getlantern/flashlight/server"
	"github.com/getlantern/flashlight/statreporter"
	"github.com/getlantern/flashlight/util"
)

const (
	CloudConfigPollInterval = 1 * time.Minute

	cloudflare  = "cloudflare"
	etag        = "ETag"
	ifNoneMatch = "If-None-Match"
)

var (
	log                 = golog.LoggerFor("flashlight.config")
	m                   *yamlconf.Manager
	lastCloudConfigETag = ""
)

type Config struct {
	Version       int
	CloudConfig   string
	CloudConfigCA string
	Addr          string
	Role          string
	InstanceId    string
	Country       string
	CpuProfile    string
	MemProfile    string
	UIAddr        string // UI HTTP server address
	AutoReport    *bool  // Report anonymous usage to GA
	Stats         *statreporter.Config
	Server        *server.ServerConfig
	Client        *client.ClientConfig
	ProxiedSites  *proxiedsites.Config // List of proxied site domains that get routed through Lantern rather than accessed directly
	TrustedCAs    []*CA
}

// CA represents a certificate authority
type CA struct {
	CommonName string
	Cert       string // PEM-encoded
}

// Init initializes the configuration system.
func Init() (*Config, error) {
	configPath, err := InConfigDir("lantern.yaml")
	if err != nil {
		return nil, err
	}
	m = &yamlconf.Manager{
		FilePath:         configPath,
		FilePollInterval: 1 * time.Second,
		ConfigServerAddr: *configaddr,
		EmptyConfig: func() yamlconf.Config {
			return &Config{}
		},
		OneTimeSetup: func(ycfg yamlconf.Config) error {
			cfg := ycfg.(*Config)
			return cfg.applyFlags()
		},
		CustomPoll: func(currentCfg yamlconf.Config) (mutate func(yamlconf.Config) error, waitTime time.Duration, err error) {
			// By default, do nothing
			mutate = func(ycfg yamlconf.Config) error {
				// do nothing
				return nil
			}
			cfg := currentCfg.(*Config)
			waitTime = cfg.cloudPollSleepTime()
			if cfg.CloudConfig == "" {
				// Config doesn't have a CloudConfig, just ignore
				return
			}

			var bytes []byte
			bytes, err = cfg.fetchCloudConfig()
			if err == nil && bytes != nil {
				mutate = func(ycfg yamlconf.Config) error {
					log.Debugf("Merging cloud configuration")
					cfg := ycfg.(*Config)
					return cfg.updateFrom(bytes)
				}
			}
			return
		},
	}
	initial, err := m.Start()
	var cfg *Config
	if err == nil {
		cfg = initial.(*Config)
		err = updateGlobals(cfg)
		if err != nil {
			return nil, err
		}
	}
	return cfg, err
}

// Run runs the configuration system.
func Run(updateHandler func(updated *Config)) error {
	for {
		next := m.Next()
		nextCfg := next.(*Config)
		err := updateGlobals(nextCfg)
		if err != nil {
			return err
		}
		updateHandler(nextCfg)
	}
}

func updateGlobals(cfg *Config) error {
	globals.InstanceId = cfg.InstanceId
	loc := &geolookup.City{}
	loc.Country.IsoCode = cfg.Country
	globals.SetLocation(loc)
	err := globals.SetTrustedCAs(cfg.TrustedCACerts())
	if err != nil {
		return fmt.Errorf("Unable to configure trusted CAs: %s", err)
	}
	return nil
}

// Update updates the configuration using the given mutator function.
func Update(mutate func(cfg *Config) error) error {
	return m.Update(func(ycfg yamlconf.Config) error {
		return mutate(ycfg.(*Config))
	})
}

// InConfigDir returns the path to the given filename inside of the configdir.
func InConfigDir(filename string) (string, error) {
	cdir := *configdir
	if cdir == "" {
		cdir = appdir.General("Lantern")
	}
	log.Debugf("Placing configuration in %v", cdir)
	if _, err := os.Stat(cdir); err != nil {
		if os.IsNotExist(err) {
			// Create config dir
			if err := os.MkdirAll(cdir, 0755); err != nil {
				return "", fmt.Errorf("Unable to create configdir at %s: %s", cdir, err)
			}
		}
	}
	return filepath.Join(cdir, filename), nil
}

// TrustedCACerts returns a slice of PEM-encoded certs for the trusted CAs
func (cfg *Config) TrustedCACerts() []string {
	certs := make([]string, 0, len(cfg.TrustedCAs))
	for _, ca := range cfg.TrustedCAs {
		certs = append(certs, ca.Cert)
	}
	return certs
}

// GetVersion implements the method from interface yamlconf.Config
func (cfg *Config) GetVersion() int {
	return cfg.Version
}

// SetVersion implements the method from interface yamlconf.Config
func (cfg *Config) SetVersion(version int) {
	cfg.Version = version
}

// ApplyDefaults implements the method from interface yamlconf.Config
//
// ApplyDefaults populates default values on a Config to make sure that we have
// a minimum viable config for running.  As new settings are added to
// flashlight, this function should be updated to provide sensible defaults for
// those settings.
func (cfg *Config) ApplyDefaults() {
	if cfg.Role == "" {
		cfg.Role = "client"
	}

	if cfg.Addr == "" {
		cfg.Addr = "localhost:8787"
	}

	if cfg.UIAddr == "" {
		cfg.UIAddr = "localhost:16823"
	}

	if cfg.CloudConfig == "" {
		cfg.CloudConfig = "https://s3.amazonaws.com/lantern_config/cloud.2.0.0-nl.yaml.gz"
	}

	// Default country
	if cfg.Country == "" {
		cfg.Country = *country
	}

	// Make sure we always have a stats config
	if cfg.Stats == nil {
		cfg.Stats = &statreporter.Config{}
	}

	if cfg.Stats.StatshubAddr == "" {
		cfg.Stats.StatshubAddr = *statshubAddr
	}

	if cfg.Client != nil && cfg.Role == "client" {
		cfg.applyClientDefaults()
	}

	if cfg.ProxiedSites == nil {
		log.Debugf("Adding empty proxiedsites")
		cfg.ProxiedSites = &proxiedsites.Config{
			Delta: &proxiedsites.Delta{
				Additions: []string{},
				Deletions: []string{},
			},
			Cloud: []string{},
		}
	}

	if cfg.ProxiedSites.Cloud == nil || len(cfg.ProxiedSites.Cloud) == 0 {
		log.Debugf("Loading default cloud proxiedsites")
		cfg.ProxiedSites.Cloud = defaultProxiedSites
	}

	if cfg.TrustedCAs == nil || len(cfg.TrustedCAs) == 0 {
		cfg.TrustedCAs = defaultTrustedCAs
	}
}

func (cfg *Config) applyClientDefaults() {
	// Make sure we always have at least one masquerade set
	if cfg.Client.MasqueradeSets == nil {
		cfg.Client.MasqueradeSets = make(map[string][]*fronted.Masquerade)
	}
	if len(cfg.Client.MasqueradeSets) == 0 {
		cfg.Client.MasqueradeSets[cloudflare] = cloudflareMasquerades
	}

	// Make sure we always have at least one server
	if cfg.Client.FrontedServers == nil {
		cfg.Client.FrontedServers = make([]*client.FrontedServerInfo, 0)
	}
	if len(cfg.Client.FrontedServers) == 0 && len(cfg.Client.ChainedServers) == 0 {
		cfg.Client.FrontedServers = []*client.FrontedServerInfo{
			&client.FrontedServerInfo{
				Host:           "nl.fallbacks.getiantem.org",
				Port:           443,
				PoolSize:       30,
				MasqueradeSet:  cloudflare,
				MaxMasquerades: 20,
				QOS:            10,
				Weight:         4000,
			},
		}

		cfg.Client.ChainedServers = make(map[string]*client.ChainedServerInfo, len(fallbacks))
		for key, fb := range fallbacks {
			cfg.Client.ChainedServers[key] = fb
		}
	}

	if cfg.AutoReport == nil {
		cfg.AutoReport = new(bool)
		*cfg.AutoReport = true
	}

	// Make sure all servers have a QOS and Weight configured
	for _, server := range cfg.Client.FrontedServers {
		if server.QOS == 0 {
			server.QOS = 5
		}
		if server.Weight == 0 {
			server.Weight = 100
		}
		if server.RedialAttempts == 0 {
			server.RedialAttempts = 2
		}
	}

	// Always make sure we have a map of ChainedServers
	if cfg.Client.ChainedServers == nil {
		cfg.Client.ChainedServers = make(map[string]*client.ChainedServerInfo)
	}

	// Sort servers so that they're always in a predictable order
	cfg.Client.SortServers()
}

func (cfg *Config) IsDownstream() bool {
	return cfg.Role == "client"
}

func (cfg *Config) IsUpstream() bool {
	return !cfg.IsDownstream()
}

func (cfg Config) cloudPollSleepTime() time.Duration {
	return time.Duration((CloudConfigPollInterval.Nanoseconds() / 2) + rand.Int63n(CloudConfigPollInterval.Nanoseconds()))
}

func (cfg Config) fetchCloudConfig() (bytes []byte, err error) {
	log.Debugf("Fetching cloud config from: %s", cfg.CloudConfig)

	if cfg.IsDownstream() {
		// Clients must always proxy the request
		if cfg.Addr == "" {
			err = fmt.Errorf("No proxyAddr")
		} else {
			bytes, err = cfg.doFetchCloudConfig(cfg.Addr)
		}
	} else {
		bytes, err = cfg.doFetchCloudConfig("")
	}
	if err != nil {
		bytes = nil
		err = fmt.Errorf("Unable to read yaml from %s: %s", cfg.CloudConfig, err)
	}
	return
}

func (cfg Config) doFetchCloudConfig(proxyAddr string) ([]byte, error) {
	log.Tracef("doFetchCloudConfig via '%s'", proxyAddr)

	if proxyAddr != "" {
		// Wait for proxy to become available
		err := waitforserver.WaitForServer("tcp", proxyAddr, 30*time.Second)
		if err != nil {
			return nil, fmt.Errorf("Proxy never came up at %v: %v", proxyAddr, err)
		}
	}

	client, err := util.HTTPClient(cfg.CloudConfigCA, proxyAddr)
	if err != nil {
		return nil, fmt.Errorf("Unable to initialize HTTP client: %s", err)
	}

	log.Debugf("Checking for cloud configuration at: %s", cfg.CloudConfig)
	req, err := http.NewRequest("GET", cfg.CloudConfig, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to construct request for cloud config at %s: %s", cfg.CloudConfig, err)
	}
	if lastCloudConfigETag != "" {
		// Don't bother fetching if unchanged
		req.Header.Set(ifNoneMatch, lastCloudConfigETag)
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Unable to fetch cloud config at %s: %s", cfg.CloudConfig, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 304 {
		log.Debugf("Config unchanged in cloud")
		return nil, nil
	} else if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Unexpected response status: %d", resp.StatusCode)
	}

	lastCloudConfigETag = resp.Header.Get(etag)
	gzReader, err := gzip.NewReader(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("Unable to open gzip reader: %s", err)
	}
	return ioutil.ReadAll(gzReader)
}

// updateFrom creates a new Config by 'merging' the given yaml into this Config.
// The masquerade sets and the collections of servers in the update yaml
// completely replace the ones in the original Config.
func (updated *Config) updateFrom(updateBytes []byte) error {
	// XXX: does this need a mutex, along with everyone that uses the config?
	oldFrontedServers := updated.Client.FrontedServers
	oldChainedServers := updated.Client.ChainedServers
	oldMasqueradeSets := updated.Client.MasqueradeSets
	updated.Client.FrontedServers = []*client.FrontedServerInfo{}
	updated.Client.ChainedServers = map[string]*client.ChainedServerInfo{}
	updated.Client.MasqueradeSets = map[string][]*fronted.Masquerade{}
	err := yaml.Unmarshal(updateBytes, updated)
	if err != nil {
		updated.Client.FrontedServers = oldFrontedServers
		updated.Client.ChainedServers = oldChainedServers
		updated.Client.MasqueradeSets = oldMasqueradeSets
		return fmt.Errorf("Unable to unmarshal YAML for update: %s", err)
	}
	// Deduplicate global proxiedsites
	if len(updated.ProxiedSites.Cloud) > 0 {
		wlDomains := make(map[string]bool)
		for _, domain := range updated.ProxiedSites.Cloud {
			wlDomains[domain] = true
		}
		updated.ProxiedSites.Cloud = make([]string, 0, len(wlDomains))
		for domain, _ := range wlDomains {
			updated.ProxiedSites.Cloud = append(updated.ProxiedSites.Cloud, domain)
		}
		sort.Strings(updated.ProxiedSites.Cloud)
	}
	return nil
}
<|endoftext|>


<commit_before>package osversion

import (
	"regexp"
	"runtime"
	"testing"
)

func TestString(t *testing.T) {
	reg := regexp.MustCompile("^([0-9])+\\.([0-9])+\\.([0-9])+.*")
	str, err := GetString()
	if err != nil {
		t.Fatal("Error getting string")
	}
	if !reg.MatchString(str) {
		t.Fatal("Improper string format: %s", str)
	}
}

func TestHumanReadable(t *testing.T) {
	switch runtime.GOOS {
	case "darwin":
		str, err := GetHumanReadable()
		if err != nil {
			t.Fatal("Error getting string")
		}
		reg := regexp.MustCompile("OS X 10\\..+")
		if !reg.MatchString(str) {
			t.Fatalf("Improper human readable format: %s", str)
		}
	case "linux":
		str, err := GetHumanReadable()
		if err != nil {
			t.Fatal("Error getting string")
		}
		reg := regexp.MustCompile(".+kernel.+")
		if !reg.MatchString(str) {
			t.Fatal("Improper human readable format: %s", str)
		}
	case "windows":
		str, err := GetHumanReadable()
		if err != nil {
			t.Fatal("Error getting string")
		}
		reg := regexp.MustCompile("Windows .+")
		if !reg.MatchString(str) {
			t.Fatal("Improper human readable format: %s", str)
		}
	default:
		t.Fatal("Unsupported OS detected: %s", runtime.GOOS)
	}
}
<commit_msg>Fixing test when no distribution detected<commit_after>package osversion

import (
	"regexp"
	"runtime"
	"testing"
)

func TestString(t *testing.T) {
	reg := regexp.MustCompile("^([0-9])+\\.([0-9])+\\.([0-9])+.*")
	str, err := GetString()
	if err != nil {
		t.Fatal("Error getting string")
	}
	if !reg.MatchString(str) {
		t.Fatalf("Improper string format: %s", str)
	}
}

func TestHumanReadable(t *testing.T) {
	switch runtime.GOOS {
	case "darwin":
		str, err := GetHumanReadable()
		if err != nil {
			t.Fatal("Error getting string")
		}
		reg := regexp.MustCompile("OS X 10\\..+")
		if !reg.MatchString(str) {
			t.Fatalf("Improper human readable format: %s", str)
		}
	case "linux":
		str, err := GetHumanReadable()
		if err != nil {
			t.Fatal("Error getting string")
		}
		reg := regexp.MustCompile(".*kernel.+")
		if !reg.MatchString(str) {
			t.Fatalf("Improper human readable format: %s", str)
		}
	case "windows":
		str, err := GetHumanReadable()
		if err != nil {
			t.Fatal("Error getting string")
		}
		reg := regexp.MustCompile("Windows .+")
		if !reg.MatchString(str) {
			t.Fatalf("Improper human readable format: %s", str)
		}
	default:
		t.Fatal("Unsupported OS detected: %s", runtime.GOOS)
	}
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"fmt"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc/creds"

	v1 "k8s.io/api/core/v1"
	k8serr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

type Secrets struct {
	logger lager.Logger

	client          kubernetes.Interface
	namespacePrefix string
}

// NewSecretLookupPaths defines how variables will be searched in the underlying secret manager
func (secrets Secrets) NewSecretLookupPaths(teamName string, pipelineName string, allowRootPath bool) []creds.SecretLookupPath {
	lookupPaths := []creds.SecretLookupPath{}
	if len(pipelineName) > 0 {
		lookupPaths = append(lookupPaths, creds.NewSecretLookupWithPrefix(secrets.namespacePrefix+teamName+":"+pipelineName+"."))
	}
	lookupPaths = append(lookupPaths, creds.NewSecretLookupWithPrefix(secrets.namespacePrefix+teamName+":"))
	if allowRootPath {
		lookupPaths = append(lookupPaths, creds.NewSecretLookupWithPrefix(secrets.namespacePrefix+":"))
	}
	return lookupPaths
}

// Get retrieves the value and expiration of an individual secret
func (secrets Secrets) Get(secretPath string) (interface{}, *time.Time, bool, error) {
	parts := strings.Split(secretPath, ":")
	if len(parts) != 2 {
		return nil, nil, false, fmt.Errorf("unable to split kubernetes secret path into [namespace]:[secret]: %s", secretPath)
	}

	var namespace = parts[0]
	var secretName = parts[1]

	secret, found, err := secrets.findSecret(namespace, secretName)
	if err != nil {
		secrets.logger.Error("failed-to-fetch-secret", err, lager.Data{
			"namespace":   namespace,
			"secret-name": secretName,
		})
		return nil, nil, false, err
	}

	if found {
		return secrets.getValueFromSecret(secret)
	}

	secrets.logger.Info("secret-not-found", lager.Data{
		"namespace":   namespace,
		"secret-name": secretName,
	})

	return nil, nil, false, nil
}

func (secrets Secrets) getValueFromSecret(secret *v1.Secret) (interface{}, *time.Time, bool, error) {
	val, found := secret.Data["value"]
	if found {
		return string(val), nil, true, nil
	}

	stringified := map[string]interface{}{}
	for k, v := range secret.Data {
		stringified[k] = string(v)
	}

	return stringified, nil, true, nil
}

func (secrets Secrets) findSecret(namespace, name string) (*v1.Secret, bool, error) {
	var secret *v1.Secret
	var err error

	secret, err = secrets.client.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})

	if err != nil && k8serr.IsNotFound(err) {
		return nil, false, nil
	} else if err != nil {
		return nil, false, err
	} else {
		return secret, true, err
	}
}
<commit_msg>atc: structure: use a different separator for k8s<commit_after>package kubernetes

import (
	"fmt"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc/creds"

	v1 "k8s.io/api/core/v1"
	k8serr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

type Secrets struct {
	logger lager.Logger

	client          kubernetes.Interface
	namespacePrefix string
}

// NewSecretLookupPaths defines how variables will be searched in the underlying secret manager
func (secrets Secrets) NewSecretLookupPaths(teamName string, pipelineName string, allowRootPath bool) []creds.SecretLookupPath {
	lookupPaths := []creds.SecretLookupPath{}
	if len(pipelineName) > 0 {
		lookupPaths = append(lookupPaths, creds.NewSecretLookupWithPrefix(secrets.namespacePrefix+teamName+"/"+pipelineName+"."))
	}
	lookupPaths = append(lookupPaths, creds.NewSecretLookupWithPrefix(secrets.namespacePrefix+teamName+"/"))
	if allowRootPath {
		lookupPaths = append(lookupPaths, creds.NewSecretLookupWithPrefix(secrets.namespacePrefix+"/"))
	}
	return lookupPaths
}

// Get retrieves the value and expiration of an individual secret
func (secrets Secrets) Get(secretPath string) (interface{}, *time.Time, bool, error) {
	parts := strings.Split(secretPath, "/")
	if len(parts) != 2 {
		return nil, nil, false, fmt.Errorf("unable to split kubernetes secret path into [namespace]/[secret]: %s", secretPath)
	}

	var namespace = parts[0]
	var secretName = parts[1]

	secret, found, err := secrets.findSecret(namespace, secretName)
	if err != nil {
		secrets.logger.Error("failed-to-fetch-secret", err, lager.Data{
			"namespace":   namespace,
			"secret-name": secretName,
		})
		return nil, nil, false, err
	}

	if found {
		return secrets.getValueFromSecret(secret)
	}

	secrets.logger.Info("secret-not-found", lager.Data{
		"namespace":   namespace,
		"secret-name": secretName,
	})

	return nil, nil, false, nil
}

func (secrets Secrets) getValueFromSecret(secret *v1.Secret) (interface{}, *time.Time, bool, error) {
	val, found := secret.Data["value"]
	if found {
		return string(val), nil, true, nil
	}

	stringified := map[string]interface{}{}
	for k, v := range secret.Data {
		stringified[k] = string(v)
	}

	return stringified, nil, true, nil
}

func (secrets Secrets) findSecret(namespace, name string) (*v1.Secret, bool, error) {
	var secret *v1.Secret
	var err error

	secret, err = secrets.client.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})

	if err != nil && k8serr.IsNotFound(err) {
		return nil, false, nil
	} else if err != nil {
		return nil, false, err
	} else {
		return secret, true, err
	}
}
<|endoftext|>


<commit_before>package daemon

import (
    "os"
    "time"
    "fmt"
    "github.com/sevlyar/go-daemon"
    "github.com/Sirupsen/logrus"
    "github.com/BluePecker/JwtAuth/server/types/token"
    "github.com/BluePecker/JwtAuth/server"
    "github.com/BluePecker/JwtAuth/storage"
    //"github.com/BluePecker/JwtAuth/server/router"
    _ "github.com/BluePecker/JwtAuth/storage/redis"
    //_ "github.com/BluePecker/JwtAuth/storage/ram"
    "github.com/dgrijalva/jwt-go"
    RouteToken "github.com/BluePecker/JwtAuth/server/router/token"
    "github.com/kataras/iris"
    "github.com/kataras/iris/core/netutil"
)

const (
    TOKEN_TTL = 2 * 3600

    VERSION = "1.0.0"

    ALLOW_LOGIN_NUM = 3
)

type Storage struct {
    Driver string
    Opts   string
}

type Security struct {
    Verify bool
    TLS    bool
    Key    string
    Cert   string
}

type Options struct {
    PidFile  string
    LogFile  string
    LogLevel string
    Port     int
    Host     string
    Daemon   bool
    Version  bool
    Security Security
    Storage  Storage
    Secret   string
}

type Daemon struct {
    Options *Options
    Front   *server.Server
    Backend *server.Server
    Storage *storage.Driver
}

type (
    CustomClaims struct {
        Device    string `json:"device"`
        Unique    string `json:"unique"`
        Timestamp int64  `json:"timestamp"`
        Addr      string `json:"addr"`
        jwt.StandardClaims
    }
)

func (d *Daemon) NewStorage() (err error) {
    conf := d.Options.Storage
    d.Storage, err = storage.New(conf.Driver, conf.Opts)
    return err
}

func (d *Daemon) NewFront() (err error) {
    d.Front = &server.Server{
        App: iris.New(),
    }
    d.Front.AddRouter(RouteToken.NewRouter(d))
    Addr := fmt.Sprintf("%s:%d", d.Options.Host, d.Options.Port)
    if !d.Options.Security.TLS && !d.Options.Security.Verify {
        err = d.Front.Run(iris.Addr(Addr))
    } else {
        runner := iris.TLS(Addr, d.Options.Security.Cert, d.Options.Security.Key)
        err = d.Front.Run(runner)
    }
    return err
}

func (d *Daemon) NewBackend() (err error) {
    d.Backend = &server.Server{}
    l, err := netutil.UNIX("/tmp/srv.sock", 0666)
    if err != nil {
        return err
    }
    err = d.Backend.Run(iris.Listener(l))
    if err == nil {
        // todo add backend router
    }
    return err
}

func (d *Daemon) Generate(req token.GenerateRequest) (string, error) {
    Claims := CustomClaims{
        req.Device,
        req.Unique,
        time.Now().Unix(),
        req.Addr,
        jwt.StandardClaims{
            ExpiresAt: time.Now().Add(time.Second * TOKEN_TTL).Unix(),
            Issuer: "shuc324@gmail.com",
        },
    }
    Token := jwt.NewWithClaims(jwt.SigningMethodHS256, Claims)
    if Signed, err := Token.SignedString([]byte(d.Options.Secret)); err != nil {
        return "", err
    } else {
        err := (*d.Storage).LKeep(req.Unique, Signed, ALLOW_LOGIN_NUM, TOKEN_TTL)
        if err != nil {
            return "", err
        }
        return Signed, err
    }
}

func (d *Daemon) Auth(req token.AuthRequest) (interface{}, error) {
    Token, err := jwt.ParseWithClaims(
        req.JsonWebToken,
        &CustomClaims{},
        func(token *jwt.Token) (interface{}, error) {
            if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
                return nil, fmt.Errorf("Unexpected signing method %v", token.Header["alg"])
            }
            return []byte(d.Options.Secret), nil
        })
    if err == nil && Token.Valid {
        if Claims, ok := Token.Claims.(*CustomClaims); ok {
            if (*d.Storage).LExist(Claims.Unique, req.JsonWebToken) {
                return Claims, nil
            }
        }
    }
    return nil, err
}

func NewDaemon(background bool, args Options) *Daemon {
    if background {
        ctx := daemon.Context{
            PidFileName: args.PidFile,
            PidFilePerm: 0644,
            LogFilePerm: 0640,
            Umask:       027,
            WorkDir:     "/",
            LogFileName: args.LogFile,
        }
        if rank, err := logrus.ParseLevel(args.LogLevel); err != nil {
            fmt.Println(err)
            os.Exit(0)
        } else {
            logrus.SetFormatter(&logrus.TextFormatter{
                TimestampFormat: "2006-01-02 15:04:05",
            })
            logrus.SetLevel(rank)
        }
        if process, err := ctx.Reborn(); err == nil {
            defer ctx.Release()
            if process != nil {
                return nil
            }
        } else {
            if err == daemon.ErrWouldBlock {
                fmt.Println("daemon already exists.")
            } else {
                fmt.Println("Unable to run: ", err)
            }
            os.Exit(0)
        }
    }
    return &Daemon{Options: &args}
}

func NewStart(args Options) {
    var err error;

    if args.Version == true {
        fmt.Printf("JwtAuth version %s.\n", VERSION)
        os.Exit(0)
    }

    Process := NewDaemon(args.Daemon, args)

    if Process == nil {
        return
    }

    if Process.Options.Secret == "" {
        fmt.Println("please specify the key.")
        os.Exit(0)
    }

    if err = Process.NewStorage(); err != nil {
        logrus.Fatal(err)
        os.Exit(0)
    }

    //go func() {
    //    if err = Process.NewBackend(); err != nil {
    //        logrus.Fatalf("backend server listen error: %s", err)
    //        os.Exit(0)
    //    }
    //}()

    if err = Process.NewFront(); err != nil {
        logrus.Fatalf("front server listen error: %s", err)
        os.Exit(0)
    }
}<commit_msg>fix bug<commit_after>package daemon

import (
    "os"
    "time"
    "fmt"
    "github.com/sevlyar/go-daemon"
    "github.com/Sirupsen/logrus"
    "github.com/BluePecker/JwtAuth/server/types/token"
    "github.com/BluePecker/JwtAuth/server"
    "github.com/BluePecker/JwtAuth/storage"
    //"github.com/BluePecker/JwtAuth/server/router"
    _ "github.com/BluePecker/JwtAuth/storage/redis"
    //_ "github.com/BluePecker/JwtAuth/storage/ram"
    "github.com/dgrijalva/jwt-go"
    RouteToken "github.com/BluePecker/JwtAuth/server/router/token"
    "github.com/kataras/iris"
    "github.com/kataras/iris/core/netutil"
)

const (
    TOKEN_TTL = 2 * 3600

    VERSION = "1.0.0"

    ALLOW_LOGIN_NUM = 3
)

type Storage struct {
    Driver string
    Opts   string
}

type Security struct {
    Verify bool
    TLS    bool
    Key    string
    Cert   string
}

type Options struct {
    PidFile  string
    LogFile  string
    LogLevel string
    Port     int
    Host     string
    Daemon   bool
    Version  bool
    Security Security
    Storage  Storage
    Secret   string
}

type Daemon struct {
    Options *Options
    Front   *server.Server
    Backend *server.Server
    Storage *storage.Driver
}

type (
    CustomClaims struct {
        Device    string `json:"device"`
        Unique    string `json:"unique"`
        Timestamp int64  `json:"timestamp"`
        Addr      string `json:"addr"`
        jwt.StandardClaims
    }
)

func (d *Daemon) NewStorage() (err error) {
    conf := d.Options.Storage
    d.Storage, err = storage.New(conf.Driver, conf.Opts)
    return err
}

func (d *Daemon) NewFront() (err error) {
    d.Front = &server.Server{
        App: iris.New(),
    }
    d.Front.AddRouter(RouteToken.NewRouter(d))
    Addr := fmt.Sprintf("%s:%d", d.Options.Host, d.Options.Port)
    if !d.Options.Security.TLS && !d.Options.Security.Verify {
        err = d.Front.Run(iris.Addr(Addr))
    } else {
        runner := iris.TLS(Addr, d.Options.Security.Cert, d.Options.Security.Key)
        err = d.Front.Run(runner)
    }
    return err
}

func (d *Daemon) NewBackend() (err error) {
    d.Backend = &server.Server{
        App: iris.New(),
    }
    // todo add backend router
    l, err := netutil.UNIX("/tmp/srv.sock", 0666)
    if err != nil {
        return err
    }
    return d.Backend.Run(iris.Listener(l))
}

func (d *Daemon) Generate(req token.GenerateRequest) (string, error) {
    Claims := CustomClaims{
        req.Device,
        req.Unique,
        time.Now().Unix(),
        req.Addr,
        jwt.StandardClaims{
            ExpiresAt: time.Now().Add(time.Second * TOKEN_TTL).Unix(),
            Issuer: "shuc324@gmail.com",
        },
    }
    Token := jwt.NewWithClaims(jwt.SigningMethodHS256, Claims)
    if Signed, err := Token.SignedString([]byte(d.Options.Secret)); err != nil {
        return "", err
    } else {
        err := (*d.Storage).LKeep(req.Unique, Signed, ALLOW_LOGIN_NUM, TOKEN_TTL)
        if err != nil {
            return "", err
        }
        return Signed, err
    }
}

func (d *Daemon) Auth(req token.AuthRequest) (interface{}, error) {
    Token, err := jwt.ParseWithClaims(
        req.JsonWebToken,
        &CustomClaims{},
        func(token *jwt.Token) (interface{}, error) {
            if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
                return nil, fmt.Errorf("Unexpected signing method %v", token.Header["alg"])
            }
            return []byte(d.Options.Secret), nil
        })
    if err == nil && Token.Valid {
        if Claims, ok := Token.Claims.(*CustomClaims); ok {
            if (*d.Storage).LExist(Claims.Unique, req.JsonWebToken) {
                return Claims, nil
            }
        }
    }
    return nil, err
}

func NewDaemon(background bool, args Options) *Daemon {
    if background {
        ctx := daemon.Context{
            PidFileName: args.PidFile,
            PidFilePerm: 0644,
            LogFilePerm: 0640,
            Umask:       027,
            WorkDir:     "/",
            LogFileName: args.LogFile,
        }
        if rank, err := logrus.ParseLevel(args.LogLevel); err != nil {
            fmt.Println(err)
            os.Exit(0)
        } else {
            logrus.SetFormatter(&logrus.TextFormatter{
                TimestampFormat: "2006-01-02 15:04:05",
            })
            logrus.SetLevel(rank)
        }
        if process, err := ctx.Reborn(); err == nil {
            defer ctx.Release()
            if process != nil {
                return nil
            }
        } else {
            if err == daemon.ErrWouldBlock {
                fmt.Println("daemon already exists.")
            } else {
                fmt.Println("Unable to run: ", err)
            }
            os.Exit(0)
        }
    }
    return &Daemon{Options: &args}
}

func NewStart(args Options) {
    var err error;

    if args.Version == true {
        fmt.Printf("JwtAuth version %s.\n", VERSION)
        os.Exit(0)
    }

    Process := NewDaemon(args.Daemon, args)

    if Process == nil {
        return
    }

    if Process.Options.Secret == "" {
        fmt.Println("please specify the key.")
        os.Exit(0)
    }

    if err = Process.NewStorage(); err != nil {
        logrus.Fatal(err)
        os.Exit(0)
    }

    //go func() {
    //    if err = Process.NewBackend(); err != nil {
    //        logrus.Fatalf("backend server listen error: %s", err)
    //        os.Exit(0)
    //    }
    //}()

    if err = Process.NewFront(); err != nil {
        logrus.Fatalf("front server listen error: %s", err)
        os.Exit(0)
    }
}<|endoftext|>


<commit_before>package gservices

import (
	"math"
	"sync/atomic"
	"time"

	"github.com/HuKeping/rbtree"
	"github.com/gfandada/gserver/logger"
)

// 定时器初始值
const DEFAULT = time.Duration(math.MaxInt64)

// 自定义信号
var singal = struct{}{}

type LocalTimerServer struct {
	Id         uint64         // 自增的序列号
	JobList    *rbtree.Rbtree // 基于内存的事件容器
	Count      uint64         // 统计已执行的任务次数
	PauseChan  chan struct{}  // 暂停
	ResumeChan chan struct{}  // 重置
	ExitChan   chan struct{}  // 退出
}

type Ijob interface {
	Notify() <-chan Job // 获取一个chan:当job被执行时，可以从chan中获取消息
	GetCount() uint64   // 获取已经执行的次数
	GetTimes() uint64   // 获取允许执行的最大次数
}

type Job struct {
	Id           uint64          // 唯一id，由timerserver生成，用来区分同一时刻的不同事件
	Times        uint64          // 允许执行的最大次数:0表示无数次
	Count        uint64          // 表示已执行的次数
	IntervalTime time.Duration   // 间隔时间：支持ns
	CreateTime   time.Time       // 创建时间
	ActionTime   time.Time       // FIXME 计算得出的本次执行时间点，会有误差
	JobHandler   MessageHandler1 // 事件函数
	Args         []interface{}   // 函数调用参数
	MsgChan      chan Job        // 消息通道，执行时，控制器通过该通道向外部传递消息
}

func NewLocalTimerServer() *LocalTimerServer {
	server := &LocalTimerServer{
		JobList:    rbtree.New(),
		PauseChan:  make(chan struct{}, 0),
		ResumeChan: make(chan struct{}, 0),
		ExitChan:   make(chan struct{}, 0),
	}
	server.start()
	logger.Info("NewLocalTimerServer %v", server)
	return server
}

func (server *LocalTimerServer) start() {
	defalutJob := Job{
		CreateTime:   time.Now(),
		IntervalTime: time.Duration(math.MaxInt64),
	}
	server.addJob(time.Now(), defalutJob.IntervalTime, 1, defalutJob.JobHandler, nil)
	go server.schedule()
	server.resume()
}

func (server *LocalTimerServer) schedule() {
	timer := time.NewTimer(DEFAULT)
	defer timer.Stop()
PAUSE:
	<-server.ResumeChan
	for {
		value := server.JobList.Min()
		job, _ := value.(*Job)
		timeout := job.ActionTime.Sub(time.Now())
		timer.Reset(timeout)
		select {
		case <-timer.C:
			server.Count++
			job.ExecWithGo(true)
			if job.Times == 0 || job.Times > job.Count {
				server.JobList.Delete(job)
				job.ActionTime = job.ActionTime.Add(job.IntervalTime)
				server.JobList.Insert(job)
			} else {
				server.removeJob(job)
			}
		case <-server.PauseChan:
			goto PAUSE
		case <-server.ExitChan:
			goto EXIT
		}
	}
EXIT:
}

func (server *LocalTimerServer) resume() {
	server.ResumeChan <- singal
}

func (server *LocalTimerServer) exit() {
	server.ExitChan <- singal
}

func (server *LocalTimerServer) pause() {
	server.PauseChan <- singal
}

// 重设指定任务的超时时间
func (server *LocalTimerServer) UpdateJobTimeout(job Ijob, timeout time.Duration) bool {
	// 1s = 1e9 ns
	if timeout.Nanoseconds() <= 0 {
		return false
	}
	now := time.Now()
	server.pause()
	defer server.resume()
	item, ok := job.(*Job)
	if !ok {
		return false
	}
	server.JobList.Delete(item)
	item.ActionTime = now.Add(timeout)
	server.JobList.Insert(item)
	logger.Debug("UpdateJobTimeout job %v newTimeout %v", job, timeout)
	return true
}

// 添加单次任务，适用于单次定时业务逻辑
func (server *LocalTimerServer) AddJobWithInterval(timeout time.Duration, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	if timeout.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobWithInterval timeout %v args %v", timeout, args)
	return server.addJob(time.Now(), timeout, 1, jobFunc, args)
}

// 添加单次任务，适用于特定时期的活动等类型的任务
func (server *LocalTimerServer) AddJobWithDeadtime(deadtime time.Time, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	now := time.Now()
	timeout := deadtime.Sub(now)
	if timeout.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobWithDeadtime deadtime %v args %v", deadtime, args)
	return server.addJob(now, timeout, 1, jobFunc, args)
}

// 添加重复任务
func (server *LocalTimerServer) AddJobRepeat(jobInterval time.Duration, times uint64, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	if jobInterval.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobRepeat jobInterval %v count %d args %v", jobInterval, times, args)
	return server.addJob(time.Now(), jobInterval, times, jobFunc, args)
}

// 移除指定的单项任务
func (server *LocalTimerServer) DelJob(job Ijob) bool {
	if job == nil {
		return false
	}
	server.pause()
	defer server.resume()
	item, ok := job.(*Job)
	if !ok {
		return false
	}
	server.removeJob(item)
	logger.Debug("DelJob %v", job)
	return true
}

// 移除指定的多项任务
func (server *LocalTimerServer) DelJobs(jobs []Ijob) {
	server.pause()
	defer server.resume()
	for _, job := range jobs {
		item, ok := job.(*Job)
		if !ok {
			continue
		}
		server.removeJob(item)
	}
	logger.Debug("DelJobs %v", jobs)
}

// 获取server当前执行次数
func (server *LocalTimerServer) GetCount() uint64 {
	return atomic.LoadUint64(&server.Count)
}

// 重置server的内部状态
func (server *LocalTimerServer) Reset() *LocalTimerServer {
	server.exit()
	server.Count = 0
	server.cleanJobs()
	server.start()
	logger.Info("gentimer Reset")
	return server
}

func (server *LocalTimerServer) cleanJobs() {
	item := server.JobList.Min()
	for item != nil {
		job, ok := item.(*Job)
		if ok {
			server.removeJob(job)
		}
		item = server.JobList.Min()
	}
}

// 获取pending的任务数量
func (server *LocalTimerServer) WaitJobs() uint {
	leng := server.JobList.Len() - 1
	if leng > 0 {
		return leng
	}
	return 0
}

func (server *LocalTimerServer) addJob(createTime time.Time, intervalTime time.Duration, jobTimes uint64, jobFunc MessageHandler1, args []interface{}) (job *Job, inserted bool) {
	inserted = true
	server.Id++
	job = &Job{
		Id:           server.Id,
		Times:        jobTimes,
		CreateTime:   createTime,
		ActionTime:   createTime.Add(intervalTime),
		IntervalTime: intervalTime,
		MsgChan:      make(chan Job, 10),
		JobHandler:   jobFunc,
		Args:         args,
	}
	server.JobList.Insert(job)
	return
}

func (server *LocalTimerServer) removeJob(job *Job) {
	server.JobList.Delete(job)
	close(job.MsgChan)
}

// 优雅的关闭
// 会依次执行一次（不按照actiontime）
func (server *LocalTimerServer) StopByGrace() {
	server.exit()
	server.immediate()
	logger.Warning("gentimer StopByGrace")
}

// 强制关闭
func (server *LocalTimerServer) StopByForce() {
	server.exit()
	server.cleanJobs()
	logger.Warning("gentimer StopByForce")
}

func (server *LocalTimerServer) immediate() {
	item := server.JobList.Min()
	for item != nil {
		job, ok := item.(*Job)
		if ok {
			atomic.AddUint64(&server.Count, 1)
			job.ExecWithGo(false)
			server.removeJob(job)
		}
		item = server.JobList.Min()
	}
}

/*************************************Job相关************************************/

/****************************实现了rbtree.Item接口****************************/

func (job Job) Less(another rbtree.Item) bool {
	item, ok := another.(*Job)
	if !ok {
		return false
	}
	if !job.ActionTime.Equal(item.ActionTime) {
		return job.ActionTime.Before(item.ActionTime)
	}
	return job.Id < item.Id
}

/********************************实现了Job接口***********************************/

func (job Job) Notify() <-chan Job {
	return job.MsgChan
}

func (job Job) GetCount() uint64 {
	return job.Count
}

func (job Job) GetTimes() uint64 {
	return job.Times
}

func (job *Job) action() {
	defer func() {
		if err := recover(); err != nil {
			logger.Error("gentimer Exec job %v error: %v", job, err)
		}
	}()
	job.JobHandler(job.Args)
	logger.Debug("action job %v ", job)
}

func (job *Job) ExecWithGo(isGo bool) {
	job.Count++
	if job.JobHandler == nil {
		return
	}
	if isGo {
		go job.action()
	} else {
		job.action()
	}
}
<commit_msg>add logger<commit_after>package gservices

import (
	"math"
	"sync/atomic"
	"time"

	"github.com/HuKeping/rbtree"
	"github.com/gfandada/gserver/logger"
)

// 定时器初始值
const DEFAULT = time.Duration(math.MaxInt64)

// 自定义信号
var singal = struct{}{}

type LocalTimerServer struct {
	Id         uint64         // 自增的序列号
	JobList    *rbtree.Rbtree // 基于内存的事件容器
	Count      uint64         // 统计已执行的任务次数
	PauseChan  chan struct{}  // 暂停
	ResumeChan chan struct{}  // 重置
	ExitChan   chan struct{}  // 退出
}

type Ijob interface {
	Notify() <-chan Job // 获取一个chan:当job被执行时，可以从chan中获取消息
	GetCount() uint64   // 获取已经执行的次数
	GetTimes() uint64   // 获取允许执行的最大次数
}

type Job struct {
	Id           uint64          // 唯一id，由timerserver生成，用来区分同一时刻的不同事件
	Times        uint64          // 允许执行的最大次数:0表示无数次
	Count        uint64          // 表示已执行的次数
	IntervalTime time.Duration   // 间隔时间：支持ns
	CreateTime   time.Time       // 创建时间
	ActionTime   time.Time       // FIXME 计算得出的本次执行时间点，会有误差
	JobHandler   MessageHandler1 // 事件函数
	Args         []interface{}   // 函数调用参数
	MsgChan      chan Job        // 消息通道，执行时，控制器通过该通道向外部传递消息
}

func NewLocalTimerServer() *LocalTimerServer {
	server := &LocalTimerServer{
		JobList:    rbtree.New(),
		PauseChan:  make(chan struct{}, 0),
		ResumeChan: make(chan struct{}, 0),
		ExitChan:   make(chan struct{}, 0),
	}
	server.start()
	logger.Info("NewLocalTimerServer %v", server)
	return server
}

func (server *LocalTimerServer) start() {
	defalutJob := Job{
		CreateTime:   time.Now(),
		IntervalTime: time.Duration(math.MaxInt64),
	}
	server.addJob(time.Now(), defalutJob.IntervalTime, 1, defalutJob.JobHandler, nil)
	go server.schedule()
	server.resume()
}

func (server *LocalTimerServer) schedule() {
	timer := time.NewTimer(DEFAULT)
	defer timer.Stop()
PAUSE:
	<-server.ResumeChan
	for {
		value := server.JobList.Min()
		job, _ := value.(*Job)
		timeout := job.ActionTime.Sub(time.Now())
		timer.Reset(timeout)
		select {
		case <-timer.C:
			server.Count++
			job.ExecWithGo(true)
			if job.Times == 0 || job.Times > job.Count {
				server.JobList.Delete(job)
				job.ActionTime = job.ActionTime.Add(job.IntervalTime)
				server.JobList.Insert(job)
			} else {
				server.removeJob(job)
			}
		case <-server.PauseChan:
			goto PAUSE
		case <-server.ExitChan:
			goto EXIT
		}
	}
EXIT:
}

func (server *LocalTimerServer) resume() {
	server.ResumeChan <- singal
}

func (server *LocalTimerServer) exit() {
	server.ExitChan <- singal
}

func (server *LocalTimerServer) pause() {
	server.PauseChan <- singal
}

// 重设指定任务的超时时间
func (server *LocalTimerServer) UpdateJobTimeout(job Ijob, timeout time.Duration) bool {
	// 1s = 1e9 ns
	if timeout.Nanoseconds() <= 0 {
		return false
	}
	now := time.Now()
	server.pause()
	defer server.resume()
	item, ok := job.(*Job)
	if !ok {
		return false
	}
	server.JobList.Delete(item)
	item.ActionTime = now.Add(timeout)
	server.JobList.Insert(item)
	logger.Debug("UpdateJobTimeout job %v newTimeout %v", job, timeout)
	return true
}

// 添加单次任务，适用于单次定时业务逻辑
func (server *LocalTimerServer) AddJobWithInterval(timeout time.Duration, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	if timeout.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobWithInterval timeout %v args %v", timeout, args)
	return server.addJob(time.Now(), timeout, 1, jobFunc, args)
}

// 添加单次任务，适用于特定时期的活动等类型的任务
func (server *LocalTimerServer) AddJobWithDeadtime(deadtime time.Time, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	now := time.Now()
	timeout := deadtime.Sub(now)
	if timeout.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobWithDeadtime deadtime %v args %v", deadtime, args)
	return server.addJob(now, timeout, 1, jobFunc, args)
}

// 添加重复任务
func (server *LocalTimerServer) AddJobRepeat(jobInterval time.Duration, times uint64, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	if jobInterval.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobRepeat jobInterval %v count %d args %v", jobInterval, times, args)
	return server.addJob(time.Now(), jobInterval, times, jobFunc, args)
}

// 移除指定的单项任务
func (server *LocalTimerServer) DelJob(job Ijob) bool {
	if job == nil {
		return false
	}
	server.pause()
	defer server.resume()
	item, ok := job.(*Job)
	if !ok {
		return false
	}
	server.removeJob(item)
	logger.Debug("DelJob %v", job)
	return true
}

// 移除指定的多项任务
func (server *LocalTimerServer) DelJobs(jobs []Ijob) {
	server.pause()
	defer server.resume()
	for _, job := range jobs {
		item, ok := job.(*Job)
		if !ok {
			continue
		}
		server.removeJob(item)
	}
	logger.Debug("DelJobs %v", jobs)
}

// 获取server当前执行次数
func (server *LocalTimerServer) GetCount() uint64 {
	return atomic.LoadUint64(&server.Count)
}

// 重置server的内部状态
func (server *LocalTimerServer) Reset() *LocalTimerServer {
	server.exit()
	server.Count = 0
	server.cleanJobs()
	server.start()
	logger.Info("gentimer Reset")
	return server
}

func (server *LocalTimerServer) cleanJobs() {
	item := server.JobList.Min()
	for item != nil {
		job, ok := item.(*Job)
		if ok {
			server.removeJob(job)
		}
		item = server.JobList.Min()
	}
}

// 获取pending的任务数量
func (server *LocalTimerServer) WaitJobs() uint {
	leng := server.JobList.Len() - 1
	if leng > 0 {
		return leng
	}
	return 0
}

func (server *LocalTimerServer) addJob(createTime time.Time, intervalTime time.Duration, jobTimes uint64, jobFunc MessageHandler1, args []interface{}) (job *Job, inserted bool) {
	inserted = true
	server.Id++
	job = &Job{
		Id:           server.Id,
		Times:        jobTimes,
		CreateTime:   createTime,
		ActionTime:   createTime.Add(intervalTime),
		IntervalTime: intervalTime,
		MsgChan:      make(chan Job, 10),
		JobHandler:   jobFunc,
		Args:         args,
	}
	server.JobList.Insert(job)
	return
}

func (server *LocalTimerServer) removeJob(job *Job) {
	server.JobList.Delete(job)
	close(job.MsgChan)
}

// 优雅的关闭
// 会依次执行一次（不按照actiontime）
func (server *LocalTimerServer) StopByGrace() {
	server.exit()
	server.immediate()
	logger.Warning("gentimer StopByGrace")
}

// 强制关闭
func (server *LocalTimerServer) StopByForce() {
	server.exit()
	server.cleanJobs()
	logger.Warning("gentimer StopByForce")
}

func (server *LocalTimerServer) immediate() {
	item := server.JobList.Min()
	for item != nil {
		job, ok := item.(*Job)
		if ok {
			atomic.AddUint64(&server.Count, 1)
			job.ExecWithGo(false)
			server.removeJob(job)
		}
		item = server.JobList.Min()
	}
}

/*************************************Job相关************************************/

/****************************实现了rbtree.Item接口****************************/

func (job Job) Less(another rbtree.Item) bool {
	item, ok := another.(*Job)
	if !ok {
		return false
	}
	if !job.ActionTime.Equal(item.ActionTime) {
		return job.ActionTime.Before(item.ActionTime)
	}
	return job.Id < item.Id
}

/********************************实现了Job接口***********************************/

func (job Job) Notify() <-chan Job {
	return job.MsgChan
}

func (job Job) GetCount() uint64 {
	return job.Count
}

func (job Job) GetTimes() uint64 {
	return job.Times
}

func (job *Job) action() {
	defer func() {
		if err := recover(); err != nil {
			logger.Error("gentimer Exec job %v error: %v", job, err)
		}
	}()
	logger.Debug("start action job %v ", job)
	job.JobHandler(job.Args)
}

func (job *Job) ExecWithGo(isGo bool) {
	job.Count++
	if job.JobHandler == nil {
		return
	}
	if isGo {
		go job.action()
	} else {
		job.action()
	}
}
<|endoftext|>


<commit_before>package nomad

import "github.com/hashicorp/nomad/nomad/structs"

// drainerShim implements the drainer.RaftApplier interface required by the
// NodeDrainer.
type drainerShim struct {
	s *Server
}

func (d drainerShim) NodeDrainComplete(nodeID string) error {
	args := &structs.NodeUpdateDrainRequest{
		NodeID:       nodeID,
		Drain:        false,
		WriteRequest: structs.WriteRequest{Region: d.s.config.Region},
	}

	_, _, err := d.s.raftApply(structs.NodeUpdateDrainRequestType, args)
	return err
}

func (d drainerShim) AllocUpdateDesiredTransition(allocs map[string]*structs.DesiredTransition, evals []*structs.Evaluation) error {
	args := &structs.AllocUpdateDesiredTransitionRequest{
		Allocs:       allocs,
		Evals:        evals,
		WriteRequest: structs.WriteRequest{Region: d.s.config.Region},
	}
	_, _, err := d.s.raftApply(structs.AllocUpdateDesiredTransitionRequestType, args)
	return err
}
<commit_msg>drainer: convert fsm errors to go errors<commit_after>package nomad

import "github.com/hashicorp/nomad/nomad/structs"

// drainerShim implements the drainer.RaftApplier interface required by the
// NodeDrainer.
type drainerShim struct {
	s *Server
}

func (d drainerShim) NodeDrainComplete(nodeID string) error {
	args := &structs.NodeUpdateDrainRequest{
		NodeID:       nodeID,
		Drain:        false,
		WriteRequest: structs.WriteRequest{Region: d.s.config.Region},
	}

	resp, _, err := d.s.raftApply(structs.NodeUpdateDrainRequestType, args)
	return d.convertApplyErrors(resp, err)
}

func (d drainerShim) AllocUpdateDesiredTransition(allocs map[string]*structs.DesiredTransition, evals []*structs.Evaluation) error {
	args := &structs.AllocUpdateDesiredTransitionRequest{
		Allocs:       allocs,
		Evals:        evals,
		WriteRequest: structs.WriteRequest{Region: d.s.config.Region},
	}
	resp, _, err := d.s.raftApply(structs.AllocUpdateDesiredTransitionRequestType, args)
	return d.convertApplyErrors(resp, err)
}

// convertApplyErrors parses the results of a raftApply and returns the index at
// which it was applied and any error that occurred. Raft Apply returns two
// separate errors, Raft library errors and user returned errors from the FSM.
// This helper, joins the errors by inspecting the applyResponse for an error.
//
// Similar to deployment watcher's convertApplyErrors
func (d drainerShim) convertApplyErrors(applyResp interface{}, err error) error {
	if applyResp != nil {
		if fsmErr, ok := applyResp.(error); ok && fsmErr != nil {
			return fsmErr
		}
	}
	return err
}
<|endoftext|>


<commit_before>package haaasd

import (
	"bytes"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"os"
	"os/exec"
	"time"
)

func NewHaproxy(role string, properties *Config, application string, platform string, version string) *Haproxy {
	if version == "" {
		version = "1.4.22"
	}
	return &Haproxy{
		Role:             role,
		Application: application,
		Platform:    platform,
		properties:  properties,
		Version:     version,
	}
}

type Haproxy struct {
	Role        string
	Application string
	Platform    string
	Version     string
	properties  *Config
	State       int
}

const (
	SUCCESS int = iota
	UNCHANGED int = iota
	ERR_SYSLOG int = iota
	ERR_CONF int = iota
	ERR_RELOAD int = iota
)

// ApplyConfiguration write the new configuration and reload
// A rollback is called on failure
func (hap *Haproxy) ApplyConfiguration(data *EventMessage) (int, error) {
	hap.createSkeleton(data.Correlationid)

	newConf := data.Conf
	path := hap.confPath()

	// Check conf diff
	oldConf, err := ioutil.ReadFile(path)
	if log.GetLevel() == log.DebugLevel {
		hap.dumpConfiguration(hap.NewDebugPath(), newConf, data)
	}
	if bytes.Equal(oldConf, newConf) {
		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"application": data.Application,
			"plateform":   data.Platform,
		}).Debug("Unchanged configuration")
		return UNCHANGED, nil
	}

	// Archive previous configuration
	archivePath := hap.confArchivePath()
	os.Rename(path, archivePath)
	log.WithFields(
		log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"application": data.Application,
			"plateform":   data.Platform,
			"archivePath": archivePath,
		}).Info("Old configuration saved")
	err = ioutil.WriteFile(path, newConf, 0644)
	if err != nil {
		return ERR_CONF, err
	}
	log.WithField("path", path).Info("New configuration written")

	// Reload haproxy
	err = hap.reload(data.Correlationid)
	if err != nil {
		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"application": data.Application,
			"plateform":   data.Platform,
		}).WithError(err).Error("Reload failed")
		hap.dumpConfiguration(hap.NewErrorPath(), newConf, data)
		err = hap.rollback(data.Correlationid)
		return ERR_RELOAD, err
	}
	// Write syslog fragment
	fragmentPath := hap.syslogFragmentPath()
	err = ioutil.WriteFile(fragmentPath, data.SyslogFragment, 0644)
	if err != nil {
		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"application": data.Application,
			"plateform":   data.Platform,
		}).WithError(err).Error("Failed to write syslog fragment")
		// TODO Should we rollback on syslog error ?
		return ERR_SYSLOG, err
	}
	log.WithFields(log.Fields{
		"correlationId": data.Correlationid,
		"role": hap.Role,
		"application": data.Application,
		"plateform":   data.Platform,
		"content" : data.SyslogFragment,
		"filename": fragmentPath,
	}).Debug("Write syslog fragment")

	return SUCCESS, nil
}

// dumpConfiguration dumps the new configuration file with context for debugging purpose
func (hap *Haproxy) dumpConfiguration(filename string, newConf []byte, data *EventMessage) {

	f, err2 := os.Create(filename)
	defer f.Close()
	if err2 == nil {
		f.WriteString("================================================================\n")
		f.WriteString(fmt.Sprintf("application: %s\n", data.Application))
		f.WriteString(fmt.Sprintf("platform: %s\n", data.Platform))
		f.WriteString(fmt.Sprintf("correlationid: %s\n", data.Correlationid))
		f.WriteString("================================================================\n")
		f.Write(newConf)
		f.Sync()

		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"filename": filename,
			"application": data.Application,
			"platform": data.Platform,
		}).Info("Dump configuration")
	}
}

// confPath give the path of the configuration file given an application context
// It returns the absolute path to the file
func (hap *Haproxy) confPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/Config"
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/hap" + hap.Application + hap.Platform + ".conf"
}

// confPath give the path of the archived configuration file given an application context
func (hap *Haproxy) confArchivePath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/version-1"
	// It returns the absolute path to the file
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/hap" + hap.Application + hap.Platform + ".conf"
}

// NewErrorPath gives a unique path the error file given the hap context
// It returns the full path to the file
func (hap *Haproxy) NewErrorPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/errors"
	os.MkdirAll(baseDir, 0755)
	prefix := time.Now().Format("20060102150405")
	return baseDir + "/" + prefix + "_" + hap.Application + hap.Platform + ".log"
}

func (hap *Haproxy) NewDebugPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/dump"
	os.MkdirAll(baseDir, 0755)
	prefix := time.Now().Format("20060102150405")
	return baseDir + "/" + prefix + "_" + hap.Application + hap.Platform + ".log"
}

// reload calls external shell script to reload haproxy
// It returns error if the reload fails
func (hap *Haproxy) reload(correlationId string) error {

	reloadScript := hap.getReloadScript()
	cmd, err := exec.Command("sh", reloadScript, "reload", "-y").Output()
	if err != nil {
		log.WithError(err).Error("Error reloading")
	}
	log.WithFields(log.Fields{
		"correlationId" : correlationId,
		"role": hap.Role,
		"application": hap.Application,
		"platform": hap.Platform,
		"reloadScript": reloadScript,
	}).WithField("cmd", cmd).Debug("Reload succeeded")
	return err
}

// rollbac reverts configuration files and call for reload
func (hap *Haproxy) rollback(correlationId string) error {
	lastConf := hap.confArchivePath()
	if _, err := os.Stat(lastConf); os.IsNotExist(err) {
		return errors.New("No configuration file to rollback")
	}
	os.Rename(lastConf, hap.confPath())
	hap.reload(correlationId)
	return nil
}

// createSkeleton creates the directory tree for a new haproxy context
func (hap *Haproxy) createSkeleton(correlationId string) error {
	baseDir := hap.properties.HapHome + "/" + hap.Application

	createDirectory(correlationId, baseDir + "/Config")
	createDirectory(correlationId, baseDir + "/logs/" + hap.Application + hap.Platform)
	createDirectory(correlationId, baseDir + "/scripts")
	createDirectory(correlationId, baseDir + "/version-1")

	updateSymlink(correlationId, hap.getHapctlFilename(), hap.getReloadScript())
	updateSymlink(correlationId, hap.getHapBinary(), baseDir + "/Config/haproxy")

	return nil
}

// confPath give the path of the configuration file given an application context
// It returns the absolute path to the file
func (hap *Haproxy) syslogFragmentPath() string {
	baseDir := hap.properties.HapHome + "/SYSLOG/Config/syslog.conf.d"
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/syslog" + hap.Application + hap.Platform + ".conf"
}

// updateSymlink create or update a symlink
func updateSymlink(correlationId, oldname string, newname string) {
	newLink := true
	if _, err := os.Stat(newname); err == nil {
		os.Remove(newname)
		newLink = false
	}
	err := os.Symlink(oldname, newname)
	if err != nil {
		log.WithError(err).WithFields(log.Fields{
			"correlationId" : correlationId,
			"path": newname,
		}).Error("Symlink failed")
	}

	if newLink {
		log.WithFields(log.Fields{
			"correlationId" : correlationId,
			"path": newname,
		}).Info("Symlink created")
	}
}

// createDirectory recursively creates directory if it doesn't exists
func createDirectory(correlationId string, dir string) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.WithError(err).WithFields(log.Fields{
				"correlationId" : correlationId,
				"dir": dir,
			}).Error("Failed to create")
		} else {
			log.WithFields(log.Fields{
				"correlationId" : correlationId,
				"dir": dir,
			}).Info("Directory created")
		}
	}
}

// getHapctlFilename return the path to the vsc hapctl shell script
// This script is provided
func (hap *Haproxy) getHapctlFilename() string {
	return "/HOME/uxwadm/scripts/hapctl_unif"
}

// getReloadScript calculates reload script path given the hap context
// It returns the full script path
func (hap *Haproxy) getReloadScript() string {
	return fmt.Sprintf("%s/%s/scripts/hapctl%s%s", hap.properties.HapHome, hap.Application, hap.Application, hap.Platform)
}

// getHapBinary calculates the haproxy binary to use given the expected version
// It returns the full path to the haproxy binary
func (hap *Haproxy) getHapBinary() string {
	return fmt.Sprintf("/export/product/haproxy/product/%s/bin/haproxy", hap.Version)
}
<commit_msg>Add log details<commit_after>package haaasd

import (
	"bytes"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"os"
	"os/exec"
	"time"
)

func NewHaproxy(role string, properties *Config, application string, platform string, version string) *Haproxy {
	if version == "" {
		version = "1.4.22"
	}
	return &Haproxy{
		Role:             role,
		Application: application,
		Platform:    platform,
		properties:  properties,
		Version:     version,
	}
}

type Haproxy struct {
	Role        string
	Application string
	Platform    string
	Version     string
	properties  *Config
	State       int
}

const (
	SUCCESS int = iota
	UNCHANGED int = iota
	ERR_SYSLOG int = iota
	ERR_CONF int = iota
	ERR_RELOAD int = iota
)

// ApplyConfiguration write the new configuration and reload
// A rollback is called on failure
func (hap *Haproxy) ApplyConfiguration(data *EventMessage) (int, error) {
	hap.createSkeleton(data.Correlationid)

	newConf := data.Conf
	path := hap.confPath()

	// Check conf diff
	oldConf, err := ioutil.ReadFile(path)
	if log.GetLevel() == log.DebugLevel {
		hap.dumpConfiguration(hap.NewDebugPath(), newConf, data)
	}
	if bytes.Equal(oldConf, newConf) {
		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"application": data.Application,
			"plateform":   data.Platform,
		}).Debug("Unchanged configuration")
		return UNCHANGED, nil
	}

	// Archive previous configuration
	archivePath := hap.confArchivePath()
	os.Rename(path, archivePath)
	log.WithFields(
		log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"application": data.Application,
			"plateform":   data.Platform,
			"archivePath": archivePath,
		}).Info("Old configuration saved")
	err = ioutil.WriteFile(path, newConf, 0644)
	if err != nil {
		return ERR_CONF, err
	}

	log.WithFields(log.Fields{
		"correlationId": data.Correlationid,
		"role": hap.Role,
		"application": data.Application,
		"plateform":   data.Platform,
		"path", path,
	}).Info("New configuration written")

	// Reload haproxy
	err = hap.reload(data.Correlationid)
	if err != nil {
		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"application": data.Application,
			"plateform":   data.Platform,
		}).WithError(err).Error("Reload failed")
		hap.dumpConfiguration(hap.NewErrorPath(), newConf, data)
		err = hap.rollback(data.Correlationid)
		return ERR_RELOAD, err
	}
	// Write syslog fragment
	fragmentPath := hap.syslogFragmentPath()
	err = ioutil.WriteFile(fragmentPath, data.SyslogFragment, 0644)
	if err != nil {
		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"application": data.Application,
			"plateform":   data.Platform,
		}).WithError(err).Error("Failed to write syslog fragment")
		// TODO Should we rollback on syslog error ?
		return ERR_SYSLOG, err
	}
	log.WithFields(log.Fields{
		"correlationId": data.Correlationid,
		"role": hap.Role,
		"application": data.Application,
		"plateform":   data.Platform,
		"content" : data.SyslogFragment,
		"filename": fragmentPath,
	}).Debug("Write syslog fragment")

	return SUCCESS, nil
}

// dumpConfiguration dumps the new configuration file with context for debugging purpose
func (hap *Haproxy) dumpConfiguration(filename string, newConf []byte, data *EventMessage) {

	f, err2 := os.Create(filename)
	defer f.Close()
	if err2 == nil {
		f.WriteString("================================================================\n")
		f.WriteString(fmt.Sprintf("application: %s\n", data.Application))
		f.WriteString(fmt.Sprintf("platform: %s\n", data.Platform))
		f.WriteString(fmt.Sprintf("correlationid: %s\n", data.Correlationid))
		f.WriteString("================================================================\n")
		f.Write(newConf)
		f.Sync()

		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"role": hap.Role,
			"filename": filename,
			"application": data.Application,
			"platform": data.Platform,
		}).Info("Dump configuration")
	}
}

// confPath give the path of the configuration file given an application context
// It returns the absolute path to the file
func (hap *Haproxy) confPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/Config"
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/hap" + hap.Application + hap.Platform + ".conf"
}

// confPath give the path of the archived configuration file given an application context
func (hap *Haproxy) confArchivePath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/version-1"
	// It returns the absolute path to the file
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/hap" + hap.Application + hap.Platform + ".conf"
}

// NewErrorPath gives a unique path the error file given the hap context
// It returns the full path to the file
func (hap *Haproxy) NewErrorPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/errors"
	os.MkdirAll(baseDir, 0755)
	prefix := time.Now().Format("20060102150405")
	return baseDir + "/" + prefix + "_" + hap.Application + hap.Platform + ".log"
}

func (hap *Haproxy) NewDebugPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/dump"
	os.MkdirAll(baseDir, 0755)
	prefix := time.Now().Format("20060102150405")
	return baseDir + "/" + prefix + "_" + hap.Application + hap.Platform + ".log"
}

// reload calls external shell script to reload haproxy
// It returns error if the reload fails
func (hap *Haproxy) reload(correlationId string) error {

	reloadScript := hap.getReloadScript()
	cmd, err := exec.Command("sh", reloadScript, "reload", "-y").Output()
	if err != nil {
		log.WithError(err).Error("Error reloading")
	}
	log.WithFields(log.Fields{
		"correlationId" : correlationId,
		"role": hap.Role,
		"application": hap.Application,
		"platform": hap.Platform,
		"reloadScript": reloadScript,
	}).WithField("cmd", cmd).Debug("Reload succeeded")
	return err
}

// rollbac reverts configuration files and call for reload
func (hap *Haproxy) rollback(correlationId string) error {
	lastConf := hap.confArchivePath()
	if _, err := os.Stat(lastConf); os.IsNotExist(err) {
		return errors.New("No configuration file to rollback")
	}
	os.Rename(lastConf, hap.confPath())
	hap.reload(correlationId)
	return nil
}

// createSkeleton creates the directory tree for a new haproxy context
func (hap *Haproxy) createSkeleton(correlationId string) error {
	baseDir := hap.properties.HapHome + "/" + hap.Application

	createDirectory(correlationId, baseDir + "/Config")
	createDirectory(correlationId, baseDir + "/logs/" + hap.Application + hap.Platform)
	createDirectory(correlationId, baseDir + "/scripts")
	createDirectory(correlationId, baseDir + "/version-1")

	updateSymlink(correlationId, hap.getHapctlFilename(), hap.getReloadScript())
	updateSymlink(correlationId, hap.getHapBinary(), baseDir + "/Config/haproxy")

	return nil
}

// confPath give the path of the configuration file given an application context
// It returns the absolute path to the file
func (hap *Haproxy) syslogFragmentPath() string {
	baseDir := hap.properties.HapHome + "/SYSLOG/Config/syslog.conf.d"
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/syslog" + hap.Application + hap.Platform + ".conf"
}

// updateSymlink create or update a symlink
func updateSymlink(correlationId, oldname string, newname string) {
	newLink := true
	if _, err := os.Stat(newname); err == nil {
		os.Remove(newname)
		newLink = false
	}
	err := os.Symlink(oldname, newname)
	if err != nil {
		log.WithError(err).WithFields(log.Fields{
			"correlationId" : correlationId,
			"path": newname,
		}).Error("Symlink failed")
	}

	if newLink {
		log.WithFields(log.Fields{
			"correlationId" : correlationId,
			"path": newname,
		}).Info("Symlink created")
	}
}

// createDirectory recursively creates directory if it doesn't exists
func createDirectory(correlationId string, dir string) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.WithError(err).WithFields(log.Fields{
				"correlationId" : correlationId,
				"dir": dir,
			}).Error("Failed to create")
		} else {
			log.WithFields(log.Fields{
				"correlationId" : correlationId,
				"dir": dir,
			}).Info("Directory created")
		}
	}
}

// getHapctlFilename return the path to the vsc hapctl shell script
// This script is provided
func (hap *Haproxy) getHapctlFilename() string {
	return "/HOME/uxwadm/scripts/hapctl_unif"
}

// getReloadScript calculates reload script path given the hap context
// It returns the full script path
func (hap *Haproxy) getReloadScript() string {
	return fmt.Sprintf("%s/%s/scripts/hapctl%s%s", hap.properties.HapHome, hap.Application, hap.Application, hap.Platform)
}

// getHapBinary calculates the haproxy binary to use given the expected version
// It returns the full path to the haproxy binary
func (hap *Haproxy) getHapBinary() string {
	return fmt.Sprintf("/export/product/haproxy/product/%s/bin/haproxy", hap.Version)
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "math"
import "math/big"

func getTestCases() []int {
	testCases := []int{}
	var numberOfLines int
	fmt.Scanf("%d", &numberOfLines)

	if numberOfLines > 0 {
		for i := 0; i < numberOfLines; i++ {
			var testCase int
			fmt.Scanf("%d", &testCase)
			testCases = append(testCases, testCase)
		}
	}
	return testCases
}

func isPerfectSquare(number int) bool {
	floatNumber := math.Sqrt(float64(number))
	// intNumber := float64(math.Trunc(floatNumber))
	intNumber := big.NewFloat(floatNumber)
	// isPerfectSquare := (intNumber == floatNumber)
	isPerfectSquare := intNumber.IsInt()
	fmt.Println(number)
	fmt.Println(floatNumber)
	fmt.Println(intNumber)
	fmt.Println(isPerfectSquare)
	return isPerfectSquare
}

func isFibonacciNumber(number int) bool {
	fiveByXSquare := 5 * number * number
	isFibonacciNumber := isPerfectSquare(fiveByXSquare+4) || isPerfectSquare(fiveByXSquare-4)
	return isFibonacciNumber
}

func main() {

	// https://www.hackerrank.com/challenges/is-fibo
	// testCases := getTestCases()
	testCases := []int{7778742049}
	// testCases := []int{5,7,8}

	for _, element := range testCases {
		if isFibonacciNumber(element) {
			fmt.Println("IsFibo")
		} else {
			fmt.Println("IsNotFibo")
		}
	}
}
<commit_msg>Read test cases as float64 insted of int.<commit_after>package main

import "fmt"
import "math"
import "math/big"

func getTestCases() []float64 {
	testCases := []float64{}
	var numberOfLines int
	fmt.Scanf("%d", &numberOfLines)

	if numberOfLines > 0 {
		for i := 0; i < numberOfLines; i++ {
			var testCase float64
			fmt.Scanf("%d", &testCase)
			testCases = append(testCases, testCase)
		}
	}
	return testCases
}

func isPerfectSquare(number float64) bool {
	floatNumber := math.Sqrt(float64(number))
	// intNumber := float64(math.Trunc(floatNumber))
	intNumber := big.NewFloat(floatNumber)
	// isPerfectSquare := (intNumber == floatNumber)
	isPerfectSquare := intNumber.IsInt()
	fmt.Println(number)
	fmt.Println(floatNumber)
	fmt.Println(intNumber)
	fmt.Println(isPerfectSquare)
	return isPerfectSquare
}

func isFibonacciNumber(number float64) bool {
	fiveByXSquare := float64(5 * number * number)
	isFibonacciNumber := isPerfectSquare(fiveByXSquare+4) || isPerfectSquare(fiveByXSquare-4)
	return isFibonacciNumber
}

func main() {

	// https://www.hackerrank.com/challenges/is-fibo
	// testCases := getTestCases()
	testCases := []float64{7778742049}
	// testCases := []float64{5,7,8}
	for _, element := range testCases {
		if isFibonacciNumber(element) {
			fmt.Println("IsFibo")
		} else {
			fmt.Println("IsNotFibo")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"cydev.ru/hath"
	"github.com/pivotal-golang/bytefmt"
)

var (
	count      int64
	dbpath     string
	sizeMax    int64
	sizeMin    int64
	resMax     int
	resMin     int
	workers    int
	generate   bool
	cpus       int
	collect    bool
	bulkSize   int64
	onlyOpen   bool
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
)

func init() {
	flag.Int64Var(&count, "count", 100, "files to generate")
	flag.Int64Var(&bulkSize, "bulk", 10000, "bulk size")
	flag.Int64Var(&sizeMax, "size-max", 1024*100, "maximum file size in bytes")
	flag.Int64Var(&sizeMin, "size-min", 1024*5, "minimum file size in bytes")
	flag.IntVar(&resMax, "res-max", 1980, "maximum ephemeral resolution")
	flag.IntVar(&resMin, "res-min", 500, "minumum ephemeral resolution")
	flag.BoolVar(&generate, "generate", false, "generate data")
	flag.BoolVar(&collect, "collect", true, "collect old files")
	flag.BoolVar(&onlyOpen, "only-open", false, "only open db")
	flag.StringVar(&dbpath, "dbfile", "db.bolt", "working directory")
	flag.IntVar(&cpus, "cpus", runtime.GOMAXPROCS(0), "cpu to use")
}

func main() {
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()
	flag.Parse()
	runtime.GOMAXPROCS(cpus)
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	g := hath.FileGenerator{
		SizeMax:       sizeMax,
		SizeMin:       sizeMin,
		ResolutionMax: resMax,
		ResolutionMin: resMin,
		TimeDelta:     20,
	}
	db, err := hath.NewDB(dbpath)
	defer db.Close()
	if err != nil {
		log.Fatal(err)
	}
	d, err := g.NewFake().Marshal()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("average file info is", len(d), "bytes")
	log.Printf("%x", d)
	if onlyOpen {
		log.Println("only open. Waiting for 10s")
		count = int64(db.GetFilesCount())
		start := time.Now()
		n, err := db.GetOldFilesCount(time.Now().Add(time.Second * -10))
		if err != nil {
			log.Fatal(err)
		}
		log.Println("got", n)
		end := time.Now()
		duration := end.Sub(start)
		rate := float64(count) / duration.Seconds()
		log.Printf("Scanned %d for %v at rate %f per second\n", count, duration, rate)
		time.Sleep(10 * time.Second)
	}
	if generate {
		log.Println("generating", count, "files")

		fmt.Printf("%+v\n", g)
		var i int64
		files := make([]hath.File, count)
		for i = 0; i < count; i++ {
			files[i] = g.NewFake()
		}
		start := time.Now()
		if count < bulkSize {
			log.Println("writing")
			if err := db.AddBatch(files); err != nil {
				log.Fatal(err)
			}
		} else {
			log.Println("writing in bulks")
			for i = 0; i+bulkSize < count; i += bulkSize {
				bulkstart := time.Now()

				if err := db.AddBatch(files[i : i+bulkSize]); err != nil {
					log.Fatal(err)
				}

				log.Println("from", i, "to", i+bulkSize, time.Now().Sub(bulkstart))
			}
			log.Println("from", i+bulkSize, "to", count)
			if err := db.AddBatch(files[i:]); err != nil {
				log.Fatal(err)
			}
		}
		end := time.Now()
		duration := end.Sub(start)
		rate := float64(count) / duration.Seconds()
		fmt.Printf("OK for %v at rate %f per second\n", duration, rate)
	}
	if collect {
		log.Println("collecting")
		start := time.Now()
		n, err := db.Collect(time.Now().Add(-time.Second))
		if err != nil {
			log.Fatal(err)
		}

		end := time.Now()
		duration := end.Sub(start)
		rate := float64(n) / duration.Seconds()
		fmt.Printf("Removed %d for %v at rate %f per second\n", n, duration, rate)
	}
	log.Println(count, "is rought", bytefmt.ByteSize(hath.GetRoughCacheSize(count)))
	log.Println("OK")
}
<commit_msg>added only-memory test<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"cydev.ru/hath"
	"github.com/pivotal-golang/bytefmt"
)

var (
	count      int64
	dbpath     string
	sizeMax    int64
	sizeMin    int64
	resMax     int
	resMin     int
	workers    int
	generate   bool
	cpus       int
	collect    bool
	bulkSize   int64
	onlyOpen   bool
	onlyMemory bool
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
)

func init() {
	flag.Int64Var(&count, "count", 10000, "files to generate")
	flag.Int64Var(&bulkSize, "bulk", 10000, "bulk size")
	flag.Int64Var(&sizeMax, "size-max", 1024*100, "maximum file size in bytes")
	flag.Int64Var(&sizeMin, "size-min", 1024*5, "minimum file size in bytes")
	flag.IntVar(&resMax, "res-max", 1980, "maximum ephemeral resolution")
	flag.IntVar(&resMin, "res-min", 500, "minumum ephemeral resolution")
	flag.BoolVar(&generate, "generate", false, "generate data")
	flag.BoolVar(&collect, "collect", true, "collect old files")
	flag.BoolVar(&onlyOpen, "only-open", false, "only open db")
	flag.BoolVar(&onlyMemory, "only-memory", false, "only load to memory")
	flag.StringVar(&dbpath, "dbfile", "db.bolt", "working directory")
	flag.IntVar(&cpus, "cpus", runtime.GOMAXPROCS(0), "cpu to use")
}

func main() {
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()
	flag.Parse()
	runtime.GOMAXPROCS(cpus)
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	g := hath.FileGenerator{
		SizeMax:       sizeMax,
		SizeMin:       sizeMin,
		ResolutionMax: resMax,
		ResolutionMin: resMin,
		TimeDelta:     20,
	}
	if onlyMemory {
		log.Println("allocating")
		var files = make([]hath.File, count)
		log.Println("generating")
		for i := range files {
			files[i] = g.NewFake()
		}
		log.Println("generated", len(files))
		fmt.Scanln()
		start := time.Now()
		log.Println("iterating...")
		var count int
		for _, f := range files {
			if f.Static {
				count++
			}
		}
		end := time.Now()
		duration := end.Sub(start)
		rate := float64(len(files)) / duration.Seconds()
		fmt.Println("static", count)
		fmt.Printf("OK for %v at rate %f per second\n", duration, rate)
		log.Println("iterated")
		os.Exit(0)
	}
	db, err := hath.NewDB(dbpath)
	defer db.Close()
	if err != nil {
		log.Fatal(err)
	}
	d, err := g.NewFake().Marshal()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("average file info is", len(d), "bytes")
	log.Printf("%x", d)
	if onlyOpen {
		log.Println("only open. Waiting for 10s")
		count = int64(db.Count())
		start := time.Now()
		n, err := db.GetOldFilesCount(time.Now().Add(time.Second * -10))
		if err != nil {
			log.Fatal(err)
		}
		log.Println("got", n)
		end := time.Now()
		duration := end.Sub(start)
		rate := float64(count) / duration.Seconds()
		log.Printf("Scanned %d for %v at rate %f per second\n", count, duration, rate)
		time.Sleep(10 * time.Second)
	}
	if generate {
		log.Println("generating", count, "files")

		fmt.Printf("%+v\n", g)
		var i int64
		files := make([]hath.File, count)
		for i = 0; i < count; i++ {
			files[i] = g.NewFake()
		}
		start := time.Now()
		if count < bulkSize {
			log.Println("writing")
			if err := db.AddBatch(files); err != nil {
				log.Fatal(err)
			}
		} else {
			log.Println("writing in bulks")
			for i = 0; i+bulkSize < count; i += bulkSize {
				bulkstart := time.Now()

				if err := db.AddBatch(files[i : i+bulkSize]); err != nil {
					log.Fatal(err)
				}

				log.Println("from", i, "to", i+bulkSize, time.Now().Sub(bulkstart))
			}
			log.Println("from", i+bulkSize, "to", count)
			if err := db.AddBatch(files[i:]); err != nil {
				log.Fatal(err)
			}
		}
		end := time.Now()
		duration := end.Sub(start)
		rate := float64(count) / duration.Seconds()
		fmt.Printf("OK for %v at rate %f per second\n", duration, rate)
	}
	// if collect {
	// 	log.Println("collecting")
	// 	start := time.Now()
	// 	n, err := db.Collect(time.Now().Add(-time.Second))
	// 	if err != nil {
	// 		log.Fatal(err)
	// 	}
	//
	// 	end := time.Now()
	// 	duration := end.Sub(start)
	// 	rate := float64(n) / duration.Seconds()
	// 	fmt.Printf("Removed %d for %v at rate %f per second\n", n, duration, rate)
	// }
	log.Println(count, "is rought", bytefmt.ByteSize(hath.GetRoughCacheSize(count)))
	log.Println("OK")
}
<|endoftext|>


<commit_before>package main

import (
	"bitmm/bitfinex"
	// "github.com/davecgh/go-spew/spew"
	"testing"
)

func TestCalcOrderParams(t *testing.T) {
	maxPos = 100
	minEdge = 0.02
	exitPercent = 0.25
	var params []bitfinex.OrderParams
	// Test long postions
	params = calculateOrderParams(100-MINO/2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 1 {
		t.Fatal("Should only create one order")
	}
	params = calculateOrderParams(100-MINO*2, 2.00, 0.01)
	// spew.Dump(params)
	if len(params) != 3 {
		t.Fatal("Should create three orders")
	}
	params = calculateOrderParams(100*2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 1 {
		t.Fatal("Should only create one order")
	}
	params = calculateOrderParams(MINO/2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 2 {
		t.Fatal("Should create two orders")
	}
	// Test short positions
	params = calculateOrderParams(-100+MINO/2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 1 {
		t.Fatal("Should only create one order")
	}
	params = calculateOrderParams(-100+MINO*2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 3 {
		t.Fatal("Should create three orders")
	}
	params = calculateOrderParams(-100*2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 1 {
		t.Fatal("Should only create one order")
	}
	params = calculateOrderParams(-MINO/2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 2 {
		t.Fatal("Should create two orders")
	}
}
<commit_msg>fix broken test<commit_after>package main

import (
	"bitmm/bitfinex"
	"code.google.com/p/gcfg"
	// "github.com/davecgh/go-spew/spew"
	"testing"
)

func TestCalcOrderParams(t *testing.T) {
	// Get config info
	err := gcfg.ReadFileInto(&cfg, "bitmm.gcfg")
	if err != nil {
		t.Fatal(err)
	}

	var params []bitfinex.OrderParams
	// Test long postions
	params = calculateOrderParams(cfg.Sec.MaxPos-cfg.Sec.MinPos/2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 1 {
		t.Fatal("Should only create one order")
	}
	params = calculateOrderParams(cfg.Sec.MaxPos-cfg.Sec.MinPos*2, 2.00, 0.01)
	// spew.Dump(params)
	if len(params) != 3 {
		t.Fatal("Should create three orders")
	}
	params = calculateOrderParams(cfg.Sec.MaxPos*2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 1 {
		t.Fatal("Should only create one order")
	}
	params = calculateOrderParams(cfg.Sec.MinPos/2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 2 {
		t.Fatal("Should create two orders")
	}
	// Test short positions
	params = calculateOrderParams(-cfg.Sec.MaxPos+cfg.Sec.MinPos/2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 1 {
		t.Fatal("Should only create one order")
	}
	params = calculateOrderParams(-cfg.Sec.MaxPos+cfg.Sec.MinPos*2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 3 {
		t.Fatal("Should create three orders")
	}
	params = calculateOrderParams(-cfg.Sec.MaxPos*2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 1 {
		t.Fatal("Should only create one order")
	}
	params = calculateOrderParams(-cfg.Sec.MinPos/2, 2.00, 0.04)
	// spew.Dump(params)
	if len(params) != 2 {
		t.Fatal("Should create two orders")
	}
}
<|endoftext|>