<commit_before>package conf

import (
	"io/ioutil"
	"os"

	"github.com/imdario/mergo"
	"gopkg.in/yaml.v2"
)

// C is the exported global configuration variable
var C Conf

// Conf is the struct containing the configuration of the server
type Conf struct {
	NameServer    string  `yaml:"name_server" form:"name_server"`
	UploadDir     string  `yaml:"upload_dir" form:"upload_dir"`
	DB            string  `yaml:"db" form:"db"`
	Port          int     `yaml:"port" form:"port"`
	UniURILength  int     `yaml:"uniuri_length" form:"uniuri_length"`
	KeyLength     int     `yaml:"key_length" form:"key_length"`
	SizeLimit     int64   `yaml:"size_limit" form:"size_limit"`
	DiskQuota     float64 `yaml:"disk_quota" form:"disk_quota"`
	NoWeb         bool    `yaml:"no_web" form:"no_web"`
	FullDoc       bool    `yaml:"fulldoc" form:"fulldoc"`
	LogLevel      string  `yaml:"loglevel" form:"loglevel"`
	Stats         bool    `yaml:"stats" form:"stats"`
	SensitiveMode bool    `yaml:"sensitive_mode" form:"sensitive_mode"`
	ServeHTTPS    bool    `yaml:"serve_https" form:"serve_https"`
	SSLCert       string  `yaml:"ssl_cert" form:"ssl_cert"`
	SSLPrivKey    string  `yaml:"ssl_private_key" form:"ssl_private_key"`
	AppendPort    bool    `yaml:"append_port" form:"append_port"`
}

// NewDefault returns a Conf instance filled with default values
func NewDefault() Conf {
	return Conf{
		UploadDir:    "up/",
		DB:           "goploader.db",
		Port:         8080,
		UniURILength: 10,
		SizeLimit:    20,
		DiskQuota:    0,
		KeyLength:    16,
		LogLevel:     "info",
	}
}

// Validate validates that an unparsed configuration is valid.
func (c *Conf) Validate() map[string]string {
	errors := make(map[string]string)
	if c.NameServer == "" {
		errors["name_server"] = "This field is required."
	}
	if c.ServeHTTPS {
		if c.SSLCert == "" {
			errors["ssl_cert"] = "This field is required if you serve https."
		}
		if c.SSLPrivKey == "" {
			errors["ssl_private_key"] = "This field is required if you serve https."
		}
	}
	return errors
}

// FillDefaults fills the zero value fields in the UnparsedConf with default values
func (c *Conf) FillDefaults() error {
	return mergo.Merge(c, NewDefault())
}

// Load loads the given fp (file path) to the C global configuration variable.
func Load(fp string, verbose bool) error {
	var err error
	var conf []byte
	if conf, err = ioutil.ReadFile(fp); err != nil {
		return err
	}
	if err = yaml.Unmarshal(conf, &C); err != nil {
		return err
	}
	if err = C.FillDefaults(); err != nil {
		return err
	}
	if _, err := os.Stat(fp); os.IsNotExist(err) {
		if err = os.Mkdir(fp, 0777); err != nil {
			return err
		}
	} else if err != nil {
		return err
	}
	return nil
}
<commit_msg>Fixing server not creating the upload directory<commit_after>package conf

import (
	"io/ioutil"
	"os"

	"github.com/imdario/mergo"
	"gopkg.in/yaml.v2"
)

// C is the exported global configuration variable
var C Conf

// Conf is the struct containing the configuration of the server
type Conf struct {
	NameServer    string  `yaml:"name_server" form:"name_server"`
	UploadDir     string  `yaml:"upload_dir" form:"upload_dir"`
	DB            string  `yaml:"db" form:"db"`
	Port          int     `yaml:"port" form:"port"`
	UniURILength  int     `yaml:"uniuri_length" form:"uniuri_length"`
	KeyLength     int     `yaml:"key_length" form:"key_length"`
	SizeLimit     int64   `yaml:"size_limit" form:"size_limit"`
	DiskQuota     float64 `yaml:"disk_quota" form:"disk_quota"`
	NoWeb         bool    `yaml:"no_web" form:"no_web"`
	FullDoc       bool    `yaml:"fulldoc" form:"fulldoc"`
	LogLevel      string  `yaml:"loglevel" form:"loglevel"`
	Stats         bool    `yaml:"stats" form:"stats"`
	SensitiveMode bool    `yaml:"sensitive_mode" form:"sensitive_mode"`
	ServeHTTPS    bool    `yaml:"serve_https" form:"serve_https"`
	SSLCert       string  `yaml:"ssl_cert" form:"ssl_cert"`
	SSLPrivKey    string  `yaml:"ssl_private_key" form:"ssl_private_key"`
	AppendPort    bool    `yaml:"append_port" form:"append_port"`
}

// NewDefault returns a Conf instance filled with default values
func NewDefault() Conf {
	return Conf{
		UploadDir:    "up/",
		DB:           "goploader.db",
		Port:         8080,
		UniURILength: 10,
		SizeLimit:    20,
		DiskQuota:    0,
		KeyLength:    16,
		LogLevel:     "info",
	}
}

// Validate validates that an unparsed configuration is valid.
func (c *Conf) Validate() map[string]string {
	errors := make(map[string]string)
	if c.NameServer == "" {
		errors["name_server"] = "This field is required."
	}
	if c.ServeHTTPS {
		if c.SSLCert == "" {
			errors["ssl_cert"] = "This field is required if you serve https."
		}
		if c.SSLPrivKey == "" {
			errors["ssl_private_key"] = "This field is required if you serve https."
		}
	}
	return errors
}

// FillDefaults fills the zero value fields in the UnparsedConf with default values
func (c *Conf) FillDefaults() error {
	return mergo.Merge(c, NewDefault())
}

// Load loads the given fp (file path) to the C global configuration variable.
func Load(fp string, verbose bool) error {
	var err error
	var conf []byte
	if conf, err = ioutil.ReadFile(fp); err != nil {
		return err
	}
	if err = yaml.Unmarshal(conf, &C); err != nil {
		return err
	}
	if err = C.FillDefaults(); err != nil {
		return err
	}
	if _, err := os.Stat(C.UploadDir); os.IsNotExist(err) {
		if err = os.Mkdir(C.UploadDir, 0777); err != nil {
			return err
		}
	} else if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// package idletiming provides mechanisms for adding idle timeouts to net.Conn
// and net.Listener.
package idletiming

import (
	"net"
	"sync/atomic"
	"time"
)

// Conn creates a new net.Conn wrapping the given net.Conn that times out after
// the specified period. Read and Write calls will timeout if they take longer
// than the indicated
//
// idleTimeout specifies how long to wait for inactivity before considering
// connection idle.
//
// onClose is an optional function to call after the connection has been closed,
// whether or not that was due to the connection idling.
func Conn(conn net.Conn, idleTimeout time.Duration, onClose func()) *IdleTimingConn {
	c := &IdleTimingConn{
		conn:             conn,
		idleTimeout:      idleTimeout,
		halfIdleTimeout:  time.Duration(idleTimeout.Nanoseconds() / 2),
		activeCh:         make(chan bool, 10),
		closedCh:         make(chan bool, 10),
		lastActivityTime: int64(time.Now().UnixNano()),
	}

	go func() {
		if onClose != nil {
			defer onClose()
		}

		timer := time.NewTimer(idleTimeout)
		defer timer.Stop()
		for {
			select {
			case <-c.activeCh:
				// We're active, continue
				timer.Reset(idleTimeout)
				atomic.StoreInt64(&c.lastActivityTime, time.Now().UnixNano())
				continue
			case <-timer.C:
				//c.Close()
				return
			case <-c.closedCh:
				c.Close()
				return
			}
		}
	}()

	return c
}

// IdleTimingConn is a net.Conn that wraps another net.Conn and that times out
// if idle for more than idleTimeout.
type IdleTimingConn struct {
	conn             net.Conn
	idleTimeout      time.Duration
	halfIdleTimeout  time.Duration
	readDeadline     *time.Time
	writeDeadline    *time.Time
	activeCh         chan bool
	closedCh         chan bool
	lastActivityTime int64
}

// TimesOutIn returns how much time is left before this connection will time
// out, assuming there is no further activity.
func (c *IdleTimingConn) TimesOutIn() time.Duration {
	return c.TimesOutAt().Sub(time.Now())
}

// TimesOutAt returns the time at which this connection will time out, assuming
// there is no further activity
func (c *IdleTimingConn) TimesOutAt() time.Time {
	return time.Unix(0, c.lastActivityTime).Add(c.idleTimeout)
}

// Read implements the method from io.Reader
func (c *IdleTimingConn) Read(b []byte) (int, error) {
	totalN := 0
	// Continually read while we can, always setting a deadline that's less than
	// our idleTimeout so that we can update our active status before we hit the
	// idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if c.readDeadline != nil && !maxDeadline.Before(*c.readDeadline) {
			// Caller's deadline is before ours, use it
			c.conn.SetReadDeadline(*c.readDeadline)
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			c.conn.SetReadDeadline(maxDeadline)
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			timedOut := isTimeout(err)
			if timedOut {
				// Ignore timeouts when using deadline based on IdleTimeout
				err = nil
			}
			if n == 0 || !timedOut {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Write implements the method from io.Reader
func (c *IdleTimingConn) Write(b []byte) (int, error) {
	totalN := 0
	// Continually write while we can, always setting a deadline that's less
	// than our idleTimeout so that we can update our active status before we
	// hit the idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if c.writeDeadline != nil && !maxDeadline.Before(*c.writeDeadline) {
			// Caller's deadline is before ours, use it
			c.conn.SetWriteDeadline(*c.writeDeadline)
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			c.conn.SetWriteDeadline(maxDeadline)
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			timedOut := isTimeout(err)
			if timedOut {
				// Ignore timeouts when using deadline based on IdleTimeout
				err = nil
			}
			if n == 0 || !timedOut {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

func (c *IdleTimingConn) Close() error {
	c.closedCh <- true
	return c.conn.Close()
}

func (c *IdleTimingConn) LocalAddr() net.Addr {
	return c.conn.LocalAddr()
}

func (c *IdleTimingConn) RemoteAddr() net.Addr {
	return c.conn.RemoteAddr()
}

func (c *IdleTimingConn) SetDeadline(t time.Time) error {
	c.SetReadDeadline(t)
	c.SetWriteDeadline(t)
	return nil
}

func (c *IdleTimingConn) SetReadDeadline(t time.Time) error {
	c.readDeadline = &t
	return nil
}

func (c *IdleTimingConn) SetWriteDeadline(t time.Time) error {
	c.writeDeadline = &t
	return nil
}

func (c *IdleTimingConn) markActive(n int) bool {
	if n > 0 {
		c.activeCh <- true
		return true
	} else {
		return false
	}
}

func isTimeout(err error) bool {
	if netErr, ok := err.(net.Error); ok {
		return netErr.Timeout()
	}
	return false
}
<commit_msg>getlantern/lantern#2121 Reduced CPU usage in waitforserver by increasing sleep time on each loop<commit_after>// package idletiming provides mechanisms for adding idle timeouts to net.Conn
// and net.Listener.
package idletiming

import (
	"net"
	"sync/atomic"
	"time"
)

// Conn creates a new net.Conn wrapping the given net.Conn that times out after
// the specified period. Read and Write calls will timeout if they take longer
// than the indicated
//
// idleTimeout specifies how long to wait for inactivity before considering
// connection idle.
//
// onClose is an optional function to call after the connection has been closed,
// whether or not that was due to the connection idling.
func Conn(conn net.Conn, idleTimeout time.Duration, onClose func()) *IdleTimingConn {
	c := &IdleTimingConn{
		conn:             conn,
		idleTimeout:      idleTimeout,
		halfIdleTimeout:  time.Duration(idleTimeout.Nanoseconds() / 2),
		activeCh:         make(chan bool, 10),
		closedCh:         make(chan bool, 10),
		lastActivityTime: int64(time.Now().UnixNano()),
	}

	go func() {
		if onClose != nil {
			defer onClose()
		}

		timer := time.NewTimer(idleTimeout)
		defer timer.Stop()
		for {
			select {
			case <-c.activeCh:
				// We're active, continue
				timer.Reset(idleTimeout)
				atomic.StoreInt64(&c.lastActivityTime, time.Now().UnixNano())
				continue
			case <-timer.C:
				return
			case <-c.closedCh:
				c.Close()
				return
			}
		}
	}()

	return c
}

// IdleTimingConn is a net.Conn that wraps another net.Conn and that times out
// if idle for more than idleTimeout.
type IdleTimingConn struct {
	conn             net.Conn
	idleTimeout      time.Duration
	halfIdleTimeout  time.Duration
	readDeadline     *time.Time
	writeDeadline    *time.Time
	activeCh         chan bool
	closedCh         chan bool
	lastActivityTime int64
}

// TimesOutIn returns how much time is left before this connection will time
// out, assuming there is no further activity.
func (c *IdleTimingConn) TimesOutIn() time.Duration {
	return c.TimesOutAt().Sub(time.Now())
}

// TimesOutAt returns the time at which this connection will time out, assuming
// there is no further activity
func (c *IdleTimingConn) TimesOutAt() time.Time {
	return time.Unix(0, c.lastActivityTime).Add(c.idleTimeout)
}

// Read implements the method from io.Reader
func (c *IdleTimingConn) Read(b []byte) (int, error) {
	totalN := 0
	// Continually read while we can, always setting a deadline that's less than
	// our idleTimeout so that we can update our active status before we hit the
	// idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if c.readDeadline != nil && !maxDeadline.Before(*c.readDeadline) {
			// Caller's deadline is before ours, use it
			c.conn.SetReadDeadline(*c.readDeadline)
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			c.conn.SetReadDeadline(maxDeadline)
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			timedOut := isTimeout(err)
			if timedOut {
				// Ignore timeouts when using deadline based on IdleTimeout
				err = nil
			}
			if n == 0 || !timedOut {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Write implements the method from io.Reader
func (c *IdleTimingConn) Write(b []byte) (int, error) {
	totalN := 0
	// Continually write while we can, always setting a deadline that's less
	// than our idleTimeout so that we can update our active status before we
	// hit the idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if c.writeDeadline != nil && !maxDeadline.Before(*c.writeDeadline) {
			// Caller's deadline is before ours, use it
			c.conn.SetWriteDeadline(*c.writeDeadline)
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			c.conn.SetWriteDeadline(maxDeadline)
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			timedOut := isTimeout(err)
			if timedOut {
				// Ignore timeouts when using deadline based on IdleTimeout
				err = nil
			}
			if n == 0 || !timedOut {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

func (c *IdleTimingConn) Close() error {
	c.closedCh <- true
	return c.conn.Close()
}

func (c *IdleTimingConn) LocalAddr() net.Addr {
	return c.conn.LocalAddr()
}

func (c *IdleTimingConn) RemoteAddr() net.Addr {
	return c.conn.RemoteAddr()
}

func (c *IdleTimingConn) SetDeadline(t time.Time) error {
	c.SetReadDeadline(t)
	c.SetWriteDeadline(t)
	return nil
}

func (c *IdleTimingConn) SetReadDeadline(t time.Time) error {
	c.readDeadline = &t
	return nil
}

func (c *IdleTimingConn) SetWriteDeadline(t time.Time) error {
	c.writeDeadline = &t
	return nil
}

func (c *IdleTimingConn) markActive(n int) bool {
	if n > 0 {
		c.activeCh <- true
		return true
	} else {
		return false
	}
}

func isTimeout(err error) bool {
	if netErr, ok := err.(net.Error); ok {
		return netErr.Timeout()
	}
	return false
}
<|endoftext|>


<commit_before>package cloudflare

import (
	"encoding/json"
	"time"

	"github.com/pkg/errors"
)

// PageRuleTarget is the target to evaluate on a request.
//
// Currently Target must always be "url" and Operator must be "matches". Value
// is the URL pattern to match against.
type PageRuleTarget struct {
	Target     string `json:"target"`
	Constraint struct {
		Operator string `json:"operator"`
		Value    string `json:"value"`
	} `json:"constraint"`
}

/*
PageRuleAction is the action to take when the target is matched.

Valid IDs are:
  always_online
  always_use_https
  automatic_https_rewrites
  browser_cache_ttl
  browser_check
  bypass_cache_on_cookie
  cache_by_device_type
  cache_deception_armor
  cache_level
  cache_on_cookie
  disable_apps
  disable_performance
  disable_railgun
  disable_security
  edge_cache_ttl
  email_obfuscation
  explicit_cache_control
  forwarding_url
  host_header_override
  ip_geolocation
  minify
  mirage
  opportunistic_encryption
  origin_error_page_pass_thru
  polish
  resolve_override
  respect_strong_etag
  response_buffering
  rocket_loader
  security_level
  server_side_exclude
  sort_query_string_for_cache
  ssl
  true_client_ip_header
  waf
*/
type PageRuleAction struct {
	ID    string      `json:"id"`
	Value interface{} `json:"value"`
}

// PageRuleActions maps API action IDs to human-readable strings.
var PageRuleActions = map[string]string{
	"always_online":               "Always Online",               // Value of type string
	"always_use_https":            "Always Use HTTPS",            // Value of type interface{}
	"automatic_https_rewrites":    "Automatic HTTPS Rewrites",    // Value of type string
	"browser_cache_ttl":           "Browser Cache TTL",           // Value of type int
	"browser_check":               "Browser Integrity Check",     // Value of type string
	"bypass_cache_on_cookie":      "Bypass Cache on Cookie",      // Value of type string
	"cache_by_device_type":        "Cache By Device Type",        // Value of type string
	"cache_deception_armor":       "Cache Deception Armor",       // Value of type string
	"cache_level":                 "Cache Level",                 // Value of type string
	"cache_on_cookie":             "Cache On Cookie",             // Value of type string
	"disable_apps":                "Disable Apps",                // Value of type interface{}
	"disable_performance":         "Disable Performance",         // Value of type interface{}
	"disable_railgun":             "Disable Railgun",             // Value of type string
	"disable_security":            "Disable Security",            // Value of type interface{}
	"edge_cache_ttl":              "Edge Cache TTL",              // Value of type int
	"email_obfuscation":           "Email Obfuscation",           // Value of type string
	"explicit_cache_control":      "Origin Cache Control",        // Value of type string
	"forwarding_url":              "Forwarding URL",              // Value of type map[string]interface
	"host_header_override":        "Host Header Override",        // Value of type string
	"ip_geolocation":              "IP Geolocation Header",       // Value of type string
	"minify":                      "Minify",                      // Value of type map[string]interface
	"mirage":                      "Mirage",                      // Value of type string
	"opportunistic_encryption":    "Opportunistic Encryption",    // Value of type string
	"origin_error_page_pass_thru": "Origin Error Page Pass-thru", // Value of type string
	"polish":                      "Polish",                      // Value of type string
	"resolve_override":            "Resolve Override",            // Value of type string
	"respect_strong_etag":         "Respect Strong ETags",        // Value of type string
	"response_buffering":          "Response Buffering",          // Value of type string
	"rocket_loader":               "Rocker Loader",               // Value of type string
	"security_level":              "Security Level",              // Value of type string
	"server_side_exclude":         "Server Side Excludes",        // Value of type string
	"sort_query_string_for_cache": "Query String Sort",           // Value of type string
	"ssl":                         "SSL",                         // Value of type string
	"true_client_ip_header":       "True Client IP Header",       // Value of type string
	"waf":                         "Web Application Firewall",    // Value of type string
}

// PageRule describes a Page Rule.
type PageRule struct {
	ID         string           `json:"id,omitempty"`
	Targets    []PageRuleTarget `json:"targets"`
	Actions    []PageRuleAction `json:"actions"`
	Priority   int              `json:"priority"`
	Status     string           `json:"status"`
	ModifiedOn time.Time        `json:"modified_on,omitempty"`
	CreatedOn  time.Time        `json:"created_on,omitempty"`
}

// PageRuleDetailResponse is the API response, containing a single PageRule.
type PageRuleDetailResponse struct {
	Success  bool     `json:"success"`
	Errors   []string `json:"errors"`
	Messages []string `json:"messages"`
	Result   PageRule `json:"result"`
}

// PageRulesResponse is the API response, containing an array of PageRules.
type PageRulesResponse struct {
	Success  bool       `json:"success"`
	Errors   []string   `json:"errors"`
	Messages []string   `json:"messages"`
	Result   []PageRule `json:"result"`
}

// CreatePageRule creates a new Page Rule for a zone.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-create-a-page-rule
func (api *API) CreatePageRule(zoneID string, rule PageRule) (*PageRule, error) {
	uri := "/zones/" + zoneID + "/pagerules"
	res, err := api.makeRequest("POST", uri, rule)
	if err != nil {
		return nil, errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return nil, errors.Wrap(err, errUnmarshalError)
	}
	return &r.Result, nil
}

// ListPageRules returns all Page Rules for a zone.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-list-page-rules
func (api *API) ListPageRules(zoneID string) ([]PageRule, error) {
	uri := "/zones/" + zoneID + "/pagerules"
	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return []PageRule{}, errors.Wrap(err, errMakeRequestError)
	}
	var r PageRulesResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return []PageRule{}, errors.Wrap(err, errUnmarshalError)
	}
	return r.Result, nil
}

// PageRule fetches detail about one Page Rule for a zone.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-page-rule-details
func (api *API) PageRule(zoneID, ruleID string) (PageRule, error) {
	uri := "/zones/" + zoneID + "/pagerules/" + ruleID
	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return PageRule{}, errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return PageRule{}, errors.Wrap(err, errUnmarshalError)
	}
	return r.Result, nil
}

// ChangePageRule lets you change individual settings for a Page Rule. This is
// in contrast to UpdatePageRule which replaces the entire Page Rule.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-change-a-page-rule
func (api *API) ChangePageRule(zoneID, ruleID string, rule PageRule) error {
	uri := "/zones/" + zoneID + "/pagerules/" + ruleID
	res, err := api.makeRequest("PATCH", uri, rule)
	if err != nil {
		return errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}
	return nil
}

// UpdatePageRule lets you replace a Page Rule. This is in contrast to
// ChangePageRule which lets you change individual settings.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-update-a-page-rule
func (api *API) UpdatePageRule(zoneID, ruleID string, rule PageRule) error {
	uri := "/zones/" + zoneID + "/pagerules/" + ruleID
	res, err := api.makeRequest("PUT", uri, rule)
	if err != nil {
		return errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}
	return nil
}

// DeletePageRule deletes a Page Rule for a zone.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-delete-a-page-rule
func (api *API) DeletePageRule(zoneID, ruleID string) error {
	uri := "/zones/" + zoneID + "/pagerules/" + ruleID
	res, err := api.makeRequest("DELETE", uri, nil)
	if err != nil {
		return errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}
	return nil
}
<commit_msg>page_rules: Add base support for `cache_key_fields` (#433)<commit_after>package cloudflare

import (
	"encoding/json"
	"time"

	"github.com/pkg/errors"
)

// PageRuleTarget is the target to evaluate on a request.
//
// Currently Target must always be "url" and Operator must be "matches". Value
// is the URL pattern to match against.
type PageRuleTarget struct {
	Target     string `json:"target"`
	Constraint struct {
		Operator string `json:"operator"`
		Value    string `json:"value"`
	} `json:"constraint"`
}

/*
PageRuleAction is the action to take when the target is matched.

Valid IDs are:
  always_online
  always_use_https
  automatic_https_rewrites
  browser_cache_ttl
  browser_check
  bypass_cache_on_cookie
  cache_by_device_type
  cache_deception_armor
  cache_level
  cache_key_fields
  cache_on_cookie
  disable_apps
  disable_performance
  disable_railgun
  disable_security
  edge_cache_ttl
  email_obfuscation
  explicit_cache_control
  forwarding_url
  host_header_override
  ip_geolocation
  minify
  mirage
  opportunistic_encryption
  origin_error_page_pass_thru
  polish
  resolve_override
  respect_strong_etag
  response_buffering
  rocket_loader
  security_level
  server_side_exclude
  sort_query_string_for_cache
  ssl
  true_client_ip_header
  waf
*/
type PageRuleAction struct {
	ID    string      `json:"id"`
	Value interface{} `json:"value"`
}

// PageRuleActions maps API action IDs to human-readable strings.
var PageRuleActions = map[string]string{
	"always_online":               "Always Online",               // Value of type string
	"always_use_https":            "Always Use HTTPS",            // Value of type interface{}
	"automatic_https_rewrites":    "Automatic HTTPS Rewrites",    // Value of type string
	"browser_cache_ttl":           "Browser Cache TTL",           // Value of type int
	"browser_check":               "Browser Integrity Check",     // Value of type string
	"bypass_cache_on_cookie":      "Bypass Cache on Cookie",      // Value of type string
	"cache_by_device_type":        "Cache By Device Type",        // Value of type string
	"cache_deception_armor":       "Cache Deception Armor",       // Value of type string
	"cache_level":                 "Cache Level",                 // Value of type string
	"cache_key_fields":            "Custom Cache Key",            // Value of type map[string]interface
	"cache_on_cookie":             "Cache On Cookie",             // Value of type string
	"disable_apps":                "Disable Apps",                // Value of type interface{}
	"disable_performance":         "Disable Performance",         // Value of type interface{}
	"disable_railgun":             "Disable Railgun",             // Value of type string
	"disable_security":            "Disable Security",            // Value of type interface{}
	"edge_cache_ttl":              "Edge Cache TTL",              // Value of type int
	"email_obfuscation":           "Email Obfuscation",           // Value of type string
	"explicit_cache_control":      "Origin Cache Control",        // Value of type string
	"forwarding_url":              "Forwarding URL",              // Value of type map[string]interface
	"host_header_override":        "Host Header Override",        // Value of type string
	"ip_geolocation":              "IP Geolocation Header",       // Value of type string
	"minify":                      "Minify",                      // Value of type map[string]interface
	"mirage":                      "Mirage",                      // Value of type string
	"opportunistic_encryption":    "Opportunistic Encryption",    // Value of type string
	"origin_error_page_pass_thru": "Origin Error Page Pass-thru", // Value of type string
	"polish":                      "Polish",                      // Value of type string
	"resolve_override":            "Resolve Override",            // Value of type string
	"respect_strong_etag":         "Respect Strong ETags",        // Value of type string
	"response_buffering":          "Response Buffering",          // Value of type string
	"rocket_loader":               "Rocker Loader",               // Value of type string
	"security_level":              "Security Level",              // Value of type string
	"server_side_exclude":         "Server Side Excludes",        // Value of type string
	"sort_query_string_for_cache": "Query String Sort",           // Value of type string
	"ssl":                         "SSL",                         // Value of type string
	"true_client_ip_header":       "True Client IP Header",       // Value of type string
	"waf":                         "Web Application Firewall",    // Value of type string
}

// PageRule describes a Page Rule.
type PageRule struct {
	ID         string           `json:"id,omitempty"`
	Targets    []PageRuleTarget `json:"targets"`
	Actions    []PageRuleAction `json:"actions"`
	Priority   int              `json:"priority"`
	Status     string           `json:"status"`
	ModifiedOn time.Time        `json:"modified_on,omitempty"`
	CreatedOn  time.Time        `json:"created_on,omitempty"`
}

// PageRuleDetailResponse is the API response, containing a single PageRule.
type PageRuleDetailResponse struct {
	Success  bool     `json:"success"`
	Errors   []string `json:"errors"`
	Messages []string `json:"messages"`
	Result   PageRule `json:"result"`
}

// PageRulesResponse is the API response, containing an array of PageRules.
type PageRulesResponse struct {
	Success  bool       `json:"success"`
	Errors   []string   `json:"errors"`
	Messages []string   `json:"messages"`
	Result   []PageRule `json:"result"`
}

// CreatePageRule creates a new Page Rule for a zone.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-create-a-page-rule
func (api *API) CreatePageRule(zoneID string, rule PageRule) (*PageRule, error) {
	uri := "/zones/" + zoneID + "/pagerules"
	res, err := api.makeRequest("POST", uri, rule)
	if err != nil {
		return nil, errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return nil, errors.Wrap(err, errUnmarshalError)
	}
	return &r.Result, nil
}

// ListPageRules returns all Page Rules for a zone.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-list-page-rules
func (api *API) ListPageRules(zoneID string) ([]PageRule, error) {
	uri := "/zones/" + zoneID + "/pagerules"
	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return []PageRule{}, errors.Wrap(err, errMakeRequestError)
	}
	var r PageRulesResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return []PageRule{}, errors.Wrap(err, errUnmarshalError)
	}
	return r.Result, nil
}

// PageRule fetches detail about one Page Rule for a zone.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-page-rule-details
func (api *API) PageRule(zoneID, ruleID string) (PageRule, error) {
	uri := "/zones/" + zoneID + "/pagerules/" + ruleID
	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return PageRule{}, errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return PageRule{}, errors.Wrap(err, errUnmarshalError)
	}
	return r.Result, nil
}

// ChangePageRule lets you change individual settings for a Page Rule. This is
// in contrast to UpdatePageRule which replaces the entire Page Rule.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-change-a-page-rule
func (api *API) ChangePageRule(zoneID, ruleID string, rule PageRule) error {
	uri := "/zones/" + zoneID + "/pagerules/" + ruleID
	res, err := api.makeRequest("PATCH", uri, rule)
	if err != nil {
		return errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}
	return nil
}

// UpdatePageRule lets you replace a Page Rule. This is in contrast to
// ChangePageRule which lets you change individual settings.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-update-a-page-rule
func (api *API) UpdatePageRule(zoneID, ruleID string, rule PageRule) error {
	uri := "/zones/" + zoneID + "/pagerules/" + ruleID
	res, err := api.makeRequest("PUT", uri, rule)
	if err != nil {
		return errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}
	return nil
}

// DeletePageRule deletes a Page Rule for a zone.
//
// API reference: https://api.cloudflare.com/#page-rules-for-a-zone-delete-a-page-rule
func (api *API) DeletePageRule(zoneID, ruleID string) error {
	uri := "/zones/" + zoneID + "/pagerules/" + ruleID
	res, err := api.makeRequest("DELETE", uri, nil)
	if err != nil {
		return errors.Wrap(err, errMakeRequestError)
	}
	var r PageRuleDetailResponse
	err = json.Unmarshal(res, &r)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"

	rpio "github.com/stianeikeland/go-rpio"

	"github.com/alittlebrighter/rpi-garage-doors/commands"
	"github.com/alittlebrighter/rpi-garage-doors/util"
)

const (
	httpGet  = "GET"
	httpPost = "POST"
)

func main() {
	if err := rpio.Open(); err != nil {
		log.Fatalf("Error: %v\n", err)
	}
	defer rpio.Close()

	controllers := []commands.GarageDoorController{}

	// takes one command line argument specifying the configuration file path
	configFile := "/etc/garage-doors.conf"
	if len(os.Args) == 2 {
		configFile = os.Args[1]
	}
	conf := util.ParseConfig(configFile)

	for _, door := range conf.Controllers.Garage_doors.Gpio_pins.Bcm {
		controllers = append(controllers,
			commands.ControllerFactory(door, conf.Controllers.Garage_doors.Trigger_time, conf.Controllers.Garage_doors.Force_time))
	}

	cpuCount := runtime.NumCPU()
	if cpuCount < len(controllers) {
		runtime.GOMAXPROCS(cpuCount)
	} else {
		runtime.GOMAXPROCS(len(controllers))
	}

	http.HandleFunc(conf.Endpoints.Paths.Control, func(w http.ResponseWriter, r *http.Request) {
		if r.Method != httpPost {
			fmt.Fprintf(w, "HTTP Method "+r.Method+" not valid for this endpoint.")
			return
		}

		door, doorErr := strconv.Atoi(r.PostFormValue("door"))
		if doorErr != nil || door >= len(controllers) {
			fmt.Fprintf(w, "Error parsing door number.")
			return
		}

		force, forceErr := strconv.ParseBool(r.PostFormValue("force"))
		if forceErr != nil {
			fmt.Fprintf(w, "Error parsing force value.")
			return
		}

		log.Println("Opening door " + r.PostFormValue("door") + ", force: " + r.PostFormValue("force"))

		go controllers[door].Trigger(force)

		fmt.Fprintf(w, "door: %d, force: %v", door, force)
	})

	log.Println("Starting server at " + conf.Endpoints.Host)
	log.Fatal(http.ListenAndServe(conf.Endpoints.Host, nil))
}
<commit_msg>stop requiring force parameter<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"

	rpio "github.com/stianeikeland/go-rpio"

	"github.com/alittlebrighter/rpi-garage-doors/commands"
	"github.com/alittlebrighter/rpi-garage-doors/util"
)

const (
	httpGet  = "GET"
	httpPost = "POST"
)

func main() {
	if err := rpio.Open(); err != nil {
		log.Fatalf("Error: %v\n", err)
	}
	defer rpio.Close()

	controllers := []commands.GarageDoorController{}

	// takes one command line argument specifying the configuration file path
	configFile := "/etc/garage-doors.conf"
	if len(os.Args) == 2 {
		configFile = os.Args[1]
	}
	conf := util.ParseConfig(configFile)

	for _, door := range conf.Controllers.Garage_doors.Gpio_pins.Bcm {
		controllers = append(controllers,
			commands.ControllerFactory(door, conf.Controllers.Garage_doors.Trigger_time, conf.Controllers.Garage_doors.Force_time))
	}

	cpuCount := runtime.NumCPU()
	if cpuCount < len(controllers) {
		runtime.GOMAXPROCS(cpuCount)
	} else {
		runtime.GOMAXPROCS(len(controllers))
	}

	http.HandleFunc(conf.Endpoints.Paths.Control, func(w http.ResponseWriter, r *http.Request) {
		if r.Method != httpPost {
			fmt.Fprintf(w, "HTTP Method "+r.Method+" not valid for this endpoint.")
			return
		}

		door, doorErr := strconv.Atoi(r.PostFormValue("door"))
		if doorErr != nil || door >= len(controllers) {
			fmt.Fprintf(w, "Error parsing door number.")
			return
		}

		force, forceErr := strconv.ParseBool(r.PostFormValue("force"))
		if forceErr != nil {
			force = false
			return
		}

		log.Println("Opening door " + r.PostFormValue("door") + ", force: " + r.PostFormValue("force"))

		go controllers[door].Trigger(force)

		fmt.Fprintf(w, "door: %d, force: %v", door, force)
	})

	log.Println("Starting server at " + conf.Endpoints.Host)
	log.Fatal(http.ListenAndServe(conf.Endpoints.Host, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"
	"testing"
	"time"
)

func tic() time.Time {
	return time.Now()
}

func toc(ts time.Time) time.Duration {
	return time.Now().Sub(ts)
}

func TestInit(t *testing.T) {
	var d, w int64
	d = 75
	w = 700
	cms := NewCMSketch(d, w)
	if cms.Width != w {
		t.Errorf("Width: %d != %d\n", w, cms.Width)
	}
	if cms.Depth != d {
		t.Errorf("Depth: %d != %d\n", d, cms.Depth)
	}
}

func TestInsert(t *testing.T) {
	cms := NewCMSketch(2, 5)
	r := rand.New(rand.NewSource(0))
	rh := RandomHashes(r, 2)
	cms.Hash = rh
	if rh[0].Apply(1)%5 != 2 {
		t.Errorf("hash[0](1) %% 5 != 2\n")
	}
	if rh[1].Apply(1)%5 != 3 {
		t.Errorf("hash[0](1) %% 5 != 2\n")
	}
	fmt.Printf("%v\n", cms)
	t.Logf("Before:\n%v\n", cms.Counter.String())
	cms.UpdateSerial(1, 1)
	t.Logf("After:\n%v\n", cms.Counter.String())
	if cms.Counter.Read(0, 2) != 1 {
		t.Errorf("Bad first row\n")
	}
	if cms.Counter.Read(1, 3) != 1 {
		t.Errorf("Bad second row\n")
	}
}

//loss: implements a generic loss function for scoring the implementation.
func loss(cj, qj float64) float64 {
	return (cj - qj) * (cj - qj)
}

//makeCMS: Initialize the CMSKETCH to standard values for test
func makeCMS(r *rand.Rand) *CMSketch {
	var Depth, Width int64
	Depth = 80
	Width = 160
	hslice := RandomHashes(r, Depth)
	cms := NewCMSketch(Depth, Width)
	cms.Hash = hslice
	return &cms
}

//makeCMSparams: Initialize the CMSKETCH to parameters that are passed
func makeCMSparams(r *rand.Rand, Depth int64, Width int64) *CMSketch {
	hslice := RandomHashes(r, Depth)
	cms := NewCMSketch(Depth, Width)
	cms.Hash = hslice
	return &cms
}

//makeZipfer: Initialize the stream of random elements for the tests.
func makeZipfer(r *rand.Rand) *rand.Zipf {
	//Make the zipf distribution of random input
	var s, v float64
	var imax uint64
	s = 1.2
	v = 1.0
	imax = 2 << 10
	zipfer := rand.NewZipf(r, s, v, imax)
	return zipfer
}

//TestSpeed: Test the speed for Serial insertions
func TestSpeed(t *testing.T) {
	src := rand.NewSource(0)
	r := rand.New(src)
	cms := *makeCMS(r)
	var efactor, numElements int64
	efactor = 10
	numElements = cms.Depth * cms.Width * efactor
	zipfer := makeZipfer(r)
	log.Printf("Inserting\n")
	ts := time.Now()
	var j, z int64
	for j = 0; j < numElements; j++ {
		z = int64(zipfer.Uint64())
		cms.UpdateSerial(z, 1)
	}
	te := time.Now().Sub(ts)
	fmt.Printf("time Serial: %v\n", te)
	numProcs := runtime.NumCPU()
	runtime.GOMAXPROCS(numProcs)
	ts = time.Now()
	for j = 0; j < numElements; j++ {
		z = int64(zipfer.Uint64())
		cms.UpdateDepthParallel(z, 1, int64(numProcs))
	}
	te = time.Now().Sub(ts)
	fmt.Printf("time parallel: %v\n", te)
}

//TestAccuracy: Run a test storing the right answers
//compute the loss using the function loss
func TestAccuracy(t *testing.T) {
	src := rand.NewSource(0)
	r := rand.New(src)
	cms := *makeCMS(r)
	var efactor, numElements int64
	efactor = 10
	numElements = cms.Depth * cms.Width * efactor
	zipfer := makeZipfer(r)

	//Use set to store the exact answers
	set := make(map[int64]int64)
	log.Printf("Inserting\n")
	ts := time.Now()
	var j, z int64
	for j = 0; j < numElements; j++ {
		z = int64(zipfer.Uint64())
		set[z] += 1
		//fmt.Println(z)
		cms.UpdateSerial(z, 1)
	}
	te := time.Now().Sub(ts)
	fmt.Printf("time accuracy: %v\n", te)
	//fmt.Printf("%s\n", cms.Counter.String())
	var qj int64 //approximate answers
	var totalLoss float64
	for j, cj := range set {
		qj = cms.PointQuery(j)
		//fmt.Printf("results:%d %d %d %f\n", j, qj, cj, float64(qj)/float64(cj))
		totalLoss += loss(float64(cj), float64(qj))
	}
	fmt.Printf("Total Loss: %f/%d\n", totalLoss, numElements)
}

//drawZipf: initialize a zipf distribution generator and draw batchsize samples.
//Not for use in external modules because it does not set the seed properly.
func drawZipf(batchsize int64) []int64 {
	src := rand.NewSource(0)
	r := rand.New(src)
	zipfer := makeZipfer(r)

	elements := make([]int64, batchsize)
	var i, z int64
	var ts time.Time
	var te time.Duration
	ts = tic()
	for i = 0; i < batchsize; i++ {
		z = int64(zipfer.Uint64())
		elements[i] = z
	}
	te = toc(ts)
	fmt.Printf("time zipfer: %s\n", te)
	return elements
}

//TestBatchInsert: Compare the data structures after making batch inserts to one
//where we have made serial insertions to test that they produce the same result
func TestBatchInsert(t *testing.T) {
	var batchsize int64
	batchsize = 1000000
	elements := drawZipf(batchsize)
	src := rand.NewSource(4)
	r := rand.New(src)
	cms := makeCMS(r)
	ts := tic()
	for _, z := range elements {
		cms.UpdateSerial(z, 1)
	}
	te := toc(ts)
	fmt.Printf("time single insertions: %v\n", te)
	t.Logf("cms:\n%v\n", cms)
	batchcms := cms.Clone()
	ch := make(chan int, cms.Depth)
	NumCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(NumCPU)
	ts = tic()
	batchcms.BatchUpdate(elements, int64(NumCPU), false, ch)
	te = toc(ts)
	fmt.Printf("time batch insertions: %v\n", te)
	result := cms.Equal(&batchcms)
	if !result {
		t.Errorf("the sketches did not come up equal\n")
		t.Logf("batchcms:\n%v\n", batchcms)
	}
	t.Logf("Working on sorted batch updates")
	sbatchcms := cms.Clone()
	ts = tic()
	sbatchcms.BatchUpdate(elements, int64(NumCPU), true, ch)
	te = toc(ts)
	fmt.Printf("time sorted batch insertions: %v\n", te)
	result = cms.Equal(&sbatchcms)
	if !result {
		t.Errorf("sortedbatch sketch != regular sketch\n")
		t.Logf("sbatchcms:\n%v\n", sbatchcms)
	}
}

func sampleZipf(zipfer *rand.Zipf, batchsize int64, elements []int64) {
	var i int64
	var z int64
	for i = 0; i < batchsize; i++ {
		z = int64(zipfer.Uint64())
		elements[i] = z
	}
}

func BenchmarkSequeInsert(b *testing.B) {
	var batchsize int64
	batchsize = 1000000
	src := rand.NewSource(4)
	r := rand.New(src)
	cms := makeCMS(r)
	zipfer := makeZipfer(r)
	elements := make([]int64, batchsize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		sampleZipf(zipfer, batchsize, elements)
		for _, z := range elements {
			cms.UpdateSerial(z, 1)
		}
	}

}

func benchmarkBatchInsert(gomaxprocs int, batchsize int64, b *testing.B) {
	runtime.GOMAXPROCS(gomaxprocs)
	sort := false
	src := rand.NewSource(4)
	r := rand.New(src)
	cms := makeCMSparams(r, 160, 200)
	ch := make(chan int, cms.Depth)
	zipfer := makeZipfer(r)
	elements := make([]int64, batchsize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		sampleZipf(zipfer, batchsize, elements)
		cms.BatchUpdate(elements, int64(gomaxprocs), sort, ch)
	}
}

var batchsize int64 = 1000000

func BenchmarkBatchInsert2(b *testing.B)  { benchmarkBatchInsert(2, batchsize, b) }
func BenchmarkBatchInsert4(b *testing.B)  { benchmarkBatchInsert(4, batchsize, b) }
func BenchmarkBatchInsert6(b *testing.B)  { benchmarkBatchInsert(6, batchsize, b) }
func BenchmarkBatchInsert8(b *testing.B)  { benchmarkBatchInsert(8, batchsize, b) }
func BenchmarkBatchInsert10(b *testing.B) { benchmarkBatchInsert(10, batchsize, b) }
func BenchmarkBatchInsert12(b *testing.B) { benchmarkBatchInsert(12, batchsize, b) }
func BenchmarkBatchInsert14(b *testing.B) { benchmarkBatchInsert(14, batchsize, b) }
func BenchmarkBatchInsert16(b *testing.B) { benchmarkBatchInsert(16, batchsize, b) }
func TestDecrement(t *testing.T) {
	t.Logf("Decrement not implemented yet")
}
<commit_msg>Modified benchmark to be more parameterized.<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"
	"testing"
	"time"
)

func tic() time.Time {
	return time.Now()
}

func toc(ts time.Time) time.Duration {
	return time.Now().Sub(ts)
}

func TestInit(t *testing.T) {
	var d, w int64
	d = 75
	w = 700
	cms := NewCMSketch(d, w)
	if cms.Width != w {
		t.Errorf("Width: %d != %d\n", w, cms.Width)
	}
	if cms.Depth != d {
		t.Errorf("Depth: %d != %d\n", d, cms.Depth)
	}
}

func TestInsert(t *testing.T) {
	cms := NewCMSketch(2, 5)
	r := rand.New(rand.NewSource(0))
	rh := RandomHashes(r, 2)
	cms.Hash = rh
	if rh[0].Apply(1)%5 != 2 {
		t.Errorf("hash[0](1) %% 5 != 2\n")
	}
	if rh[1].Apply(1)%5 != 3 {
		t.Errorf("hash[0](1) %% 5 != 2\n")
	}
	fmt.Printf("%v\n", cms)
	t.Logf("Before:\n%v\n", cms.Counter.String())
	cms.UpdateSerial(1, 1)
	t.Logf("After:\n%v\n", cms.Counter.String())
	if cms.Counter.Read(0, 2) != 1 {
		t.Errorf("Bad first row\n")
	}
	if cms.Counter.Read(1, 3) != 1 {
		t.Errorf("Bad second row\n")
	}
}

//loss: implements a generic loss function for scoring the implementation.
func loss(cj, qj float64) float64 {
	return (cj - qj) * (cj - qj)
}

//makeCMS: Initialize the CMSKETCH to standard values for test
func makeCMS(r *rand.Rand) *CMSketch {
	var Depth, Width int64
	Depth = 80
	Width = 160
	hslice := RandomHashes(r, Depth)
	cms := NewCMSketch(Depth, Width)
	cms.Hash = hslice
	return &cms
}

//makeCMSparams: Initialize the CMSKETCH to parameters that are passed
func makeCMSparams(r *rand.Rand, Depth int64, Width int64) *CMSketch {
	hslice := RandomHashes(r, Depth)
	cms := NewCMSketch(Depth, Width)
	cms.Hash = hslice
	return &cms
}

//makeZipfer: Initialize the stream of random elements for the tests.
func makeZipfer(r *rand.Rand) *rand.Zipf {
	//Make the zipf distribution of random input
	var s, v float64
	var imax uint64
	s = 1.2
	v = 1.0
	imax = 2 << 10
	zipfer := rand.NewZipf(r, s, v, imax)
	return zipfer
}

//TestSpeed: Test the speed for Serial insertions
func TestSpeed(t *testing.T) {
	src := rand.NewSource(0)
	r := rand.New(src)
	cms := *makeCMS(r)
	var efactor, numElements int64
	efactor = 10
	numElements = cms.Depth * cms.Width * efactor
	zipfer := makeZipfer(r)
	log.Printf("Inserting\n")
	ts := time.Now()
	var j, z int64
	for j = 0; j < numElements; j++ {
		z = int64(zipfer.Uint64())
		cms.UpdateSerial(z, 1)
	}
	te := time.Now().Sub(ts)
	fmt.Printf("time Serial: %v\n", te)
	numProcs := runtime.NumCPU()
	runtime.GOMAXPROCS(numProcs)
	ts = time.Now()
	for j = 0; j < numElements; j++ {
		z = int64(zipfer.Uint64())
		cms.UpdateDepthParallel(z, 1, int64(numProcs))
	}
	te = time.Now().Sub(ts)
	fmt.Printf("time parallel: %v\n", te)
}

//TestAccuracy: Run a test storing the right answers
//compute the loss using the function loss
func TestAccuracy(t *testing.T) {
	src := rand.NewSource(0)
	r := rand.New(src)
	cms := *makeCMS(r)
	var efactor, numElements int64
	efactor = 10
	numElements = cms.Depth * cms.Width * efactor
	zipfer := makeZipfer(r)

	//Use set to store the exact answers
	set := make(map[int64]int64)
	log.Printf("Inserting\n")
	ts := time.Now()
	var j, z int64
	for j = 0; j < numElements; j++ {
		z = int64(zipfer.Uint64())
		set[z] += 1
		//fmt.Println(z)
		cms.UpdateSerial(z, 1)
	}
	te := time.Now().Sub(ts)
	fmt.Printf("time accuracy: %v\n", te)
	//fmt.Printf("%s\n", cms.Counter.String())
	var qj int64 //approximate answers
	var totalLoss float64
	for j, cj := range set {
		qj = cms.PointQuery(j)
		//fmt.Printf("results:%d %d %d %f\n", j, qj, cj, float64(qj)/float64(cj))
		totalLoss += loss(float64(cj), float64(qj))
	}
	fmt.Printf("Total Loss: %f/%d\n", totalLoss, numElements)
}

//drawZipf: initialize a zipf distribution generator and draw batchsize samples.
//Not for use in external modules because it does not set the seed properly.
func drawZipf(batchsize int64) []int64 {
	src := rand.NewSource(0)
	r := rand.New(src)
	zipfer := makeZipfer(r)

	elements := make([]int64, batchsize)
	var i, z int64
	var ts time.Time
	var te time.Duration
	ts = tic()
	for i = 0; i < batchsize; i++ {
		z = int64(zipfer.Uint64())
		elements[i] = z
	}
	te = toc(ts)
	fmt.Printf("time zipfer: %s\n", te)
	return elements
}

//TestBatchInsert: Compare the data structures after making batch inserts to one
//where we have made serial insertions to test that they produce the same result
func TestBatchInsert(t *testing.T) {
	var batchsize int64
	batchsize = 1000000
	elements := drawZipf(batchsize)
	src := rand.NewSource(4)
	r := rand.New(src)
	cms := makeCMS(r)
	ts := tic()
	for _, z := range elements {
		cms.UpdateSerial(z, 1)
	}
	te := toc(ts)
	fmt.Printf("time single insertions: %v\n", te)
	t.Logf("cms:\n%v\n", cms)
	batchcms := cms.Clone()
	ch := make(chan int, cms.Depth)
	NumCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(NumCPU)
	ts = tic()
	batchcms.BatchUpdate(elements, int64(NumCPU), false, ch)
	te = toc(ts)
	fmt.Printf("time batch insertions: %v\n", te)
	result := cms.Equal(&batchcms)
	if !result {
		t.Errorf("the sketches did not come up equal\n")
		t.Logf("batchcms:\n%v\n", batchcms)
	}
	t.Logf("Working on sorted batch updates")
	sbatchcms := cms.Clone()
	ts = tic()
	sbatchcms.BatchUpdate(elements, int64(NumCPU), true, ch)
	te = toc(ts)
	fmt.Printf("time sorted batch insertions: %v\n", te)
	result = cms.Equal(&sbatchcms)
	if !result {
		t.Errorf("sortedbatch sketch != regular sketch\n")
		t.Logf("sbatchcms:\n%v\n", sbatchcms)
	}
}

func sampleZipf(zipfer *rand.Zipf, batchsize int64, elements []int64) {
	var i int64
	var z int64
	for i = 0; i < batchsize; i++ {
		z = int64(zipfer.Uint64())
		elements[i] = z
	}
}

var cmsDepth int64 = 2 << 10
var cmsWidth int64 = 2 << 13
var batchsize int64 = 2 << 15

func BenchmarkSequeInsert(b *testing.B) {
	src := rand.NewSource(4)
	r := rand.New(src)
	cms := makeCMSparams(r, cmsDepth, cmsWidth)
	zipfer := makeZipfer(r)
	elements := make([]int64, batchsize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		sampleZipf(zipfer, batchsize, elements)
		for _, z := range elements {
			cms.UpdateSerial(z, 1)
		}
	}

}

func benchmarkBatchInsert(gomaxprocs int, batchsize int64, b *testing.B) {
	runtime.GOMAXPROCS(gomaxprocs)
	sort := false
	src := rand.NewSource(4)
	r := rand.New(src)
	cms := makeCMSparams(r, cmsDepth, cmsWidth)
	ch := make(chan int, cms.Depth)
	zipfer := makeZipfer(r)
	elements := make([]int64, batchsize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		sampleZipf(zipfer, batchsize, elements)
		cms.BatchUpdate(elements, int64(gomaxprocs), sort, ch)
	}
}

func BenchmarkBatchInsert2(b *testing.B) { benchmarkBatchInsert(2, batchsize, b) }
func BenchmarkBatchInsert4(b *testing.B) { benchmarkBatchInsert(4, batchsize, b) }
func BenchmarkBatchInsert6(b *testing.B) { benchmarkBatchInsert(6, batchsize, b) }
func BenchmarkBatchInsert8(b *testing.B) { benchmarkBatchInsert(8, batchsize, b) }

//func BenchmarkBatchInsert10(b *testing.B) { benchmarkBatchInsert(10, batchsize, b) }
//func BenchmarkBatchInsert12(b *testing.B) { benchmarkBatchInsert(12, batchsize, b) }
//func BenchmarkBatchInsert14(b *testing.B) { benchmarkBatchInsert(14, batchsize, b) }
//func BenchmarkBatchInsert16(b *testing.B) { benchmarkBatchInsert(16, batchsize, b) }
func TestDecrement(t *testing.T) {
	t.Logf("Decrement not implemented yet")
}
<|endoftext|>


<commit_before>package soy

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"code.google.com/p/go.exp/fsnotify"
	"github.com/robfig/soy/data"
	"github.com/robfig/soy/parse"
	"github.com/robfig/soy/parsepasses"
	"github.com/robfig/soy/soyhtml"
	"github.com/robfig/soy/template"
)

// Logger is used to print notifications and compile errors when using the
// "WatchFiles" feature.
var Logger = log.New(os.Stderr, "[soy] ", 0)

type soyFile struct{ name, content string }

// Bundle is a collection of soy content (templates and globals).  It acts as
// input for the soy compiler.
type Bundle struct {
	files   []soyFile
	globals data.Map
	err     error
	watcher *fsnotify.Watcher
}

// NewBundle returns an empty bundle.
func NewBundle() *Bundle {
	return &Bundle{globals: make(data.Map)}
}

// WatchFiles tells soy to watch any template files added to this bundle,
// re-compile as necessary, and propagate the updates to your tofu.  It should
// be called once, before adding any files.
func (b *Bundle) WatchFiles(watch bool) *Bundle {
	if watch && b.err == nil && b.watcher == nil {
		b.watcher, b.err = fsnotify.NewWatcher()
	}
	return b
}

// AddTemplateDir adds all *.soy files found within the given directory
// (including sub-directories) to the bundle.
func (b *Bundle) AddTemplateDir(root string) *Bundle {
	var err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		if !strings.HasSuffix(path, ".soy") {
			return nil
		}
		b.AddTemplateFile(path)
		return nil
	})
	if err != nil {
		b.err = err
	}
	return b
}

// AddTemplateFile adds the given soy template file text to this bundle.
// If WatchFiles is on, it will be subsequently watched for updates.
func (b *Bundle) AddTemplateFile(filename string) *Bundle {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		b.err = err
	}
	if b.err == nil && b.watcher != nil {
		b.err = b.watcher.Watch(filename)
	}
	return b.AddTemplateString(filename, string(content))
}

// AddTemplateString adds the given template to the bundle. The name is only
// used for error messages - it does not need to be provided nor does it need to
// be a real filename.
func (b *Bundle) AddTemplateString(filename, soyfile string) *Bundle {
	b.files = append(b.files, soyFile{filename, soyfile})
	return b
}

// AddGlobalsFile opens and parses the given filename for Soy globals, and adds
// the resulting data map to the bundle.
func (b *Bundle) AddGlobalsFile(filename string) *Bundle {
	var f, err = os.Open(filename)
	if err != nil {
		b.err = err
		return b
	}
	globals, err := ParseGlobals(f)
	if err != nil {
		b.err = err
	}
	return b.AddGlobalsMap(globals)
}

func (b *Bundle) AddGlobalsMap(globals data.Map) *Bundle {
	for k, v := range globals {
		if existing, ok := b.globals[k]; ok {
			b.err = fmt.Errorf("global %q already defined as %q", k, existing)
			return b
		}
		b.globals[k] = v
	}
	return b
}

// Compile parses all of the soy files in this bundle, verifies a number of
// rules about data references, and returns the completed template registry.
func (b *Bundle) Compile() (*template.Registry, error) {
	if b.err != nil {
		return nil, b.err
	}

	// Compile all the soy (globals are already parsed)
	var registry = template.Registry{}
	for _, soyfile := range b.files {
		var tree, err = parse.SoyFile(soyfile.name, soyfile.content, b.globals)
		if err != nil {
			return nil, err
		}
		if err = registry.Add(tree); err != nil {
			return nil, err
		}
	}

	// Apply the post-parse processing
	var err = parsepasses.CheckDataRefs(registry)
	if err != nil {
		return nil, err
	}

	if b.watcher != nil {
		go b.recompiler(&registry)
	}
	return &registry, nil
}

// CompileToTofu returns a soyhtml.Tofu object that allows you to render soy
// templates to HTML.
func (b *Bundle) CompileToTofu() (*soyhtml.Tofu, error) {
	var registry, err = b.Compile()
	// TODO: Verify all used funcs exist and have the right # args.
	return soyhtml.NewTofu(registry), err
}

func (b *Bundle) recompiler(reg *template.Registry) {
	for {
		select {
		case ev := <-b.watcher.Event:
			// If it's a rename, then fsnotify has removed the watch.
			// Add it back, after a delay.
			if ev.IsRename() || ev.IsDelete() {
				time.Sleep(10 * time.Millisecond)
				if err := b.watcher.Watch(ev.Name); err != nil {
					Logger.Println(err)
				}
			}

			// Recompile all the soy.
			var bundle = NewBundle().
				AddGlobalsMap(b.globals)
			for _, soyfile := range b.files {
				bundle.AddTemplateFile(soyfile.name)
			}
			var registry, err = bundle.Compile()
			if err != nil {
				Logger.Println(err)
				continue
			}

			// update the existing template registry.
			// (this is not goroutine-safe, but that seems ok for a development aid,
			// as long as it works in practice)
			*registry = *registry
			Logger.Printf("update successful (%v)", ev)

		case err := <-b.watcher.Error:
			// Nothing to do with errors
			Logger.Println(err)
		}
	}
}
<commit_msg>Fix soy hot-reload.  How embarrassing.<commit_after>package soy

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"code.google.com/p/go.exp/fsnotify"
	"github.com/robfig/soy/data"
	"github.com/robfig/soy/parse"
	"github.com/robfig/soy/parsepasses"
	"github.com/robfig/soy/soyhtml"
	"github.com/robfig/soy/template"
)

// Logger is used to print notifications and compile errors when using the
// "WatchFiles" feature.
var Logger = log.New(os.Stderr, "[soy] ", 0)

type soyFile struct{ name, content string }

// Bundle is a collection of soy content (templates and globals).  It acts as
// input for the soy compiler.
type Bundle struct {
	files   []soyFile
	globals data.Map
	err     error
	watcher *fsnotify.Watcher
}

// NewBundle returns an empty bundle.
func NewBundle() *Bundle {
	return &Bundle{globals: make(data.Map)}
}

// WatchFiles tells soy to watch any template files added to this bundle,
// re-compile as necessary, and propagate the updates to your tofu.  It should
// be called once, before adding any files.
func (b *Bundle) WatchFiles(watch bool) *Bundle {
	if watch && b.err == nil && b.watcher == nil {
		b.watcher, b.err = fsnotify.NewWatcher()
	}
	return b
}

// AddTemplateDir adds all *.soy files found within the given directory
// (including sub-directories) to the bundle.
func (b *Bundle) AddTemplateDir(root string) *Bundle {
	var err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		if !strings.HasSuffix(path, ".soy") {
			return nil
		}
		b.AddTemplateFile(path)
		return nil
	})
	if err != nil {
		b.err = err
	}
	return b
}

// AddTemplateFile adds the given soy template file text to this bundle.
// If WatchFiles is on, it will be subsequently watched for updates.
func (b *Bundle) AddTemplateFile(filename string) *Bundle {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		b.err = err
	}
	if b.err == nil && b.watcher != nil {
		b.err = b.watcher.Watch(filename)
	}
	return b.AddTemplateString(filename, string(content))
}

// AddTemplateString adds the given template to the bundle. The name is only
// used for error messages - it does not need to be provided nor does it need to
// be a real filename.
func (b *Bundle) AddTemplateString(filename, soyfile string) *Bundle {
	b.files = append(b.files, soyFile{filename, soyfile})
	return b
}

// AddGlobalsFile opens and parses the given filename for Soy globals, and adds
// the resulting data map to the bundle.
func (b *Bundle) AddGlobalsFile(filename string) *Bundle {
	var f, err = os.Open(filename)
	if err != nil {
		b.err = err
		return b
	}
	globals, err := ParseGlobals(f)
	if err != nil {
		b.err = err
	}
	return b.AddGlobalsMap(globals)
}

func (b *Bundle) AddGlobalsMap(globals data.Map) *Bundle {
	for k, v := range globals {
		if existing, ok := b.globals[k]; ok {
			b.err = fmt.Errorf("global %q already defined as %q", k, existing)
			return b
		}
		b.globals[k] = v
	}
	return b
}

// Compile parses all of the soy files in this bundle, verifies a number of
// rules about data references, and returns the completed template registry.
func (b *Bundle) Compile() (*template.Registry, error) {
	if b.err != nil {
		return nil, b.err
	}

	// Compile all the soy (globals are already parsed)
	var registry = template.Registry{}
	for _, soyfile := range b.files {
		var tree, err = parse.SoyFile(soyfile.name, soyfile.content, b.globals)
		if err != nil {
			return nil, err
		}
		if err = registry.Add(tree); err != nil {
			return nil, err
		}
	}

	// Apply the post-parse processing
	var err = parsepasses.CheckDataRefs(registry)
	if err != nil {
		return nil, err
	}

	if b.watcher != nil {
		go b.recompiler(&registry)
	}
	return &registry, nil
}

// CompileToTofu returns a soyhtml.Tofu object that allows you to render soy
// templates to HTML.
func (b *Bundle) CompileToTofu() (*soyhtml.Tofu, error) {
	var registry, err = b.Compile()
	// TODO: Verify all used funcs exist and have the right # args.
	return soyhtml.NewTofu(registry), err
}

func (b *Bundle) recompiler(reg *template.Registry) {
	for {
		select {
		case ev := <-b.watcher.Event:
			// If it's a rename, then fsnotify has removed the watch.
			// Add it back, after a delay.
			if ev.IsRename() || ev.IsDelete() {
				time.Sleep(10 * time.Millisecond)
				if err := b.watcher.Watch(ev.Name); err != nil {
					Logger.Println(err)
				}
			}

			// Recompile all the soy.
			var bundle = NewBundle().
				AddGlobalsMap(b.globals)
			for _, soyfile := range b.files {
				bundle.AddTemplateFile(soyfile.name)
			}
			var registry, err = bundle.Compile()
			if err != nil {
				Logger.Println(err)
				continue
			}

			// update the existing template registry.
			// (this is not goroutine-safe, but that seems ok for a development aid,
			// as long as it works in practice)
			*reg = *registry
			Logger.Printf("update successful (%v)", ev)

		case err := <-b.watcher.Error:
			// Nothing to do with errors
			Logger.Println(err)
		}
	}
}
<|endoftext|>


<commit_before>package cluster

import (
	"fmt"
	"net"
	"testing"
	"time"
)

func Test_NewServiceOpenClose(t *testing.T) {
	ml := mustNewMockListener()
	ms := &mockStore{}
	s := NewService(ml, ms)
	if s == nil {
		t.Fatalf("failed to create cluster service")
	}

	if err := s.Open(); err != nil {
		t.Fatalf("failed to open cluster service")
	}
	if err := s.Close(); err != nil {
		t.Fatalf("failed to close cluster service")
	}
}

func Test_SetAPIPeer(t *testing.T) {
	raftAddr, apiAddr := "localhost:4002", "localhost:4001"

	s, _, ms := mustNewOpenService()
	defer s.Close()
	if err := s.SetPeer(raftAddr, apiAddr); err != nil {
		t.Fatalf("failed to set peer: %s", err.Error())
	}

	if ms.peers[raftAddr] != apiAddr {
		t.Fatalf("peer not set correctly, exp %s, got %s", apiAddr, ms.peers[raftAddr])
	}
}

func Test_SerAPIPeerNetwork(t *testing.T) {
	t.Skip("service not responding correctly")

	raftAddr, apiAddr := "localhost:4002", "localhost:4001"

	s, _, ms := mustNewOpenService()
	defer s.Close()

	raddr, err := net.ResolveTCPAddr("tcp", s.Addr())
	if err != nil {
		t.Fatalf("failed to resolve remote uster ervice address: %s", err.Error())
	}

	conn, err := net.DialTCP("tcp4", nil, raddr)
	if err != nil {
		t.Fatalf("failed to connect to remote cluster service: %s", err.Error())
	}
	conn.Write([]byte(fmt.Sprintf(`{"%s": "%d"}`, raftAddr, apiAddr)))
	if err != nil {
		t.Fatalf("failed to write to remote cluster service: %s", err.Error())
	}
	// XXX Check response

	if ms.peers[raftAddr] != apiAddr {
		t.Fatalf("peer not set correctly, exp %s, got %s", apiAddr, ms.peers[raftAddr])
	}
}

func mustNewOpenService() (*Service, *mockListener, *mockStore) {
	ml := mustNewMockListener()
	ms := newMockStore()
	s := NewService(ml, ms)
	if err := s.Open(); err != nil {
		panic("failed to open new service")
	}
	return s, ml, ms
}

type mockListener struct {
	ln net.Listener
}

func mustNewMockListener() *mockListener {
	ln, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		panic("failed to create mock listener")
	}
	return &mockListener{
		ln: ln,
	}
}

func (ml *mockListener) Accept() (c net.Conn, err error) {
	return ml.ln.Accept()
}

func (ml *mockListener) Addr() net.Addr {
	return ml.ln.Addr()
}

func (ml *mockListener) Close() (err error) {
	return ml.ln.Close()
}

func (ml *mockListener) Dial(addr string, t time.Duration) (net.Conn, error) {
	return nil, nil
}

type mockStore struct {
	leader string
	peers  map[string]string
}

func newMockStore() *mockStore {
	return &mockStore{
		peers: make(map[string]string),
	}
}

func (ms *mockStore) Leader() string {
	return ms.leader
}

func (ms *mockStore) UpdateAPIPeers(peers map[string]string) error {
	for k, v := range peers {
		ms.peers[k] = v
	}
	return nil
}
<commit_msg>'go vet' fixes<commit_after>package cluster

import (
	"fmt"
	"net"
	"testing"
	"time"
)

func Test_NewServiceOpenClose(t *testing.T) {
	ml := mustNewMockListener()
	ms := &mockStore{}
	s := NewService(ml, ms)
	if s == nil {
		t.Fatalf("failed to create cluster service")
	}

	if err := s.Open(); err != nil {
		t.Fatalf("failed to open cluster service")
	}
	if err := s.Close(); err != nil {
		t.Fatalf("failed to close cluster service")
	}
}

func Test_SetAPIPeer(t *testing.T) {
	raftAddr, apiAddr := "localhost:4002", "localhost:4001"

	s, _, ms := mustNewOpenService()
	defer s.Close()
	if err := s.SetPeer(raftAddr, apiAddr); err != nil {
		t.Fatalf("failed to set peer: %s", err.Error())
	}

	if ms.peers[raftAddr] != apiAddr {
		t.Fatalf("peer not set correctly, exp %s, got %s", apiAddr, ms.peers[raftAddr])
	}
}

func Test_SerAPIPeerNetwork(t *testing.T) {
	t.Skip("remote service not responding correctly")

	raftAddr, apiAddr := "localhost:4002", "localhost:4001"

	s, _, ms := mustNewOpenService()
	defer s.Close()

	raddr, err := net.ResolveTCPAddr("tcp", s.Addr())
	if err != nil {
		t.Fatalf("failed to resolve remote uster ervice address: %s", err.Error())
	}

	conn, err := net.DialTCP("tcp4", nil, raddr)
	if err != nil {
		t.Fatalf("failed to connect to remote cluster service: %s", err.Error())
	}
	conn.Write([]byte(fmt.Sprintf(`{"%s": "%s"}`, raftAddr, apiAddr)))
	if err != nil {
		t.Fatalf("failed to write to remote cluster service: %s", err.Error())
	}
	// XXX Check response

	if ms.peers[raftAddr] != apiAddr {
		t.Fatalf("peer not set correctly, exp %s, got %s", apiAddr, ms.peers[raftAddr])
	}
}

func mustNewOpenService() (*Service, *mockListener, *mockStore) {
	ml := mustNewMockListener()
	ms := newMockStore()
	s := NewService(ml, ms)
	if err := s.Open(); err != nil {
		panic("failed to open new service")
	}
	return s, ml, ms
}

type mockListener struct {
	ln net.Listener
}

func mustNewMockListener() *mockListener {
	ln, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		panic("failed to create mock listener")
	}
	return &mockListener{
		ln: ln,
	}
}

func (ml *mockListener) Accept() (c net.Conn, err error) {
	return ml.ln.Accept()
}

func (ml *mockListener) Addr() net.Addr {
	return ml.ln.Addr()
}

func (ml *mockListener) Close() (err error) {
	return ml.ln.Close()
}

func (ml *mockListener) Dial(addr string, t time.Duration) (net.Conn, error) {
	return nil, nil
}

type mockStore struct {
	leader string
	peers  map[string]string
}

func newMockStore() *mockStore {
	return &mockStore{
		peers: make(map[string]string),
	}
}

func (ms *mockStore) Leader() string {
	return ms.leader
}

func (ms *mockStore) UpdateAPIPeers(peers map[string]string) error {
	for k, v := range peers {
		ms.peers[k] = v
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/concourse/atc"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/restart"
	"github.com/tedsuo/ifrit/sigmon"
)

type WorkerCommand struct {
	Name string   `long:"name" description:"The name to set for the worker during registration. If not specified, the hostname will be used."`
	Tags []string `long:"tag" description:"A tag to set during registration. Can be specified multiple times."`

	WorkDir string `long:"work-dir" required:"true" description:"Directory in which to place container data."`

	BindIP   IPFlag `long:"bind-ip"   default:"0.0.0.0" description:"IP address on which to listen for the Garden server."`
	BindPort uint16 `long:"bind-port" default:"7777"    description:"Port on which to listen for the Garden server."`

	PeerIP IPFlag `long:"peer-ip" description:"IP used to reach this worker from the ATC nodes. If omitted, the worker will be forwarded through the SSH connection to the TSA."`

	TSA BeaconConfig `group:"TSA Configuration" namespace:"tsa"`

	Baggageclaim struct {
		BindIP   IPFlag `long:"bind-ip"   default:"0.0.0.0" description:"IP address on which to listen for API traffic."`
		BindPort uint16 `long:"bind-port" default:"7788" description:"Port on which to listen for API traffic."`

		ReapInterval time.Duration `long:"reap-interval" default:"10s" description:"Interval on which to reap expired volumes."`
	} `group:"Baggageclaim Configuration" namespace:"baggageclaim"`

	Metrics struct {
		YellerAPIKey      string `long:"yeller-api-key"     description:"Yeller API key. If specified, all errors logged will be emitted."`
		YellerEnvironment string `long:"yeller-environment" description:"Environment to tag on all Yeller events emitted."`
	} `group:"Metrics & Diagnostics"`
}

func (cmd *WorkerCommand) Execute(args []string) error {
	logger := lager.NewLogger("worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	worker, gardenRunner, err := cmd.gardenRunner(logger.Session("garden"), args)
	if err != nil {
		return err
	}

	baggageclaimRunner, err := cmd.baggageclaimRunner(logger.Session("baggageclaim"))
	if err != nil {
		return err
	}

	worker.BaggageclaimURL = fmt.Sprintf("http://%s:%d", cmd.PeerIP, cmd.Baggageclaim.BindPort)

	members := grouper.Members{
		{
			Name:   "garden",
			Runner: gardenRunner,
		},
		{
			Name:   "baggageclaim",
			Runner: baggageclaimRunner,
		},
	}

	if cmd.TSA.WorkerPrivateKey != "" {
		members = append(members, grouper.Member{
			Name:   "beacon",
			Runner: cmd.beaconRunner(logger.Session("beacon"), worker),
		})
	}

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, members))

	return <-ifrit.Invoke(runner).Wait()
}

func (cmd *WorkerCommand) bindAddr() string {
	return fmt.Sprintf("%s:%d", cmd.BindIP, cmd.BindPort)
}

func (cmd *WorkerCommand) workerName() (string, error) {
	if cmd.Name != "" {
		return cmd.Name, nil
	}

	return os.Hostname()
}

func (cmd *WorkerCommand) beaconRunner(logger lager.Logger, worker atc.Worker) ifrit.Runner {
	beacon := Beacon{
		Logger: logger,
		Config: cmd.TSA,
	}

	var beaconRunner ifrit.RunFunc
	if cmd.PeerIP != "" {
		worker.GardenAddr = fmt.Sprintf("%s:%d", cmd.PeerIP, cmd.BindPort)
		beaconRunner = beacon.Register
	} else {
		worker.GardenAddr = fmt.Sprintf("%s:%d", cmd.BindIP, cmd.BindPort)
		beaconRunner = beacon.Forward
	}

	beacon.Worker = worker

	return restart.Restarter{
		Runner: beaconRunner,
		Load: func(prevRunner ifrit.Runner, prevErr error) ifrit.Runner {
			if _, ok := prevErr.(*ssh.ExitError); !ok {
				logger.Error("restarting", prevErr)
				time.Sleep(5 * time.Second)
				return beaconRunner
			}

			return nil
		},
	}
}
<commit_msg>fix baggage claim forwarding<commit_after>package main

import (
	"fmt"
	"os"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/concourse/atc"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/restart"
	"github.com/tedsuo/ifrit/sigmon"
)

type WorkerCommand struct {
	Name string   `long:"name" description:"The name to set for the worker during registration. If not specified, the hostname will be used."`
	Tags []string `long:"tag" description:"A tag to set during registration. Can be specified multiple times."`

	WorkDir string `long:"work-dir" required:"true" description:"Directory in which to place container data."`

	BindIP   IPFlag `long:"bind-ip"   default:"0.0.0.0" description:"IP address on which to listen for the Garden server."`
	BindPort uint16 `long:"bind-port" default:"7777"    description:"Port on which to listen for the Garden server."`

	PeerIP IPFlag `long:"peer-ip" description:"IP used to reach this worker from the ATC nodes. If omitted, the worker will be forwarded through the SSH connection to the TSA."`

	TSA BeaconConfig `group:"TSA Configuration" namespace:"tsa"`

	Baggageclaim struct {
		BindIP   IPFlag `long:"bind-ip"   default:"0.0.0.0" description:"IP address on which to listen for API traffic."`
		BindPort uint16 `long:"bind-port" default:"7788" description:"Port on which to listen for API traffic."`

		ReapInterval time.Duration `long:"reap-interval" default:"10s" description:"Interval on which to reap expired volumes."`
	} `group:"Baggageclaim Configuration" namespace:"baggageclaim"`

	Metrics struct {
		YellerAPIKey      string `long:"yeller-api-key"     description:"Yeller API key. If specified, all errors logged will be emitted."`
		YellerEnvironment string `long:"yeller-environment" description:"Environment to tag on all Yeller events emitted."`
	} `group:"Metrics & Diagnostics"`
}

func (cmd *WorkerCommand) Execute(args []string) error {
	logger := lager.NewLogger("worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	worker, gardenRunner, err := cmd.gardenRunner(logger.Session("garden"), args)
	if err != nil {
		return err
	}

	baggageclaimRunner, err := cmd.baggageclaimRunner(logger.Session("baggageclaim"))
	if err != nil {
		return err
	}

	members := grouper.Members{
		{
			Name:   "garden",
			Runner: gardenRunner,
		},
		{
			Name:   "baggageclaim",
			Runner: baggageclaimRunner,
		},
	}

	if cmd.TSA.WorkerPrivateKey != "" {
		members = append(members, grouper.Member{
			Name:   "beacon",
			Runner: cmd.beaconRunner(logger.Session("beacon"), worker),
		})
	}

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, members))

	return <-ifrit.Invoke(runner).Wait()
}

func (cmd *WorkerCommand) bindAddr() string {
	return fmt.Sprintf("%s:%d", cmd.BindIP, cmd.BindPort)
}

func (cmd *WorkerCommand) workerName() (string, error) {
	if cmd.Name != "" {
		return cmd.Name, nil
	}

	return os.Hostname()
}

func (cmd *WorkerCommand) beaconRunner(logger lager.Logger, worker atc.Worker) ifrit.Runner {
	beacon := Beacon{
		Logger: logger,
		Config: cmd.TSA,
	}

	var beaconRunner ifrit.RunFunc
	if cmd.PeerIP != "" {
		worker.GardenAddr = fmt.Sprintf("%s:%d", cmd.PeerIP, cmd.BindPort)
		worker.BaggageclaimURL = fmt.Sprintf("http://%s:%d", cmd.PeerIP, cmd.Baggageclaim.BindPort)
		beaconRunner = beacon.Register
	} else {
		worker.GardenAddr = fmt.Sprintf("%s:%d", cmd.BindIP, cmd.BindPort)
		worker.BaggageclaimURL = fmt.Sprintf("http://%s:%d", cmd.Baggageclaim.BindIP, cmd.Baggageclaim.BindPort)
		beaconRunner = beacon.Forward
	}

	beacon.Worker = worker

	return restart.Restarter{
		Runner: beaconRunner,
		Load: func(prevRunner ifrit.Runner, prevErr error) ifrit.Runner {
			if _, ok := prevErr.(*ssh.ExitError); !ok {
				logger.Error("restarting", prevErr)
				time.Sleep(5 * time.Second)
				return beaconRunner
			}

			return nil
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"regexp"
	"strings"

	"github.com/tkrajina/golongfuncs/internal"
)

func main() {
	ty := make([]string, len(internal.AllTypes))
	for n := range internal.AllTypes {
		ty[n] = string(internal.AllTypes[n])
	}

	var ignoreRegexp, types string

	var params internal.CmdParams
	flag.StringVar(&types, "type", string(internal.Lines), "Type of stats, valid types are: "+strings.Join(ty, ", "))
	flag.Float64Var(&params.Treshold, "treshold", 0, "Min value, functions with value less than this will be ignored")
	flag.IntVar(&params.MinLines, "min-lines", 10, "Functions shorter than this will be ignored")
	flag.IntVar(&params.Top, "top", 25, "Show only top n functions")
	flag.BoolVar(&params.IncludeTests, "include-tests", false, "Include tests")
	flag.BoolVar(&params.IncludeVendor, "include-vendor", false, "Include vendored files")
	flag.StringVar(&ignoreRegexp, "ignore", "", "Regexp for files/directories to ignore")
	flag.Parse()

	paths := flag.Args()
	if len(paths) == 0 {
		paths = append(paths, "./...")
	}

	prepareParams(&params, types, ignoreRegexp)
	stats := internal.Do(params, paths)
	printStats(params, stats)
}

func prepareParams(params *internal.CmdParams, types, ignoreRegexp string) {
	var err error
	params.Types, err = internal.ParseTypes(types)
	if err != nil {
		internal.PrintUsage("Invalid type(s) '%s'", types)
	}
	if len(ignoreRegexp) > 0 {
		r, err := regexp.Compile(ignoreRegexp)
		if err != nil {
			internal.PrintUsage("Invalid ignore regexp '%s'", ignoreRegexp)
		}
		params.Ignore = r
	}
}

func printStats(params internal.CmdParams, stats []internal.FunctionStats) {
	count := 0
	for _, st := range stats {
		val, err := st.Get(params.Types[0])
		if err != nil {
			internal.PrintUsage("Invalid type %s\n", params.Types[0])
		}
		lines, _ := st.Get(internal.Lines)
		if val >= params.Treshold && int(lines) >= params.MinLines {
			fmt.Printf("%40s %-40s", shortenTo(st.FuncWithRecv(), 40), shortenTo(st.Location, 40))
			printSingleStat(params.Types[0], val)
			count += 1
			if len(params.Types) > 1 {
				for i := 1; i < len(params.Types); i++ {
					val, _ := st.Get(params.Types[i])
					printSingleStat(params.Types[i], val)
				}
			}
			fmt.Println()
		}
		if count >= params.Top {
			return
		}
	}
}

func shortenTo(str string, l int) string {
	if len(str) > l {
		return "..." + str[len(str)-l+5:]
	}
	return str
}

func printSingleStat(ty internal.FuncMeasurement, val float64) {
	format := fmt.Sprintf("%%%ds", len(string(ty))+8)
	fmt.Printf(format, fmt.Sprintf("%s=%.1f", ty, val))
}
<commit_msg>Change defaults<commit_after>package main

import (
	"flag"
	"fmt"
	"regexp"
	"strings"

	"github.com/tkrajina/golongfuncs/internal"
)

func main() {
	ty := make([]string, len(internal.AllTypes))
	for n := range internal.AllTypes {
		ty[n] = string(internal.AllTypes[n])
	}

	var ignoreRegexp, types string

	var params internal.CmdParams
	flag.StringVar(&types, "type", fmt.Sprintf("%s,%s,%s", internal.Lines, internal.Complexity, internal.MaxNesting), "Type of stats, valid types are: "+strings.Join(ty, ", "))
	flag.Float64Var(&params.Treshold, "treshold", 0, "Min value, functions with value less than this will be ignored")
	flag.IntVar(&params.MinLines, "min-lines", 10, "Functions shorter than this will be ignored")
	flag.IntVar(&params.Top, "top", 25, "Show only top n functions")
	flag.BoolVar(&params.IncludeTests, "include-tests", false, "Include tests")
	flag.BoolVar(&params.IncludeVendor, "include-vendor", false, "Include vendored files")
	flag.StringVar(&ignoreRegexp, "ignore", "", "Regexp for files/directories to ignore")
	flag.Parse()

	paths := flag.Args()
	if len(paths) == 0 {
		paths = append(paths, "./...")
	}

	prepareParams(&params, types, ignoreRegexp)
	stats := internal.Do(params, paths)
	printStats(params, stats)
}

func prepareParams(params *internal.CmdParams, types, ignoreRegexp string) {
	var err error
	params.Types, err = internal.ParseTypes(types)
	if err != nil {
		internal.PrintUsage("Invalid type(s) '%s'", types)
	}
	if len(ignoreRegexp) > 0 {
		r, err := regexp.Compile(ignoreRegexp)
		if err != nil {
			internal.PrintUsage("Invalid ignore regexp '%s'", ignoreRegexp)
		}
		params.Ignore = r
	}
}

func printStats(params internal.CmdParams, stats []internal.FunctionStats) {
	count := 0
	for _, st := range stats {
		val, err := st.Get(params.Types[0])
		if err != nil {
			internal.PrintUsage("Invalid type %s\n", params.Types[0])
		}
		lines, _ := st.Get(internal.Lines)
		if val >= params.Treshold && int(lines) >= params.MinLines {
			fmt.Printf("%40s %-40s", shortenTo(st.FuncWithRecv(), 40), shortenTo(st.Location, 40))
			printSingleStat(params.Types[0], val)
			count += 1
			if len(params.Types) > 1 {
				for i := 1; i < len(params.Types); i++ {
					val, _ := st.Get(params.Types[i])
					printSingleStat(params.Types[i], val)
				}
			}
			fmt.Println()
		}
		if count >= params.Top {
			return
		}
	}
}

func shortenTo(str string, l int) string {
	if len(str) > l {
		return "..." + str[len(str)-l+5:]
	}
	return str
}

func printSingleStat(ty internal.FuncMeasurement, val float64) {
	format := fmt.Sprintf("%%%ds", len(string(ty))+8)
	fmt.Printf(format, fmt.Sprintf("%s=%.1f", ty, val))
}
<|endoftext|>