<commit_before>package csrf

import (
	"fmt"
	"net/http"
	"net/url"

	"github.com/pkg/errors"

	"github.com/gorilla/securecookie"
)

// CSRF token length in bytes.
const tokenLength = 32

// Context/session keys & prefixes
const (
	tokenKey     string = "gorilla.csrf.Token"
	formKey      string = "gorilla.csrf.Form"
	errorKey     string = "gorilla.csrf.Error"
	skipCheckKey string = "gorilla.csrf.Skip"
	cookieName   string = "_gorilla_csrf"
	errorPrefix  string = "gorilla/csrf: "
)

var (
	// The name value used in form fields.
	fieldName = tokenKey
	// defaultAge sets the default MaxAge for cookies.
	defaultAge = 3600 * 12
	// The default HTTP request header to inspect
	headerName = "X-CSRF-Token"
	// Idempotent (safe) methods as defined by RFC7231 section 4.2.2.
	safeMethods = []string{"GET", "HEAD", "OPTIONS", "TRACE"}
)

// TemplateTag provides a default template tag - e.g. {{ .csrfField }} - for use
// with the TemplateField function.
var TemplateTag = "csrfField"

var (
	// ErrNoReferer is returned when a HTTPS request provides an empty Referer
	// header.
	ErrNoReferer = errors.New("referer not supplied")
	// ErrBadReferer is returned when the scheme & host in the URL do not match
	// the supplied Referer header.
	ErrBadReferer = errors.New("referer invalid")
	// ErrNoToken is returned if no CSRF token is supplied in the request.
	ErrNoToken = errors.New("CSRF token not found in request")
	// ErrBadToken is returned if the CSRF token in the request does not match
	// the token in the session, or is otherwise malformed.
	ErrBadToken = errors.New("CSRF token invalid")
)

type csrf struct {
	h    http.Handler
	sc   *securecookie.SecureCookie
	st   store
	opts options
}

// options contains the optional settings for the CSRF middleware.
type options struct {
	MaxAge int
	Domain string
	Path   string
	// Note that the function and field names match the case of the associated
	// http.Cookie field instead of the "correct" HTTPOnly name that golint suggests.
	HttpOnly      bool
	Secure        bool
	RequestHeader string
	FieldName     string
	ErrorHandler  http.Handler
	CookieName    string
}

// Protect is HTTP middleware that provides Cross-Site Request Forgery
// protection.
//
// It securely generates a masked (unique-per-request) token that
// can be embedded in the HTTP response (e.g. form field or HTTP header).
// The original (unmasked) token is stored in the session, which is inaccessible
// by an attacker (provided you are using HTTPS). Subsequent requests are
// expected to include this token, which is compared against the session token.
// Requests that do not provide a matching token are served with a HTTP 403
// 'Forbidden' error response.
//
// Example:
//	package main
//
//	import (
//		"github.com/elithrar/protect"
//		"github.com/gorilla/mux"
//	)
//
//	func main() {
//	  r := mux.NewRouter()
//
//	  mux.HandlerFunc("/signup", GetSignupForm)
//	  // POST requests without a valid token will return a HTTP 403 Forbidden.
//	  mux.HandlerFunc("/signup/post", PostSignupForm)
//
//	  // Add the middleware to your router.
//	  http.ListenAndServe(":8000",
//            // Note that the authentication key provided should be 32 bytes
//            // long and persist across application restarts.
//			  csrf.Protect([]byte("32-byte-long-auth-key"))(r))
//	}
//
//	func GetSignupForm(w http.ResponseWriter, r *http.Request) {
//		// signup_form.tmpl just needs a {{ .csrfField }} template tag for
//		// csrf.TemplateField to inject the CSRF token into. Easy!
//		t.ExecuteTemplate(w, "signup_form.tmpl", map[string]interface{
//			csrf.TemplateTag: csrf.TemplateField(r),
//		})
//		// We could also retrieve the token directly from csrf.Token(r) and
//		// set it in the request header - w.Header.Set("X-CSRF-Token", token)
//		// This is useful if your sending JSON to clients or a front-end JavaScript
//		// framework.
//	}
//
func Protect(authKey []byte, opts ...Option) func(http.Handler) http.Handler {
	return func(h http.Handler) http.Handler {
		cs := parseOptions(h, opts...)

		// Set the defaults if no options have been specified
		if cs.opts.ErrorHandler == nil {
			cs.opts.ErrorHandler = http.HandlerFunc(unauthorizedHandler)
		}

		if cs.opts.MaxAge < 0 {
			// Default of 12 hours
			cs.opts.MaxAge = defaultAge
		}

		if cs.opts.FieldName == "" {
			cs.opts.FieldName = fieldName
		}

		if cs.opts.CookieName == "" {
			cs.opts.CookieName = cookieName
		}

		if cs.opts.RequestHeader == "" {
			cs.opts.RequestHeader = headerName
		}

		// Create an authenticated securecookie instance.
		if cs.sc == nil {
			cs.sc = securecookie.New(authKey, nil)
			// Use JSON serialization (faster than one-off gob encoding)
			cs.sc.SetSerializer(securecookie.JSONEncoder{})
			// Set the MaxAge of the underlying securecookie.
			cs.sc.MaxAge(cs.opts.MaxAge)
		}

		if cs.st == nil {
			// Default to the cookieStore
			cs.st = &cookieStore{
				name:     cs.opts.CookieName,
				maxAge:   cs.opts.MaxAge,
				secure:   cs.opts.Secure,
				httpOnly: cs.opts.HttpOnly,
				path:     cs.opts.Path,
				domain:   cs.opts.Domain,
				sc:       cs.sc,
			}
		}

		return cs
	}
}

// Implements http.Handler for the csrf type.
func (cs *csrf) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Skip the check if directed to. This should always be a bool.
	if val, err := contextGet(r, skipCheckKey); err == nil {
		if skip, ok := val.(bool); ok {
			if skip {
				cs.h.ServeHTTP(w, r)
				return
			}
		}
	}

	// Retrieve the token from the session.
	// An error represents either a cookie that failed HMAC validation
	// or that doesn't exist.
	realToken, err := cs.st.Get(r)
	if err != nil || len(realToken) != tokenLength {
		// If there was an error retrieving the token, the token doesn't exist
		// yet, or it's the wrong length, generate a new token.
		// Note that the new token will (correctly) fail validation downstream
		// as it will no longer match the request token.
		realToken, err = generateRandomBytes(tokenLength)
		if err != nil {
			r = envError(r, err)
			cs.opts.ErrorHandler.ServeHTTP(w, r)
			return
		}

		// Save the new (real) token in the session store.
		err = cs.st.Save(realToken, w)
		if err != nil {
			r = envError(r, err)
			cs.opts.ErrorHandler.ServeHTTP(w, r)
			return
		}
	}

	// Save the masked token to the request context
	r = contextSave(r, tokenKey, mask(realToken, r))
	// Save the field name to the request context
	r = contextSave(r, formKey, cs.opts.FieldName)

	// HTTP methods not defined as idempotent ("safe") under RFC7231 require
	// inspection.
	if !contains(safeMethods, r.Method) {
		// Enforce an origin check for HTTPS connections. As per the Django CSRF
		// implementation (https://goo.gl/vKA7GE) the Referer header is almost
		// always present for same-domain HTTP requests.
		if r.URL.Scheme == "https" {
			// Fetch the Referer value. Call the error handler if it's empty or
			// otherwise fails to parse.
			referer, err := url.Parse(r.Referer())
			if err != nil || referer.String() == "" {
				r = envError(r, ErrNoReferer)
				cs.opts.ErrorHandler.ServeHTTP(w, r)
				return
			}

			if sameOrigin(r.URL, referer) == false {
				r = envError(r, ErrBadReferer)
				cs.opts.ErrorHandler.ServeHTTP(w, r)
				return
			}
		}

		// If the token returned from the session store is nil for non-idempotent
		// ("unsafe") methods, call the error handler.
		if realToken == nil {
			r = envError(r, ErrNoToken)
			cs.opts.ErrorHandler.ServeHTTP(w, r)
			return
		}

		// Retrieve the combined token (pad + masked) token and unmask it.
		requestToken := unmask(cs.requestToken(r))

		// Compare the request token against the real token
		if !compareTokens(requestToken, realToken) {
			r = envError(r, ErrBadToken)
			cs.opts.ErrorHandler.ServeHTTP(w, r)
			return
		}

	}

	// Set the Vary: Cookie header to protect clients from caching the response.
	w.Header().Add("Vary", "Cookie")

	// Call the wrapped handler/router on success.
	cs.h.ServeHTTP(w, r)
	// Clear the request context after the handler has completed.
	contextClear(r)
}

// unauthorizedhandler sets a HTTP 403 Forbidden status and writes the
// CSRF failure reason to the response.
func unauthorizedHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, fmt.Sprintf("%s - %s",
		http.StatusText(http.StatusForbidden), FailureReason(r)),
		http.StatusForbidden)
	return
}
<commit_msg>[docs] Improve commented code (#46)<commit_after>package csrf

import (
	"fmt"
	"net/http"
	"net/url"

	"github.com/pkg/errors"

	"github.com/gorilla/securecookie"
)

// CSRF token length in bytes.
const tokenLength = 32

// Context/session keys & prefixes
const (
	tokenKey     string = "gorilla.csrf.Token"
	formKey      string = "gorilla.csrf.Form"
	errorKey     string = "gorilla.csrf.Error"
	skipCheckKey string = "gorilla.csrf.Skip"
	cookieName   string = "_gorilla_csrf"
	errorPrefix  string = "gorilla/csrf: "
)

var (
	// The name value used in form fields.
	fieldName = tokenKey
	// defaultAge sets the default MaxAge for cookies.
	defaultAge = 3600 * 12
	// The default HTTP request header to inspect
	headerName = "X-CSRF-Token"
	// Idempotent (safe) methods as defined by RFC7231 section 4.2.2.
	safeMethods = []string{"GET", "HEAD", "OPTIONS", "TRACE"}
)

// TemplateTag provides a default template tag - e.g. {{ .csrfField }} - for use
// with the TemplateField function.
var TemplateTag = "csrfField"

var (
	// ErrNoReferer is returned when a HTTPS request provides an empty Referer
	// header.
	ErrNoReferer = errors.New("referer not supplied")
	// ErrBadReferer is returned when the scheme & host in the URL do not match
	// the supplied Referer header.
	ErrBadReferer = errors.New("referer invalid")
	// ErrNoToken is returned if no CSRF token is supplied in the request.
	ErrNoToken = errors.New("CSRF token not found in request")
	// ErrBadToken is returned if the CSRF token in the request does not match
	// the token in the session, or is otherwise malformed.
	ErrBadToken = errors.New("CSRF token invalid")
)

type csrf struct {
	h    http.Handler
	sc   *securecookie.SecureCookie
	st   store
	opts options
}

// options contains the optional settings for the CSRF middleware.
type options struct {
	MaxAge int
	Domain string
	Path   string
	// Note that the function and field names match the case of the associated
	// http.Cookie field instead of the "correct" HTTPOnly name that golint suggests.
	HttpOnly      bool
	Secure        bool
	RequestHeader string
	FieldName     string
	ErrorHandler  http.Handler
	CookieName    string
}

// Protect is HTTP middleware that provides Cross-Site Request Forgery
// protection.
//
// It securely generates a masked (unique-per-request) token that
// can be embedded in the HTTP response (e.g. form field or HTTP header).
// The original (unmasked) token is stored in the session, which is inaccessible
// by an attacker (provided you are using HTTPS). Subsequent requests are
// expected to include this token, which is compared against the session token.
// Requests that do not provide a matching token are served with a HTTP 403
// 'Forbidden' error response.
//
// Example:
//	package main
//
//	import (
//		"github.com/elithrar/protect"
//		"github.com/gorilla/mux"
//	)
//
//	func main() {
//	  r := mux.NewRouter()
//
//	  mux.HandlerFunc("/signup", GetSignupForm)
//	  // POST requests without a valid token will return a HTTP 403 Forbidden.
//	  mux.HandlerFunc("/signup/post", PostSignupForm)
//
//	  // Add the middleware to your router.
//	  http.ListenAndServe(":8000",
//            // Note that the authentication key provided should be 32 bytes
//            // long and persist across application restarts.
//			  csrf.Protect([]byte("32-byte-long-auth-key"))(r))
//	}
//
//	func GetSignupForm(w http.ResponseWriter, r *http.Request) {
//		// signup_form.tmpl just needs a {{ .csrfField }} template tag for
//		// csrf.TemplateField to inject the CSRF token into. Easy!
//		t.ExecuteTemplate(w, "signup_form.tmpl", map[string]interface{}{
//			csrf.TemplateTag: csrf.TemplateField(r),
//		})
//		// We could also retrieve the token directly from csrf.Token(r) and
//		// set it in the request header - w.Header.Set("X-CSRF-Token", token)
//		// This is useful if your sending JSON to clients or a front-end JavaScript
//		// framework.
//	}
//
func Protect(authKey []byte, opts ...Option) func(http.Handler) http.Handler {
	return func(h http.Handler) http.Handler {
		cs := parseOptions(h, opts...)

		// Set the defaults if no options have been specified
		if cs.opts.ErrorHandler == nil {
			cs.opts.ErrorHandler = http.HandlerFunc(unauthorizedHandler)
		}

		if cs.opts.MaxAge < 0 {
			// Default of 12 hours
			cs.opts.MaxAge = defaultAge
		}

		if cs.opts.FieldName == "" {
			cs.opts.FieldName = fieldName
		}

		if cs.opts.CookieName == "" {
			cs.opts.CookieName = cookieName
		}

		if cs.opts.RequestHeader == "" {
			cs.opts.RequestHeader = headerName
		}

		// Create an authenticated securecookie instance.
		if cs.sc == nil {
			cs.sc = securecookie.New(authKey, nil)
			// Use JSON serialization (faster than one-off gob encoding)
			cs.sc.SetSerializer(securecookie.JSONEncoder{})
			// Set the MaxAge of the underlying securecookie.
			cs.sc.MaxAge(cs.opts.MaxAge)
		}

		if cs.st == nil {
			// Default to the cookieStore
			cs.st = &cookieStore{
				name:     cs.opts.CookieName,
				maxAge:   cs.opts.MaxAge,
				secure:   cs.opts.Secure,
				httpOnly: cs.opts.HttpOnly,
				path:     cs.opts.Path,
				domain:   cs.opts.Domain,
				sc:       cs.sc,
			}
		}

		return cs
	}
}

// Implements http.Handler for the csrf type.
func (cs *csrf) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Skip the check if directed to. This should always be a bool.
	if val, err := contextGet(r, skipCheckKey); err == nil {
		if skip, ok := val.(bool); ok {
			if skip {
				cs.h.ServeHTTP(w, r)
				return
			}
		}
	}

	// Retrieve the token from the session.
	// An error represents either a cookie that failed HMAC validation
	// or that doesn't exist.
	realToken, err := cs.st.Get(r)
	if err != nil || len(realToken) != tokenLength {
		// If there was an error retrieving the token, the token doesn't exist
		// yet, or it's the wrong length, generate a new token.
		// Note that the new token will (correctly) fail validation downstream
		// as it will no longer match the request token.
		realToken, err = generateRandomBytes(tokenLength)
		if err != nil {
			r = envError(r, err)
			cs.opts.ErrorHandler.ServeHTTP(w, r)
			return
		}

		// Save the new (real) token in the session store.
		err = cs.st.Save(realToken, w)
		if err != nil {
			r = envError(r, err)
			cs.opts.ErrorHandler.ServeHTTP(w, r)
			return
		}
	}

	// Save the masked token to the request context
	r = contextSave(r, tokenKey, mask(realToken, r))
	// Save the field name to the request context
	r = contextSave(r, formKey, cs.opts.FieldName)

	// HTTP methods not defined as idempotent ("safe") under RFC7231 require
	// inspection.
	if !contains(safeMethods, r.Method) {
		// Enforce an origin check for HTTPS connections. As per the Django CSRF
		// implementation (https://goo.gl/vKA7GE) the Referer header is almost
		// always present for same-domain HTTP requests.
		if r.URL.Scheme == "https" {
			// Fetch the Referer value. Call the error handler if it's empty or
			// otherwise fails to parse.
			referer, err := url.Parse(r.Referer())
			if err != nil || referer.String() == "" {
				r = envError(r, ErrNoReferer)
				cs.opts.ErrorHandler.ServeHTTP(w, r)
				return
			}

			if sameOrigin(r.URL, referer) == false {
				r = envError(r, ErrBadReferer)
				cs.opts.ErrorHandler.ServeHTTP(w, r)
				return
			}
		}

		// If the token returned from the session store is nil for non-idempotent
		// ("unsafe") methods, call the error handler.
		if realToken == nil {
			r = envError(r, ErrNoToken)
			cs.opts.ErrorHandler.ServeHTTP(w, r)
			return
		}

		// Retrieve the combined token (pad + masked) token and unmask it.
		requestToken := unmask(cs.requestToken(r))

		// Compare the request token against the real token
		if !compareTokens(requestToken, realToken) {
			r = envError(r, ErrBadToken)
			cs.opts.ErrorHandler.ServeHTTP(w, r)
			return
		}

	}

	// Set the Vary: Cookie header to protect clients from caching the response.
	w.Header().Add("Vary", "Cookie")

	// Call the wrapped handler/router on success.
	cs.h.ServeHTTP(w, r)
	// Clear the request context after the handler has completed.
	contextClear(r)
}

// unauthorizedhandler sets a HTTP 403 Forbidden status and writes the
// CSRF failure reason to the response.
func unauthorizedHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, fmt.Sprintf("%s - %s",
		http.StatusText(http.StatusForbidden), FailureReason(r)),
		http.StatusForbidden)
	return
}
<|endoftext|>


<commit_before>package myslave

import (
	"fmt"
	"time"

	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/checkpoint"
	"github.com/funkygao/dbus/pkg/checkpoint/state/binlog"
	czk "github.com/funkygao/dbus/pkg/checkpoint/store/zk"
	"github.com/funkygao/dbus/pkg/model"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/golib/sync2"
	conf "github.com/funkygao/jsconf"
	mylog "github.com/ngaut/log"
	"github.com/siddontang/go-mysql/client"
	"github.com/siddontang/go-mysql/replication"
)

// MySlave is a mimic mysql slave that replicates binlog from mysql master using IO thread.
type MySlave struct {
	c     *conf.Conf
	r     *replication.BinlogSyncer
	p     checkpoint.Checkpoint
	m     *slaveMetrics
	z     *zk.ZkZone
	conn  *client.Conn
	state *binlog.BinlogState

	name string

	Predicate func(schema, table string) bool
	GTID      bool // global tx id

	dsn          string
	masterAddr   string
	host         string
	port         uint16
	user, passwd string

	dbAllowed  map[string]struct{}
	dbExcluded map[string]struct{}

	started   sync2.AtomicBool
	errors    chan error
	rowsEvent chan *model.RowsEvent
}

// New creates a MySlave instance.
func New(dsn string) *MySlave {
	// github.com/siddontang/go-mysql is using github.com/ngaut/log
	mylog.SetLevel(mylog.LOG_LEVEL_ERROR)
	if err := mylog.SetOutputByName("myslave.log"); err != nil {
		panic(err)
	}

	return &MySlave{
		dsn:        dsn,
		dbExcluded: map[string]struct{}{},
		dbAllowed:  map[string]struct{}{},
		state:      binlog.New(dsn),
	}
}

// LoadConfig initialize internal state according to the config section.
func (m *MySlave) LoadConfig(config *conf.Conf) *MySlave {
	m.c = config

	var err error
	var dbs []string
	var zone string
	zone, m.host, m.port, m.user, m.passwd, dbs, err = ParseDSN(m.dsn)
	if m.user == "" || zone == "" || m.host == "" || m.port == 0 || err != nil {
		panic("invalid dsn")
	}
	m.masterAddr = fmt.Sprintf("%s:%d", m.host, m.port)

	for _, db := range dbs {
		m.dbAllowed[db] = struct{}{}
	}
	for _, db := range config.StringList("db_excluded", nil) {
		m.dbExcluded[db] = struct{}{}
	}
	if len(m.dbAllowed) > 0 && len(m.dbExcluded) > 0 {
		panic("db_excluded and db allowed cannot be set at the same time")
	}
	m.setupPredicate()

	m.name = m.c.String("name", m.masterAddr)
	m.GTID = m.c.Bool("GTID", false)
	if m.GTID {
		panic("GTID not implemented")
	}

	m.m = newMetrics(m.host, m.port)
	m.z = engine.Globals().GetOrRegisterZkzone(zone)
	m.p = czk.New(m.z, m.state, m.masterAddr, m.c.Duration("pos_commit_interval", time.Second))

	return m
}

// MarkAsProcessed notifies the positioner that a certain binlog event
// has been successfully processed and should be committed.
func (m *MySlave) MarkAsProcessed(r *model.RowsEvent) error {
	if !r.IsStmtEnd() {
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | Log_name           | Pos       | Event_type | Server_id | End_log_pos | Info                                                                        |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | 8623306-bin.008517 | 117732471 | Query      |  81413306 |   117732534 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117732534 | Table_map  |  81413306 |   117732590 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117732590 | Write_rows |  81413306 |   117735750 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117735750 | Table_map  |  81413306 |   117735810 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117735810 | Write_rows |  81413306 |   117744020 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117744020 | Write_rows |  81413306 |   117752230 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117752230 | Write_rows |  81413306 |   117755340 | table_id: 2968 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117755340 | Xid        |  81413306 |   117755371 | COMMIT /* xid=2243095633 */                                                 |
		// | 8623306-bin.008517 | 117755371 | Gtid       |  81413306 |   117755419 | SET @@SESSION.GTID_NEXT= 'a0f36bb1-fdbb-11e5-8413-a0369f7c3bb4:16393135780' |
		// | 8623306-bin.008517 | 117755419 | Query      |  81413306 |   117755482 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117755482 | Table_map  |  81413306 |   117755538 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117755538 | Write_rows |  81413306 |   117758123 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117758123 | Table_map  |  81413306 |   117758183 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117758183 | Write_rows |  81413306 |   117766393 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117766393 | Write_rows |  81413306 |   117772728 | table_id: 2968 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117772728 | Xid        |  81413306 |   117772759 | COMMIT /* xid=2243095636 */                                                 |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		//
		// Pos:117735810 will not be checkpointed
		// Pos:117752230 will be checkpointed
		//
		// otherwise when restart and resume from last checkpoint:
		// invalid table id 2413, no correspond table map event
		return nil
	}

	return m.commitPosition(r.Log, r.Position)
}

func (m *MySlave) commitPosition(file string, offset uint32) error {
	m.state.File = file
	m.state.Offset = offset
	return m.p.Commit(m.state)
}

// Events returns the iterator of mysql binlog rows event.
func (m *MySlave) Events() <-chan *model.RowsEvent {
	return m.rowsEvent
}

// Errors returns the iterator of unexpected errors.
func (m *MySlave) Errors() <-chan error {
	return m.errors
}
<commit_msg>setup logger once<commit_after>package myslave

import (
	"fmt"
	"sync"
	"time"

	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/checkpoint"
	"github.com/funkygao/dbus/pkg/checkpoint/state/binlog"
	czk "github.com/funkygao/dbus/pkg/checkpoint/store/zk"
	"github.com/funkygao/dbus/pkg/model"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/golib/sync2"
	conf "github.com/funkygao/jsconf"
	mylog "github.com/ngaut/log"
	"github.com/siddontang/go-mysql/client"
	"github.com/siddontang/go-mysql/replication"
)

// MySlave is a mimic mysql slave that replicates binlog from mysql master using IO thread.
type MySlave struct {
	c     *conf.Conf
	r     *replication.BinlogSyncer
	p     checkpoint.Checkpoint
	m     *slaveMetrics
	z     *zk.ZkZone
	conn  *client.Conn
	state *binlog.BinlogState

	name string

	Predicate func(schema, table string) bool
	GTID      bool // global tx id

	dsn          string
	masterAddr   string
	host         string
	port         uint16
	user, passwd string

	dbAllowed  map[string]struct{}
	dbExcluded map[string]struct{}

	started   sync2.AtomicBool
	errors    chan error
	rowsEvent chan *model.RowsEvent
}

var setupLogger sync.Once

// New creates a MySlave instance.
func New(dsn string) *MySlave {
	setupLogger.Do(func() {
		// github.com/siddontang/go-mysql is using github.com/ngaut/log
		mylog.SetLevel(mylog.LOG_LEVEL_ERROR)
		if err := mylog.SetOutputByName("myslave.log"); err != nil {
			panic(err)
		}
	})

	return &MySlave{
		dsn:        dsn,
		dbExcluded: map[string]struct{}{},
		dbAllowed:  map[string]struct{}{},
		state:      binlog.New(dsn),
	}
}

// LoadConfig initialize internal state according to the config section.
func (m *MySlave) LoadConfig(config *conf.Conf) *MySlave {
	m.c = config

	var err error
	var dbs []string
	var zone string
	zone, m.host, m.port, m.user, m.passwd, dbs, err = ParseDSN(m.dsn)
	if m.user == "" || zone == "" || m.host == "" || m.port == 0 || err != nil {
		panic("invalid dsn")
	}
	m.masterAddr = fmt.Sprintf("%s:%d", m.host, m.port)

	for _, db := range dbs {
		m.dbAllowed[db] = struct{}{}
	}
	for _, db := range config.StringList("db_excluded", nil) {
		m.dbExcluded[db] = struct{}{}
	}
	if len(m.dbAllowed) > 0 && len(m.dbExcluded) > 0 {
		panic("db_excluded and db allowed cannot be set at the same time")
	}
	m.setupPredicate()

	m.name = m.c.String("name", m.masterAddr)
	m.GTID = m.c.Bool("GTID", false)
	if m.GTID {
		panic("GTID not implemented")
	}

	m.m = newMetrics(m.host, m.port)
	m.z = engine.Globals().GetOrRegisterZkzone(zone)
	m.p = czk.New(m.z, m.state, m.masterAddr, m.c.Duration("pos_commit_interval", time.Second))

	return m
}

// MarkAsProcessed notifies the positioner that a certain binlog event
// has been successfully processed and should be committed.
func (m *MySlave) MarkAsProcessed(r *model.RowsEvent) error {
	if !r.IsStmtEnd() {
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | Log_name           | Pos       | Event_type | Server_id | End_log_pos | Info                                                                        |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | 8623306-bin.008517 | 117732471 | Query      |  81413306 |   117732534 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117732534 | Table_map  |  81413306 |   117732590 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117732590 | Write_rows |  81413306 |   117735750 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117735750 | Table_map  |  81413306 |   117735810 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117735810 | Write_rows |  81413306 |   117744020 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117744020 | Write_rows |  81413306 |   117752230 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117752230 | Write_rows |  81413306 |   117755340 | table_id: 2968 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117755340 | Xid        |  81413306 |   117755371 | COMMIT /* xid=2243095633 */                                                 |
		// | 8623306-bin.008517 | 117755371 | Gtid       |  81413306 |   117755419 | SET @@SESSION.GTID_NEXT= 'a0f36bb1-fdbb-11e5-8413-a0369f7c3bb4:16393135780' |
		// | 8623306-bin.008517 | 117755419 | Query      |  81413306 |   117755482 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117755482 | Table_map  |  81413306 |   117755538 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117755538 | Write_rows |  81413306 |   117758123 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117758123 | Table_map  |  81413306 |   117758183 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117758183 | Write_rows |  81413306 |   117766393 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117766393 | Write_rows |  81413306 |   117772728 | table_id: 2968 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117772728 | Xid        |  81413306 |   117772759 | COMMIT /* xid=2243095636 */                                                 |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		//
		// Pos:117735810 will not be checkpointed
		// Pos:117752230 will be checkpointed
		//
		// otherwise when restart and resume from last checkpoint:
		// invalid table id 2413, no correspond table map event
		return nil
	}

	return m.commitPosition(r.Log, r.Position)
}

func (m *MySlave) commitPosition(file string, offset uint32) error {
	m.state.File = file
	m.state.Offset = offset
	return m.p.Commit(m.state)
}

// Events returns the iterator of mysql binlog rows event.
func (m *MySlave) Events() <-chan *model.RowsEvent {
	return m.rowsEvent
}

// Errors returns the iterator of unexpected errors.
func (m *MySlave) Errors() <-chan error {
	return m.errors
}
<|endoftext|>


<commit_before>package cb

import (
	"github.com/afex/hystrix-go/hystrix"
	"github.com/afex/hystrix-go/hystrix/metric_collector"
	"github.com/afex/hystrix-go/plugins"
	"github.com/asaskevich/govalidator"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

const (
	statsdPrefix = "hystrix"
	pluginName   = "cb"
)

// Config represents the Body Limit configuration
type Config struct {
	hystrix.CommandConfig
	Name      string `json:"name"`
	Predicate string `json:"predicate"`
}

func init() {
	plugin.RegisterEventHook(plugin.StartupEvent, onStartup)
	plugin.RegisterEventHook(plugin.AdminAPIStartupEvent, onAdminAPIStartup)
	plugin.RegisterPlugin(pluginName, plugin.Plugin{
		Action:   setupCB,
		Validate: validateConfig,
	})
}

func setupCB(def *proxy.RouterDefinition, rawConfig plugin.Config) error {
	var c Config
	err := plugin.Decode(rawConfig, &c)
	if err != nil {
		return err
	}

	log.WithFields(log.Fields{
		"plugin_event": plugin.SetupEvent,
		"plugin":       pluginName,
		"name":         c.Name,
	}).Debug("Configuring cb plugin")

	hystrix.ConfigureCommand(c.Name, hystrix.CommandConfig{
		Timeout:               c.Timeout,
		MaxConcurrentRequests: c.MaxConcurrentRequests,
		ErrorPercentThreshold: c.ErrorPercentThreshold,
		SleepWindow:           c.SleepWindow,
	})

	def.AddMiddleware(NewCBMiddleware(c))
	return nil
}

func validateConfig(rawConfig plugin.Config) (bool, error) {
	var config Config
	err := plugin.Decode(rawConfig, &config)
	if err != nil {
		return false, err
	}

	return govalidator.ValidateStruct(config)
}

func onAdminAPIStartup(event interface{}) error {
	logger := log.WithFields(log.Fields{
		"plugin_event": plugin.AdminAPIStartupEvent,
		"plugin":       pluginName,
	})

	e, ok := event.(plugin.OnAdminAPIStartup)
	if !ok {
		return errors.New("Could not convert event to admin startup type")
	}

	logger.Debug("Registering hystrix stream endpoint")
	hystrixStreamHandler := hystrix.NewStreamHandler()
	hystrixStreamHandler.Start()

	e.Router.GET("/hystrix", hystrixStreamHandler.ServeHTTP)
	return nil
}

func onStartup(event interface{}) error {
	logger := log.WithFields(log.Fields{
		"plugin_event": plugin.StartupEvent,
		"plugin":       pluginName,
	})

	e, ok := event.(plugin.OnStartup)
	if !ok {
		return errors.New("Could not convert event to startup type")
	}

	logger.WithField("metrics_dsn", e.Config.Stats.DSN).Debug("Statsd metrics enabled")
	c, err := plugins.InitializeStatsdCollector(&plugins.StatsdCollectorConfig{
		StatsdAddr: e.Config.Stats.DSN,
		Prefix:     statsdPrefix,
	})
	if err != nil {
		return errors.Wrap(err, "could not initialize statsd client")
	}

	metricCollector.Registry.Register(c.NewStatsdCollector)
	logger.Debug("Metrics enabled")

	return nil
}
<commit_msg>Fix circuit breaker plugin statsd collector<commit_after>package cb

import (
	"net/url"
	"strings"

	"github.com/afex/hystrix-go/hystrix"
	"github.com/afex/hystrix-go/hystrix/metric_collector"
	"github.com/afex/hystrix-go/plugins"
	"github.com/asaskevich/govalidator"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

const (
	statsdPrefix = "hystrix"
	pluginName   = "cb"
)

// Config represents the Body Limit configuration
type Config struct {
	hystrix.CommandConfig
	Name      string `json:"name"`
	Predicate string `json:"predicate"`
}

func init() {
	plugin.RegisterEventHook(plugin.StartupEvent, onStartup)
	plugin.RegisterEventHook(plugin.AdminAPIStartupEvent, onAdminAPIStartup)
	plugin.RegisterPlugin(pluginName, plugin.Plugin{
		Action:   setupCB,
		Validate: validateConfig,
	})
}

func setupCB(def *proxy.RouterDefinition, rawConfig plugin.Config) error {
	var c Config
	err := plugin.Decode(rawConfig, &c)
	if err != nil {
		return err
	}

	log.WithFields(log.Fields{
		"plugin_event": plugin.SetupEvent,
		"plugin":       pluginName,
		"name":         c.Name,
	}).Debug("Configuring cb plugin")

	hystrix.ConfigureCommand(c.Name, hystrix.CommandConfig{
		Timeout:               c.Timeout,
		MaxConcurrentRequests: c.MaxConcurrentRequests,
		ErrorPercentThreshold: c.ErrorPercentThreshold,
		SleepWindow:           c.SleepWindow,
	})

	def.AddMiddleware(NewCBMiddleware(c))
	return nil
}

func validateConfig(rawConfig plugin.Config) (bool, error) {
	var config Config
	err := plugin.Decode(rawConfig, &config)
	if err != nil {
		return false, err
	}

	return govalidator.ValidateStruct(config)
}

func onAdminAPIStartup(event interface{}) error {
	logger := log.WithFields(log.Fields{
		"plugin_event": plugin.AdminAPIStartupEvent,
		"plugin":       pluginName,
	})

	e, ok := event.(plugin.OnAdminAPIStartup)
	if !ok {
		return errors.New("Could not convert event to admin startup type")
	}

	logger.Debug("Registering hystrix stream endpoint")
	hystrixStreamHandler := hystrix.NewStreamHandler()
	hystrixStreamHandler.Start()

	e.Router.GET("/hystrix", hystrixStreamHandler.ServeHTTP)
	return nil
}

func onStartup(event interface{}) error {
	logger := log.WithFields(log.Fields{
		"plugin_event": plugin.StartupEvent,
		"plugin":       pluginName,
	})

	e, ok := event.(plugin.OnStartup)
	if !ok {
		return errors.New("Could not convert event to startup type")
	}

	logger.WithField("metrics_dsn", e.Config.Stats.DSN).Debug("Statsd metrics enabled")

	dsnURL, err := url.Parse(e.Config.Stats.DSN)
	if err != nil {
		return errors.Wrap(err, "could not parse stats dsn")
	}

	c, err := plugins.InitializeStatsdCollector(&plugins.StatsdCollectorConfig{
		StatsdAddr: dsnURL.Host,
		Prefix:     strings.Trim(dsnURL.Path, "/"),
	})
	if err != nil {
		return errors.Wrap(err, "could not initialize statsd client")
	}

	metricCollector.Registry.Register(c.NewStatsdCollector)
	logger.Debug("Metrics enabled")

	return nil
}
<|endoftext|>


<commit_before>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.7.2"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 5.7.3-dev (#1480)<commit_after>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.7.3-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package service

import (
	goflag "flag"
	"fmt"
	"strconv"

	flag "github.com/spf13/pflag"
)

type Service interface {
	Run()
	// TODO(mpolednik): uncomment initOptions when all services implement it.
	// AddFlags()

}

type ServiceListen struct {
	Name string
	Host string
	Port int
}

// TODO(mpolednik): remove this when all affected services use new flag handling.
func NewServiceListen(name string, host *string, port *int) *ServiceListen {
	return &ServiceListen{
		Name: name,
		Host: *host,
		Port: *port,
	}
}

func (service *ServiceListen) Address() string {
	return fmt.Sprintf("%s:%s", service.Host, strconv.Itoa(service.Port))
}

func (service *ServiceListen) InitFlags() {
	flag.CommandLine.AddGoFlagSet(goflag.CommandLine)
}

func (service *ServiceListen) AddCommonFlags() {
	flag.StringVar(&service.Host, "listen", service.Host, "Address where to listen on")
	flag.IntVar(&service.Port, "port", service.Port, "Port to listen on")
}
<commit_msg>extend Service interface to require AddFlags<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package service

import (
	goflag "flag"
	"fmt"
	"strconv"

	flag "github.com/spf13/pflag"
)

type Service interface {
	Run()
	AddFlags()
}

type ServiceListen struct {
	Name string
	Host string
	Port int
}

func (service *ServiceListen) Address() string {
	return fmt.Sprintf("%s:%s", service.Host, strconv.Itoa(service.Port))
}

func (service *ServiceListen) InitFlags() {
	flag.CommandLine.AddGoFlagSet(goflag.CommandLine)
}

func (service *ServiceListen) AddCommonFlags() {
	flag.StringVar(&service.Host, "listen", service.Host, "Address where to listen on")
	flag.IntVar(&service.Port, "port", service.Port, "Port to listen on")
}
<|endoftext|>


<commit_before>package stdcopy

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"

	"github.com/Sirupsen/logrus"
)

// StdType is the type of standard stream
// a writer can multiplex to.
type StdType byte

const (
	// Stdin represents standard input stream type.
	Stdin StdType = iota
	// Stdout represents standard output stream type.
	Stdout
	// Stderr represents standard error steam type.
	Stderr

	stdWriterPrefixLen = 8
	stdWriterFdIndex   = 0
	stdWriterSizeIndex = 4

	startingBufLen = 32*1024 + stdWriterPrefixLen + 1
)

// stdWriter is wrapper of io.Writer with extra customized info.
type stdWriter struct {
	io.Writer
	prefix byte
}

// Write sends the buffer to the underneath writer.
// It insert the prefix header before the buffer,
// so stdcopy.StdCopy knows where to multiplex the output.
// It makes stdWriter to implement io.Writer.
func (w *stdWriter) Write(buf []byte) (n int, err error) {
	if w == nil || w.Writer == nil {
		return 0, errors.New("Writer not instantiated")
	}
	if buf == nil {
		return 0, nil
	}

	header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix}
	binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(buf)))

	line := append(header[:], buf...)

	n, err = w.Writer.Write(line)
	n -= stdWriterPrefixLen

	if n < 0 {
		n = 0
	}
	return
}

// NewStdWriter instantiates a new Writer.
// Everything written to it will be encapsulated using a custom format,
// and written to the underlying `w` stream.
// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection.
// `t` indicates the id of the stream to encapsulate.
// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr.
func NewStdWriter(w io.Writer, t StdType) io.Writer {
	return &stdWriter{
		Writer: w,
		prefix: byte(t),
	}
}

// StdCopy is a modified version of io.Copy.
//
// StdCopy will demultiplex `src`, assuming that it contains two streams,
// previously multiplexed together using a StdWriter instance.
// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`.
//
// StdCopy will read until it hits EOF on `src`. It will then return a nil error.
// In other words: if `err` is non nil, it indicates a real underlying error.
//
// `written` will hold the total number of bytes written to `dstout` and `dsterr`.
func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) {
	var (
		buf       = make([]byte, startingBufLen)
		bufLen    = len(buf)
		nr, nw    int
		er, ew    error
		out       io.Writer
		frameSize int
	)

	for {
		// Make sure we have at least a full header
		for nr < stdWriterPrefixLen {
			var nr2 int
			nr2, er = src.Read(buf[nr:])
			nr += nr2
			if er == io.EOF {
				if nr < stdWriterPrefixLen {
					logrus.Debugf("Corrupted prefix: %v", buf[:nr])
					return written, nil
				}
				break
			}
			if er != nil {
				logrus.Debugf("Error reading header: %s", er)
				return 0, er
			}
		}

		// Check the first byte to know where to write
		switch StdType(buf[stdWriterFdIndex]) {
		case Stdin:
			fallthrough
		case Stdout:
			// Write on stdout
			out = dstout
		case Stderr:
			// Write on stderr
			out = dsterr
		default:
			logrus.Debugf("Error selecting output fd: (%d)", buf[stdWriterFdIndex])
			return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex])
		}

		// Retrieve the size of the frame
		frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4]))
		logrus.Debugf("framesize: %d", frameSize)

		// Check if the buffer is big enough to read the frame.
		// Extend it if necessary.
		if frameSize+stdWriterPrefixLen > bufLen {
			logrus.Debugf("Extending buffer cap by %d (was %d)", frameSize+stdWriterPrefixLen-bufLen+1, len(buf))
			buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...)
			bufLen = len(buf)
		}

		// While the amount of bytes read is less than the size of the frame + header, we keep reading
		for nr < frameSize+stdWriterPrefixLen {
			var nr2 int
			nr2, er = src.Read(buf[nr:])
			nr += nr2
			if er == io.EOF {
				if nr < frameSize+stdWriterPrefixLen {
					logrus.Debugf("Corrupted frame: %v", buf[stdWriterPrefixLen:nr])
					return written, nil
				}
				break
			}
			if er != nil {
				logrus.Debugf("Error reading frame: %s", er)
				return 0, er
			}
		}

		// Write the retrieved frame (without header)
		nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen])
		if ew != nil {
			logrus.Debugf("Error writing frame: %s", ew)
			return 0, ew
		}
		// If the frame has not been fully written: error
		if nw != frameSize {
			logrus.Debugf("Error Short Write: (%d on %d)", nw, frameSize)
			return 0, io.ErrShortWrite
		}
		written += int64(nw)

		// Move the rest of the buffer to the beginning
		copy(buf, buf[frameSize+stdWriterPrefixLen:])
		// Move the index
		nr -= frameSize + stdWriterPrefixLen
	}
}
<commit_msg>Optimizations for StdWriter<commit_after>package stdcopy

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"sync"

	"github.com/Sirupsen/logrus"
)

// StdType is the type of standard stream
// a writer can multiplex to.
type StdType byte

const (
	// Stdin represents standard input stream type.
	Stdin StdType = iota
	// Stdout represents standard output stream type.
	Stdout
	// Stderr represents standard error steam type.
	Stderr

	stdWriterPrefixLen = 8
	stdWriterFdIndex   = 0
	stdWriterSizeIndex = 4

	startingBufLen = 32*1024 + stdWriterPrefixLen + 1
)

var bufPool = &sync.Pool{New: func() interface{} { return bytes.NewBuffer(nil) }}

// stdWriter is wrapper of io.Writer with extra customized info.
type stdWriter struct {
	io.Writer
	prefix byte
}

// Write sends the buffer to the underneath writer.
// It inserts the prefix header before the buffer,
// so stdcopy.StdCopy knows where to multiplex the output.
// It makes stdWriter to implement io.Writer.
func (w *stdWriter) Write(p []byte) (n int, err error) {
	if w == nil || w.Writer == nil {
		return 0, errors.New("Writer not instantiated")
	}
	if p == nil {
		return 0, nil
	}

	header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix}
	binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(p)))
	buf := bufPool.Get().(*bytes.Buffer)
	buf.Write(header[:])
	buf.Write(p)

	n, err = w.Writer.Write(buf.Bytes())
	n -= stdWriterPrefixLen
	if n < 0 {
		n = 0
	}

	buf.Reset()
	bufPool.Put(buf)
	return
}

// NewStdWriter instantiates a new Writer.
// Everything written to it will be encapsulated using a custom format,
// and written to the underlying `w` stream.
// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection.
// `t` indicates the id of the stream to encapsulate.
// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr.
func NewStdWriter(w io.Writer, t StdType) io.Writer {
	return &stdWriter{
		Writer: w,
		prefix: byte(t),
	}
}

// StdCopy is a modified version of io.Copy.
//
// StdCopy will demultiplex `src`, assuming that it contains two streams,
// previously multiplexed together using a StdWriter instance.
// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`.
//
// StdCopy will read until it hits EOF on `src`. It will then return a nil error.
// In other words: if `err` is non nil, it indicates a real underlying error.
//
// `written` will hold the total number of bytes written to `dstout` and `dsterr`.
func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) {
	var (
		buf       = make([]byte, startingBufLen)
		bufLen    = len(buf)
		nr, nw    int
		er, ew    error
		out       io.Writer
		frameSize int
	)

	for {
		// Make sure we have at least a full header
		for nr < stdWriterPrefixLen {
			var nr2 int
			nr2, er = src.Read(buf[nr:])
			nr += nr2
			if er == io.EOF {
				if nr < stdWriterPrefixLen {
					logrus.Debugf("Corrupted prefix: %v", buf[:nr])
					return written, nil
				}
				break
			}
			if er != nil {
				logrus.Debugf("Error reading header: %s", er)
				return 0, er
			}
		}

		// Check the first byte to know where to write
		switch StdType(buf[stdWriterFdIndex]) {
		case Stdin:
			fallthrough
		case Stdout:
			// Write on stdout
			out = dstout
		case Stderr:
			// Write on stderr
			out = dsterr
		default:
			logrus.Debugf("Error selecting output fd: (%d)", buf[stdWriterFdIndex])
			return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex])
		}

		// Retrieve the size of the frame
		frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4]))
		logrus.Debugf("framesize: %d", frameSize)

		// Check if the buffer is big enough to read the frame.
		// Extend it if necessary.
		if frameSize+stdWriterPrefixLen > bufLen {
			logrus.Debugf("Extending buffer cap by %d (was %d)", frameSize+stdWriterPrefixLen-bufLen+1, len(buf))
			buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...)
			bufLen = len(buf)
		}

		// While the amount of bytes read is less than the size of the frame + header, we keep reading
		for nr < frameSize+stdWriterPrefixLen {
			var nr2 int
			nr2, er = src.Read(buf[nr:])
			nr += nr2
			if er == io.EOF {
				if nr < frameSize+stdWriterPrefixLen {
					logrus.Debugf("Corrupted frame: %v", buf[stdWriterPrefixLen:nr])
					return written, nil
				}
				break
			}
			if er != nil {
				logrus.Debugf("Error reading frame: %s", er)
				return 0, er
			}
		}

		// Write the retrieved frame (without header)
		nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen])
		if ew != nil {
			logrus.Debugf("Error writing frame: %s", ew)
			return 0, ew
		}
		// If the frame has not been fully written: error
		if nw != frameSize {
			logrus.Debugf("Error Short Write: (%d on %d)", nw, frameSize)
			return 0, io.ErrShortWrite
		}
		written += int64(nw)

		// Move the rest of the buffer to the beginning
		copy(buf, buf[frameSize+stdWriterPrefixLen:])
		// Move the index
		nr -= frameSize + stdWriterPrefixLen
	}
}
<|endoftext|>


<commit_before>package version

import (
	"fmt"
	"github.com/fsouza/go-dockerclient"
	"os/exec"
	"strings"
)

// MariaDBDefaultVersion is the default version we use in the db container
const MariaDBDefaultVersion = "10.2"

// VERSION is supplied with the git committish this is built from
var VERSION = ""

// IMPORTANT: These versions are overridden by version ldflags specifications VERSION_VARIABLES in the Makefile

// DdevVersion is the current version of ddev, by default the git committish (should be current git tag)
var DdevVersion = "v0.0.0-overridden-by-make" // Note that this is overridden by make

// SentryDSN is the ddev-specific key for the Sentry service.
// It is compiled in using link-time variables
var SentryDSN = ""

// DockerVersionConstraint is the current minimum version of docker required for ddev.
// See https://godoc.org/github.com/Masterminds/semver#hdr-Checking_Version_Constraints
// for examples defining version constraints.
// REMEMBER TO CHANGE docs/index.md if you touch this!
var DockerVersionConstraint = ">= 18.06.0-ce"

// DockerComposeVersionConstraint is the current minimum version of docker-compose required for ddev.
// REMEMBER TO CHANGE docs/index.md if you touch this!
var DockerComposeVersionConstraint = ">= 1.21.0"

// DockerComposeFileFormatVersion is the compose version to be used
var DockerComposeFileFormatVersion = "3.6"

// WebImg defines the default web image used for applications.
var WebImg = "drud/ddev-webserver"

// WebTag defines the default web image tag for drud dev
var WebTag = "20190116_update_web_versions_drush" // Note that this can be overridden by make

// DBImg defines the default db image used for applications.
var DBImg = "drud/ddev-dbserver"

// BaseDBTag is the main tag, DBTag is constructed from it
var BaseDBTag = "20190205_bump_maria_10_1_version"

// DBAImg defines the default phpmyadmin image tag used for applications.
var DBAImg = "drud/phpmyadmin"

// DBATag defines the default phpmyadmin image tag used for applications.
var DBATag = "v1.5.2" // Note that this can be overridden by make

// BgsyncImg defines the default bgsync image tag used for applications.
var BgsyncImg = "drud/ddev-bgsync"

// BgsyncTag defines the default phpmyadmin image tag used for applications.
var BgsyncTag = "v1.5.2" // Note that this can be overridden by make

// RouterImage defines the image used for the router.
var RouterImage = "drud/ddev-router"

// RouterTag defines the tag used for the router.
var RouterTag = "v1.5.2" // Note that this can be overridden by make

var SSHAuthImage = "drud/ddev-ssh-agent"

var SSHAuthTag = "20190125_ddev_auth_ssh_show_key"

// COMMIT is the actual committish, supplied by make
var COMMIT = "COMMIT should be overridden"

// BUILDINFO is information with date and context, supplied by make
var BUILDINFO = "BUILDINFO should have new info"

// DockerVersion is cached version of docker
var DockerVersion = ""

// DockerComposeVersion is filled with the version we find for docker-compose
var DockerComposeVersion = ""

// DDevTLD defines the tld to use for DDev site URLs.
const DDevTLD = "ddev.local"

// GetVersionInfo returns a map containing the version info defined above.
func GetVersionInfo() map[string]string {
	var err error
	versionInfo := make(map[string]string)

	versionInfo["cli"] = DdevVersion
	versionInfo["web"] = GetWebImage()
	versionInfo["db"] = GetDBImage()
	versionInfo["dba"] = GetDBAImage()
	versionInfo["bgsync"] = BgsyncImg + ":" + BgsyncTag
	versionInfo["router"] = RouterImage + ":" + RouterTag
	versionInfo["ddev-ssh-agent"] = SSHAuthImage + ":" + SSHAuthTag
	versionInfo["commit"] = COMMIT
	versionInfo["domain"] = DDevTLD
	versionInfo["build info"] = BUILDINFO
	if versionInfo["docker"], err = GetDockerVersion(); err != nil {
		versionInfo["docker"] = fmt.Sprintf("failed to GetDockerVersion(): %v", err)
	}
	if versionInfo["docker-compose"], err = GetDockerComposeVersion(); err != nil {
		versionInfo["docker-compose"] = fmt.Sprintf("failed to GetDockerComposeVersion(): %v", err)
	}

	return versionInfo
}

// GetWebImage returns the correctly formatted web image:tag reference
func GetWebImage() string {
	return fmt.Sprintf("%s:%s", WebImg, WebTag)
}

// GetDBImage returns the correctly formatted db image:tag reference
func GetDBImage(mariaDBVersion ...string) string {
	version := MariaDBDefaultVersion
	if len(mariaDBVersion) > 0 {
		version = mariaDBVersion[0]
	}
	return fmt.Sprintf("%s:%s", DBImg, BaseDBTag+"-"+version)
}

// GetDBAImage returns the correctly formatted dba image:tag reference
func GetDBAImage() string {
	return fmt.Sprintf("%s:%s", DBAImg, DBATag)
}

// GetDBAImage returns the correctly formatted dba image:tag reference
func GetBgsyncImage() string {
	return fmt.Sprintf("%s:%s", BgsyncImg, BgsyncTag)
}

// GetDockerComposeVersion runs docker-compose -v to get the current version
func GetDockerComposeVersion() (string, error) {
	if DockerComposeVersion != "" {
		return DockerComposeVersion, nil
	}

	executableName := "docker-compose"

	path, err := exec.LookPath(executableName)
	if err != nil {
		return "", fmt.Errorf("no docker-compose")
	}

	out, err := exec.Command(path, "version", "--short").Output()
	if err != nil {
		return "", err
	}

	v := string(out)
	DockerComposeVersion = strings.TrimSpace(v)
	return DockerComposeVersion, nil
}

// GetDockerVersion gets the cached or api-sourced version of docker engine
func GetDockerVersion() (string, error) {
	if DockerVersion != "" {
		return DockerVersion, nil
	}
	var client *docker.Client
	var err error
	if client, err = docker.NewClientFromEnv(); err != nil {
		return "", err
	}

	v, err := client.Version()
	if err != nil {
		return "", err
	}
	DockerVersion = v.Get("Version")
	return DockerVersion, nil
}
<commit_msg>Fix version constraint to have a pre, fixes #1431 (#1433)<commit_after>package version

import (
	"fmt"
	"github.com/fsouza/go-dockerclient"
	"os/exec"
	"strings"
)

// MariaDBDefaultVersion is the default version we use in the db container
const MariaDBDefaultVersion = "10.2"

// VERSION is supplied with the git committish this is built from
var VERSION = ""

// IMPORTANT: These versions are overridden by version ldflags specifications VERSION_VARIABLES in the Makefile

// DdevVersion is the current version of ddev, by default the git committish (should be current git tag)
var DdevVersion = "v0.0.0-overridden-by-make" // Note that this is overridden by make

// SentryDSN is the ddev-specific key for the Sentry service.
// It is compiled in using link-time variables
var SentryDSN = ""

// DockerVersionConstraint is the current minimum version of docker required for ddev.
// See https://godoc.org/github.com/Masterminds/semver#hdr-Checking_Version_Constraints
// for examples defining version constraints.
// REMEMBER TO CHANGE docs/index.md if you touch this!
// The constraint MUST HAVE a -pre of some kind on it for successful comparison.
// See https://github.com/drud/ddev/pull/738.. and regression https://github.com/drud/ddev/issues/1431
var DockerVersionConstraint = ">= 18.06.1-alpha1"

// DockerComposeVersionConstraint is the current minimum version of docker-compose required for ddev.
// REMEMBER TO CHANGE docs/index.md if you touch this!
// The constraint MUST HAVE a -pre of some kind on it for successful comparison.
// See https://github.com/drud/ddev/pull/738.. and regression https://github.com/drud/ddev/issues/1431
var DockerComposeVersionConstraint = ">= 1.21.0-alpha1"

// DockerComposeFileFormatVersion is the compose version to be used
var DockerComposeFileFormatVersion = "3.6"

// WebImg defines the default web image used for applications.
var WebImg = "drud/ddev-webserver"

// WebTag defines the default web image tag for drud dev
var WebTag = "20190116_update_web_versions_drush" // Note that this can be overridden by make

// DBImg defines the default db image used for applications.
var DBImg = "drud/ddev-dbserver"

// BaseDBTag is the main tag, DBTag is constructed from it
var BaseDBTag = "20190205_bump_maria_10_1_version"

// DBAImg defines the default phpmyadmin image tag used for applications.
var DBAImg = "drud/phpmyadmin"

// DBATag defines the default phpmyadmin image tag used for applications.
var DBATag = "v1.5.2" // Note that this can be overridden by make

// BgsyncImg defines the default bgsync image tag used for applications.
var BgsyncImg = "drud/ddev-bgsync"

// BgsyncTag defines the default phpmyadmin image tag used for applications.
var BgsyncTag = "v1.5.2" // Note that this can be overridden by make

// RouterImage defines the image used for the router.
var RouterImage = "drud/ddev-router"

// RouterTag defines the tag used for the router.
var RouterTag = "v1.5.2" // Note that this can be overridden by make

var SSHAuthImage = "drud/ddev-ssh-agent"

var SSHAuthTag = "20190125_ddev_auth_ssh_show_key"

// COMMIT is the actual committish, supplied by make
var COMMIT = "COMMIT should be overridden"

// BUILDINFO is information with date and context, supplied by make
var BUILDINFO = "BUILDINFO should have new info"

// DockerVersion is cached version of docker
var DockerVersion = ""

// DockerComposeVersion is filled with the version we find for docker-compose
var DockerComposeVersion = ""

// DDevTLD defines the tld to use for DDev site URLs.
const DDevTLD = "ddev.local"

// GetVersionInfo returns a map containing the version info defined above.
func GetVersionInfo() map[string]string {
	var err error
	versionInfo := make(map[string]string)

	versionInfo["cli"] = DdevVersion
	versionInfo["web"] = GetWebImage()
	versionInfo["db"] = GetDBImage()
	versionInfo["dba"] = GetDBAImage()
	versionInfo["bgsync"] = BgsyncImg + ":" + BgsyncTag
	versionInfo["router"] = RouterImage + ":" + RouterTag
	versionInfo["ddev-ssh-agent"] = SSHAuthImage + ":" + SSHAuthTag
	versionInfo["commit"] = COMMIT
	versionInfo["domain"] = DDevTLD
	versionInfo["build info"] = BUILDINFO
	if versionInfo["docker"], err = GetDockerVersion(); err != nil {
		versionInfo["docker"] = fmt.Sprintf("failed to GetDockerVersion(): %v", err)
	}
	if versionInfo["docker-compose"], err = GetDockerComposeVersion(); err != nil {
		versionInfo["docker-compose"] = fmt.Sprintf("failed to GetDockerComposeVersion(): %v", err)
	}

	return versionInfo
}

// GetWebImage returns the correctly formatted web image:tag reference
func GetWebImage() string {
	return fmt.Sprintf("%s:%s", WebImg, WebTag)
}

// GetDBImage returns the correctly formatted db image:tag reference
func GetDBImage(mariaDBVersion ...string) string {
	version := MariaDBDefaultVersion
	if len(mariaDBVersion) > 0 {
		version = mariaDBVersion[0]
	}
	return fmt.Sprintf("%s:%s", DBImg, BaseDBTag+"-"+version)
}

// GetDBAImage returns the correctly formatted dba image:tag reference
func GetDBAImage() string {
	return fmt.Sprintf("%s:%s", DBAImg, DBATag)
}

// GetDBAImage returns the correctly formatted dba image:tag reference
func GetBgsyncImage() string {
	return fmt.Sprintf("%s:%s", BgsyncImg, BgsyncTag)
}

// GetDockerComposeVersion runs docker-compose -v to get the current version
func GetDockerComposeVersion() (string, error) {
	if DockerComposeVersion != "" {
		return DockerComposeVersion, nil
	}

	executableName := "docker-compose"

	path, err := exec.LookPath(executableName)
	if err != nil {
		return "", fmt.Errorf("no docker-compose")
	}

	out, err := exec.Command(path, "version", "--short").Output()
	if err != nil {
		return "", err
	}

	v := string(out)
	DockerComposeVersion = strings.TrimSpace(v)
	return DockerComposeVersion, nil
}

// GetDockerVersion gets the cached or api-sourced version of docker engine
func GetDockerVersion() (string, error) {
	if DockerVersion != "" {
		return DockerVersion, nil
	}
	var client *docker.Client
	var err error
	if client, err = docker.NewClientFromEnv(); err != nil {
		return "", err
	}

	v, err := client.Version()
	if err != nil {
		return "", err
	}
	DockerVersion = v.Get("Version")
	return DockerVersion, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package juju

import (
	"github.com/flaviamissi/go-elb/aws"
	"github.com/flaviamissi/go-elb/elb"
	"github.com/flaviamissi/go-elb/elb/elbtest"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/queue"
	"github.com/globocom/tsuru/testing"
	"labix.org/v2/mgo/bson"
	"launchpad.net/gocheck"
	"sort"
)

type ELBSuite struct {
	server      *elbtest.Server
	client      *elb.ELB
	conn        *db.Storage
	cName       string
	provisioner *testing.FakeProvisioner
}

var _ = gocheck.Suite(&ELBSuite{})

func (s *ELBSuite) SetUpSuite(c *gocheck.C) {
	var err error
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "juju_elb_tests")
	s.conn, err = db.Conn()
	s.server, err = elbtest.NewServer()
	c.Assert(err, gocheck.IsNil)
	config.Set("juju:elb-endpoint", s.server.URL())
	config.Set("juju:use-elb", true)
	region := aws.SAEast
	region.ELBEndpoint = s.server.URL()
	s.client = elb.New(aws.Auth{AccessKey: "some", SecretKey: "thing"}, region)
	c.Assert(err, gocheck.IsNil)
	s.cName = "juju_test_elbs"
	config.Set("juju:elb-collection", s.cName)
	config.Set("juju:elb-avail-zones", []interface{}{"my-zone-1a", "my-zone-1b"})
	config.Set("aws:access-key-id", "access")
	config.Set("aws:secret-access-key", "s3cr3t")
	config.Set("git:host", "git.tsuru.io")
	config.Set("queue", "fake")
	config.Set("juju:units-collection", "juju_units_test_elb")
	s.provisioner = testing.NewFakeProvisioner()
	app.Provisioner = s.provisioner
}

func (s *ELBSuite) TearDownSuite(c *gocheck.C) {
	config.Unset("juju:use-elb")
	s.conn.Collection("juju_units_test_elb").Database.DropDatabase()
	s.server.Quit()
	queue.Preempt()
}

func (s *ELBSuite) TestGetCollection(c *gocheck.C) {
	manager := ELBManager{}
	coll := manager.collection()
	other := s.conn.Collection(s.cName)
	c.Assert(coll, gocheck.DeepEquals, other)
}

func (s *ELBSuite) TestGetELBClient(c *gocheck.C) {
	manager := ELBManager{}
	elb := manager.elb()
	c.Assert(elb.ELBEndpoint, gocheck.Equals, s.server.URL())
}

func (s *ELBSuite) TestCreateELB(c *gocheck.C) {
	app := testing.NewFakeApp("together", "gotthard", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer s.client.DeleteLoadBalancer(app.GetName())
	defer manager.collection().Remove(bson.M{"name": app.GetName()})
	resp, err := s.client.DescribeLoadBalancers("together")
	c.Assert(err, gocheck.IsNil)
	c.Assert(resp.LoadBalancerDescriptions, gocheck.HasLen, 1)
	c.Assert(resp.LoadBalancerDescriptions[0].ListenerDescriptions, gocheck.HasLen, 1)
	listener := resp.LoadBalancerDescriptions[0].ListenerDescriptions[0].Listener
	c.Assert(listener.InstancePort, gocheck.Equals, 80)
	c.Assert(listener.LoadBalancerPort, gocheck.Equals, 80)
	c.Assert(listener.InstanceProtocol, gocheck.Equals, "HTTP")
	c.Assert(listener.Protocol, gocheck.Equals, "HTTP")
	c.Assert(listener.SSLCertificateId, gocheck.Equals, "")
	dnsName := resp.LoadBalancerDescriptions[0].DNSName
	var lb loadBalancer
	err = s.conn.Collection(s.cName).Find(bson.M{"name": app.GetName()}).One(&lb)
	c.Assert(err, gocheck.IsNil)
	c.Assert(lb.DNSName, gocheck.Equals, dnsName)
}

func (s *ELBSuite) TestCreateELBUsingVPC(c *gocheck.C) {
	old, _ := config.Get("juju:elb-avail-zones")
	config.Unset("juju:elb-avail-zones")
	config.Set("juju:elb-use-vpc", true)
	config.Set("juju:elb-vpc-subnets", []string{"subnet-a4a3a2a1", "subnet-002200"})
	config.Set("juju:elb-vpc-secgroups", []string{"sg-0900"})
	defer func() {
		config.Set("juju:elb-avail-zones", old)
		config.Unset("juju:elb-use-vpc")
		config.Unset("juju:elb-vpc-subnets")
		config.Unset("juju:elb-vpc-secgroups")
	}()
	app := testing.NewFakeApp("relax", "who", 1)
	manager := ELBManager{}
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer s.client.DeleteLoadBalancer(app.GetName())
	defer manager.collection().Remove(bson.M{"name": app.GetName()})
	resp, err := s.client.DescribeLoadBalancers(app.GetName())
	c.Assert(err, gocheck.IsNil)
	c.Assert(resp.LoadBalancerDescriptions, gocheck.HasLen, 1)
	lbd := resp.LoadBalancerDescriptions[0]
	c.Assert(lbd.Subnets, gocheck.DeepEquals, []string{"subnet-a4a3a2a1", "subnet-002200"})
	c.Assert(lbd.SecurityGroups, gocheck.DeepEquals, []string{"sg-0900"})
	c.Assert(lbd.Scheme, gocheck.Equals, "internal")
	c.Assert(lbd.AvailZones, gocheck.HasLen, 0)
}

func (s *ELBSuite) TestDestroyELB(c *gocheck.C) {
	app := testing.NewFakeApp("blue", "who", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer s.client.DeleteLoadBalancer(app.GetName())                 // sanity
	defer manager.collection().Remove(bson.M{"name": app.GetName()}) // sanity
	err = manager.Destroy(app)
	c.Assert(err, gocheck.IsNil)
	_, err = s.client.DescribeLoadBalancers(app.GetName())
	c.Assert(err, gocheck.NotNil)
	c.Assert(err, gocheck.ErrorMatches, `^.*\(LoadBalancerNotFound\)$`)
	n, err := manager.collection().Find(bson.M{"name": app.GetName()}).Count()
	c.Assert(err, gocheck.IsNil)
	c.Assert(n, gocheck.Equals, 0)
}

func (s *ELBSuite) TestRegisterUnit(c *gocheck.C) {
	id1 := s.server.NewInstance()
	defer s.server.RemoveInstance(id1)
	id2 := s.server.NewInstance()
	defer s.server.RemoveInstance(id2)
	app := testing.NewFakeApp("fooled", "who", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer manager.Destroy(app)
	err = manager.Register(app, provision.Unit{InstanceId: id1}, provision.Unit{InstanceId: id2})
	c.Assert(err, gocheck.IsNil)
	resp, err := s.client.DescribeLoadBalancers(app.GetName())
	c.Assert(err, gocheck.IsNil)
	c.Assert(resp.LoadBalancerDescriptions, gocheck.HasLen, 1)
	c.Assert(resp.LoadBalancerDescriptions[0].Instances, gocheck.HasLen, 2)
	instances := resp.LoadBalancerDescriptions[0].Instances
	ids := []string{instances[0].InstanceId, instances[1].InstanceId}
	sort.Strings(ids)
	expected := []string{id1, id2}
	sort.Strings(expected)
	c.Assert(ids, gocheck.DeepEquals, expected)
}

func (s *ELBSuite) TestDeregisterUnit(c *gocheck.C) {
	id1 := s.server.NewInstance()
	defer s.server.RemoveInstance(id1)
	id2 := s.server.NewInstance()
	defer s.server.RemoveInstance(id2)
	unit1 := provision.Unit{InstanceId: id1}
	unit2 := provision.Unit{InstanceId: id2}
	app := testing.NewFakeApp("dirty", "who", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer manager.Destroy(app)
	err = manager.Register(app, unit1, unit2)
	c.Assert(err, gocheck.IsNil)
	err = manager.Deregister(app, unit1, unit2)
	c.Assert(err, gocheck.IsNil)
	resp, err := s.client.DescribeLoadBalancers(app.GetName())
	c.Assert(err, gocheck.IsNil)
	c.Assert(resp.LoadBalancerDescriptions, gocheck.HasLen, 1)
	c.Assert(resp.LoadBalancerDescriptions[0].Instances, gocheck.HasLen, 0)
}

func (s *ELBSuite) TestAddr(c *gocheck.C) {
	app := testing.NewFakeApp("enough", "who", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer manager.Destroy(app)
	var lb loadBalancer
	err = manager.collection().Find(bson.M{"name": app.GetName()}).One(&lb)
	c.Assert(err, gocheck.IsNil)
	addr, err := manager.Addr(app)
	c.Assert(err, gocheck.IsNil)
	c.Assert(addr, gocheck.Equals, lb.DNSName)
}

func (s *ELBSuite) TestAddrUnknownLoadBalancer(c *gocheck.C) {
	app := testing.NewFakeApp("five", "who", 1)
	manager := ELBManager{}
	addr, err := manager.Addr(app)
	c.Assert(addr, gocheck.Equals, "")
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "not found")
}

func (s *ELBSuite) TestELBInstanceHealer(c *gocheck.C) {
	lb := "elbtest"
	instance := s.server.NewInstance()
	defer s.server.RemoveInstance(instance)
	s.server.NewLoadBalancer(lb)
	defer s.server.RemoveLoadBalancer(lb)
	s.server.RegisterInstance(instance, lb)
	defer s.server.DeregisterInstance(instance, lb)
	a := app.App{
		Name:  "elbtest",
		Units: []app.Unit{{InstanceId: instance, State: "error", Name: "elbtest/0"}},
	}
	storage, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	err = storage.Apps().Insert(a)
	c.Assert(err, gocheck.IsNil)
	defer storage.Apps().Remove(bson.M{"name": a.Name})
	s.provisioner.Provision(&a)
	defer s.provisioner.Destroy(&a)
	state := elb.InstanceState{
		Description: "Instance has failed at least the UnhealthyThreshold number of health checks consecutively.",
		State:       "OutOfService",
		ReasonCode:  "Instance",
		InstanceId:  instance,
	}
	s.server.ChangeInstanceState(lb, state)
	healer := elbInstanceHealer{}
	err = healer.Heal()
	c.Assert(err, gocheck.IsNil)
	err = a.Get()
	c.Assert(err, gocheck.IsNil)
	c.Assert(a.Units, gocheck.HasLen, 1)
	c.Assert(a.Units[0].InstanceId, gocheck.Not(gocheck.Equals), instance)
	queue.Preempt()
}

func (s *ELBSuite) TestELBInstanceHealerInstallingUnit(c *gocheck.C) {
	lb := "elbtest"
	instance := s.server.NewInstance()
	defer s.server.RemoveInstance(instance)
	s.server.NewLoadBalancer(lb)
	defer s.server.RemoveLoadBalancer(lb)
	s.server.RegisterInstance(instance, lb)
	defer s.server.DeregisterInstance(instance, lb)
	a := app.App{
		Name:  "elbtest",
		Units: []app.Unit{{InstanceId: instance, State: "installing", Name: "elbtest/0"}},
	}
	storage, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	err = storage.Apps().Insert(a)
	c.Assert(err, gocheck.IsNil)
	defer storage.Apps().Remove(bson.M{"name": a.Name})
	s.provisioner.Provision(&a)
	defer s.provisioner.Destroy(&a)
	state := elb.InstanceState{
		Description: "Instance has failed at least the UnhealthyThreshold number of health checks consecutively.",
		State:       "OutOfService",
		ReasonCode:  "Instance",
		InstanceId:  instance,
	}
	s.server.ChangeInstanceState(lb, state)
	healer := elbInstanceHealer{}
	err = healer.Heal()
	c.Assert(err, gocheck.IsNil)
	err = a.Get()
	c.Assert(err, gocheck.IsNil)
	c.Assert(a.Units, gocheck.HasLen, 1)
	c.Assert(a.Units[0].InstanceId, gocheck.Equals, instance)
}
<commit_msg>provision/juju: fix test<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package juju

import (
	"github.com/flaviamissi/go-elb/aws"
	"github.com/flaviamissi/go-elb/elb"
	"github.com/flaviamissi/go-elb/elb/elbtest"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/queue"
	"github.com/globocom/tsuru/testing"
	"labix.org/v2/mgo/bson"
	"launchpad.net/gocheck"
	"sort"
)

type ELBSuite struct {
	server      *elbtest.Server
	client      *elb.ELB
	conn        *db.Storage
	cName       string
	provisioner *testing.FakeProvisioner
}

var _ = gocheck.Suite(&ELBSuite{})

func (s *ELBSuite) SetUpSuite(c *gocheck.C) {
	var err error
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "juju_elb_tests")
	s.conn, err = db.Conn()
	s.server, err = elbtest.NewServer()
	c.Assert(err, gocheck.IsNil)
	config.Set("juju:elb-endpoint", s.server.URL())
	config.Set("juju:use-elb", true)
	region := aws.SAEast
	region.ELBEndpoint = s.server.URL()
	s.client = elb.New(aws.Auth{AccessKey: "some", SecretKey: "thing"}, region)
	c.Assert(err, gocheck.IsNil)
	s.cName = "juju_test_elbs"
	config.Set("juju:elb-collection", s.cName)
	config.Set("juju:elb-avail-zones", []interface{}{"my-zone-1a", "my-zone-1b"})
	config.Set("aws:access-key-id", "access")
	config.Set("aws:secret-access-key", "s3cr3t")
	config.Set("git:host", "git.tsuru.io")
	config.Set("queue", "fake")
	config.Set("juju:units-collection", "juju_units_test_elb")
	s.provisioner = testing.NewFakeProvisioner()
	app.Provisioner = s.provisioner
}

func (s *ELBSuite) TearDownSuite(c *gocheck.C) {
	config.Unset("juju:use-elb")
	s.conn.Collection("juju_units_test_elb").Database.DropDatabase()
	s.server.Quit()
	queue.Preempt()
}

func (s *ELBSuite) TestGetCollection(c *gocheck.C) {
	manager := ELBManager{}
	coll := manager.collection()
	other := s.conn.Collection(s.cName)
	c.Assert(coll.FullName, gocheck.Equals, other.FullName)
}

func (s *ELBSuite) TestGetELBClient(c *gocheck.C) {
	manager := ELBManager{}
	elb := manager.elb()
	c.Assert(elb.ELBEndpoint, gocheck.Equals, s.server.URL())
}

func (s *ELBSuite) TestCreateELB(c *gocheck.C) {
	app := testing.NewFakeApp("together", "gotthard", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer s.client.DeleteLoadBalancer(app.GetName())
	defer manager.collection().Remove(bson.M{"name": app.GetName()})
	resp, err := s.client.DescribeLoadBalancers("together")
	c.Assert(err, gocheck.IsNil)
	c.Assert(resp.LoadBalancerDescriptions, gocheck.HasLen, 1)
	c.Assert(resp.LoadBalancerDescriptions[0].ListenerDescriptions, gocheck.HasLen, 1)
	listener := resp.LoadBalancerDescriptions[0].ListenerDescriptions[0].Listener
	c.Assert(listener.InstancePort, gocheck.Equals, 80)
	c.Assert(listener.LoadBalancerPort, gocheck.Equals, 80)
	c.Assert(listener.InstanceProtocol, gocheck.Equals, "HTTP")
	c.Assert(listener.Protocol, gocheck.Equals, "HTTP")
	c.Assert(listener.SSLCertificateId, gocheck.Equals, "")
	dnsName := resp.LoadBalancerDescriptions[0].DNSName
	var lb loadBalancer
	err = s.conn.Collection(s.cName).Find(bson.M{"name": app.GetName()}).One(&lb)
	c.Assert(err, gocheck.IsNil)
	c.Assert(lb.DNSName, gocheck.Equals, dnsName)
}

func (s *ELBSuite) TestCreateELBUsingVPC(c *gocheck.C) {
	old, _ := config.Get("juju:elb-avail-zones")
	config.Unset("juju:elb-avail-zones")
	config.Set("juju:elb-use-vpc", true)
	config.Set("juju:elb-vpc-subnets", []string{"subnet-a4a3a2a1", "subnet-002200"})
	config.Set("juju:elb-vpc-secgroups", []string{"sg-0900"})
	defer func() {
		config.Set("juju:elb-avail-zones", old)
		config.Unset("juju:elb-use-vpc")
		config.Unset("juju:elb-vpc-subnets")
		config.Unset("juju:elb-vpc-secgroups")
	}()
	app := testing.NewFakeApp("relax", "who", 1)
	manager := ELBManager{}
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer s.client.DeleteLoadBalancer(app.GetName())
	defer manager.collection().Remove(bson.M{"name": app.GetName()})
	resp, err := s.client.DescribeLoadBalancers(app.GetName())
	c.Assert(err, gocheck.IsNil)
	c.Assert(resp.LoadBalancerDescriptions, gocheck.HasLen, 1)
	lbd := resp.LoadBalancerDescriptions[0]
	c.Assert(lbd.Subnets, gocheck.DeepEquals, []string{"subnet-a4a3a2a1", "subnet-002200"})
	c.Assert(lbd.SecurityGroups, gocheck.DeepEquals, []string{"sg-0900"})
	c.Assert(lbd.Scheme, gocheck.Equals, "internal")
	c.Assert(lbd.AvailZones, gocheck.HasLen, 0)
}

func (s *ELBSuite) TestDestroyELB(c *gocheck.C) {
	app := testing.NewFakeApp("blue", "who", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer s.client.DeleteLoadBalancer(app.GetName())                 // sanity
	defer manager.collection().Remove(bson.M{"name": app.GetName()}) // sanity
	err = manager.Destroy(app)
	c.Assert(err, gocheck.IsNil)
	_, err = s.client.DescribeLoadBalancers(app.GetName())
	c.Assert(err, gocheck.NotNil)
	c.Assert(err, gocheck.ErrorMatches, `^.*\(LoadBalancerNotFound\)$`)
	n, err := manager.collection().Find(bson.M{"name": app.GetName()}).Count()
	c.Assert(err, gocheck.IsNil)
	c.Assert(n, gocheck.Equals, 0)
}

func (s *ELBSuite) TestRegisterUnit(c *gocheck.C) {
	id1 := s.server.NewInstance()
	defer s.server.RemoveInstance(id1)
	id2 := s.server.NewInstance()
	defer s.server.RemoveInstance(id2)
	app := testing.NewFakeApp("fooled", "who", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer manager.Destroy(app)
	err = manager.Register(app, provision.Unit{InstanceId: id1}, provision.Unit{InstanceId: id2})
	c.Assert(err, gocheck.IsNil)
	resp, err := s.client.DescribeLoadBalancers(app.GetName())
	c.Assert(err, gocheck.IsNil)
	c.Assert(resp.LoadBalancerDescriptions, gocheck.HasLen, 1)
	c.Assert(resp.LoadBalancerDescriptions[0].Instances, gocheck.HasLen, 2)
	instances := resp.LoadBalancerDescriptions[0].Instances
	ids := []string{instances[0].InstanceId, instances[1].InstanceId}
	sort.Strings(ids)
	expected := []string{id1, id2}
	sort.Strings(expected)
	c.Assert(ids, gocheck.DeepEquals, expected)
}

func (s *ELBSuite) TestDeregisterUnit(c *gocheck.C) {
	id1 := s.server.NewInstance()
	defer s.server.RemoveInstance(id1)
	id2 := s.server.NewInstance()
	defer s.server.RemoveInstance(id2)
	unit1 := provision.Unit{InstanceId: id1}
	unit2 := provision.Unit{InstanceId: id2}
	app := testing.NewFakeApp("dirty", "who", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer manager.Destroy(app)
	err = manager.Register(app, unit1, unit2)
	c.Assert(err, gocheck.IsNil)
	err = manager.Deregister(app, unit1, unit2)
	c.Assert(err, gocheck.IsNil)
	resp, err := s.client.DescribeLoadBalancers(app.GetName())
	c.Assert(err, gocheck.IsNil)
	c.Assert(resp.LoadBalancerDescriptions, gocheck.HasLen, 1)
	c.Assert(resp.LoadBalancerDescriptions[0].Instances, gocheck.HasLen, 0)
}

func (s *ELBSuite) TestAddr(c *gocheck.C) {
	app := testing.NewFakeApp("enough", "who", 1)
	manager := ELBManager{}
	manager.e = s.client
	err := manager.Create(app)
	c.Assert(err, gocheck.IsNil)
	defer manager.Destroy(app)
	var lb loadBalancer
	err = manager.collection().Find(bson.M{"name": app.GetName()}).One(&lb)
	c.Assert(err, gocheck.IsNil)
	addr, err := manager.Addr(app)
	c.Assert(err, gocheck.IsNil)
	c.Assert(addr, gocheck.Equals, lb.DNSName)
}

func (s *ELBSuite) TestAddrUnknownLoadBalancer(c *gocheck.C) {
	app := testing.NewFakeApp("five", "who", 1)
	manager := ELBManager{}
	addr, err := manager.Addr(app)
	c.Assert(addr, gocheck.Equals, "")
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "not found")
}

func (s *ELBSuite) TestELBInstanceHealer(c *gocheck.C) {
	lb := "elbtest"
	instance := s.server.NewInstance()
	defer s.server.RemoveInstance(instance)
	s.server.NewLoadBalancer(lb)
	defer s.server.RemoveLoadBalancer(lb)
	s.server.RegisterInstance(instance, lb)
	defer s.server.DeregisterInstance(instance, lb)
	a := app.App{
		Name:  "elbtest",
		Units: []app.Unit{{InstanceId: instance, State: "error", Name: "elbtest/0"}},
	}
	storage, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	err = storage.Apps().Insert(a)
	c.Assert(err, gocheck.IsNil)
	defer storage.Apps().Remove(bson.M{"name": a.Name})
	s.provisioner.Provision(&a)
	defer s.provisioner.Destroy(&a)
	state := elb.InstanceState{
		Description: "Instance has failed at least the UnhealthyThreshold number of health checks consecutively.",
		State:       "OutOfService",
		ReasonCode:  "Instance",
		InstanceId:  instance,
	}
	s.server.ChangeInstanceState(lb, state)
	healer := elbInstanceHealer{}
	err = healer.Heal()
	c.Assert(err, gocheck.IsNil)
	err = a.Get()
	c.Assert(err, gocheck.IsNil)
	c.Assert(a.Units, gocheck.HasLen, 1)
	c.Assert(a.Units[0].InstanceId, gocheck.Not(gocheck.Equals), instance)
	queue.Preempt()
}

func (s *ELBSuite) TestELBInstanceHealerInstallingUnit(c *gocheck.C) {
	lb := "elbtest"
	instance := s.server.NewInstance()
	defer s.server.RemoveInstance(instance)
	s.server.NewLoadBalancer(lb)
	defer s.server.RemoveLoadBalancer(lb)
	s.server.RegisterInstance(instance, lb)
	defer s.server.DeregisterInstance(instance, lb)
	a := app.App{
		Name:  "elbtest",
		Units: []app.Unit{{InstanceId: instance, State: "installing", Name: "elbtest/0"}},
	}
	storage, err := db.Conn()
	c.Assert(err, gocheck.IsNil)
	err = storage.Apps().Insert(a)
	c.Assert(err, gocheck.IsNil)
	defer storage.Apps().Remove(bson.M{"name": a.Name})
	s.provisioner.Provision(&a)
	defer s.provisioner.Destroy(&a)
	state := elb.InstanceState{
		Description: "Instance has failed at least the UnhealthyThreshold number of health checks consecutively.",
		State:       "OutOfService",
		ReasonCode:  "Instance",
		InstanceId:  instance,
	}
	s.server.ChangeInstanceState(lb, state)
	healer := elbInstanceHealer{}
	err = healer.Heal()
	c.Assert(err, gocheck.IsNil)
	err = a.Get()
	c.Assert(err, gocheck.IsNil)
	c.Assert(a.Units, gocheck.HasLen, 1)
	c.Assert(a.Units[0].InstanceId, gocheck.Equals, instance)
}
<|endoftext|>


<commit_before>package start

import (
	"os"
	"testing"

	flag "github.com/ogier/pflag"

	. "github.com/smartystreets/goconvey/convey"
)

func TestParse(t *testing.T) {
	// ContinueOnError is required when running goconvey as server; otherwise, unrecognized
	// flags that are passed to the test executable will cause an error:
	// "unknown shorthand flag: 't' in -test.v=true"
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	os.Args = []string{os.Args[0], "--cmdline=FromCmdLine", "anargument", "anotherarg"}

	Convey("When setting up some flag variables", t, func() {
		var stringFlag string
		var cmdlineStringFlag string
		flag.StringVarP(&stringFlag, "astring", "a", "From default", "A string flag")
		flag.StringVarP(&cmdlineStringFlag, "cmdline", "c", "From command line", "A string flag")
		intFlag := flag.IntP("anint", "i", 23, "An integer flag")
		boolFlag := flag.BoolP("anewbool", "b", true, "A boolean flag")

		UseConfigFile("test/test.toml")
		os.Setenv("START_ASTRING", "From Environment Variable")
		Parse()
		Convey("Then Parse() should find the correct values from config file, env var, or default. (Restriction: passing the command line flags is not possible with automated calls to go test)", func() {
			So(cmdlineStringFlag, ShouldEqual, "FromCmdLine")
			So(stringFlag, ShouldEqual, "From Environment Variable")
			So(*intFlag, ShouldEqual, 42)    // from config file
			So(*boolFlag, ShouldEqual, true) // from default

		})
		Reset(func() {
			os.Setenv("START_ASTRING", "")
		})
	})

}

func TestUp(t *testing.T) {
	var first int
	var second int
	var global int
	var params []string

	GlobalInit = func() error {
		SetDescription("Testing testcmd")
		return nil
	}

	// ContinueOnError is required when running goconvey as server; otherwise, unrecognized
	// flags that are passed to the test executable will cause an error:
	// "unknown shorthand flag: 't' in -test.v=true"
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	// To suppress warnings resulting from standard flags -test and -json,
	// read -t and -j into dummy flags.
	// The flags used for actual testing must not use -t or -j shorthands.
	var testflag string
	var jsonflag string
	flag.StringVarP(&testflag, "t", "t", "t", "t")
	flag.StringVarP(&jsonflag, "j", "j", "j", "j")

	flag.IntVarP(&first, "first", "f", 1, "The first flag")
	flag.IntVarP(&second, "second", "s", 2, "The second flag")
	flag.IntVarP(&global, "global", "g", 3, "The global flag")

	os.Args = []string{os.Args[0], "testcmd", "--first=10", "-s20", "arg1", "arg2"}

	Commands = make(CommandMap) // clear the commands map for this test
	Add(&Command{
		Name:  "testcmd",
		Flags: []string{"first", "second"},
		Cmd: func(cmd *Command) error {
			params = cmd.Args
			return nil
		},
	})

	Convey("The test command should read all flags and parameters.", t, func() {
		Up()
		So(first, ShouldEqual, 10)
		So(second, ShouldEqual, 20)
		So(global, ShouldEqual, 3)
		So(params[0], ShouldEqual, "arg1")
		So(params[1], ShouldEqual, "arg2")
		So(description, ShouldEqual, "Testing testcmd")
	})
}
<commit_msg>Tests adapted to the changes in start.go<commit_after>package start

import (
	"os"
	"testing"

	flag "github.com/ogier/pflag"

	. "github.com/smartystreets/goconvey/convey"
)

func TestParse(t *testing.T) {
	// ContinueOnError is required when running goconvey as server; otherwise, unrecognized
	// flags that are passed to the test executable will cause an error:
	// "unknown shorthand flag: 't' in -test.v=true"
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	os.Args = []string{os.Args[0], "--cmdline=FromCmdLine", "anargument", "anotherarg"}

	Convey("When setting up some flag variables", t, func() {
		var stringFlag string
		var cmdlineStringFlag string
		flag.StringVarP(&stringFlag, "astring", "a", "From default", "A string flag")
		flag.StringVarP(&cmdlineStringFlag, "cmdline", "c", "From command line", "A string flag")
		intFlag := flag.IntP("anint", "i", 23, "An integer flag")
		boolFlag := flag.BoolP("anewbool", "b", true, "A boolean flag")

		SetConfigFile("test/test.toml")
		os.Setenv("START_ASTRING", "From Environment Variable")
		Parse()
		Convey("Then Parse() should find the correct values from config file, env var, or default. (Restriction: passing the command line flags is not possible with automated calls to go test)", func() {
			So(cmdlineStringFlag, ShouldEqual, "FromCmdLine")
			So(stringFlag, ShouldEqual, "From Environment Variable")
			So(*intFlag, ShouldEqual, 42)    // from config file
			So(*boolFlag, ShouldEqual, true) // from default

		})
		Reset(func() {
			os.Setenv("START_ASTRING", "")
		})
	})

}

func TestUp(t *testing.T) {
	var first int
	var second int
	var global int
	var params []string

	SetInitFunc(func() error {
		SetDescription("Testing testcmd")
		return nil
	})

	// ContinueOnError is required when running goconvey as server; otherwise, unrecognized
	// flags that are passed to the test executable will cause an error:
	// "unknown shorthand flag: 't' in -test.v=true"
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	// To suppress warnings resulting from standard flags -test and -json,
	// read -t and -j into dummy flags.
	// The flags used for actual testing must not use -t or -j shorthands.
	var testflag string
	var jsonflag string
	flag.StringVarP(&testflag, "t", "t", "t", "t")
	flag.StringVarP(&jsonflag, "j", "j", "j", "j")

	flag.IntVarP(&first, "first", "f", 1, "The first flag")
	flag.IntVarP(&second, "second", "s", 2, "The second flag")
	flag.IntVarP(&global, "global", "g", 3, "The global flag")

	os.Args = []string{os.Args[0], "testcmd", "--first=10", "-s20", "arg1", "arg2"}

	Commands = make(CommandMap) // clear the commands map for this test
	Add(&Command{
		Name:  "testcmd",
		Flags: []string{"first", "second"},
		Cmd: func(cmd *Command) error {
			params = cmd.Args
			return nil
		},
	})

	Up()
	Convey("The test command should read all flags and parameters.", t, func() {
		So(first, ShouldEqual, 10)
		So(second, ShouldEqual, 20)
		So(global, ShouldEqual, 3)
		So(params[0], ShouldEqual, "arg1")
		So(params[1], ShouldEqual, "arg2")
		So(description, ShouldEqual, "Testing testcmd")
	})
	Convey("The test command should contain only its private flags.", t, func() {
		cmd := Commands["testcmd"]
		So(len(cmd.Flags), ShouldEqual, 2)
		So(cmd.Flags[0], ShouldEqual, "first")
		So(cmd.Flags[1], ShouldEqual, "second")
	})
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package clnt

import "plan9/p"

// Removes the file associated with the Fid. Returns nil if the
// operation is successful.
func (clnt *Clnt) Remove(fid *Fid) *p.Error {
	tc := p.NewFcall(clnt.Msize);
	err := p.PackTremove(tc, fid.Fid);
	if err != nil {
		return err
	}

	rc, err := clnt.rpc(tc);
	clnt.fidpool.putId(fid.Fid);

	if rc.Type == p.Rerror {
		return &p.Error{rc.Error, int(rc.Errornum)}
	}

	return err;
}

// Removes the named file. Returns nil if the operation is successful.
func (clnt *Clnt) FRemove(path string) *p.Error {
	var err *p.Error;
	fid, err := clnt.FWalk(path);
	if err != nil {
		return err
	}

	err = clnt.Remove(fid);
	if err != nil {
		clnt.Clunk(fid)
	}

	return err;
}
<commit_msg>clunk after remove (otherwise the fid is reused improperly)<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package clnt

import "plan9/p"

// Removes the file associated with the Fid. Returns nil if the
// operation is successful.
func (clnt *Clnt) Remove(fid *Fid) *p.Error {
	tc := p.NewFcall(clnt.Msize);
	err := p.PackTremove(tc, fid.Fid);
	if err != nil {
		return err
	}

	rc, err := clnt.rpc(tc);
	clnt.fidpool.putId(fid.Fid);

	if rc.Type == p.Rerror {
		return &p.Error{rc.Error, int(rc.Errornum)}
	}

	return err;
}

// Removes the named file. Returns nil if the operation is successful.
func (clnt *Clnt) FRemove(path string) *p.Error {
	var err *p.Error;
	fid, err := clnt.FWalk(path);
	if err != nil {
		return err
	}

	err = clnt.Remove(fid);
	clnt.Clunk(fid) // Remove always clunks

	return err;
}
<|endoftext|>


<commit_before>package stats

import (
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/rcrowley/go-metrics"
)

type Manager interface {
	Publisher
	Fetch(name string) Stats
	Reset()
}

type Publisher interface {
	Add(name string, dur time.Duration)
	Error(name string)
}

type Stats interface {
	Processed() int64
	Errors() int64
	Min() int64
	Mean() float64
	P95() float64
	Max() int64
	StdDev() float64
}

const (
	DefaultSampleSize = 1000
	Latency           = "Latency"
	TaskWait          = "TaskWait"
)

//
// base
//

type base struct {
	sync.Mutex
	stats      map[string]*baseStats
	sampleSize int
}

func (b *base) Add(name string, dur time.Duration) {
	b.metrics(name).time.Update(int64(dur))
}

func (b *base) Error(name string) {
	b.metrics(name).errors.Inc(1)
}

func (b *base) Fetch(name string) Stats {
	return b.metrics(name)
}

func (b *base) Reset() {
	for _, s := range b.stats {
		s.time.Clear()
		s.errors.Clear()
	}
}

func (b *base) init() {
	b.stats = make(map[string]*baseStats)
}

func (b *base) metrics(name string) *baseStats {
	if _, ok := b.stats[name]; !ok {
		b.Lock()
		defer b.Unlock()

		// Double checking being protected by mutex
		if s, ok := b.stats[name]; ok {
			return s
		}

		if b.sampleSize == 0 {
			b.sampleSize = DefaultSampleSize
		}
		b.stats[name] = &baseStats{
			name:   name,
			time:   metrics.NewHistogram(metrics.NewUniformSample(b.sampleSize)),
			errors: metrics.NewCounter(),
		}
	}

	return b.stats[name]
}

//
// baseStats
//

type baseStats struct {
	name   string
	time   metrics.Histogram
	errors metrics.Counter
}

func (s *baseStats) Processed() int64 {
	return s.time.Count()
}

func (s *baseStats) Errors() int64 {
	return s.errors.Count()
}

func (s *baseStats) Min() int64 {
	return s.time.Min()
}

func (s *baseStats) Mean() float64 {
	return s.time.Mean()
}

func (s *baseStats) P95() float64 {
	return s.time.Percentile(0.95)
}

func (s *baseStats) Max() int64 {
	return s.time.Max()
}

func (s *baseStats) StdDev() float64 {
	return s.time.StdDev()
}

func (s *baseStats) String() string {
	return fmt.Sprintf("%s statistics:\n"+
		"Processed: %10d\n"+
		"Errors:    %10d\n"+
		"Min:       %10s\n"+
		"Mean:      %10s\n"+
		"95%%:       %10s\n"+
		"Max:       %10s\n"+
		"StdDev:    %10s",
		s.name,
		s.time.Count(),
		s.errors.Count(),
		formatDuration(float64(s.time.Min())),
		formatDuration(s.time.Mean()),
		formatDuration(s.time.Percentile(0.95)),
		formatDuration(float64(s.time.Max())),
		formatDuration(s.time.StdDev()),
	)
}

func (s *baseStats) snapshot() *baseSnapshot {
	return &baseSnapshot{
		Timestamp:    time.Now().UTC().Unix(),
		ProcessedVal: s.time.Count(),
		ErrorsVal:    s.errors.Count(),
		MinVal:       round(float64(s.time.Min())/1000000, 6),
		MeanVal:      round(s.time.Mean()/1000000, 6),
		P95Val:       round(s.time.Percentile(0.95)/1000000, 6),
		MaxVal:       round(float64(s.time.Max())/1000000, 6),
		StdDevVal:    round(s.time.StdDev()/1000000, 6),
	}
}

//
// baseSnapshot
//

type baseSnapshot struct {
	timestamp int64
	processed int64
	errors    int64
	min       float64
	mean      float64
	p95       float64
	max       float64
	stddev    float64
}

// Implements json.Marshaler
func (s *baseSnapshot) MarshalJSON() ([]byte, error) {
	return []byte(fmt.Sprintf("[%d,%d,%d,%.6f,%.6f,%.6f,%.6f,%.6f]",
		s.timestamp, s.processed, s.errors, s.min,
		s.mean, s.p95, s.max, s.stddev)), nil
}

//
// Helpers
//

func formatDuration(dur float64) string {
	switch {
	case dur < 1000:
		return fmt.Sprintf("%10.0fns", dur)
	case dur < 1000000:
		return fmt.Sprintf("%10.3fμs", dur/1000)
	case dur < 1000000000:
		return fmt.Sprintf("%10.3fms", dur/1000000)
	default:
		return fmt.Sprintf("%10.3fs", dur/1000000000)
	}
}

func round(num float64, decimals int) float64 {
	pow := math.Pow(10, float64(decimals))
	return float64(int(num*pow+0.5)) / pow
}
<commit_msg>Fix stats snapshot creation<commit_after>package stats

import (
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/rcrowley/go-metrics"
)

type Manager interface {
	Publisher
	Fetch(name string) Stats
	Reset()
}

type Publisher interface {
	Add(name string, dur time.Duration)
	Error(name string)
}

type Stats interface {
	Processed() int64
	Errors() int64
	Min() int64
	Mean() float64
	P95() float64
	Max() int64
	StdDev() float64
}

const (
	DefaultSampleSize = 1000
	Latency           = "Latency"
	TaskWait          = "TaskWait"
)

//
// base
//

type base struct {
	sync.Mutex
	stats      map[string]*baseStats
	sampleSize int
}

func (b *base) Add(name string, dur time.Duration) {
	b.metrics(name).time.Update(int64(dur))
}

func (b *base) Error(name string) {
	b.metrics(name).errors.Inc(1)
}

func (b *base) Fetch(name string) Stats {
	return b.metrics(name)
}

func (b *base) Reset() {
	for _, s := range b.stats {
		s.time.Clear()
		s.errors.Clear()
	}
}

func (b *base) init() {
	b.stats = make(map[string]*baseStats)
}

func (b *base) metrics(name string) *baseStats {
	if _, ok := b.stats[name]; !ok {
		b.Lock()
		defer b.Unlock()

		// Double checking being protected by mutex
		if s, ok := b.stats[name]; ok {
			return s
		}

		if b.sampleSize == 0 {
			b.sampleSize = DefaultSampleSize
		}
		b.stats[name] = &baseStats{
			name:   name,
			time:   metrics.NewHistogram(metrics.NewUniformSample(b.sampleSize)),
			errors: metrics.NewCounter(),
		}
	}

	return b.stats[name]
}

//
// baseStats
//

type baseStats struct {
	name   string
	time   metrics.Histogram
	errors metrics.Counter
}

func (s *baseStats) Processed() int64 {
	return s.time.Count()
}

func (s *baseStats) Errors() int64 {
	return s.errors.Count()
}

func (s *baseStats) Min() int64 {
	return s.time.Min()
}

func (s *baseStats) Mean() float64 {
	return s.time.Mean()
}

func (s *baseStats) P95() float64 {
	return s.time.Percentile(0.95)
}

func (s *baseStats) Max() int64 {
	return s.time.Max()
}

func (s *baseStats) StdDev() float64 {
	return s.time.StdDev()
}

func (s *baseStats) String() string {
	return fmt.Sprintf("%s statistics:\n"+
		"Processed: %10d\n"+
		"Errors:    %10d\n"+
		"Min:       %10s\n"+
		"Mean:      %10s\n"+
		"95%%:       %10s\n"+
		"Max:       %10s\n"+
		"StdDev:    %10s",
		s.name,
		s.time.Count(),
		s.errors.Count(),
		formatDuration(float64(s.time.Min())),
		formatDuration(s.time.Mean()),
		formatDuration(s.time.Percentile(0.95)),
		formatDuration(float64(s.time.Max())),
		formatDuration(s.time.StdDev()),
	)
}

func (s *baseStats) snapshot() *baseSnapshot {
	return &baseSnapshot{
		timestamp: time.Now().UTC().Unix(),
		processed: s.time.Count(),
		errors:    s.errors.Count(),
		min:       round(float64(s.time.Min())/1000000, 6),
		mean:      round(s.time.Mean()/1000000, 6),
		p95:       round(s.time.Percentile(0.95)/1000000, 6),
		max:       round(float64(s.time.Max())/1000000, 6),
		stddev:    round(s.time.StdDev()/1000000, 6),
	}
}

//
// baseSnapshot
//

type baseSnapshot struct {
	timestamp int64
	processed int64
	errors    int64
	min       float64
	mean      float64
	p95       float64
	max       float64
	stddev    float64
}

// Implements json.Marshaler
func (s *baseSnapshot) MarshalJSON() ([]byte, error) {
	return []byte(fmt.Sprintf("[%d,%d,%d,%.6f,%.6f,%.6f,%.6f,%.6f]",
		s.timestamp, s.processed, s.errors, s.min,
		s.mean, s.p95, s.max, s.stddev)), nil
}

//
// Helpers
//

func formatDuration(dur float64) string {
	switch {
	case dur < 1000:
		return fmt.Sprintf("%10.0fns", dur)
	case dur < 1000000:
		return fmt.Sprintf("%10.3fμs", dur/1000)
	case dur < 1000000000:
		return fmt.Sprintf("%10.3fms", dur/1000000)
	default:
		return fmt.Sprintf("%10.3fs", dur/1000000000)
	}
}

func round(num float64, decimals int) float64 {
	pow := math.Pow(10, float64(decimals))
	return float64(int(num*pow+0.5)) / pow
}
<|endoftext|>