<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package simplepush

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/mux"
)

func NewEndpointHandler() (h *EndpointHandler) {
	h = &EndpointHandler{mux: mux.NewRouter()}
	h.mux.HandleFunc("/update/{key}", h.UpdateHandler)
	return h
}

type EndpointHandlerConfig struct {
	MaxDataLen  int  `toml:"max_data_len" env:"max_data_len"`
	AlwaysRoute bool `toml:"always_route" env:"always_route"`
	Listener    TCPListenerConfig
}

type EndpointHandler struct {
	app         *Application
	logger      *SimpleLogger
	metrics     Statistician
	store       Store
	router      Router
	pinger      PropPinger
	balancer    Balancer
	hostname    string
	tokenKey    []byte
	listener    net.Listener
	server      *ServeCloser
	mux         *mux.Router
	url         string
	maxConns    int
	maxDataLen  int
	alwaysRoute bool
	closeOnce   Once
}

func (h *EndpointHandler) ConfigStruct() interface{} {
	return &EndpointHandlerConfig{
		MaxDataLen:  4096,
		AlwaysRoute: false,
		Listener: TCPListenerConfig{
			Addr:            ":8081",
			MaxConns:        1000,
			KeepAlivePeriod: "3m",
		},
	}
}

func (h *EndpointHandler) Init(app *Application, config interface{}) (err error) {
	conf := config.(*EndpointHandlerConfig)
	h.setApp(app)

	if h.listener, err = conf.Listener.Listen(); err != nil {
		h.logger.Panic("handlers_endpoint", "Could not attach update listener",
			LogFields{"error": err.Error()})
		return err
	}

	var scheme string
	if conf.Listener.UseTLS() {
		scheme = "https"
	} else {
		scheme = "http"
	}
	host, port := HostPort(h.listener, app)
	h.url = CanonicalURL(scheme, host, port)

	h.maxConns = conf.Listener.MaxConns
	h.setMaxDataLen(conf.MaxDataLen)
	h.alwaysRoute = conf.AlwaysRoute

	return nil
}

func (h *EndpointHandler) Listener() net.Listener { return h.listener }
func (h *EndpointHandler) MaxConns() int          { return h.maxConns }
func (h *EndpointHandler) URL() string            { return h.url }
func (h *EndpointHandler) ServeMux() ServeMux     { return (*RouteMux)(h.mux) }

// setApp sets the parent application for this update handler.
func (h *EndpointHandler) setApp(app *Application) {
	h.app = app
	h.logger = app.Logger()
	h.metrics = app.Metrics()
	h.store = app.Store()
	h.router = app.Router()
	h.pinger = app.PropPinger()
	h.tokenKey = app.TokenKey()
	h.server = NewServeCloser(&http.Server{
		ConnState: func(c net.Conn, state http.ConnState) {
			if state == http.StateNew {
				h.metrics.Increment("endpoint.socket.connect")
			} else if state == http.StateClosed {
				h.metrics.Increment("endpoint.socket.disconnect")
			}
		},
		Handler: &LogHandler{h.mux, h.logger},
		ErrorLog: log.New(&LogWriter{
			Logger: h.logger,
			Name:   "handlers_endpoint",
			Level:  ERROR,
		}, "", 0),
	})
}

// setMaxDataLen sets the maximum data length to v
func (h *EndpointHandler) setMaxDataLen(v int) {
	h.maxDataLen = v
}

func (h *EndpointHandler) Start(errChan chan<- error) {
	if h.logger.ShouldLog(INFO) {
		h.logger.Info("handlers_endpoint", "Starting update server",
			LogFields{"url": h.url})
	}
	errChan <- h.server.Serve(h.listener)
}

func (h *EndpointHandler) decodePK(token string) (key string, err error) {
	if len(token) == 0 {
		return "", fmt.Errorf("Missing primary key")
	}
	if len(h.tokenKey) == 0 {
		return token, nil
	}
	bpk, err := Decode(h.tokenKey, token)
	if err != nil {
		return "", err
	}
	return string(bytes.TrimSpace(bpk)), nil
}

func (h *EndpointHandler) resolvePK(token string) (uaid, chid string, err error) {
	pk, err := h.decodePK(token)
	if err != nil {
		err = fmt.Errorf("Error decoding primary key: %s", err)
		return "", "", err
	}
	if !validPK(pk) {
		err = fmt.Errorf("Invalid primary key: %q", pk)
		return "", "", err
	}
	if uaid, chid, err = h.store.KeyToIDs(pk); err != nil {
		return "", "", err
	}
	return uaid, chid, nil
}

func (h *EndpointHandler) doPropPing(uaid string, version int64, data string) (ok bool, err error) {
	if h.pinger == nil {
		return false, nil
	}
	if ok, err = h.pinger.Send(uaid, version, data); err != nil {
		return false, fmt.Errorf("Could not send proprietary ping: %s", err)
	}
	if !ok {
		return false, nil
	}
	/* if this is a GCM connected host, boot vers immediately to GCM
	 */
	return h.pinger.CanBypassWebsocket(), nil
}

// getUpdateParams extracts the update version and data from req.
func (h *EndpointHandler) getUpdateParams(req *http.Request) (version int64, data string, err error) {
	if req.Header.Get("Content-Type") == "" {
		req.Header.Set("Content-Type",
			"application/x-www-form-urlencoded")
	}
	svers := req.FormValue("version")
	if svers != "" {
		if version, err = strconv.ParseInt(svers, 10, 64); err != nil || version < 0 {
			return 0, "", ErrBadVersion
		}
	} else {
		version = timeNow().UTC().Unix()
	}

	data = req.FormValue("data")
	if len(data) > h.maxDataLen {
		return 0, "", ErrDataTooLong
	}
	return
}

// -- REST
func (h *EndpointHandler) UpdateHandler(resp http.ResponseWriter, req *http.Request) {
	// Handle the version updates.
	timer := timeNow()
	requestID := req.Header.Get(HeaderID)
	logWarning := h.logger.ShouldLog(WARNING)
	var (
		err        error
		updateSent bool
		version    int64
		uaid, chid string
	)

	defer func() {
		now := timeNow()
		if h.logger.ShouldLog(DEBUG) {
			h.logger.Debug("handlers_endpoint", "+++++++++++++ DONE +++",
				LogFields{"rid": requestID})
		}
		if h.logger.ShouldLog(INFO) {
			h.logger.Info("handlers_endpoint", "Client Update complete", LogFields{
				"rid":        requestID,
				"uaid":       uaid,
				"chid":       chid,
				"successful": strconv.FormatBool(updateSent)})
		}
		if updateSent {
			h.metrics.Timer("updates.handled", now.Sub(timer))
		}
	}()

	if h.logger.ShouldLog(INFO) {
		h.logger.Info("handlers_endpoint", "Handling Update",
			LogFields{"rid": requestID})
	}

	if req.Method != "PUT" {
		writeJSON(resp, http.StatusMethodNotAllowed, []byte(`"Method Not Allowed"`))
		h.metrics.Increment("updates.appserver.invalid")
		return
	}

	version, data, err := h.getUpdateParams(req)
	if err != nil {
		if err == ErrDataTooLong {
			if logWarning {
				h.logger.Warn("handlers_endpoint", "Data too large, rejecting request",
					LogFields{"rid": requestID})
			}
			writeJSON(resp, http.StatusRequestEntityTooLarge, []byte(fmt.Sprintf(
				`"Data exceeds max length of %d bytes"`, h.maxDataLen)))
			h.metrics.Increment("updates.appserver.toolong")
			return
		}
		writeJSON(resp, http.StatusBadRequest, []byte(`"Invalid Version"`))
		h.metrics.Increment("updates.appserver.invalid")
		return
	}

	// TODO:
	// is there a magic flag for proxyable endpoints?
	// e.g. update/p/gcm/LSoC or something?
	// (Note, this would allow us to use smarter FE proxies.)
	token := mux.Vars(req)["key"]
	if uaid, chid, err = h.resolvePK(token); err != nil {
		if logWarning {
			h.logger.Warn("handlers_endpoint", "Invalid primary key for update",
				LogFields{"error": err.Error(), "rid": requestID, "token": token})
		}
		writeJSON(resp, http.StatusNotFound, []byte(`"Invalid Token"`))
		h.metrics.Increment("updates.appserver.invalid")
		return
	}

	// At this point we should have a valid endpoint in the URL
	h.metrics.Increment("updates.appserver.incoming")

	// is there a Proprietary Ping for this?
	updateSent, err = h.doPropPing(uaid, version, data)
	if err != nil {
		if logWarning {
			h.logger.Warn("handlers_endpoint", "Could not send proprietary ping",
				LogFields{"rid": requestID, "uaid": uaid, "error": err.Error()})
		}
	} else if updateSent {
		// Neat! Might as well return.
		h.metrics.Increment("updates.appserver.received")
		writeSuccess(resp)
		return
	}

	if h.logger.ShouldLog(INFO) {
		h.logger.Info("handlers_endpoint", "setting version for ChannelID",
			LogFields{"rid": requestID, "uaid": uaid, "chid": chid,
				"version": strconv.FormatInt(version, 10)})
	}

	if err = h.store.Update(uaid, chid, version); err != nil {
		if logWarning {
			h.logger.Warn("handlers_endpoint", "Could not update channel", LogFields{
				"rid":     requestID,
				"uaid":    uaid,
				"chid":    chid,
				"version": strconv.FormatInt(version, 10),
				"error":   err.Error()})
		}
		status, _ := ErrToStatus(err)
		h.metrics.Increment("updates.appserver.error")
		writeJSON(resp, status, []byte(`"Could not update channel version"`))
		return
	}

	cn, _ := resp.(http.CloseNotifier)
	if !h.deliver(cn, uaid, chid, version, requestID, data) {
		// We've accepted the valid endpoint, stored the data for
		// eventual pickup by the client, but failed to deliver to
		// the client via routing.
		writeJSON(resp, http.StatusAccepted, []byte("{}"))
		return
	}

	writeSuccess(resp)
	updateSent = true
	return
}

// deliver routes an incoming update to the appropriate server.
func (h *EndpointHandler) deliver(cn http.CloseNotifier, uaid, chid string,
	version int64, requestID string, data string) (delivered bool) {

	worker, workerConnected := h.app.GetWorker(uaid)
	var routingTime time.Duration

	// Always route to other servers first, in case we're holding open a stale
	// connection and the client has already reconnected to a different server.
	if h.alwaysRoute || !workerConnected {
		h.metrics.Increment("updates.routed.outgoing")
		// Abort routing if the connection goes away.
		var cancelSignal <-chan bool
		if cn != nil {
			cancelSignal = cn.CloseNotify()
		}
		// Route the update.
		startTime := timeNow().UTC()
		delivered, _ = h.router.Route(cancelSignal, uaid, chid, version,
			startTime, requestID, data)
		routingTime = timeNow().UTC().Sub(startTime)
	}

	// If we delivered the message
	if delivered {
		h.metrics.Increment("router.broadcast.hit")
		h.metrics.Timer("updates.routed.hits", routingTime)
		// If we're not always routing, we're done now
		if !h.alwaysRoute {
			h.metrics.Increment("updates.appserver.received")
			return true
		}
	} else if !workerConnected {
		// Worker is not connected and routing failed
		h.metrics.Increment("router.broadcast.miss")
		h.metrics.Timer("updates.routed.misses", routingTime)
		h.metrics.Increment("updates.appserver.rejected")
		return false
	}

	// Possible conditions at this point:
	// Router delivered, but alwaysRoute is true
	// Router didn't deliver, alwaysRoute true or false

	// Try local delivery
	if err := worker.Send(chid, version, data); err == nil {
		// Local delivery worked, avoid incrementing received if we
		// also already delivered it
		if !delivered {
			h.metrics.Increment("updates.appserver.received")
		}
		delivered = true
	} else if !delivered {
		// Local delivery failed *and* routing failed
		h.metrics.Increment("updates.appserver.rejected")
	}
	return delivered
}

func (h *EndpointHandler) Close() error {
	return h.closeOnce.Do(h.close)
}

func (h *EndpointHandler) close() (err error) {
	if h.logger.ShouldLog(INFO) {
		h.logger.Info("handlers_endpoint", "Closing update handler",
			LogFields{"url": h.url})
	}
	if err = h.listener.Close(); err != nil && h.logger.ShouldLog(ERROR) {
		h.logger.Error("handlers_endpoint", "Error closing update listener",
			LogFields{"error": err.Error(), "url": h.url})
	}
	h.server.Close()
	return
}

func validPK(pk string) bool {
	for i := 0; i < len(pk); i++ {
		b := pk[i]
		if b >= 'A' && b <= 'Z' {
			b += 'a' - 'A'
		}
		// Accept bin64 && UUID encoding
		if (b < 'a' || b > 'z') && (b < '0' || b > '9') && b != '_' && b != '.' && b != '=' && b != '-' {
			return false
		}
	}
	return true
}

func writeJSON(resp http.ResponseWriter, status int, data []byte) {
	resp.Header().Set("Content-Type", "application/json")
	resp.WriteHeader(status)
	resp.Write(data)
}

func writeSuccess(resp http.ResponseWriter) {
	writeJSON(resp, http.StatusOK, []byte("{}"))
}

// o4fs
// vim: set tabstab=4 softtabstop=4 shiftwidth=4 noexpandtab
<commit_msg>Clean up logic<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package simplepush

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/mux"
)

func NewEndpointHandler() (h *EndpointHandler) {
	h = &EndpointHandler{mux: mux.NewRouter()}
	h.mux.HandleFunc("/update/{key}", h.UpdateHandler)
	return h
}

type EndpointHandlerConfig struct {
	MaxDataLen  int  `toml:"max_data_len" env:"max_data_len"`
	AlwaysRoute bool `toml:"always_route" env:"always_route"`
	Listener    TCPListenerConfig
}

type EndpointHandler struct {
	app         *Application
	logger      *SimpleLogger
	metrics     Statistician
	store       Store
	router      Router
	pinger      PropPinger
	balancer    Balancer
	hostname    string
	tokenKey    []byte
	listener    net.Listener
	server      *ServeCloser
	mux         *mux.Router
	url         string
	maxConns    int
	maxDataLen  int
	alwaysRoute bool
	closeOnce   Once
}

func (h *EndpointHandler) ConfigStruct() interface{} {
	return &EndpointHandlerConfig{
		MaxDataLen:  4096,
		AlwaysRoute: false,
		Listener: TCPListenerConfig{
			Addr:            ":8081",
			MaxConns:        1000,
			KeepAlivePeriod: "3m",
		},
	}
}

func (h *EndpointHandler) Init(app *Application, config interface{}) (err error) {
	conf := config.(*EndpointHandlerConfig)
	h.setApp(app)

	if h.listener, err = conf.Listener.Listen(); err != nil {
		h.logger.Panic("handlers_endpoint", "Could not attach update listener",
			LogFields{"error": err.Error()})
		return err
	}

	var scheme string
	if conf.Listener.UseTLS() {
		scheme = "https"
	} else {
		scheme = "http"
	}
	host, port := HostPort(h.listener, app)
	h.url = CanonicalURL(scheme, host, port)

	h.maxConns = conf.Listener.MaxConns
	h.setMaxDataLen(conf.MaxDataLen)
	h.alwaysRoute = conf.AlwaysRoute

	return nil
}

func (h *EndpointHandler) Listener() net.Listener { return h.listener }
func (h *EndpointHandler) MaxConns() int          { return h.maxConns }
func (h *EndpointHandler) URL() string            { return h.url }
func (h *EndpointHandler) ServeMux() ServeMux     { return (*RouteMux)(h.mux) }

// setApp sets the parent application for this update handler.
func (h *EndpointHandler) setApp(app *Application) {
	h.app = app
	h.logger = app.Logger()
	h.metrics = app.Metrics()
	h.store = app.Store()
	h.router = app.Router()
	h.pinger = app.PropPinger()
	h.tokenKey = app.TokenKey()
	h.server = NewServeCloser(&http.Server{
		ConnState: func(c net.Conn, state http.ConnState) {
			if state == http.StateNew {
				h.metrics.Increment("endpoint.socket.connect")
			} else if state == http.StateClosed {
				h.metrics.Increment("endpoint.socket.disconnect")
			}
		},
		Handler: &LogHandler{h.mux, h.logger},
		ErrorLog: log.New(&LogWriter{
			Logger: h.logger,
			Name:   "handlers_endpoint",
			Level:  ERROR,
		}, "", 0),
	})
}

// setMaxDataLen sets the maximum data length to v
func (h *EndpointHandler) setMaxDataLen(v int) {
	h.maxDataLen = v
}

func (h *EndpointHandler) Start(errChan chan<- error) {
	if h.logger.ShouldLog(INFO) {
		h.logger.Info("handlers_endpoint", "Starting update server",
			LogFields{"url": h.url})
	}
	errChan <- h.server.Serve(h.listener)
}

func (h *EndpointHandler) decodePK(token string) (key string, err error) {
	if len(token) == 0 {
		return "", fmt.Errorf("Missing primary key")
	}
	if len(h.tokenKey) == 0 {
		return token, nil
	}
	bpk, err := Decode(h.tokenKey, token)
	if err != nil {
		return "", err
	}
	return string(bytes.TrimSpace(bpk)), nil
}

func (h *EndpointHandler) resolvePK(token string) (uaid, chid string, err error) {
	pk, err := h.decodePK(token)
	if err != nil {
		err = fmt.Errorf("Error decoding primary key: %s", err)
		return "", "", err
	}
	if !validPK(pk) {
		err = fmt.Errorf("Invalid primary key: %q", pk)
		return "", "", err
	}
	if uaid, chid, err = h.store.KeyToIDs(pk); err != nil {
		return "", "", err
	}
	return uaid, chid, nil
}

func (h *EndpointHandler) doPropPing(uaid string, version int64, data string) (ok bool, err error) {
	if h.pinger == nil {
		return false, nil
	}
	if ok, err = h.pinger.Send(uaid, version, data); err != nil {
		return false, fmt.Errorf("Could not send proprietary ping: %s", err)
	}
	if !ok {
		return false, nil
	}
	/* if this is a GCM connected host, boot vers immediately to GCM
	 */
	return h.pinger.CanBypassWebsocket(), nil
}

// getUpdateParams extracts the update version and data from req.
func (h *EndpointHandler) getUpdateParams(req *http.Request) (version int64, data string, err error) {
	if req.Header.Get("Content-Type") == "" {
		req.Header.Set("Content-Type",
			"application/x-www-form-urlencoded")
	}
	svers := req.FormValue("version")
	if svers != "" {
		if version, err = strconv.ParseInt(svers, 10, 64); err != nil || version < 0 {
			return 0, "", ErrBadVersion
		}
	} else {
		version = timeNow().UTC().Unix()
	}

	data = req.FormValue("data")
	if len(data) > h.maxDataLen {
		return 0, "", ErrDataTooLong
	}
	return
}

// -- REST
func (h *EndpointHandler) UpdateHandler(resp http.ResponseWriter, req *http.Request) {
	// Handle the version updates.
	timer := timeNow()
	requestID := req.Header.Get(HeaderID)
	logWarning := h.logger.ShouldLog(WARNING)
	var (
		err        error
		updateSent bool
		version    int64
		uaid, chid string
	)

	defer func() {
		now := timeNow()
		if h.logger.ShouldLog(DEBUG) {
			h.logger.Debug("handlers_endpoint", "+++++++++++++ DONE +++",
				LogFields{"rid": requestID})
		}
		if h.logger.ShouldLog(INFO) {
			h.logger.Info("handlers_endpoint", "Client Update complete", LogFields{
				"rid":        requestID,
				"uaid":       uaid,
				"chid":       chid,
				"successful": strconv.FormatBool(updateSent)})
		}
		if updateSent {
			h.metrics.Timer("updates.handled", now.Sub(timer))
		}
	}()

	if h.logger.ShouldLog(INFO) {
		h.logger.Info("handlers_endpoint", "Handling Update",
			LogFields{"rid": requestID})
	}

	if req.Method != "PUT" {
		writeJSON(resp, http.StatusMethodNotAllowed, []byte(`"Method Not Allowed"`))
		h.metrics.Increment("updates.appserver.invalid")
		return
	}

	version, data, err := h.getUpdateParams(req)
	if err != nil {
		if err == ErrDataTooLong {
			if logWarning {
				h.logger.Warn("handlers_endpoint", "Data too large, rejecting request",
					LogFields{"rid": requestID})
			}
			writeJSON(resp, http.StatusRequestEntityTooLarge, []byte(fmt.Sprintf(
				`"Data exceeds max length of %d bytes"`, h.maxDataLen)))
			h.metrics.Increment("updates.appserver.toolong")
			return
		}
		writeJSON(resp, http.StatusBadRequest, []byte(`"Invalid Version"`))
		h.metrics.Increment("updates.appserver.invalid")
		return
	}

	// TODO:
	// is there a magic flag for proxyable endpoints?
	// e.g. update/p/gcm/LSoC or something?
	// (Note, this would allow us to use smarter FE proxies.)
	token := mux.Vars(req)["key"]
	if uaid, chid, err = h.resolvePK(token); err != nil {
		if logWarning {
			h.logger.Warn("handlers_endpoint", "Invalid primary key for update",
				LogFields{"error": err.Error(), "rid": requestID, "token": token})
		}
		writeJSON(resp, http.StatusNotFound, []byte(`"Invalid Token"`))
		h.metrics.Increment("updates.appserver.invalid")
		return
	}

	// At this point we should have a valid endpoint in the URL
	h.metrics.Increment("updates.appserver.incoming")

	// is there a Proprietary Ping for this?
	updateSent, err = h.doPropPing(uaid, version, data)
	if err != nil {
		if logWarning {
			h.logger.Warn("handlers_endpoint", "Could not send proprietary ping",
				LogFields{"rid": requestID, "uaid": uaid, "error": err.Error()})
		}
	} else if updateSent {
		// Neat! Might as well return.
		h.metrics.Increment("updates.appserver.received")
		writeSuccess(resp)
		return
	}

	if h.logger.ShouldLog(INFO) {
		h.logger.Info("handlers_endpoint", "setting version for ChannelID",
			LogFields{"rid": requestID, "uaid": uaid, "chid": chid,
				"version": strconv.FormatInt(version, 10)})
	}

	if err = h.store.Update(uaid, chid, version); err != nil {
		if logWarning {
			h.logger.Warn("handlers_endpoint", "Could not update channel", LogFields{
				"rid":     requestID,
				"uaid":    uaid,
				"chid":    chid,
				"version": strconv.FormatInt(version, 10),
				"error":   err.Error()})
		}
		status, _ := ErrToStatus(err)
		h.metrics.Increment("updates.appserver.error")
		writeJSON(resp, status, []byte(`"Could not update channel version"`))
		return
	}

	cn, _ := resp.(http.CloseNotifier)
	if !h.deliver(cn, uaid, chid, version, requestID, data) {
		// We've accepted the valid endpoint, stored the data for
		// eventual pickup by the client, but failed to deliver to
		// the client via routing.
		writeJSON(resp, http.StatusAccepted, []byte("{}"))
		return
	}

	writeSuccess(resp)
	updateSent = true
	return
}

// deliver routes an incoming update to the appropriate server.
func (h *EndpointHandler) deliver(cn http.CloseNotifier, uaid, chid string,
	version int64, requestID string, data string) (delivered bool) {

	worker, workerConnected := h.app.GetWorker(uaid)
	var routingTime time.Duration

	shouldRoute := h.alwaysRoute || !workerConnected

	// Always route to other servers first, in case we're holding open a stale
	// connection and the client has already reconnected to a different server.
	if shouldRoute {
		h.metrics.Increment("updates.routed.outgoing")
		// Abort routing if the connection goes away.
		var cancelSignal <-chan bool
		if cn != nil {
			cancelSignal = cn.CloseNotify()
		}
		// Route the update.
		startTime := timeNow().UTC()
		delivered, _ = h.router.Route(cancelSignal, uaid, chid, version,
			startTime, requestID, data)
		routingTime = timeNow().UTC().Sub(startTime)

		// Increment appropriate metrics
		if delivered {
			h.metrics.Increment("router.broadcast.hit")
			h.metrics.Timer("updates.routed.hits", routingTime)
		} else {
			h.metrics.Increment("router.broadcast.miss")
			h.metrics.Timer("updates.routed.misses", routingTime)
		}
	}

	// Should we attempt local delivery? Only if the worker is connected
	// and we either always route, or failed to remote deliver
	shouldLocalDeliver := workerConnected && (h.alwaysRoute || !delivered)

	if shouldLocalDeliver {
		if err := worker.Send(chid, version, data); err == nil {
			delivered = true
		}
	}

	// Increment the appropriate final metric whether deliver did or
	// did not work
	if delivered {
		h.metrics.Increment("updates.appserver.received")
	} else {
		h.metrics.Increment("updates.appserver.rejected")
	}

	return delivered
}

func (h *EndpointHandler) Close() error {
	return h.closeOnce.Do(h.close)
}

func (h *EndpointHandler) close() (err error) {
	if h.logger.ShouldLog(INFO) {
		h.logger.Info("handlers_endpoint", "Closing update handler",
			LogFields{"url": h.url})
	}
	if err = h.listener.Close(); err != nil && h.logger.ShouldLog(ERROR) {
		h.logger.Error("handlers_endpoint", "Error closing update listener",
			LogFields{"error": err.Error(), "url": h.url})
	}
	h.server.Close()
	return
}

func validPK(pk string) bool {
	for i := 0; i < len(pk); i++ {
		b := pk[i]
		if b >= 'A' && b <= 'Z' {
			b += 'a' - 'A'
		}
		// Accept bin64 && UUID encoding
		if (b < 'a' || b > 'z') && (b < '0' || b > '9') && b != '_' && b != '.' && b != '=' && b != '-' {
			return false
		}
	}
	return true
}

func writeJSON(resp http.ResponseWriter, status int, data []byte) {
	resp.Header().Set("Content-Type", "application/json")
	resp.WriteHeader(status)
	resp.Write(data)
}

func writeSuccess(resp http.ResponseWriter) {
	writeJSON(resp, http.StatusOK, []byte("{}"))
}

// o4fs
// vim: set tabstab=4 softtabstop=4 shiftwidth=4 noexpandtab
<|endoftext|>


<commit_before>package integration_test

import (
	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("pushing a rails51 webpacker app with sprockets", func() {
	var app *cutlass.App
	AfterEach(func() { app = DestroyApp(app) })

	BeforeEach(func() {
		app = cutlass.New(Fixtures("rails51_webpacker"))
	})

	It("compiles assets with webpacker", func() {
		PushAppAndConfirm(app)
		Expect(app.Stdout.String()).To(ContainSubstring("Webpacker is installed"))
		Expect(app.Stdout.String()).To(ContainSubstring("Asset precompilation completed"))

		Expect(app.GetBody("/")).To(ContainSubstring("Welcome to Rails51 Webpacker!"))
		Eventually(app.Stdout.String()).Should(ContainSubstring("Cleaning assets"))
	})
})
<commit_msg>Bumps disk quota on Rails 5.1 app with sprockets<commit_after>package integration_test

import (
	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("pushing a rails51 webpacker app with sprockets", func() {
	var app *cutlass.App
	AfterEach(func() { app = DestroyApp(app) })

	BeforeEach(func() {
		app = cutlass.New(Fixtures("rails51_webpacker"))
		app.Disk = "1G"
	})

	It("compiles assets with webpacker", func() {
		PushAppAndConfirm(app)
		Expect(app.Stdout.String()).To(ContainSubstring("Webpacker is installed"))
		Expect(app.Stdout.String()).To(ContainSubstring("Asset precompilation completed"))

		Expect(app.GetBody("/")).To(ContainSubstring("Welcome to Rails51 Webpacker!"))
		Eventually(app.Stdout.String()).Should(ContainSubstring("Cleaning assets"))
	})
})
<|endoftext|>


<commit_before>package termite

import (
	"errors"
	"fmt"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/cba"
	"github.com/hanwen/termite/stats"
	"log"
	"math/rand"
	"net"
	"net/rpc"
	"strings"
	"sync"
	"time"
)

type mirrorConnection struct {
	workerAddr string // key in map.
	rpcClient  *rpc.Client
	connection net.Conn

	// For serving the Fileserver.
	reverseConnection net.Conn

	// Protected by mirrorConnections.Mutex.
	maxJobs       int
	availableJobs int

	master        *Master
	fileSetWaiter *attr.FileSetWaiter
}

func (me *mirrorConnection) Id() string {
	return me.workerAddr
}

func (me *mirrorConnection) innerFetch(start, end int, hash string) ([]byte, error) {
	req := &cba.ContentRequest{
		Hash:  hash,
		Start: start,
		End:   end,
	}
	rep := &cba.ContentResponse{}
	err := me.rpcClient.Call("Mirror.FileContent", req, rep)
	return rep.Chunk, err
}

func (me *mirrorConnection) replay(fset attr.FileSet) error {
	// Must get data before we modify the file-system, so we don't
	// leave the FS in a half-finished state.
	for _, info := range fset.Files {
		if info.Hash != "" && !me.master.cache.HasHash(info.Hash) {
			saved, err := me.master.cache.Fetch(
				func(start, end int) ([]byte, error) {
					return me.innerFetch(start, end, info.Hash)
				})
			if err == nil && saved != info.Hash {
				log.Fatalf("mirrorConnection.replay: fetch corruption got %x want %x", saved, info.Hash)
			}
			if err != nil {
				return err
			}
		}
	}
	me.master.replay(fset)
	return nil
}

func (me *mirrorConnection) Send(files []*attr.FileAttr) error {
	req := UpdateRequest{
		Files: files,
	}
	rep := UpdateResponse{}
	err := me.rpcClient.Call("Mirror.Update", &req, &rep)
	if err != nil {
		log.Println("Mirror.Update failure", err)
		return err
	}
	log.Printf("Sent pending changes to %s", me.workerAddr)
	return nil
}

// mirrorConnection manages connections from the master to the mirrors
// on the workers.
type mirrorConnections struct {
	master      *Master
	coordinator string

	keepAlive time.Duration

	wantedMaxJobs int

	stats *stats.ServerStats

	// Protects all of the below.
	sync.Mutex
	workers        map[string]bool
	mirrors        map[string]*mirrorConnection
	lastActionTime time.Time
}

func (me *mirrorConnections) fetchWorkers() (newMap map[string]bool) {
	newMap = map[string]bool{}
	client, err := rpc.DialHTTP("tcp", me.coordinator)
	if err != nil {
		log.Println("fetchWorkers: dialing coordinator:", err)
		return newMap
	}
	defer client.Close()
	req := 0
	rep := Registered{}
	err = client.Call("Coordinator.List", &req, &rep)
	if err != nil {
		log.Println("coordinator rpc error:", err)
		return newMap
	}

	for _, v := range rep.Registrations {
		newMap[v.Address] = true
	}
	if len(newMap) == 0 {
		log.Println("coordinator has no workers for us.")
	}
	return newMap
}

func (me *mirrorConnections) refreshWorkers() {
	newWorkers := me.fetchWorkers()
	if len(newWorkers) > 0 {
		me.Mutex.Lock()
		defer me.Mutex.Unlock()
		me.workers = newWorkers
	}
}

func newMirrorConnections(m *Master, coordinator string, maxJobs int) *mirrorConnections {
	me := &mirrorConnections{
		master:        m,
		wantedMaxJobs: maxJobs,
		workers:       make(map[string]bool),
		mirrors:       make(map[string]*mirrorConnection),
		coordinator:   coordinator,
		keepAlive:     time.Minute,
	}
	me.refreshStats()
	return me
}

func (me *mirrorConnections) refreshStats() {
	me.stats = stats.NewServerStats()
	me.stats.PhaseOrder = []string{"run", "send", "remote", "filewait"}
}

func (me *mirrorConnections) periodicHouseholding() {
	me.refreshWorkers()
	me.maybeDropConnections()
}

// Must be called with lock held.
func (me *mirrorConnections) availableJobs() int {
	a := 0
	for _, mc := range me.mirrors {
		if mc.availableJobs > 0 {
			a += mc.availableJobs
		}
	}
	return a
}

// Must be called with lock held.
func (me *mirrorConnections) maxJobs() int {
	a := 0
	for _, mc := range me.mirrors {
		a += mc.maxJobs
	}
	return a
}

func (me *mirrorConnections) maybeDropConnections() {
	me.Mutex.Lock()
	defer me.Mutex.Unlock()

	// Already dropped everything.
	if len(me.mirrors) == 0 {
		return
	}

	// Something is running.
	if me.availableJobs() < me.maxJobs() {
		return
	}

	if me.lastActionTime.Add(me.keepAlive).After(time.Now()) {
		return
	}

	log.Println("master inactive too long. Dropping connections.")
	me.dropConnections()
}

func (me *mirrorConnections) dropConnections() {
	for _, mc := range me.mirrors {
		mc.rpcClient.Close()
		mc.connection.Close()
		mc.reverseConnection.Close()
		me.master.attributes.RmClient(mc)
	}
	me.mirrors = make(map[string]*mirrorConnection)
	me.refreshStats()
}

// Gets a mirrorConnection to run on.  Will block if none available
func (me *mirrorConnections) find(name string) (*mirrorConnection, error) {
	me.Mutex.Lock()
	defer me.Mutex.Unlock()

	var found *mirrorConnection
	for nm, v := range me.mirrors {
		if strings.Contains(nm, name) {
			found = v
			break
		}
	}
	if found == nil {
		return nil, fmt.Errorf("No worker with name: %q. Have %v", name, me.mirrors)
	}
	found.availableJobs--
	return found, nil
}

func (me *mirrorConnections) pick() (*mirrorConnection, error) {
	me.Mutex.Lock()
	defer me.Mutex.Unlock()

	if me.availableJobs() <= 0 {
		if len(me.workers) == 0 {
			me.workers = me.fetchWorkers()
		}
		me.tryConnect()

		if me.maxJobs() == 0 {
			// Didn't connect to anything.  Should
			// probably direct the wrapper to compile
			// locally.
			return nil, errors.New("No workers found at all.")
		}
	}

	j := len(me.mirrors)
	if me.availableJobs() == 0 {
		// All workers full: schedule on a random one.
		j = rand.Intn(j)
	}

	var found *mirrorConnection
	for _, v := range me.mirrors {
		if j <= 0 || v.availableJobs > 0 {
			found = v
			break
		}
		j--
	}
	found.availableJobs--
	return found, nil
}

func (me *mirrorConnections) drop(mc *mirrorConnection, err error) {
	me.master.fileServer.attributes.RmClient(mc)

	me.Mutex.Lock()
	defer me.Mutex.Unlock()
	log.Printf("Dropping mirror %s. Reason: %s", mc.workerAddr, err)
	mc.connection.Close()
	mc.reverseConnection.Close()
	delete(me.mirrors, mc.workerAddr)
	delete(me.workers, mc.workerAddr)
}

func (me *mirrorConnections) jobDone(mc *mirrorConnection) {
	me.Mutex.Lock()
	defer me.Mutex.Unlock()

	me.lastActionTime = time.Now()
	mc.availableJobs++
}

func (me *mirrorConnections) idleWorkerAddress() string {
	cands := []string{}
	for addr := range me.workers {
		_, ok := me.mirrors[addr]
		if ok {
			continue
		}
		cands = append(cands, addr)
	}

	if len(cands) == 0 {
		return ""
	}
	return cands[rand.Intn(len(cands))]
}

// Tries to connect to one extra worker.  Must already hold mutex.
func (me *mirrorConnections) tryConnect() {
	// We want to max out capacity of each worker, as that helps
	// with cache hit rates on the worker.
	wanted := me.wantedMaxJobs - me.maxJobs()
	if wanted <= 0 {
		return
	}

	for {
		addr := me.idleWorkerAddress()
		if addr == "" {
			break
		}
		me.Mutex.Unlock()
		log.Printf("Creating mirror on %v, requesting %d jobs", addr, wanted)
		mc, err := me.master.createMirror(addr, wanted)
		me.Mutex.Lock()
		if err != nil {
			delete(me.workers, addr)
			log.Println("nonfatal error creating mirror:", err)
		} else {
			// This could happen in the unlikely event of
			// the workers having more capacity than our
			// parallelism.
			if _, ok := me.mirrors[addr]; ok {
				log.Panicf("already have this mirror: %v", addr)
			}
			mc.workerAddr = addr
			me.mirrors[addr] = mc
			me.master.fileServer.attributes.AddClient(mc)
		}
	}
}
<commit_msg>Run jobs on least-loaded mirror.<commit_after>package termite

import (
	"errors"
	"fmt"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/cba"
	"github.com/hanwen/termite/stats"
	"log"
	"math/rand"
	"net"
	"net/rpc"
	"strings"
	"sync"
	"time"
)

type mirrorConnection struct {
	workerAddr string // key in map.
	rpcClient  *rpc.Client
	connection net.Conn

	// For serving the Fileserver.
	reverseConnection net.Conn

	// Protected by mirrorConnections.Mutex.
	maxJobs       int
	availableJobs int

	master        *Master
	fileSetWaiter *attr.FileSetWaiter
}

func (me *mirrorConnection) Id() string {
	return me.workerAddr
}

func (me *mirrorConnection) innerFetch(start, end int, hash string) ([]byte, error) {
	req := &cba.ContentRequest{
		Hash:  hash,
		Start: start,
		End:   end,
	}
	rep := &cba.ContentResponse{}
	err := me.rpcClient.Call("Mirror.FileContent", req, rep)
	return rep.Chunk, err
}

func (me *mirrorConnection) replay(fset attr.FileSet) error {
	// Must get data before we modify the file-system, so we don't
	// leave the FS in a half-finished state.
	for _, info := range fset.Files {
		if info.Hash != "" && !me.master.cache.HasHash(info.Hash) {
			saved, err := me.master.cache.Fetch(
				func(start, end int) ([]byte, error) {
					return me.innerFetch(start, end, info.Hash)
				})
			if err == nil && saved != info.Hash {
				log.Fatalf("mirrorConnection.replay: fetch corruption got %x want %x", saved, info.Hash)
			}
			if err != nil {
				return err
			}
		}
	}
	me.master.replay(fset)
	return nil
}

func (me *mirrorConnection) Send(files []*attr.FileAttr) error {
	req := UpdateRequest{
		Files: files,
	}
	rep := UpdateResponse{}
	err := me.rpcClient.Call("Mirror.Update", &req, &rep)
	if err != nil {
		log.Println("Mirror.Update failure", err)
		return err
	}
	log.Printf("Sent pending changes to %s", me.workerAddr)
	return nil
}

// mirrorConnection manages connections from the master to the mirrors
// on the workers.
type mirrorConnections struct {
	master      *Master
	coordinator string

	keepAlive time.Duration

	wantedMaxJobs int

	stats *stats.ServerStats

	// Protects all of the below.
	sync.Mutex
	workers        map[string]bool
	mirrors        map[string]*mirrorConnection
	lastActionTime time.Time
}

func (me *mirrorConnections) fetchWorkers() (newMap map[string]bool) {
	newMap = map[string]bool{}
	client, err := rpc.DialHTTP("tcp", me.coordinator)
	if err != nil {
		log.Println("fetchWorkers: dialing coordinator:", err)
		return newMap
	}
	defer client.Close()
	req := 0
	rep := Registered{}
	err = client.Call("Coordinator.List", &req, &rep)
	if err != nil {
		log.Println("coordinator rpc error:", err)
		return newMap
	}

	for _, v := range rep.Registrations {
		newMap[v.Address] = true
	}
	if len(newMap) == 0 {
		log.Println("coordinator has no workers for us.")
	}
	return newMap
}

func (me *mirrorConnections) refreshWorkers() {
	newWorkers := me.fetchWorkers()
	if len(newWorkers) > 0 {
		me.Mutex.Lock()
		defer me.Mutex.Unlock()
		me.workers = newWorkers
	}
}

func newMirrorConnections(m *Master, coordinator string, maxJobs int) *mirrorConnections {
	me := &mirrorConnections{
		master:        m,
		wantedMaxJobs: maxJobs,
		workers:       make(map[string]bool),
		mirrors:       make(map[string]*mirrorConnection),
		coordinator:   coordinator,
		keepAlive:     time.Minute,
	}
	me.refreshStats()
	return me
}

func (me *mirrorConnections) refreshStats() {
	me.stats = stats.NewServerStats()
	me.stats.PhaseOrder = []string{"run", "send", "remote", "filewait"}
}

func (me *mirrorConnections) periodicHouseholding() {
	me.refreshWorkers()
	me.maybeDropConnections()
}

// Must be called with lock held.
func (me *mirrorConnections) availableJobs() int {
	a := 0
	for _, mc := range me.mirrors {
		if mc.availableJobs > 0 {
			a += mc.availableJobs
		}
	}
	return a
}

// Must be called with lock held.
func (me *mirrorConnections) maxJobs() int {
	a := 0
	for _, mc := range me.mirrors {
		a += mc.maxJobs
	}
	return a
}

func (me *mirrorConnections) maybeDropConnections() {
	me.Mutex.Lock()
	defer me.Mutex.Unlock()

	// Already dropped everything.
	if len(me.mirrors) == 0 {
		return
	}

	// Something is running.
	if me.availableJobs() < me.maxJobs() {
		return
	}

	if me.lastActionTime.Add(me.keepAlive).After(time.Now()) {
		return
	}

	log.Println("master inactive too long. Dropping connections.")
	me.dropConnections()
}

func (me *mirrorConnections) dropConnections() {
	for _, mc := range me.mirrors {
		mc.rpcClient.Close()
		mc.connection.Close()
		mc.reverseConnection.Close()
		me.master.attributes.RmClient(mc)
	}
	me.mirrors = make(map[string]*mirrorConnection)
	me.refreshStats()
}

// Gets a mirrorConnection to run on.  Will block if none available
func (me *mirrorConnections) find(name string) (*mirrorConnection, error) {
	me.Mutex.Lock()
	defer me.Mutex.Unlock()

	var found *mirrorConnection
	for nm, v := range me.mirrors {
		if strings.Contains(nm, name) {
			found = v
			break
		}
	}
	if found == nil {
		return nil, fmt.Errorf("No worker with name: %q. Have %v", name, me.mirrors)
	}
	found.availableJobs--
	return found, nil
}

func (me *mirrorConnections) pick() (*mirrorConnection, error) {
	me.Mutex.Lock()
	defer me.Mutex.Unlock()

	if me.availableJobs() <= 0 {
		if len(me.workers) == 0 {
			me.workers = me.fetchWorkers()
		}
		me.tryConnect()

		if me.maxJobs() == 0 {
			// Didn't connect to anything.  Should
			// probably direct the wrapper to compile
			// locally.
			return nil, errors.New("No workers found at all.")
		}
	}

	maxAvail := -1e9
	var maxAvailMirror *mirrorConnection
	for _, v := range me.mirrors {
		if v.availableJobs > 0 {
			v.availableJobs--
			return v, nil
		}
		l := float64(v.availableJobs) / float64(v.maxJobs)
		if l > maxAvail {
			maxAvailMirror = v
			maxAvail = l
		}
	}

	maxAvailMirror.availableJobs--
	return maxAvailMirror, nil
}

func (me *mirrorConnections) drop(mc *mirrorConnection, err error) {
	me.master.fileServer.attributes.RmClient(mc)

	me.Mutex.Lock()
	defer me.Mutex.Unlock()
	log.Printf("Dropping mirror %s. Reason: %s", mc.workerAddr, err)
	mc.connection.Close()
	mc.reverseConnection.Close()
	delete(me.mirrors, mc.workerAddr)
	delete(me.workers, mc.workerAddr)
}

func (me *mirrorConnections) jobDone(mc *mirrorConnection) {
	me.Mutex.Lock()
	defer me.Mutex.Unlock()

	me.lastActionTime = time.Now()
	mc.availableJobs++
}

func (me *mirrorConnections) idleWorkerAddress() string {
	cands := []string{}
	for addr := range me.workers {
		_, ok := me.mirrors[addr]
		if ok {
			continue
		}
		cands = append(cands, addr)
	}

	if len(cands) == 0 {
		return ""
	}
	return cands[rand.Intn(len(cands))]
}

// Tries to connect to one extra worker.  Must already hold mutex.
func (me *mirrorConnections) tryConnect() {
	// We want to max out capacity of each worker, as that helps
	// with cache hit rates on the worker.
	wanted := me.wantedMaxJobs - me.maxJobs()
	if wanted <= 0 {
		return
	}

	for {
		addr := me.idleWorkerAddress()
		if addr == "" {
			break
		}
		me.Mutex.Unlock()
		log.Printf("Creating mirror on %v, requesting %d jobs", addr, wanted)
		mc, err := me.master.createMirror(addr, wanted)
		me.Mutex.Lock()
		if err != nil {
			delete(me.workers, addr)
			log.Println("nonfatal error creating mirror:", err)
		} else {
			// This could happen in the unlikely event of
			// the workers having more capacity than our
			// parallelism.
			if _, ok := me.mirrors[addr]; ok {
				log.Panicf("already have this mirror: %v", addr)
			}
			mc.workerAddr = addr
			me.mirrors[addr] = mc
			me.master.fileServer.attributes.AddClient(mc)
		}
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type ServerOptions struct {
	cpuprofile *string
	v          VolumeServerOptions
}

var (
	serverOptions    ServerOptions
	masterOptions    MasterOptions
	filerOptions     FilerOptions
	s3Options        S3Options
	msgBrokerOptions MessageBrokerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -dir=/tmp -volume.max=5 -ip=server_name",
	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers acts exactly the same as starting them separately.
  So other volume servers can connect to this master server also.

  Optionally, a filer server can be started.
  Also optionally, a S3 gateway can be started.

  `,
}

var (
	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption     = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")

	// pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	isStartingFiler     = cmdServer.Flag.Bool("filer", false, "whether to start filer")
	isStartingS3        = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
	isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")

	serverWhiteList []string

	False = false
)

func init() {
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")

	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")

	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")
	filerOptions.metricsHttpPort = cmdServer.Flag.Int("filer.metricsPort", 0, "Prometheus metrics listen port")

	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 1024, "limit file size to avoid out of memory")
	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
	serverOptions.v.metricsHttpPort = cmdServer.Flag.Int("volume.metricsPort", 0, "Prometheus metrics listen port")

	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
	s3Options.metricsHttpPort = cmdServer.Flag.Int("s3.metricsPort", 0, "Prometheus metrics listen port")

	msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")

}

func runServer(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	util.LoadConfiguration("master", false)

	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *isStartingS3 {
		*isStartingFiler = true
	}
	if *isStartingMsgBroker {
		*isStartingFiler = true
	}

	_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
	peers := strings.Join(peerList, ",")
	masterOptions.peers = &peers

	masterOptions.ip = serverIp
	masterOptions.ipBind = serverBindIp
	filerOptions.masters = &peers
	filerOptions.ip = serverIp
	filerOptions.bindIp = serverBindIp
	serverOptions.v.ip = serverIp
	serverOptions.v.bindIp = serverBindIp
	serverOptions.v.masters = &peers
	serverOptions.v.idleConnectionTimeout = serverTimeout
	serverOptions.v.dataCenter = serverDataCenter
	serverOptions.v.rack = serverRack
	msgBrokerOptions.ip = serverIp

	// serverOptions.v.pulseSeconds = pulseSeconds
	// masterOptions.pulseSeconds = pulseSeconds

	masterOptions.whiteList = serverWhiteListOption

	filerOptions.dataCenter = serverDataCenter
	filerOptions.disableHttp = serverDisableHttp
	masterOptions.disableHttp = serverDisableHttp

	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
	s3Options.filer = &filerAddress
	msgBrokerOptions.filer = &filerAddress

	if *filerOptions.defaultReplicaPlacement == "" {
		*filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	folders := strings.Split(*volumeDataFolders, ",")

	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
	}

	if *masterOptions.metaFolder == "" {
		*masterOptions.metaFolder = folders[0]
	}
	if err := util.TestFolderWritable(util.ResolvePath(*masterOptions.metaFolder)); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
	}
	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			time.Sleep(1 * time.Second)

			filerOptions.startFiler()

		}()
	}

	if *isStartingS3 {
		go func() {
			time.Sleep(2 * time.Second)

			s3Options.startS3Server()

		}()
	}

	if *isStartingMsgBroker {
		go func() {
			time.Sleep(2 * time.Second)
			msgBrokerOptions.startQueueServer()
		}()
	}

	// start volume server
	{
		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)

	}

	startMaster(masterOptions, serverWhiteList)

	return true
}
<commit_msg>consolidate to one metricsPort in "weed server" mode<commit_after>package command

import (
	"fmt"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type ServerOptions struct {
	cpuprofile *string
	v          VolumeServerOptions
}

var (
	serverOptions    ServerOptions
	masterOptions    MasterOptions
	filerOptions     FilerOptions
	s3Options        S3Options
	msgBrokerOptions MessageBrokerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -dir=/tmp -volume.max=5 -ip=server_name",
	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers acts exactly the same as starting them separately.
  So other volume servers can connect to this master server also.

  Optionally, a filer server can be started.
  Also optionally, a S3 gateway can be started.

  `,
}

var (
	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption     = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")
	serverMetricsHttpPort     = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")

	// pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	isStartingFiler     = cmdServer.Flag.Bool("filer", false, "whether to start filer")
	isStartingS3        = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
	isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")

	serverWhiteList []string

	False = false
)

func init() {
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")

	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")

	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")

	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 1024, "limit file size to avoid out of memory")
	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")

	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")

	msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")

}

func runServer(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	util.LoadConfiguration("master", false)

	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *isStartingS3 {
		*isStartingFiler = true
	}
	if *isStartingMsgBroker {
		*isStartingFiler = true
	}

	_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
	peers := strings.Join(peerList, ",")
	masterOptions.peers = &peers

	// ip address
	masterOptions.ip = serverIp
	masterOptions.ipBind = serverBindIp
	filerOptions.masters = &peers
	filerOptions.ip = serverIp
	filerOptions.bindIp = serverBindIp
	serverOptions.v.ip = serverIp
	serverOptions.v.bindIp = serverBindIp
	serverOptions.v.masters = &peers
	serverOptions.v.idleConnectionTimeout = serverTimeout
	serverOptions.v.dataCenter = serverDataCenter
	serverOptions.v.rack = serverRack
	msgBrokerOptions.ip = serverIp

	// metrics port
	filerOptions.metricsHttpPort = serverMetricsHttpPort
	serverOptions.v.metricsHttpPort = serverMetricsHttpPort
	s3Options.metricsHttpPort = serverMetricsHttpPort

	// serverOptions.v.pulseSeconds = pulseSeconds
	// masterOptions.pulseSeconds = pulseSeconds

	masterOptions.whiteList = serverWhiteListOption

	filerOptions.dataCenter = serverDataCenter
	filerOptions.disableHttp = serverDisableHttp
	masterOptions.disableHttp = serverDisableHttp

	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
	s3Options.filer = &filerAddress
	msgBrokerOptions.filer = &filerAddress

	if *filerOptions.defaultReplicaPlacement == "" {
		*filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	folders := strings.Split(*volumeDataFolders, ",")

	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
	}

	if *masterOptions.metaFolder == "" {
		*masterOptions.metaFolder = folders[0]
	}
	if err := util.TestFolderWritable(util.ResolvePath(*masterOptions.metaFolder)); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
	}
	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			time.Sleep(1 * time.Second)

			filerOptions.startFiler()

		}()
	}

	if *isStartingS3 {
		go func() {
			time.Sleep(2 * time.Second)

			s3Options.startS3Server()

		}()
	}

	if *isStartingMsgBroker {
		go func() {
			time.Sleep(2 * time.Second)
			msgBrokerOptions.startQueueServer()
		}()
	}

	// start volume server
	{
		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)

	}

	startMaster(masterOptions, serverWhiteList)

	return true
}
<|endoftext|>


<commit_before>package util

const (
	VERSION = "0.71 beta"
)
<commit_msg>change to 0.71 version<commit_after>package util

const (
	VERSION = "0.71"
)
<|endoftext|>


<commit_before><commit_msg>replicant-call benchmark<commit_after><|endoftext|>


<commit_before>package etcd

import (
	"log"
	"time"

	"strings"

	"github.com/coreos/etcd/client"
	"github.com/fsouza/go-dockerclient/external/golang.org/x/net/context"
	"github.com/vsco/dcdr/cli/api/stores"
	"github.com/vsco/dcdr/config"
)

type ETCDStore struct {
	kv         client.KeysAPI
	ctx        context.Context
	getOpts    *client.GetOptions
	setOpts    *client.SetOptions
	deleteOpts *client.DeleteOptions
}

var DefaultEndpoints = []string{"http://127.0.0.1:2379"}

func DefaultETCDlStore(cfg *config.Config) (client.KeysAPI, error) {
	endpoints := DefaultEndpoints

	if len(cfg.Etcd.Endpoints) > 0 {
		endpoints = cfg.Etcd.Endpoints
	}

	ecfg := client.Config{
		Endpoints:               endpoints,
		Transport:               client.DefaultTransport,
		HeaderTimeoutPerRequest: time.Second,
	}

	c, err := client.New(ecfg)

	if err != nil {
		log.Fatal(err)
	}

	return client.NewKeysAPI(c), nil
}

func New(cfg *config.Config) stores.StoreIFace {
	kv, _ := DefaultETCDlStore(cfg)

	es := &ETCDStore{
		kv:  kv,
		ctx: context.Background(),
	}

	return es
}

func (s *ETCDStore) Get(key string) (*stores.KVByte, error) {
	resp, err := s.kv.Get(s.ctx, key, s.getOpts)

	if err != nil {
		switch err.(client.Error).Code {
		case client.ErrorCodeKeyNotFound:
			return nil, nil
		default:
			return nil, err
		}
	}

	return toKVByte(resp.Node), nil
}

func (s *ETCDStore) Set(key string, bts []byte) error {
	_, err := s.kv.Set(s.ctx, key, string(bts), s.setOpts)

	return err
}

func (s *ETCDStore) Delete(key string) error {
	_, err := s.kv.Delete(s.ctx, key, s.deleteOpts)

	switch err.(client.Error).Code {
	case client.ErrorCodeKeyNotFound:
		return nil
	default:
		return err
	}
}

func (s *ETCDStore) List(prefix string) (stores.KVBytes, error) {
	opts := &client.GetOptions{
		Recursive: true,
		Sort:      true,
		Quorum:    true,
	}

	resp, err := s.kv.Get(s.ctx, prefix, opts)

	if err != nil {
		switch err.(client.Error).Code {
		case client.ErrorCodeKeyNotFound:
			return nil, nil
		default:
			return nil, err
		}
	}

	kvbs := FlattenToKVBytes(resp.Node, make(stores.KVBytes, 0))

	return kvbs, nil
}

func FlattenToKVBytes(n *client.Node, nodes stores.KVBytes) stores.KVBytes {
	if n.Dir {
		for _, nd := range n.Nodes {
			nodes = FlattenToKVBytes(nd, nodes)
		}
	} else {
		nodes = append(nodes, toKVByte(n))
	}

	return nodes
}

func toKVByte(n *client.Node) *stores.KVByte {
	return &stores.KVByte{
		// remove leading slash as it adds an empty
		// hash entry when exploded to JSON.
		Key:   strings.TrimPrefix(n.Key, "/"),
		Bytes: []byte(n.Value),
	}
}
<commit_msg>Handle etcd cluster errors gracefully<commit_after>package etcd

import (
	"log"
	"time"

	"strings"

	"github.com/coreos/etcd/client"
	"github.com/fsouza/go-dockerclient/external/golang.org/x/net/context"
	"github.com/vsco/dcdr/cli/api/stores"
	"github.com/vsco/dcdr/config"
)

type ETCDStore struct {
	kv         client.KeysAPI
	ctx        context.Context
	getOpts    *client.GetOptions
	setOpts    *client.SetOptions
	deleteOpts *client.DeleteOptions
}

var DefaultEndpoints = []string{"http://127.0.0.1:2379"}

func DefaultETCDlStore(cfg *config.Config) (client.KeysAPI, error) {
	endpoints := DefaultEndpoints

	if len(cfg.Etcd.Endpoints) > 0 {
		endpoints = cfg.Etcd.Endpoints
	}

	ecfg := client.Config{
		Endpoints:               endpoints,
		Transport:               client.DefaultTransport,
		HeaderTimeoutPerRequest: time.Second,
	}

	c, err := client.New(ecfg)

	if err != nil {
		log.Fatal(err)
	}

	return client.NewKeysAPI(c), nil
}

func New(cfg *config.Config) stores.StoreIFace {
	kv, _ := DefaultETCDlStore(cfg)

	es := &ETCDStore{
		kv:  kv,
		ctx: context.Background(),
	}

	return es
}

func (s *ETCDStore) Get(key string) (*stores.KVByte, error) {
	resp, err := s.kv.Get(s.ctx, key, s.getOpts)

	if err != nil {
		return nil, etcdError(err)
	}

	return toKVByte(resp.Node), nil
}

func (s *ETCDStore) Set(key string, bts []byte) error {
	_, err := s.kv.Set(s.ctx, key, string(bts), s.setOpts)

	return err
}

func (s *ETCDStore) Delete(key string) error {
	_, err := s.kv.Delete(s.ctx, key, s.deleteOpts)

	return etcdError(err)
}

func (s *ETCDStore) List(prefix string) (stores.KVBytes, error) {
	opts := &client.GetOptions{
		Recursive: true,
		Sort:      true,
		Quorum:    true,
	}

	resp, err := s.kv.Get(s.ctx, prefix, opts)

	if err != nil {
		return nil, etcdError(err)
	}

	kvbs := FlattenToKVBytes(resp.Node, make(stores.KVBytes, 0))

	return kvbs, nil
}

func FlattenToKVBytes(n *client.Node, nodes stores.KVBytes) stores.KVBytes {
	if n.Dir {
		for _, nd := range n.Nodes {
			nodes = FlattenToKVBytes(nd, nodes)
		}
	} else {
		nodes = append(nodes, toKVByte(n))
	}

	return nodes
}

func toKVByte(n *client.Node) *stores.KVByte {
	return &stores.KVByte{
		// remove leading slash as it adds an empty
		// hash entry when exploded to JSON.
		Key:   strings.TrimPrefix(n.Key, "/"),
		Bytes: []byte(n.Value),
	}
}

func etcdError(err error) error {
	switch err.(type) {
	case client.Error:
		if err.(client.Error).Code == client.ErrorCodeKeyNotFound {
			return nil
		}

		return err
	default:
		return err
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"sync"

	"git.mayflower.de/vaillant-team/docker-ls/cli/docker-ls/response"
	"git.mayflower.de/vaillant-team/docker-ls/lib"
)

type repositoriesCmd struct {
	flags *flag.FlagSet
	cfg   *Config
}

func (r *repositoriesCmd) execute(argv []string) (err error) {
	libCfg := lib.NewConfig()
	libCfg.BindToFlags(r.flags)

	r.cfg = newConfig()
	r.cfg.bindToFlags(r.flags, OPTIONS_FULL)

	err = r.flags.Parse(argv)
	if err != nil {
		return
	}

	if len(r.flags.Args()) != 0 {
		r.flags.Usage()
		os.Exit(1)
	}

	registryApi := lib.NewRegistryApi(libCfg)
	var resp sortable

	switch {
	case r.cfg.recursionLevel >= 1:
		resp, err = r.listLevel1(registryApi)

	case r.cfg.recursionLevel == 0:
		resp, err = r.listLevel0(registryApi)
	}

	if err != nil {
		return
	}

	resp.Sort()
	err = serializeToStdout(resp, r.cfg)

	if r.cfg.statistics {
		dumpStatistics(registryApi.GetStatistics())
	}

	return
}

func (r *repositoriesCmd) listLevel0(api lib.RegistryApi) (resp *response.RepositoriesL0, err error) {
	progress := NewProgressIndicator(r.cfg)
	progress.Start("requesting list")

	result := api.ListRepositories()
	resp = response.NewRepositoriesL0()

	progress.Progress()

	for repository := range result.Repositories() {
		resp.AddRepository(repository)
	}

	err = result.LastError()

	progress.Finish("done")
	return
}

func (r *repositoriesCmd) listLevel1(api lib.RegistryApi) (resp *response.RepositoriesL1, err error) {
	progress := NewProgressIndicator(r.cfg)
	progress.Start("requesting list")

	repositoriesResult := api.ListRepositories()
	resp = response.NewRepositoriesL1()
	progress.Progress()

	errors := make(chan error)

	go func() {
		var wait sync.WaitGroup

		for repository := range repositoriesResult.Repositories() {
			wait.Add(1)

			go func(repository lib.Repository) {
				tagsResult := api.ListTags(repository.Name())
				progress.Progress()
				tagsL0 := response.NewTagsL0(repository.Name())

				for tag := range tagsResult.Tags() {
					tagsL0.AddTag(tag)
				}

				resp.AddTags(tagsL0)

				if err := tagsResult.LastError(); err != nil {
					errors <- err
				}

				wait.Done()
			}(repository)
		}

		if err := repositoriesResult.LastError(); err != nil {
			errors <- err
		}

		wait.Wait()

		close(errors)
	}()

	for nextError := range errors {
		if err == nil {
			err = nextError
		}
	}

	progress.Finish("done")
	return
}

func newRepositoriesCmd(name string) (cmd *repositoriesCmd) {
	cmd = &repositoriesCmd{
		flags: flag.NewFlagSet(name, flag.ExitOnError),
	}

	cmd.flags.Usage = commandUsage(name, "", "List all repositories.", cmd.flags)

	return
}
<commit_msg>More accurate progress reporting für repository listings.<commit_after>package main

import (
	"flag"
	"os"
	"sync"

	"git.mayflower.de/vaillant-team/docker-ls/cli/docker-ls/response"
	"git.mayflower.de/vaillant-team/docker-ls/lib"
)

type repositoriesCmd struct {
	flags *flag.FlagSet
	cfg   *Config
}

func (r *repositoriesCmd) execute(argv []string) (err error) {
	libCfg := lib.NewConfig()
	libCfg.BindToFlags(r.flags)

	r.cfg = newConfig()
	r.cfg.bindToFlags(r.flags, OPTIONS_FULL)

	err = r.flags.Parse(argv)
	if err != nil {
		return
	}

	if len(r.flags.Args()) != 0 {
		r.flags.Usage()
		os.Exit(1)
	}

	registryApi := lib.NewRegistryApi(libCfg)
	var resp sortable

	switch {
	case r.cfg.recursionLevel >= 1:
		resp, err = r.listLevel1(registryApi)

	case r.cfg.recursionLevel == 0:
		resp, err = r.listLevel0(registryApi)
	}

	if err != nil {
		return
	}

	resp.Sort()
	err = serializeToStdout(resp, r.cfg)

	if r.cfg.statistics {
		dumpStatistics(registryApi.GetStatistics())
	}

	return
}

func (r *repositoriesCmd) listLevel0(api lib.RegistryApi) (resp *response.RepositoriesL0, err error) {
	progress := NewProgressIndicator(r.cfg)
	progress.Start("requesting list")

	result := api.ListRepositories()
	resp = response.NewRepositoriesL0()

	progress.Progress()

	for repository := range result.Repositories() {
		resp.AddRepository(repository)
	}

	err = result.LastError()

	progress.Finish("done")
	return
}

func (r *repositoriesCmd) listLevel1(api lib.RegistryApi) (resp *response.RepositoriesL1, err error) {
	progress := NewProgressIndicator(r.cfg)
	progress.Start("requesting list")

	repositoriesResult := api.ListRepositories()
	resp = response.NewRepositoriesL1()
	progress.Progress()

	errors := make(chan error)

	go func() {
		var wait sync.WaitGroup

		for repository := range repositoriesResult.Repositories() {
			wait.Add(1)

			go func(repository lib.Repository) {
				tagsResult := api.ListTags(repository.Name())
				tagsL0 := response.NewTagsL0(repository.Name())

				for tag := range tagsResult.Tags() {
					tagsL0.AddTag(tag)
				}

				progress.Progress()
				resp.AddTags(tagsL0)

				if err := tagsResult.LastError(); err != nil {
					errors <- err
				}

				wait.Done()
			}(repository)
		}

		if err := repositoriesResult.LastError(); err != nil {
			errors <- err
		}

		wait.Wait()

		close(errors)
	}()

	for nextError := range errors {
		if err == nil {
			err = nextError
		}
	}

	progress.Finish("done")
	return
}

func newRepositoriesCmd(name string) (cmd *repositoriesCmd) {
	cmd = &repositoriesCmd{
		flags: flag.NewFlagSet(name, flag.ExitOnError),
	}

	cmd.flags.Usage = commandUsage(name, "", "List all repositories.", cmd.flags)

	return
}
<|endoftext|>


<commit_before>package tlcore

import (
	"os"
	"testing"
)
import "bytes"

/*
This file contains all core testing functions.

*/

const numChunks = 8

//Test a simple put & get
func TestSimple(t *testing.T) {
	m := NewMap("testdb/", numChunks)
	defer os.RemoveAll("testdb/")
	defer m.Close()

	m.Set([]byte("k1"), []byte("v1"))
	rval, _ := m.Get([]byte("k1"))
	if !bytes.Equal([]byte("v1"), rval) {
		t.Fatal("Error: value mismatch")
	}
}

//Test a simple put & get (after a map close())
func TestSimpleRestore(t *testing.T) {
	defer os.RemoveAll("testdb/")
	{
		m := NewMap("testdb/", numChunks)
		m.Set([]byte("k1"), []byte("v1"))
		m.Close()
	}
	{
		m := OpenMap("testdb/")
		rval, _ := m.Get([]byte("k1"))
		if !bytes.Equal([]byte("v1"), rval) {
			t.Fatal("Error: value mismatch")
		}
		m.Close()
	}
}

/*
//Test lots of simple put & get
func TestSimpleLots(t *testing.T) {
	defer os.RemoveAll("testdb/")
	defer os.Chdir("../")
	k := make([]byte, 4)
	v := make([]byte, 4)
	{
		db := Create("testdb")
		m1, _ := db.AllocMap("mapa1")

		for i := 0; i < 128*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			m1.Put(k, v)
		}

		for i := 0; i < 128*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			rval, _ := m1.Get(k)
			if !bytes.Equal(v, rval) {
				t.Fatal("Err 1: mismatch")
			}
		}

		db.Close()
	}
	{
		db := Open("testdb")
		m1 := db.mapsByName["mapa1"]
		for i := 0; i < 128*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			rval, _ := m1.Get(k)
			if !bytes.Equal(v, rval) {
				t.Fatal("Err 2: mismatch")
			}
		}
		db.Close()
	}
}

//Test lots of simple put & get, in a parallel way, test multi thread safety
func TestParSimpleLots(t *testing.T) {
	defer os.RemoveAll("testdb/")
	defer os.Chdir("../")
	k := make([]byte, 4)
	v := make([]byte, 4)
	{
		db := Create("testdb")
		m1, _ := db.AllocMap("mapa1")

		for tid := 0; tid < 27; tid++ {
			for i := 0; i < 1024; i++ {
				binary.LittleEndian.PutUint32(k, uint32(i*27+tid))
				binary.LittleEndian.PutUint32(v, uint32(i*27+tid))
				m1.Put(k, v)
			}
		}

		for i := 0; i < 27*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			rval, _ := m1.Get(k)
			if !bytes.Equal(v, rval) {
				t.Fatal("Err 1: mismatch")
			}
		}

		db.Close()
	}
	{
		db := Open("testdb")
		m1 := db.mapsByName["mapa1"]
		for i := 0; i < 27*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			rval, _ := m1.Get(k)
			if !bytes.Equal(v, rval) {
				t.Fatal("Err 2: mismatch")
			}
		}
		db.Close()
	}
}

//Test simple set
//Test simple del

//Test complex get, put, set, del mix

//Common functions test
func TestCmplx1(t *testing.T) {
	coreTest(100000, 11, 129, 64)
}

//Low key size: test delete operation
func TestCmplx2(t *testing.T) {
	coreTest(100000, 2, 129, 64)
}

//Large key size
func TestCmplx3(t *testing.T) {
	coreTest(10000, 130, 129, 64)
}

//Large value size
func TestCmplx4(t *testing.T) {
	coreTest(10000, 11, 555, 64)
}

//Test low value size
func TestCmplx5(t *testing.T) {
	coreTest(10000, 11, 2, 64)
}

func operate() {

}
func goOperate() {

}
func dbOperate() {

}
func checkDB() {

}

func coreTest(numOperations, maxKeySize, maxValueSize, threads int) {
	//Operate on built-in map

	goMap := make(map[string][]byte)
	var goDeletes []([]byte)
	for core := 0; core < threads; core++ {
		r := rand.New(rand.NewSource(int64(core)))
		base := make([]byte, 4)
		base2 := make([]byte, 4)
		for i := 0; i < numOperations; i++ {
			opType := 1 + r.Intn(3)
			opKeySize := r.Intn(maxKeySize-1) + 1
			opValueSize := r.Intn(maxValueSize-1) + 1
			binary.LittleEndian.PutUint32(base, uint32(r.Int31()*64)+uint32(core))
			binary.LittleEndian.PutUint32(base2, uint32(i*64+core))
			key := bytes.Repeat([]byte(base), opKeySize)
			value := bytes.Repeat([]byte(base), opValueSize)
			//fmt.Println("gomap", opType, key, value)
			switch opType {
			case OpPut:
				if _, ok := goMap[string(key)]; !ok {
					goMap[string(key)] = value
				}
			case OpDel:
				if _, ok := goMap[string(key)]; ok {
					delete(goMap, string(key))
					goDeletes = append(goDeletes, key)
				}
			case OpSet:
				if _, ok := goMap[string(key)]; ok {
					goMap[string(key)] = value
				}
			}
		}
	}
	//Operate on DB
	db := Create("testdb2")
	defer os.RemoveAll("testdb2/")
	defer os.Chdir("../")

	m1, _ := db.AllocMap("mapa1")

	var w sync.WaitGroup
	w.Add(threads)
	for core := 0; core < threads; core++ {
		go func(core int) {
			r := rand.New(rand.NewSource(int64(core)))
			base := make([]byte, 4)
			base2 := make([]byte, 4)
			for i := 0; i < numOperations; i++ {
				opType := 1 + r.Intn(3)
				opKeySize := r.Intn(maxKeySize-1) + 1
				opValueSize := r.Intn(maxValueSize-1) + 1
				binary.LittleEndian.PutUint32(base, uint32(r.Int31()*64)+uint32(core))
				binary.LittleEndian.PutUint32(base2, uint32(i*64+core))
				key := bytes.Repeat([]byte(base), opKeySize)[0:opKeySize]
				value := bytes.Repeat([]byte(base2), opValueSize)[0:opValueSize]
				//fmt.Println("db   ", opType, key, value)
				switch opType {
				case OpPut:
					m1.Put(key, value)
				case OpDel:
					m1.Delete(key)
				case OpSet:
					m1.Set(key, value)
				}
			}
			w.Done()
		}(core)
	}
	w.Wait()
	//Check map is in DB
	for key, value := range goMap {
		rval, err := m1.Get([]byte(key))
		//fmt.Println([]byte(key), value)
		if err != nil {
			panic(err)
		}
		if !bytes.Equal(rval, value) {
			panic(1)
		}
	}

	//Check deleteds aren't in DB
	fmt.Println("Tested deletes:", len(goDeletes))
	for i := 0; i < len(goDeletes); i++ {
		key := goDeletes[i]
		_, err := m1.Get([]byte(key))
		if err == nil {
			panic(2)
		}
	}
	//Close DB
	db.Close()

	{
		//Restore DB
		db := Open("testdb2")
		m1 := db.mapsByName["mapa1"]
		//Check again
		//Check map is in DB
		for key, value := range goMap {
			rval, err := m1.Get([]byte(key))
			//fmt.Println([]byte(key), value)
			if err != nil {
				panic(err)
			}
			if !bytes.Equal(rval, value) {
				panic(1)
			}
		}

		//Check deleteds aren't in DB
		fmt.Println("Tested deletes:", len(goDeletes))
		for i := 0; i < len(goDeletes); i++ {
			key := goDeletes[i]
			_, err := m1.Get([]byte(key))
			if err == nil {
				panic(2)
			}
		}
	}
}

//Test parallel complex get, put, set, del mix

//Tests operational limits: size

//Test sync/nosync file

//Bench with diferents sizes

//Bench lots of gets
func BenchmarkGet(b *testing.B) {
	defer os.RemoveAll("benchdb/")
	defer os.Chdir("../")
	if testing.Verbose() {
		fmt.Println("\tInserting", b.N, "keys...")
	}
	db := Create("benchdb/")
	m, _ := db.AllocMap("mapA")
	key := make([]byte, 4)
	lenValue := 100
	value := bytes.Repeat([]byte("X"), lenValue)
	for i := 0; i < b.N/32+1; i++ {
		binary.LittleEndian.PutUint32(key, uint32(3*i))
		binary.LittleEndian.PutUint32(value, uint32(3*i))
		err := m.Put(key, value)
		if err != nil {
			panic(err)
		}
	}
	gid := uint64(0)
	fmt.Println("get...")
	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		sum := 0
		key := make([]byte, 4)
		id := uint32(atomic.AddUint64(&gid, 1))
		fmt.Println(id)
		r := rand.New(rand.NewSource(int64(id)))
		for i := 0; pb.Next(); i++ {
			binary.LittleEndian.PutUint32(key, uint32(3*r.Intn(b.N/32+1)))
			v, err := m.Get(key)
			sum += int(v[len(v)-1])
			if err != nil || !bytes.Equal(v, v) {
				b.Fatal("Key not present", key, id, i, v, sum)
			}
		}
	})
	b.StopTimer()
	db.Close()
}

//Bench lots of gets, in a parallel way

//Bench lots of puts

//Bench lots of puts, in a parallel way
*/
<commit_msg>core - TestSimpleLots recuperado<commit_after>package tlcore

import (
	"encoding/binary"
	"os"
	"testing"
)
import "bytes"

/*
This file contains all core testing functions.

*/

const numChunks = 8

//Test a simple put & get
func TestSimple(t *testing.T) {
	m := NewMap("testdb/", numChunks)
	defer os.RemoveAll("testdb/")
	defer m.Close()

	m.Set([]byte("k1"), []byte("v1"))
	rval, _ := m.Get([]byte("k1"))
	if !bytes.Equal([]byte("v1"), rval) {
		t.Fatal("Error: value mismatch")
	}
}

//Test a simple put & get (after a map close())
func TestSimpleRestore(t *testing.T) {
	defer os.RemoveAll("testdb/")
	{
		m := NewMap("testdb/", numChunks)
		m.Set([]byte("k1"), []byte("v1"))
		m.Close()
	}
	{
		m := OpenMap("testdb/")
		rval, _ := m.Get([]byte("k1"))
		if !bytes.Equal([]byte("v1"), rval) {
			t.Fatal("Error: value mismatch")
		}
		m.Close()
	}
}

//Test lots of simple put & get before and after closing the map
func TestSimpleLots(t *testing.T) {
	defer os.RemoveAll("testdb/")
	k := make([]byte, 4)
	v := make([]byte, 4)
	{
		m1 := NewMap("testdb/", numChunks)
		for i := 0; i < 128*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			m1.Set(k, v)
		}
		for i := 0; i < 128*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			rval, _ := m1.Get(k)
			if !bytes.Equal(v, rval) {
				t.Fatal("Error 1: value mismatch")
			}
		}
		m1.Close()
	}
	{
		m2 := OpenMap("testdb/")
		for i := 0; i < 128*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			rval, _ := m2.Get(k)
			if !bytes.Equal(v, rval) {
				t.Fatal("Error 2: value mismatch")
			}
		}
		m2.Close()
	}
}

/*
//Test lots of simple put & get, in a parallel way, test multi thread safety
func TestParSimpleLots(t *testing.T) {
	defer os.RemoveAll("testdb/")
	defer os.Chdir("../")
	k := make([]byte, 4)
	v := make([]byte, 4)
	{
		db := Create("testdb")
		m1, _ := db.AllocMap("mapa1")

		for tid := 0; tid < 27; tid++ {
			for i := 0; i < 1024; i++ {
				binary.LittleEndian.PutUint32(k, uint32(i*27+tid))
				binary.LittleEndian.PutUint32(v, uint32(i*27+tid))
				m1.Put(k, v)
			}
		}

		for i := 0; i < 27*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			rval, _ := m1.Get(k)
			if !bytes.Equal(v, rval) {
				t.Fatal("Err 1: mismatch")
			}
		}

		db.Close()
	}
	{
		db := Open("testdb")
		m1 := db.mapsByName["mapa1"]
		for i := 0; i < 27*1024; i++ {
			binary.LittleEndian.PutUint32(k, uint32(i))
			binary.LittleEndian.PutUint32(v, uint32(i))
			rval, _ := m1.Get(k)
			if !bytes.Equal(v, rval) {
				t.Fatal("Err 2: mismatch")
			}
		}
		db.Close()
	}
}

//Test simple set
//Test simple del

//Test complex get, put, set, del mix

//Common functions test
func TestCmplx1(t *testing.T) {
	coreTest(100000, 11, 129, 64)
}

//Low key size: test delete operation
func TestCmplx2(t *testing.T) {
	coreTest(100000, 2, 129, 64)
}

//Large key size
func TestCmplx3(t *testing.T) {
	coreTest(10000, 130, 129, 64)
}

//Large value size
func TestCmplx4(t *testing.T) {
	coreTest(10000, 11, 555, 64)
}

//Test low value size
func TestCmplx5(t *testing.T) {
	coreTest(10000, 11, 2, 64)
}

func operate() {

}
func goOperate() {

}
func dbOperate() {

}
func checkDB() {

}

func coreTest(numOperations, maxKeySize, maxValueSize, threads int) {
	//Operate on built-in map

	goMap := make(map[string][]byte)
	var goDeletes []([]byte)
	for core := 0; core < threads; core++ {
		r := rand.New(rand.NewSource(int64(core)))
		base := make([]byte, 4)
		base2 := make([]byte, 4)
		for i := 0; i < numOperations; i++ {
			opType := 1 + r.Intn(3)
			opKeySize := r.Intn(maxKeySize-1) + 1
			opValueSize := r.Intn(maxValueSize-1) + 1
			binary.LittleEndian.PutUint32(base, uint32(r.Int31()*64)+uint32(core))
			binary.LittleEndian.PutUint32(base2, uint32(i*64+core))
			key := bytes.Repeat([]byte(base), opKeySize)
			value := bytes.Repeat([]byte(base), opValueSize)
			//fmt.Println("gomap", opType, key, value)
			switch opType {
			case OpPut:
				if _, ok := goMap[string(key)]; !ok {
					goMap[string(key)] = value
				}
			case OpDel:
				if _, ok := goMap[string(key)]; ok {
					delete(goMap, string(key))
					goDeletes = append(goDeletes, key)
				}
			case OpSet:
				if _, ok := goMap[string(key)]; ok {
					goMap[string(key)] = value
				}
			}
		}
	}
	//Operate on DB
	db := Create("testdb2")
	defer os.RemoveAll("testdb2/")
	defer os.Chdir("../")

	m1, _ := db.AllocMap("mapa1")

	var w sync.WaitGroup
	w.Add(threads)
	for core := 0; core < threads; core++ {
		go func(core int) {
			r := rand.New(rand.NewSource(int64(core)))
			base := make([]byte, 4)
			base2 := make([]byte, 4)
			for i := 0; i < numOperations; i++ {
				opType := 1 + r.Intn(3)
				opKeySize := r.Intn(maxKeySize-1) + 1
				opValueSize := r.Intn(maxValueSize-1) + 1
				binary.LittleEndian.PutUint32(base, uint32(r.Int31()*64)+uint32(core))
				binary.LittleEndian.PutUint32(base2, uint32(i*64+core))
				key := bytes.Repeat([]byte(base), opKeySize)[0:opKeySize]
				value := bytes.Repeat([]byte(base2), opValueSize)[0:opValueSize]
				//fmt.Println("db   ", opType, key, value)
				switch opType {
				case OpPut:
					m1.Put(key, value)
				case OpDel:
					m1.Delete(key)
				case OpSet:
					m1.Set(key, value)
				}
			}
			w.Done()
		}(core)
	}
	w.Wait()
	//Check map is in DB
	for key, value := range goMap {
		rval, err := m1.Get([]byte(key))
		//fmt.Println([]byte(key), value)
		if err != nil {
			panic(err)
		}
		if !bytes.Equal(rval, value) {
			panic(1)
		}
	}

	//Check deleteds aren't in DB
	fmt.Println("Tested deletes:", len(goDeletes))
	for i := 0; i < len(goDeletes); i++ {
		key := goDeletes[i]
		_, err := m1.Get([]byte(key))
		if err == nil {
			panic(2)
		}
	}
	//Close DB
	db.Close()

	{
		//Restore DB
		db := Open("testdb2")
		m1 := db.mapsByName["mapa1"]
		//Check again
		//Check map is in DB
		for key, value := range goMap {
			rval, err := m1.Get([]byte(key))
			//fmt.Println([]byte(key), value)
			if err != nil {
				panic(err)
			}
			if !bytes.Equal(rval, value) {
				panic(1)
			}
		}

		//Check deleteds aren't in DB
		fmt.Println("Tested deletes:", len(goDeletes))
		for i := 0; i < len(goDeletes); i++ {
			key := goDeletes[i]
			_, err := m1.Get([]byte(key))
			if err == nil {
				panic(2)
			}
		}
	}
}

//Test parallel complex get, put, set, del mix

//Tests operational limits: size

//Test sync/nosync file

//Bench with diferents sizes

//Bench lots of gets
func BenchmarkGet(b *testing.B) {
	defer os.RemoveAll("benchdb/")
	defer os.Chdir("../")
	if testing.Verbose() {
		fmt.Println("\tInserting", b.N, "keys...")
	}
	db := Create("benchdb/")
	m, _ := db.AllocMap("mapA")
	key := make([]byte, 4)
	lenValue := 100
	value := bytes.Repeat([]byte("X"), lenValue)
	for i := 0; i < b.N/32+1; i++ {
		binary.LittleEndian.PutUint32(key, uint32(3*i))
		binary.LittleEndian.PutUint32(value, uint32(3*i))
		err := m.Put(key, value)
		if err != nil {
			panic(err)
		}
	}
	gid := uint64(0)
	fmt.Println("get...")
	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		sum := 0
		key := make([]byte, 4)
		id := uint32(atomic.AddUint64(&gid, 1))
		fmt.Println(id)
		r := rand.New(rand.NewSource(int64(id)))
		for i := 0; pb.Next(); i++ {
			binary.LittleEndian.PutUint32(key, uint32(3*r.Intn(b.N/32+1)))
			v, err := m.Get(key)
			sum += int(v[len(v)-1])
			if err != nil || !bytes.Equal(v, v) {
				b.Fatal("Key not present", key, id, i, v, sum)
			}
		}
	})
	b.StopTimer()
	db.Close()
}

//Bench lots of gets, in a parallel way

//Bench lots of puts

//Bench lots of puts, in a parallel way
*/
<|endoftext|>


<commit_before>package wikifier
<commit_msg>html{}<commit_after>package wikifier

type htmlBlock struct {
	*parserBlock
}

func newHTMLBlock(name string, b *parserBlock) block {
	return &fmtBlock{parserBlock: b}
}

func (b *htmlBlock) html(page *Page, el element) {
	el.setMeta("noIndent", true)
	el.setMeta("noTags", true)
	for _, item := range b.visiblePosContent() {
		el.add(item.content)
	}
}
<|endoftext|>


<commit_before>package main

// Validates the execution of a process

type ExecutionValidation struct {
	Fatal        bool   // If matched, should we abort the (sequence of) operation(s)?
	MustContain  bool   // Should this be in there?
	OutputStream int    // 1 = standard output, 2 error output
	Text         string // Text to match
}

// Must contain XYZ
func newExecutionValidationStandardOutputMustContain(txt string) *ExecutionValidation {
	return &ExecutionValidation{
		Fatal:        true,
		MustContain:  true,
		Text:         txt,
		OutputStream: 1,
	}
}
<commit_msg>Abstract constructor<commit_after>package main

// Validates the execution of a process

type ExecutionValidation struct {
	Fatal        bool   // If matched, should we abort the (sequence of) operation(s)?
	MustContain  bool   // Should this be in there?
	OutputStream int    // 1 = standard output, 2 error output
	Text         string // Text to match
}

// Must contain XYZ
func newExecutionValidation(txt string, fatal bool, mustContain bool, outputStream int) *ExecutionValidation {
	// Validate stream
	if outputStream != 1 && outputStream != 2 {
		return nil
	}

	// Must have text
	if len(txt) < 1 {
		return nil
	}

	return &ExecutionValidation{
		Fatal:        true,
		MustContain:  true,
		Text:         txt,
		OutputStream: 1,
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package executor

import (
	"math/rand"

	. "github.com/pingcap/check"
	"github.com/pingcap/tidb/parser/mysql"
	"github.com/pingcap/tidb/planner/core"
	"github.com/pingcap/tidb/types"
	"github.com/pingcap/tidb/util/chunk"
)

var _ = Suite(&testSuiteJoiner{})

type testSuiteJoiner struct{}

func (s *testSuiteJoiner) SetUpSuite(c *C) {
}

func (s *testSuiteJoiner) TestRequiredRows(c *C) {
	joinTypes := []core.JoinType{core.InnerJoin, core.LeftOuterJoin, core.RightOuterJoin}
	lTypes := [][]byte{
		{mysql.TypeLong},
		{mysql.TypeFloat},
		{mysql.TypeLong, mysql.TypeFloat},
	}
	rTypes := lTypes

	convertTypes := func(mysqlTypes []byte) []*types.FieldType {
		fieldTypes := make([]*types.FieldType, 0, len(mysqlTypes))
		for _, t := range mysqlTypes {
			fieldTypes = append(fieldTypes, types.NewFieldType(t))
		}
		return fieldTypes
	}

	for _, joinType := range joinTypes {
		for _, ltype := range lTypes {
			for _, rtype := range rTypes {
				maxChunkSize := defaultCtx().GetSessionVars().MaxChunkSize
				lfields := convertTypes(ltype)
				rfields := convertTypes(rtype)
				outerRow := genTestChunk(maxChunkSize, 1, lfields).GetRow(0)
				innerChk := genTestChunk(maxChunkSize, maxChunkSize, rfields)
				var defaultInner []types.Datum
				for i, f := range rfields {
					defaultInner = append(defaultInner, innerChk.GetRow(0).GetDatum(i, f))
				}
				joiner := newJoiner(defaultCtx(), joinType, false, defaultInner, nil, lfields, rfields, nil)

				fields := make([]*types.FieldType, 0, len(lfields)+len(rfields))
				fields = append(fields, rfields...)
				fields = append(fields, lfields...)
				result := chunk.New(fields, maxChunkSize, maxChunkSize)

				for i := 0; i < 10; i++ {
					required := rand.Int()%maxChunkSize + 1
					result.SetRequiredRows(required, maxChunkSize)
					result.Reset()
					it := chunk.NewIterator4Chunk(innerChk)
					it.Begin()
					_, _, err := joiner.tryToMatchInners(outerRow, it, result)
					c.Assert(err, IsNil)
					c.Assert(result.NumRows(), Equals, required)
				}
			}
		}
	}
}

func genTestChunk(maxChunkSize int, numRows int, fields []*types.FieldType) *chunk.Chunk {
	chk := chunk.New(fields, maxChunkSize, maxChunkSize)
	for numRows > 0 {
		numRows--
		for col, field := range fields {
			switch field.Tp {
			case mysql.TypeLong:
				chk.AppendInt64(col, 0)
			case mysql.TypeFloat:
				chk.AppendFloat32(col, 0)
			default:
				panic("not support")
			}
		}
	}
	return chk
}
<commit_msg>executor: migrate test-infra to testify for joiner_test.go (#32454)<commit_after>// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package executor

import (
	"math/rand"
	"testing"

	"github.com/pingcap/tidb/parser/mysql"
	"github.com/pingcap/tidb/planner/core"
	"github.com/pingcap/tidb/types"
	"github.com/pingcap/tidb/util/chunk"
	"github.com/stretchr/testify/require"
)

func TestRequiredRows(t *testing.T) {
	joinTypes := []core.JoinType{core.InnerJoin, core.LeftOuterJoin, core.RightOuterJoin}
	lTypes := [][]byte{
		{mysql.TypeLong},
		{mysql.TypeFloat},
		{mysql.TypeLong, mysql.TypeFloat},
	}
	rTypes := lTypes

	convertTypes := func(mysqlTypes []byte) []*types.FieldType {
		fieldTypes := make([]*types.FieldType, 0, len(mysqlTypes))
		for _, t := range mysqlTypes {
			fieldTypes = append(fieldTypes, types.NewFieldType(t))
		}
		return fieldTypes
	}

	for _, joinType := range joinTypes {
		for _, ltype := range lTypes {
			for _, rtype := range rTypes {
				maxChunkSize := defaultCtx().GetSessionVars().MaxChunkSize
				lfields := convertTypes(ltype)
				rfields := convertTypes(rtype)
				outerRow := genTestChunk(maxChunkSize, 1, lfields).GetRow(0)
				innerChk := genTestChunk(maxChunkSize, maxChunkSize, rfields)
				var defaultInner []types.Datum
				for i, f := range rfields {
					defaultInner = append(defaultInner, innerChk.GetRow(0).GetDatum(i, f))
				}
				joiner := newJoiner(defaultCtx(), joinType, false, defaultInner, nil, lfields, rfields, nil)

				fields := make([]*types.FieldType, 0, len(lfields)+len(rfields))
				fields = append(fields, rfields...)
				fields = append(fields, lfields...)
				result := chunk.New(fields, maxChunkSize, maxChunkSize)

				for i := 0; i < 10; i++ {
					required := rand.Int()%maxChunkSize + 1
					result.SetRequiredRows(required, maxChunkSize)
					result.Reset()
					it := chunk.NewIterator4Chunk(innerChk)
					it.Begin()
					_, _, err := joiner.tryToMatchInners(outerRow, it, result)
					require.NoError(t, err)
					require.Equal(t, required, result.NumRows())
				}
			}
		}
	}
}

func genTestChunk(maxChunkSize int, numRows int, fields []*types.FieldType) *chunk.Chunk {
	chk := chunk.New(fields, maxChunkSize, maxChunkSize)
	for numRows > 0 {
		numRows--
		for col, field := range fields {
			switch field.Tp {
			case mysql.TypeLong:
				chk.AppendInt64(col, 0)
			case mysql.TypeFloat:
				chk.AppendFloat32(col, 0)
			default:
				panic("not support")
			}
		}
	}
	return chk
}
<|endoftext|>