<commit_before>// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mobile

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"log"
	"net"
	"net/http"
	"path"
	"time"

	"github.com/google/martian"
	"github.com/google/martian/api"
	"github.com/google/martian/cors"
	"github.com/google/martian/cybervillains"
	"github.com/google/martian/fifo"
	"github.com/google/martian/har"
	"github.com/google/martian/httpspec"
	mlog "github.com/google/martian/log"
	"github.com/google/martian/marbl"
	"github.com/google/martian/martianhttp"
	"github.com/google/martian/mitm"
	"github.com/google/martian/servemux"
	"github.com/google/martian/verify"

	// side-effect importing to register with JSON API
	_ "github.com/google/martian/body"
	_ "github.com/google/martian/cookie"
	_ "github.com/google/martian/failure"
	_ "github.com/google/martian/header"
	_ "github.com/google/martian/martianurl"
	_ "github.com/google/martian/method"
	_ "github.com/google/martian/pingback"
	_ "github.com/google/martian/port"
	_ "github.com/google/martian/priority"
	_ "github.com/google/martian/querystring"
	_ "github.com/google/martian/skip"
	_ "github.com/google/martian/stash"
	_ "github.com/google/martian/static"
	_ "github.com/google/martian/status"
)

func init() {
	Init()
}

// Martian is a wrapper for the initialized Martian proxy
type Martian struct {
	proxy       *martian.Proxy
	listener    net.Listener
	mux         *http.ServeMux
	started     bool
	HarLogging  bool
	TrafficPort int
	APIPort     int
	Cert        string
	Key         string
	AllowCORS   bool
}

// EnableCybervillains configures Martian to use the Cybervillians certificate.
func (m *Martian) EnableCybervillains() {
	m.Cert = cybervillains.Cert
	m.Key = cybervillains.Key
}

// NewProxy creates a new Martian struct for configuring and starting a martian.
func NewProxy() *Martian {
	return &Martian{}
}

// Start starts the proxy given the configured values of the Martian struct.
func (m *Martian) Start() {
	var err error
	m.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", m.TrafficPort))
	if err != nil {
		log.Fatal(err)
	}

	mlog.Debugf("mobile: started listener on: %v", m.listener.Addr())
	m.proxy = martian.NewProxy()
	m.mux = http.NewServeMux()

	if m.Cert != "" && m.Key != "" {
		tlsc, err := tls.X509KeyPair([]byte(m.Cert), []byte(m.Key))
		if err != nil {
			log.Fatal(err)
		}

		mlog.Debugf("mobile: loaded cert and key")

		x509c, err := x509.ParseCertificate(tlsc.Certificate[0])
		if err != nil {
			log.Fatal(err)
		}

		mlog.Debugf("mobile: parsed cert")

		mc, err := mitm.NewConfig(x509c, tlsc.PrivateKey)
		if err != nil {
			log.Fatal(err)
		}

		mc.SetValidity(12 * time.Hour)
		mc.SetOrganization("Martian Proxy")

		m.proxy.SetMITM(mc)

		m.handle("/authority.cer", martianhttp.NewAuthorityHandler(x509c))
	}

	// Forward traffic that pattern matches in m.mux before applying
	// httpspec modifiers (via modifier, specifically)
	topg := fifo.NewGroup()
	apif := servemux.NewFilter(m.mux)
	apif.SetRequestModifier(api.NewForwarder("", m.APIPort))
	topg.AddRequestModifier(apif)

	stack, fg := httpspec.NewStack("martian.mobile")
	topg.AddRequestModifier(stack)
	topg.AddResponseModifier(stack)

	m.proxy.SetRequestModifier(topg)
	m.proxy.SetResponseModifier(topg)

	if m.HarLogging {
		// add HAR logger for unmodified logs.
		uhl := har.NewLogger()
		uhmuxf := servemux.NewFilter(m.mux)
		uhmuxf.RequestWhenFalse(uhl)
		uhmuxf.ResponseWhenFalse(uhl)
		fg.AddRequestModifier(uhmuxf)
		fg.AddResponseModifier(uhmuxf)

		// add HAR logger
		hl := har.NewLogger()
		hmuxf := servemux.NewFilter(m.mux)
		hmuxf.RequestWhenFalse(hl)
		hmuxf.ResponseWhenFalse(hl)
		stack.AddRequestModifier(hmuxf)
		stack.AddResponseModifier(hmuxf)

		// Retrieve Unmodified HAR logs
		m.handle("/logs/original", har.NewExportHandler(uhl))
		m.handle("/logs/original/reset", har.NewResetHandler(uhl))

		// Retrieve HAR logs
		m.handle("/logs", har.NewExportHandler(hl))
		m.handle("/logs/reset", har.NewResetHandler(hl))
	}

	lsh := marbl.NewHandler()
	// retrieve binary marbl logs
	m.handle("/binlogs", lsh)

	lsm := marbl.NewModifier(lsh)
	muxf := servemux.NewFilter(m.mux)
	muxf.RequestWhenFalse(lsm)
	muxf.ResponseWhenFalse(lsm)
	stack.AddRequestModifier(muxf)
	stack.AddResponseModifier(muxf)

	mod := martianhttp.NewModifier()
	fg.AddRequestModifier(mod)
	fg.AddResponseModifier(mod)

	// Proxy specific handlers.
	// These handlers take precendence over proxy traffic and will not be intercepted.

	// Update modifiers.
	m.handle("/configure", mod)

	// Verify assertions.
	vh := verify.NewHandler()
	vh.SetRequestVerifier(mod)
	vh.SetResponseVerifier(mod)

	m.handle("/verify", vh)

	// Reset verifications.
	rh := verify.NewResetHandler()
	rh.SetRequestVerifier(mod)
	rh.SetResponseVerifier(mod)
	m.handle("/verify/reset", rh)

	mlog.Infof("mobile: starting Martian proxy on listener")
	go m.proxy.Serve(m.listener)

	// start the API server
	apiAddr := fmt.Sprintf(":%d", m.APIPort)
	go http.ListenAndServe(apiAddr, m.mux)
	mlog.Infof("mobile: proxy API started on %s", apiAddr)
	m.started = true
}

// IsStarted returns true if the proxy has finished starting.
func (m *Martian) IsStarted() bool {
	return m.started
}

// Shutdown tells the Proxy to close. This function returns immediately, though
// there may still be connection threads hanging around until they time out
// depending on how the OS manages them.
func (m *Martian) Shutdown() {
	mlog.Infof("mobile: shutting down proxy")
	m.listener.Close()
	m.proxy.Close()
	m.started = false
	mlog.Infof("mobile: proxy shut down")
}

// SetLogLevel sets the Martian log level (Silent = 0, Error, Info, Debug), controlling which Martian
// log calls are displayed in the console
func SetLogLevel(l int) {
	mlog.SetLevel(l)
}

func (m *Martian) handle(pattern string, handler http.Handler) {
	if m.AllowCORS {
		handler = cors.NewHandler(handler)
	}
	m.mux.Handle(pattern, handler)
	mlog.Infof("mobile: handler registered for %s", pattern)

	lhp := path.Join(fmt.Sprintf("localhost:%d", m.APIPort), pattern)
	m.mux.Handle(lhp, handler)
	mlog.Infof("mobile: handler registered for %s", lhp)
}
<commit_msg>mobile/proxy API can be served over TLS  (#203)<commit_after>// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mobile

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/google/martian"
	"github.com/google/martian/api"
	"github.com/google/martian/cors"
	"github.com/google/martian/cybervillains"
	"github.com/google/martian/fifo"
	"github.com/google/martian/har"
	"github.com/google/martian/httpspec"
	mlog "github.com/google/martian/log"
	"github.com/google/martian/marbl"
	"github.com/google/martian/martianhttp"
	"github.com/google/martian/mitm"
	"github.com/google/martian/servemux"
	"github.com/google/martian/verify"

	// side-effect importing to register with JSON API
	_ "github.com/google/martian/body"
	_ "github.com/google/martian/cookie"
	_ "github.com/google/martian/failure"
	_ "github.com/google/martian/header"
	_ "github.com/google/martian/martianurl"
	_ "github.com/google/martian/method"
	_ "github.com/google/martian/pingback"
	_ "github.com/google/martian/port"
	_ "github.com/google/martian/priority"
	_ "github.com/google/martian/querystring"
	_ "github.com/google/martian/skip"
	_ "github.com/google/martian/stash"
	_ "github.com/google/martian/static"
	_ "github.com/google/martian/status"
)

func init() {
	Init()
}

// Martian is a wrapper for the initialized Martian proxy
type Martian struct {
	proxy       *martian.Proxy
	listener    net.Listener
	mux         *http.ServeMux
	started     bool
	HARLogging  bool
	TrafficPort int
	APIPort     int
	APIOverTLS  bool
	Cert        string
	Key         string
	AllowCORS   bool
}

// EnableCybervillains configures Martian to use the Cybervillians certificate.
func (m *Martian) EnableCybervillains() {
	m.Cert = cybervillains.Cert
	m.Key = cybervillains.Key
}

// NewProxy creates a new Martian struct for configuring and starting a martian.
func NewProxy() *Martian {
	return &Martian{}
}

// Start starts the proxy given the configured values of the Martian struct.
func (m *Martian) Start() {
	var err error
	m.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", m.TrafficPort))
	if err != nil {
		log.Fatal(err)
	}

	mlog.Debugf("mobile: started listener on: %v", m.listener.Addr())
	m.proxy = martian.NewProxy()
	m.mux = http.NewServeMux()

	if m.Cert != "" && m.Key != "" {
		tlsc, err := tls.X509KeyPair([]byte(m.Cert), []byte(m.Key))
		if err != nil {
			log.Fatal(err)
		}

		mlog.Debugf("mobile: loaded cert and key")

		x509c, err := x509.ParseCertificate(tlsc.Certificate[0])
		if err != nil {
			log.Fatal(err)
		}

		mlog.Debugf("mobile: parsed cert")

		mc, err := mitm.NewConfig(x509c, tlsc.PrivateKey)
		if err != nil {
			log.Fatal(err)
		}

		mc.SetValidity(12 * time.Hour)
		mc.SetOrganization("Martian Proxy")

		m.proxy.SetMITM(mc)

		m.handle("/authority.cer", martianhttp.NewAuthorityHandler(x509c))
	}

	// Forward traffic that pattern matches in m.mux before applying
	// httpspec modifiers (via modifier, specifically)
	topg := fifo.NewGroup()
	apif := servemux.NewFilter(m.mux)
	apif.SetRequestModifier(api.NewForwarder("", m.APIPort))
	topg.AddRequestModifier(apif)

	stack, fg := httpspec.NewStack("martian.mobile")
	topg.AddRequestModifier(stack)
	topg.AddResponseModifier(stack)

	m.proxy.SetRequestModifier(topg)
	m.proxy.SetResponseModifier(topg)

	if m.HARLogging {
		// add HAR logger for unmodified logs.
		uhl := har.NewLogger()
		uhmuxf := servemux.NewFilter(m.mux)
		uhmuxf.RequestWhenFalse(uhl)
		uhmuxf.ResponseWhenFalse(uhl)
		fg.AddRequestModifier(uhmuxf)
		fg.AddResponseModifier(uhmuxf)

		// add HAR logger
		hl := har.NewLogger()
		hmuxf := servemux.NewFilter(m.mux)
		hmuxf.RequestWhenFalse(hl)
		hmuxf.ResponseWhenFalse(hl)
		stack.AddRequestModifier(hmuxf)
		stack.AddResponseModifier(hmuxf)

		// Retrieve Unmodified HAR logs
		m.handle("/logs/original", har.NewExportHandler(uhl))
		m.handle("/logs/original/reset", har.NewResetHandler(uhl))

		// Retrieve HAR logs
		m.handle("/logs", har.NewExportHandler(hl))
		m.handle("/logs/reset", har.NewResetHandler(hl))
	}

	lsh := marbl.NewHandler()
	// retrieve binary marbl logs
	m.handle("/binlogs", lsh)

	lsm := marbl.NewModifier(lsh)
	muxf := servemux.NewFilter(m.mux)
	muxf.RequestWhenFalse(lsm)
	muxf.ResponseWhenFalse(lsm)
	stack.AddRequestModifier(muxf)
	stack.AddResponseModifier(muxf)

	mod := martianhttp.NewModifier()
	fg.AddRequestModifier(mod)
	fg.AddResponseModifier(mod)

	// Proxy specific handlers.
	// These handlers take precendence over proxy traffic and will not be intercepted.

	// Update modifiers.
	m.handle("/configure", mod)

	// Verify assertions.
	vh := verify.NewHandler()
	vh.SetRequestVerifier(mod)
	vh.SetResponseVerifier(mod)

	m.handle("/verify", vh)

	// Reset verifications.
	rh := verify.NewResetHandler()
	rh.SetRequestVerifier(mod)
	rh.SetResponseVerifier(mod)
	m.handle("/verify/reset", rh)

	mlog.Infof("mobile: starting Martian proxy on listener")
	go m.proxy.Serve(m.listener)

	// start the API server
	apiAddr := fmt.Sprintf(":%d", m.APIPort)
	if m.APIOverTLS {
		if m.Cert == "" || m.Key == "" {
			log.Fatal("mobile: APIOverTLS cannot be true without valid cert and key")
		}

		cerfile, err := ioutil.TempFile("", "martian-api.cert")
		if err != nil {
			log.Fatal(err)
		}

		keyfile, err := ioutil.TempFile("", "martian-api.key")
		if err != nil {
			log.Fatal(err)
		}

		if _, err := cerfile.Write([]byte(m.Cert)); err != nil {
			log.Fatal(err)
		}

		if _, err := keyfile.Write([]byte(m.Key)); err != nil {
			log.Fatal(err)
		}

		go func() {
			http.ListenAndServeTLS(apiAddr, cerfile.Name(), keyfile.Name(), m.mux)
			defer os.Remove(cerfile.Name())
			defer os.Remove(keyfile.Name())
		}()

		mlog.Infof("mobile: proxy API started on %s over TLS", apiAddr)
	} else {
		go http.ListenAndServe(apiAddr, m.mux)
		mlog.Infof("mobile: proxy API started on %s", apiAddr)
	}

	go http.ListenAndServe(apiAddr, m.mux)
	mlog.Infof("mobile: proxy API started on %s", apiAddr)
	m.started = true
}

// IsStarted returns true if the proxy has finished starting.
func (m *Martian) IsStarted() bool {
	return m.started
}

// Shutdown tells the Proxy to close. This function returns immediately, though
// there may still be connection threads hanging around until they time out
// depending on how the OS manages them.
func (m *Martian) Shutdown() {
	mlog.Infof("mobile: shutting down proxy")
	m.listener.Close()
	m.proxy.Close()
	m.started = false
	mlog.Infof("mobile: proxy shut down")
}

// SetLogLevel sets the Martian log level (Silent = 0, Error, Info, Debug), controlling which Martian
// log calls are displayed in the console
func SetLogLevel(l int) {
	mlog.SetLevel(l)
}

func (m *Martian) handle(pattern string, handler http.Handler) {
	if m.AllowCORS {
		handler = cors.NewHandler(handler)
	}
	m.mux.Handle(pattern, handler)
	mlog.Infof("mobile: handler registered for %s", pattern)

	lhp := path.Join(fmt.Sprintf("localhost:%d", m.APIPort), pattern)
	m.mux.Handle(lhp, handler)
	mlog.Infof("mobile: handler registered for %s", lhp)
}
<|endoftext|>


<commit_before>package qlog

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"time"

	"math"
	"os"

	"github.com/gin-gonic/gin"
	"github.com/sirupsen/logrus"
)

func getGinLogger(logger ...*logrus.Entry) *logrus.Entry {
	if len(logger) == 0 {
		return logrus.StandardLogger().WithField("pkg", "gin")
	}

	return logger[0]
}

// GinLogger is the qlog logger for GIN, copy from https://github.com/toorop/gin-logrus
func GinLogger(logger ...*logrus.Entry) gin.HandlerFunc {
	log := getGinLogger(logger...)

	return func(c *gin.Context) {
		path := c.Request.URL.Path
		start := time.Now()
		c.Next()
		stop := time.Since(start)
		latency := int(math.Ceil(float64(stop.Nanoseconds()) / 1000000.0))
		statusCode := c.Writer.Status()
		clientIP := c.ClientIP()
		clientUserAgent := c.Request.UserAgent()
		referer := c.Request.Referer()
		hostname, err := os.Hostname()
		if err != nil {
			hostname = "unknown"
		}
		dataLength := c.Writer.Size()
		if dataLength < 0 {
			dataLength = 0
		}

		entry := log.WithFields(logrus.Fields{
			"hostname":   hostname,
			"statusCode": statusCode,
			"latency":    latency, // time to process
			"clientIP":   clientIP,
			"method":     c.Request.Method,
			"path":       path,
			"referer":    referer,
			"dataLength": dataLength,
			"userAgent":  clientUserAgent,
		})

		if len(c.Errors) > 0 {
			entry.Error(c.Errors.ByType(gin.ErrorTypePrivate).String())
		} else {
			msg := fmt.Sprintf("%d \"%s %s\" (%dms)", statusCode, c.Request.Method, path, latency)
			if statusCode > 499 {
				entry.Error(msg)
			} else if statusCode > 399 {
				entry.Warn(msg)
			} else {
				entry.Trace(msg)
			}
		}
	}
}

type teeReadCloser struct {
	io.Reader
	io.Closer
}

func TeeReadCloser(rc io.ReadCloser, w io.Writer) io.ReadCloser {
	tee := io.TeeReader(rc, w)
	return &teeReadCloser{tee, rc}
}

type ginResponseMultiWriter struct {
	gin.ResponseWriter
	mw io.Writer
}

const (
	trimmedMsgLength = 16 * 1024
)

func GinMultiWriter(gw gin.ResponseWriter, w io.Writer) gin.ResponseWriter {
	mw := io.MultiWriter(gw, w)

	return &ginResponseMultiWriter{gw, mw}
}

func (w *ginResponseMultiWriter) Write(p []byte) (n int, err error) {
	return w.mw.Write(p)
}

func GinAPILogger(logger ...*logrus.Entry) gin.HandlerFunc {
	log := getGinLogger(logger...)

	return func(c *gin.Context) {
		start := time.Now()

		var bufReq, bufRsp bytes.Buffer

		c.Request.Body = TeeReadCloser(c.Request.Body, &bufReq)
		c.Writer = GinMultiWriter(c.Writer, &bufRsp)

		c.Next()
		stop := time.Since(start)
		latency := int(math.Ceil(float64(stop.Nanoseconds()) / 1000000.0))
		statusCode := c.Writer.Status()
		clientIP := c.ClientIP()

		entry := log.WithFields(logrus.Fields{
			"statusCode": statusCode,
			"latency":    latency, // time to process
			"clientIP":   clientIP,
			"userAgent":  c.Request.UserAgent(),
		})

		var msg bytes.Buffer
		msg.WriteString(c.Request.Method)
		msg.WriteString(" ")
		msg.WriteString(c.Request.RequestURI)

		reqLen := bufReq.Len()

		if reqLen > 0 {
			msg.WriteString(" req:")

			if reqLen > trimmedMsgLength {
				msg.WriteString(fmt.Sprintf("[%d]", reqLen))
				msg.Write(bufReq.Bytes()[:trimmedMsgLength-12])
				msg.WriteString("...")
			} else {
				msg.Write(bufReq.Bytes())
			}
		}

		rspLen := bufRsp.Len()
		if rspLen > 0 {
			contentType := c.Writer.Header().Get("Content-Type")
			if strings.HasPrefix(contentType, "application/json") {
				msg.WriteString(" rsp:")

				if rspLen > trimmedMsgLength {
					msg.WriteString(fmt.Sprintf("[%d]", rspLen))
					msg.Write(bufRsp.Bytes()[:trimmedMsgLength-12])
					msg.WriteString("...")
				} else {
					msg.Write(bufRsp.Bytes())
				}
			} else {
				msg.WriteString(fmt.Sprintf(" rsp: len=%d, content_type=%s", rspLen, contentType))
			}
		}

		entry.Debug(msg.String())
	}
}
<commit_msg>ginlogger: check request content type<commit_after>package qlog

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"time"

	"math"
	"os"

	"github.com/gin-gonic/gin"
	"github.com/sirupsen/logrus"
)

func getGinLogger(logger ...*logrus.Entry) *logrus.Entry {
	if len(logger) == 0 {
		return logrus.StandardLogger().WithField("pkg", "gin")
	}

	return logger[0]
}

// GinLogger is the qlog logger for GIN, copy from https://github.com/toorop/gin-logrus
func GinLogger(logger ...*logrus.Entry) gin.HandlerFunc {
	log := getGinLogger(logger...)

	return func(c *gin.Context) {
		path := c.Request.URL.Path
		start := time.Now()
		c.Next()
		stop := time.Since(start)
		latency := int(math.Ceil(float64(stop.Nanoseconds()) / 1000000.0))
		statusCode := c.Writer.Status()
		clientIP := c.ClientIP()
		clientUserAgent := c.Request.UserAgent()
		referer := c.Request.Referer()
		hostname, err := os.Hostname()
		if err != nil {
			hostname = "unknown"
		}
		dataLength := c.Writer.Size()
		if dataLength < 0 {
			dataLength = 0
		}

		entry := log.WithFields(logrus.Fields{
			"hostname":   hostname,
			"statusCode": statusCode,
			"latency":    latency, // time to process
			"clientIP":   clientIP,
			"method":     c.Request.Method,
			"path":       path,
			"referer":    referer,
			"dataLength": dataLength,
			"userAgent":  clientUserAgent,
		})

		if len(c.Errors) > 0 {
			entry.Error(c.Errors.ByType(gin.ErrorTypePrivate).String())
		} else {
			msg := fmt.Sprintf("%d \"%s %s\" (%dms)", statusCode, c.Request.Method, path, latency)
			if statusCode > 499 {
				entry.Error(msg)
			} else if statusCode > 399 {
				entry.Warn(msg)
			} else {
				entry.Trace(msg)
			}
		}
	}
}

type teeReadCloser struct {
	io.Reader
	io.Closer
}

func TeeReadCloser(rc io.ReadCloser, w io.Writer) io.ReadCloser {
	tee := io.TeeReader(rc, w)
	return &teeReadCloser{tee, rc}
}

type ginResponseMultiWriter struct {
	gin.ResponseWriter
	mw io.Writer
}

const (
	trimmedMsgLength = 16 * 1024
)

func GinMultiWriter(gw gin.ResponseWriter, w io.Writer) gin.ResponseWriter {
	mw := io.MultiWriter(gw, w)

	return &ginResponseMultiWriter{gw, mw}
}

func (w *ginResponseMultiWriter) Write(p []byte) (n int, err error) {
	return w.mw.Write(p)
}

func GinAPILogger(logger ...*logrus.Entry) gin.HandlerFunc {
	log := getGinLogger(logger...)

	return func(c *gin.Context) {
		start := time.Now()

		var bufReq, bufRsp bytes.Buffer

		c.Request.Body = TeeReadCloser(c.Request.Body, &bufReq)
		c.Writer = GinMultiWriter(c.Writer, &bufRsp)

		c.Next()
		stop := time.Since(start)
		latency := int(math.Ceil(float64(stop.Nanoseconds()) / 1000000.0))
		statusCode := c.Writer.Status()
		clientIP := c.ClientIP()

		entry := log.WithFields(logrus.Fields{
			"statusCode": statusCode,
			"latency":    latency, // time to process
			"clientIP":   clientIP,
			"userAgent":  c.Request.UserAgent(),
		})

		var msg bytes.Buffer
		msg.WriteString(c.Request.Method)
		msg.WriteString(" ")
		msg.WriteString(c.Request.RequestURI)

		reqLen := bufReq.Len()

		if reqLen > 0 {
			contentType := strings.ToLower(c.Request.Header.Get("Content-Type"))
			if strings.HasPrefix(contentType, "application/json") {
				msg.WriteString(" req:")

				if reqLen > trimmedMsgLength {
					msg.WriteString(fmt.Sprintf("[%d]", reqLen))
					msg.Write(bufReq.Bytes()[:trimmedMsgLength-12])
					msg.WriteString("...")
				} else {
					msg.Write(bufReq.Bytes())
				}
			} else {
				msg.WriteString(fmt.Sprintf(" req: len=%d, content_type=%s", reqLen, contentType))
			}
		}

		rspLen := bufRsp.Len()
		if rspLen > 0 {
			contentType := c.Writer.Header().Get("Content-Type")
			if strings.HasPrefix(contentType, "application/json") {
				msg.WriteString(" rsp:")

				if rspLen > trimmedMsgLength {
					msg.WriteString(fmt.Sprintf("[%d]", rspLen))
					msg.Write(bufRsp.Bytes()[:trimmedMsgLength-12])
					msg.WriteString("...")
				} else {
					msg.Write(bufRsp.Bytes())
				}
			} else {
				msg.WriteString(fmt.Sprintf(" rsp: len=%d, content_type=%s", rspLen, contentType))
			}
		}

		entry.Debug(msg.String())
	}
}
<|endoftext|>


<commit_before>package models

import (
	"appengine"
	"appengine/datastore"
	"fmt"
	"time"
)

type Entry struct {
	Id       int64
	Title    string // optional
	Content  string `datastore:",noindex"` // Markdown
	Datetime time.Time
	Created  time.Time
	Modified time.Time
	Tags     []string
	Public   bool
	// TODO(icco): Define a meta field that is a json hash of extra data
}

func NewEntry(title string, content string, datetime time.Time, public bool, tags []string) *Entry {
	e := new(Entry)

	// User supplied content
	e.Title = title
	e.Content = content
	e.Datetime = datetime
	e.Tags = tags
	e.Public = public

	// Computer generated content
	e.Created = time.Now()
	e.Modified = time.Now()

	return e
}

func GetEntry(c appengine.Context, id int64) (*Entry, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Filter("Id =", id)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Warningf("Error getting entry %d", id)
		return nil, err
	}
	return &entry, nil
}

func MaxId(c appengine.Context) (int64, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Id").Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		return 0, err
	}
	return entry.Id, nil
}

func AllPosts(c appengine.Context) (*[]Entry, error) {
	q := datastore.NewQuery("Entry").Order("-Datetime")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	return entries, err
}

func (e *Entry) HasId() bool {
	return (e.Id <= 0)
}

func (e *Entry) Save(c appengine.Context) error {
	var k *datastore.Key
	if e.hasId() {
		id, _ := MaxId(c)
		e.Id = id + 1
		k = datastore.NewIncompleteKey(c, "Entry", nil)
	} else {
		k = datastore.NewKey(c, "Entry", fmt.Sprintf("%d", e.Id), 0, nil)
	}

	_, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}
<commit_msg>Whoops<commit_after>package models

import (
	"appengine"
	"appengine/datastore"
	"fmt"
	"time"
)

type Entry struct {
	Id       int64
	Title    string // optional
	Content  string `datastore:",noindex"` // Markdown
	Datetime time.Time
	Created  time.Time
	Modified time.Time
	Tags     []string
	Public   bool
	// TODO(icco): Define a meta field that is a json hash of extra data
}

func NewEntry(title string, content string, datetime time.Time, public bool, tags []string) *Entry {
	e := new(Entry)

	// User supplied content
	e.Title = title
	e.Content = content
	e.Datetime = datetime
	e.Tags = tags
	e.Public = public

	// Computer generated content
	e.Created = time.Now()
	e.Modified = time.Now()

	return e
}

func GetEntry(c appengine.Context, id int64) (*Entry, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Filter("Id =", id)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Warningf("Error getting entry %d", id)
		return nil, err
	}
	return &entry, nil
}

func MaxId(c appengine.Context) (int64, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Id").Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		return 0, err
	}
	return entry.Id, nil
}

func AllPosts(c appengine.Context) (*[]Entry, error) {
	q := datastore.NewQuery("Entry").Order("-Datetime")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	return entries, err
}

func (e *Entry) HasId() bool {
	return (e.Id <= 0)
}

func (e *Entry) Save(c appengine.Context) error {
	var k *datastore.Key
	if e.HasId() {
		id, _ := MaxId(c)
		e.Id = id + 1
		k = datastore.NewIncompleteKey(c, "Entry", nil)
	} else {
		k = datastore.NewKey(c, "Entry", fmt.Sprintf("%d", e.Id), 0, nil)
	}

	_, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}
<|endoftext|>


<commit_before>package gitignore

import (
	"bufio"
	"io"
	"os"
	"path/filepath"
	"strings"
)

type IgnoreMatcher interface {
	Match(path string, isDir bool) bool
}

type DummyIgnoreMatcher bool

func (d DummyIgnoreMatcher) Match(path string, isDir bool) bool {
	return bool(d)
}

type gitIgnore struct {
	ignorePatterns scanStrategy
	acceptPatterns scanStrategy
	path           string
}

func NewGitIgnore(gitignore string, base ...string) (IgnoreMatcher, error) {
	var path string
	if len(base) > 0 {
		path = base[0]
	} else {
		path = filepath.Dir(gitignore)
	}

	file, err := os.Open(gitignore)
	if err != nil {
		return DummyIgnoreMatcher(true), err
	}
	defer file.Close()

	return NewGitIgnoreFromReader(path, file), nil
}

func NewGitIgnoreFromReader(path string, r io.Reader) IgnoreMatcher {
	g := gitIgnore{
		ignorePatterns: newIndexScanPatterns(),
		acceptPatterns: newIndexScanPatterns(),
		path:           path,
	}
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := strings.Trim(scanner.Text(), " ")
		if len(line) == 0 || strings.HasPrefix(line, "#") {
			continue
		}
		if strings.HasPrefix(line, `\#`) {
			line = strings.TrimPrefix(line, `\`)
		}

		if strings.HasPrefix(line, "!") {
			g.acceptPatterns.add(strings.TrimPrefix(line, "!"))
		} else {
			g.ignorePatterns.add(line)
		}
	}
	return g
}

func (g gitIgnore) Match(path string, isDir bool) bool {
	relativePath, err := filepath.Rel(g.path, path)
	if err != nil {
		return false
	}

	if g.acceptPatterns.match(relativePath, isDir) {
		return false
	}
	return g.ignorePatterns.match(relativePath, isDir)
}
<commit_msg>fixing<commit_after>package gitignore

import (
	"bufio"
	"io"
	"os"
	"path/filepath"
	"strings"
)

type IgnoreMatcher interface {
	Match(path string, isDir bool) bool
}

type DummyIgnoreMatcher bool

func (d DummyIgnoreMatcher) Match(path string, isDir bool) bool {
	return bool(d)
}

type gitIgnore struct {
	ignorePatterns scanStrategy
	acceptPatterns scanStrategy
	path           string
}

func NewGitIgnore(gitignore string, base ...string) (IgnoreMatcher, error) {
	var path string
	if len(base) > 0 {
		path = base[0]
	} else {
		path = filepath.Dir(gitignore)
	}

	file, err := os.Open(gitignore)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return NewGitIgnoreFromReader(path, file), nil
}

func NewGitIgnoreFromReader(path string, r io.Reader) IgnoreMatcher {
	g := gitIgnore{
		ignorePatterns: newIndexScanPatterns(),
		acceptPatterns: newIndexScanPatterns(),
		path:           path,
	}
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := strings.Trim(scanner.Text(), " ")
		if len(line) == 0 || strings.HasPrefix(line, "#") {
			continue
		}
		if strings.HasPrefix(line, `\#`) {
			line = strings.TrimPrefix(line, `\`)
		}

		if strings.HasPrefix(line, "!") {
			g.acceptPatterns.add(strings.TrimPrefix(line, "!"))
		} else {
			g.ignorePatterns.add(line)
		}
	}
	return g
}

func (g gitIgnore) Match(path string, isDir bool) bool {
	relativePath, err := filepath.Rel(g.path, path)
	if err != nil {
		return false
	}

	if g.acceptPatterns.match(relativePath, isDir) {
		return false
	}
	return g.ignorePatterns.match(relativePath, isDir)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

var thePath = flag.String("dir", "/tmp", "working directory")
var git = flag.String("git", "/usr/bin/git", "path to git")

type CommandRequest struct {
	w       http.ResponseWriter
	abspath string
	bg      bool
	after   time.Time
	cmds    []*exec.Cmd
	ch      chan bool
}

var reqch = make(chan CommandRequest, 100)
var updates = map[string]time.Time{}
var updateLock sync.Mutex

func exists(path string) (rv bool) {
	rv = true
	if _, err := os.Stat(path); os.IsNotExist(err) {
		rv = false
	}
	return
}

func runCommands(w http.ResponseWriter, bg bool,
	abspath string, cmds []*exec.Cmd) {

	var stderr io.Writer = ioutil.Discard
	var stdout io.Writer = ioutil.Discard

	if !bg {
		stderr = &bytes.Buffer{}
		stdout = &bytes.Buffer{}
	}

	for _, cmd := range cmds {
		if exists(cmd.Path) {
			log.Printf("Running %v", cmd.Args)
			fmt.Fprintf(stdout, "# Running %v\n", cmd.Args)
			fmt.Fprintf(stderr, "# Running %v\n", cmd.Args)

			cmd.Stdout = stdout
			cmd.Stderr = stderr
			cmd.Dir = abspath
			err := cmd.Run()

			if err != nil {
				log.Printf("Error running command:  %v", err)
				if !bg {
					fmt.Fprintf(stderr,
						"\n[gitmirror internal error:  %v]\n", err)
				}
			}

		}
	}

	if !bg {
		fmt.Fprintf(w, "---- stdout ----\n")
		stdout.(*bytes.Buffer).WriteTo(w)
		fmt.Fprintf(w, "\n----\n\n\n---- stderr ----\n")
		stderr.(*bytes.Buffer).WriteTo(w)
		fmt.Fprintf(w, "\n----\n")
	}
}

func shouldRun(path string, after time.Time) bool {
	updateLock.Lock()
	defer updateLock.Unlock()

	lastRun := updates[path]
	if lastRun.Before(after) {
		return true
	}
	return false
}

func didRun(path string) {
	updateLock.Lock()
	defer updateLock.Unlock()
	updates[path] = time.Now()
}

func commandRunner() {
	for r := range reqch {
		if shouldRun(r.abspath, r.after) {
			runCommands(r.w, r.bg, r.abspath, r.cmds)
			didRun(r.abspath)
		} else {
			log.Printf("Skipping redundant update: %v", r.abspath)
			if !r.bg {
				fmt.Fprintf(r.w, "Redundant request.")
			}
		}
		select {
		case r.ch <- true:
		default:
		}
	}
}

func queueCommand(w http.ResponseWriter, bg bool,
	abspath string, cmds []*exec.Cmd) chan bool {
	req := CommandRequest{w, abspath, bg, time.Now(),
		cmds, make(chan bool)}
	reqch <- req
	return req.ch
}

func updateGit(w http.ResponseWriter, section string,
	bg bool, payload []byte) bool {

	time.Sleep(5 * time.Second)

	abspath := filepath.Join(*thePath, section)

	if !exists(abspath) {
		if !bg {
			http.Error(w, "Not found", http.StatusNotFound)
		}
		return false
	}

	cmds := []*exec.Cmd{
		exec.Command(*git, "remote", "update", "-p"),
		exec.Command(*git, "gc", "--auto"),
		exec.Command(filepath.Join(abspath, "hooks/post-fetch")),
		exec.Command("bin/post-fetch"),
	}

	cmds[2].Stdin = bytes.NewBuffer(payload)
	cmds[3].Stdin = bytes.NewBuffer(payload)

	return <-queueCommand(w, bg, abspath, cmds)
}

func getPath(req *http.Request) string {
	return filepath.Clean(filepath.FromSlash(req.URL.Path))[1:]
}

func createRepo(w http.ResponseWriter, section string,
	bg bool, payload []byte) {

	p := struct {
		Repository struct {
			Owner   interface{}
			Private int
			Name    string
		}
	}{}

	err := json.Unmarshal(payload, &p)
	if err != nil {
		log.Printf("Error unmarshalling data: %v", err)
		http.Error(w, "Error parsing JSON", http.StatusInternalServerError)
		return
	}

	var ownerName string
	switch i := p.Repository.Owner.(type) {
	case string:
		ownerName = i
	case map[string]interface{}:
		ownerName = fmt.Sprintf("%v", i["name"])
	}

	repo := fmt.Sprintf("git://github.com/%v/%v.git",
		ownerName, p.Repository.Name)
	if p.Repository.Private == 1 {
		repo = fmt.Sprintf("git@github.com:%v/%v.git",
			ownerName, p.Repository.Name)
	}

	cmds := []*exec.Cmd{
		exec.Command(*git, "clone", "--mirror", "--bare", repo,
			filepath.Join(*thePath, section)),
	}

	if bg {
		w.WriteHeader(201)
	}
	runCommands(w, bg, "/tmp", cmds)
}

func doUpdate(w http.ResponseWriter, path string,
	bg bool, payload []byte) {
	if bg {
		go updateGit(w, path, bg, []byte{})
		w.WriteHeader(201)
	} else {
		updateGit(w, path, bg, []byte{})
	}
}

func handleGet(w http.ResponseWriter, req *http.Request, bg bool) {
	path := getPath(req)
	doUpdate(w, path, bg, []byte{})
}

func handlePost(w http.ResponseWriter, req *http.Request, bg bool) {
	b := []byte(req.FormValue("payload"))

	path := getPath(req)
	abspath := filepath.Join(*thePath, path)

	if exists(abspath) {
		doUpdate(w, path, bg, b)
	} else {
		createRepo(w, path, bg, b)
	}
}

func handleReq(w http.ResponseWriter, req *http.Request) {
	backgrounded := req.FormValue("bg") != "false"

	log.Printf("Handling %v %v", req.Method, req.URL.Path)

	switch req.Method {
	case "GET":
		handleGet(w, req, backgrounded)
	case "POST":
		handlePost(w, req, backgrounded)
	default:
		http.Error(w, "Method not allowed",
			http.StatusMethodNotAllowed)
	}
}

func main() {
	flag.Parse()

	go commandRunner()

	http.HandleFunc("/", handleReq)
	http.HandleFunc("/favicon.ico",
		func(w http.ResponseWriter, req *http.Request) {
			http.Error(w, "No favicon", http.StatusGone)
		})
	log.Fatal(http.ListenAndServe(":8124", nil))
}
<commit_msg>Change logging flags to avoid redundancy.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

var thePath = flag.String("dir", "/tmp", "working directory")
var git = flag.String("git", "/usr/bin/git", "path to git")

type CommandRequest struct {
	w       http.ResponseWriter
	abspath string
	bg      bool
	after   time.Time
	cmds    []*exec.Cmd
	ch      chan bool
}

var reqch = make(chan CommandRequest, 100)
var updates = map[string]time.Time{}
var updateLock sync.Mutex

func exists(path string) (rv bool) {
	rv = true
	if _, err := os.Stat(path); os.IsNotExist(err) {
		rv = false
	}
	return
}

func runCommands(w http.ResponseWriter, bg bool,
	abspath string, cmds []*exec.Cmd) {

	var stderr io.Writer = ioutil.Discard
	var stdout io.Writer = ioutil.Discard

	if !bg {
		stderr = &bytes.Buffer{}
		stdout = &bytes.Buffer{}
	}

	for _, cmd := range cmds {
		if exists(cmd.Path) {
			log.Printf("Running %v", cmd.Args)
			fmt.Fprintf(stdout, "# Running %v\n", cmd.Args)
			fmt.Fprintf(stderr, "# Running %v\n", cmd.Args)

			cmd.Stdout = stdout
			cmd.Stderr = stderr
			cmd.Dir = abspath
			err := cmd.Run()

			if err != nil {
				log.Printf("Error running command:  %v", err)
				if !bg {
					fmt.Fprintf(stderr,
						"\n[gitmirror internal error:  %v]\n", err)
				}
			}

		}
	}

	if !bg {
		fmt.Fprintf(w, "---- stdout ----\n")
		stdout.(*bytes.Buffer).WriteTo(w)
		fmt.Fprintf(w, "\n----\n\n\n---- stderr ----\n")
		stderr.(*bytes.Buffer).WriteTo(w)
		fmt.Fprintf(w, "\n----\n")
	}
}

func shouldRun(path string, after time.Time) bool {
	updateLock.Lock()
	defer updateLock.Unlock()

	lastRun := updates[path]
	if lastRun.Before(after) {
		return true
	}
	return false
}

func didRun(path string) {
	updateLock.Lock()
	defer updateLock.Unlock()
	updates[path] = time.Now()
}

func commandRunner() {
	for r := range reqch {
		if shouldRun(r.abspath, r.after) {
			runCommands(r.w, r.bg, r.abspath, r.cmds)
			didRun(r.abspath)
		} else {
			log.Printf("Skipping redundant update: %v", r.abspath)
			if !r.bg {
				fmt.Fprintf(r.w, "Redundant request.")
			}
		}
		select {
		case r.ch <- true:
		default:
		}
	}
}

func queueCommand(w http.ResponseWriter, bg bool,
	abspath string, cmds []*exec.Cmd) chan bool {
	req := CommandRequest{w, abspath, bg, time.Now(),
		cmds, make(chan bool)}
	reqch <- req
	return req.ch
}

func updateGit(w http.ResponseWriter, section string,
	bg bool, payload []byte) bool {

	time.Sleep(5 * time.Second)

	abspath := filepath.Join(*thePath, section)

	if !exists(abspath) {
		if !bg {
			http.Error(w, "Not found", http.StatusNotFound)
		}
		return false
	}

	cmds := []*exec.Cmd{
		exec.Command(*git, "remote", "update", "-p"),
		exec.Command(*git, "gc", "--auto"),
		exec.Command(filepath.Join(abspath, "hooks/post-fetch")),
		exec.Command("bin/post-fetch"),
	}

	cmds[2].Stdin = bytes.NewBuffer(payload)
	cmds[3].Stdin = bytes.NewBuffer(payload)

	return <-queueCommand(w, bg, abspath, cmds)
}

func getPath(req *http.Request) string {
	return filepath.Clean(filepath.FromSlash(req.URL.Path))[1:]
}

func createRepo(w http.ResponseWriter, section string,
	bg bool, payload []byte) {

	p := struct {
		Repository struct {
			Owner   interface{}
			Private int
			Name    string
		}
	}{}

	err := json.Unmarshal(payload, &p)
	if err != nil {
		log.Printf("Error unmarshalling data: %v", err)
		http.Error(w, "Error parsing JSON", http.StatusInternalServerError)
		return
	}

	var ownerName string
	switch i := p.Repository.Owner.(type) {
	case string:
		ownerName = i
	case map[string]interface{}:
		ownerName = fmt.Sprintf("%v", i["name"])
	}

	repo := fmt.Sprintf("git://github.com/%v/%v.git",
		ownerName, p.Repository.Name)
	if p.Repository.Private == 1 {
		repo = fmt.Sprintf("git@github.com:%v/%v.git",
			ownerName, p.Repository.Name)
	}

	cmds := []*exec.Cmd{
		exec.Command(*git, "clone", "--mirror", "--bare", repo,
			filepath.Join(*thePath, section)),
	}

	if bg {
		w.WriteHeader(201)
	}
	runCommands(w, bg, "/tmp", cmds)
}

func doUpdate(w http.ResponseWriter, path string,
	bg bool, payload []byte) {
	if bg {
		go updateGit(w, path, bg, []byte{})
		w.WriteHeader(201)
	} else {
		updateGit(w, path, bg, []byte{})
	}
}

func handleGet(w http.ResponseWriter, req *http.Request, bg bool) {
	path := getPath(req)
	doUpdate(w, path, bg, []byte{})
}

func handlePost(w http.ResponseWriter, req *http.Request, bg bool) {
	b := []byte(req.FormValue("payload"))

	path := getPath(req)
	abspath := filepath.Join(*thePath, path)

	if exists(abspath) {
		doUpdate(w, path, bg, b)
	} else {
		createRepo(w, path, bg, b)
	}
}

func handleReq(w http.ResponseWriter, req *http.Request) {
	backgrounded := req.FormValue("bg") != "false"

	log.Printf("Handling %v %v", req.Method, req.URL.Path)

	switch req.Method {
	case "GET":
		handleGet(w, req, backgrounded)
	case "POST":
		handlePost(w, req, backgrounded)
	default:
		http.Error(w, "Method not allowed",
			http.StatusMethodNotAllowed)
	}
}

func main() {
	flag.Parse()

	log.SetFlags(log.Lmicroseconds)

	go commandRunner()

	http.HandleFunc("/", handleReq)
	http.HandleFunc("/favicon.ico",
		func(w http.ResponseWriter, req *http.Request) {
			http.Error(w, "No favicon", http.StatusGone)
		})
	log.Fatal(http.ListenAndServe(":8124", nil))
}
<|endoftext|>


<commit_before>// Package glob implements globbing for elvish.
package glob

import (
	"io/ioutil"
	"os"
	"runtime"
	"unicode/utf8"
)

// TODO: Use native path separators instead of always using /.

// Glob returns a list of file names satisfying the given pattern.
func Glob(p string, cb func(string) bool) bool {
	return Parse(p).Glob(cb)
}

// Glob returns a list of file names satisfying the Pattern.
func (p Pattern) Glob(cb func(string) bool) bool {
	segs := p.Segments
	dir := ""

	// XXX: This is a hack solely for supporting globs that start with ~ in the
	// eval package.
	if p.DirOverride != "" {
		dir = p.DirOverride
	}

	if len(segs) > 0 && IsSlash(segs[0]) {
		segs = segs[1:]
		dir += "/"
	} else if runtime.GOOS == "windows" && len(segs) > 1 && IsLiteral(segs[0]) && IsSlash(segs[1]) {
		// TODO: Handle UNC.
		elem := segs[0].(Literal).Data
		if isDrive(elem) {
			segs = segs[2:]
			dir = elem + "/"
		}
	}

	return glob(segs, dir, cb)
}

func isDrive(s string) bool {
	return s[1] == ':' &&
		(('a' <= s[0] && s[1] <= 'z') || ('A' <= s[0] && s[0] <= 'Z'))
}

// glob finds all filenames matching the given Segments in the given dir, and
// calls the callback on all of them. If the callback returns false, globbing is
// interrupted, and glob returns false. Otherwise it returns true.
func glob(segs []Segment, dir string, cb func(string) bool) bool {
	// Consume non-wildcard path elements simply by following the path. This may
	// seem like an optimization, but is actually required for "." and ".." to
	// be used as path elements, as they do not appear in the result of ReadDir.
	for len(segs) > 1 && IsLiteral(segs[0]) && IsSlash(segs[1]) {
		elem := segs[0].(Literal).Data
		segs = segs[2:]
		dir += elem + "/"
		if info, err := os.Stat(dir); err != nil || !info.IsDir() {
			return true
		}
	}

	if len(segs) == 0 {
		return cb(dir)
	} else if len(segs) == 1 && IsLiteral(segs[0]) {
		path := dir + segs[0].(Literal).Data
		if _, err := os.Stat(path); err == nil {
			return cb(path)
		}
		return true
	}

	infos, err := readDir(dir)
	if err != nil {
		// XXX Silently drop the error
		return true
	}

	i := -1
	// nexti moves i to the next index in segs that is either / or ** (in other
	// words, something that matches /).
	nexti := func() {
		for i++; i < len(segs); i++ {
			if IsSlash(segs[i]) || IsWild1(segs[i], StarStar) {
				break
			}
		}
	}
	nexti()

	// Enumerate the position of the first slash. In the presense of multiple
	// **'s in the pattern, the first slash may be in any of those.
	//
	// For instance, in x**y**z, the first slash may be in the first ** or the
	// second:
	// 1) If it is in the first, then pattern is equivalent to x*/**y**z. We
	//    match directories with x* and recurse in each subdirectory with the
	//    pattern **y**z.
	// 2) If it is the in the second, we know that since the first ** can no
	//    longer contain any slashes, we treat it as * (this is done in
	//    matchElement). The pattern is now equivalent to x*y*/**z. We match
	//    directories with x*y* and recurse in each subdirectory with the
	//    pattern **z.
	//
	// The rules are:
	// 1) For each **, we treat it as */** and all previous ones as *. We match
	//    subdirectories with the part before /, and recurse in subdirectories
	//    with the pattern after /.
	// 2) If a literal / is encountered, we return after recursing in the
	//    subdirectories.
	for i < len(segs) {
		slash := IsSlash(segs[i])
		var first, rest []Segment
		if slash {
			// segs = x/y. Match dir with x, recurse on y.
			first, rest = segs[:i], segs[i+1:]
		} else {
			// segs = x**y. Match dir with x*, recurse on **y.
			first, rest = segs[:i+1], segs[i:]
		}

		for _, info := range infos {
			name := info.Name()
			if matchElement(first, name) && info.IsDir() {
				if !glob(rest, dir+name+"/", cb) {
					return false
				}
			}
		}

		if slash {
			// First slash cannot appear later than a slash in the pattern.
			return true
		}
		nexti()
	}

	// If we reach here, it is possible to have no slashes at all. Simply match
	// the entire pattern with all files.
	for _, info := range infos {
		name := info.Name()
		if matchElement(segs, name) {
			if !cb(dir + name) {
				return false
			}
		}
	}
	return true
}

// readDir is just like ioutil.ReadDir except that it treats an argument of ""
// as ".".
func readDir(dir string) ([]os.FileInfo, error) {
	if dir == "" {
		dir = "."
	}
	return ioutil.ReadDir(dir)
}

// matchElement matches a path element against segments, which may not contain
// any Slash segments. It treats StarStar segments as they are Star segments.
func matchElement(segs []Segment, name string) bool {
	if len(segs) == 0 {
		return name == ""
	}
	// If the name start with "." and the first segment is a Wild, only match
	// when MatchHidden is true.
	if len(name) > 0 && name[0] == '.' && IsWild(segs[0]) && !segs[0].(Wild).MatchHidden {
		return false
	}
segs:
	for len(segs) > 0 {
		// Find a chunk. A chunk is an optional Star followed by a run of
		// fixed-length segments (Literal and Question).
		var i int
		for i = 1; i < len(segs); i++ {
			if IsWild2(segs[i], Star, StarStar) {
				break
			}
		}

		chunk := segs[:i]
		startsWithStar := IsWild2(chunk[0], Star, StarStar)
		var startingStar Wild
		if startsWithStar {
			startingStar = chunk[0].(Wild)
			chunk = chunk[1:]
		}
		segs = segs[i:]

		// NOTE A quick path when len(segs) == 0 can be implemented: match
		// backwards.

		// Match at the current position. If this is the last chunk, we need to
		// make sure name is exhausted by the matching.
		ok, rest := matchFixedLength(chunk, name)
		if ok && (rest == "" || len(segs) > 0) {
			name = rest
			continue
		}

		if startsWithStar {
			// NOTE An optimization is to make the upper bound not len(names),
			// but rather len(names) - LB(# bytes segs can match)
			for i, r := range name {
				j := i + len(string(r))
				// Match name[:j] with the starting *, and the rest with chunk.
				if !startingStar.Match(r) {
					break
				}
				ok, rest := matchFixedLength(chunk, name[j:])
				if ok && (rest == "" || len(segs) > 0) {
					name = rest
					continue segs
				}
			}
		}
		return false
	}
	return name == ""
}

// matchFixedLength returns whether a run of fixed-length segments (Literal and
// Question) matches a prefix of name. It returns whether the match is
// successful and if if it is, the remaining part of name.
func matchFixedLength(segs []Segment, name string) (bool, string) {
	for _, seg := range segs {
		if name == "" {
			return false, ""
		}
		switch seg := seg.(type) {
		case Literal:
			n := len(seg.Data)
			if len(name) < n || name[:n] != seg.Data {
				return false, ""
			}
			name = name[n:]
		case Wild:
			if seg.Type == Question {
				r, n := utf8.DecodeRuneInString(name)
				if !seg.Match(r) {
					return false, ""
				}
				name = name[n:]
			} else {
				panic("matchFixedLength given non-question wild segment")
			}
		default:
			panic("matchFixedLength given non-literal non-wild segment")
		}
	}
	return true, name
}
<commit_msg>glob: Fix logics for isDrive on Windows.<commit_after>// Package glob implements globbing for elvish.
package glob

import (
	"io/ioutil"
	"os"
	"runtime"
	"unicode/utf8"
)

// TODO: Use native path separators instead of always using /.

// Glob returns a list of file names satisfying the given pattern.
func Glob(p string, cb func(string) bool) bool {
	return Parse(p).Glob(cb)
}

// Glob returns a list of file names satisfying the Pattern.
func (p Pattern) Glob(cb func(string) bool) bool {
	segs := p.Segments
	dir := ""

	// XXX: This is a hack solely for supporting globs that start with ~ in the
	// eval package.
	if p.DirOverride != "" {
		dir = p.DirOverride
	}

	if len(segs) > 0 && IsSlash(segs[0]) {
		segs = segs[1:]
		dir += "/"
	} else if runtime.GOOS == "windows" && len(segs) > 1 && IsLiteral(segs[0]) && IsSlash(segs[1]) {
		// TODO: Handle UNC.
		elem := segs[0].(Literal).Data
		if isDrive(elem) {
			segs = segs[2:]
			dir = elem + "/"
		}
	}

	return glob(segs, dir, cb)
}

func isDrive(s string) bool {
	return len(s) == 2 && s[1] == ':' &&
		(('a' <= s[0] && s[1] <= 'z') || ('A' <= s[0] && s[0] <= 'Z'))
}

// glob finds all filenames matching the given Segments in the given dir, and
// calls the callback on all of them. If the callback returns false, globbing is
// interrupted, and glob returns false. Otherwise it returns true.
func glob(segs []Segment, dir string, cb func(string) bool) bool {
	// Consume non-wildcard path elements simply by following the path. This may
	// seem like an optimization, but is actually required for "." and ".." to
	// be used as path elements, as they do not appear in the result of ReadDir.
	for len(segs) > 1 && IsLiteral(segs[0]) && IsSlash(segs[1]) {
		elem := segs[0].(Literal).Data
		segs = segs[2:]
		dir += elem + "/"
		if info, err := os.Stat(dir); err != nil || !info.IsDir() {
			return true
		}
	}

	if len(segs) == 0 {
		return cb(dir)
	} else if len(segs) == 1 && IsLiteral(segs[0]) {
		path := dir + segs[0].(Literal).Data
		if _, err := os.Stat(path); err == nil {
			return cb(path)
		}
		return true
	}

	infos, err := readDir(dir)
	if err != nil {
		// XXX Silently drop the error
		return true
	}

	i := -1
	// nexti moves i to the next index in segs that is either / or ** (in other
	// words, something that matches /).
	nexti := func() {
		for i++; i < len(segs); i++ {
			if IsSlash(segs[i]) || IsWild1(segs[i], StarStar) {
				break
			}
		}
	}
	nexti()

	// Enumerate the position of the first slash. In the presense of multiple
	// **'s in the pattern, the first slash may be in any of those.
	//
	// For instance, in x**y**z, the first slash may be in the first ** or the
	// second:
	// 1) If it is in the first, then pattern is equivalent to x*/**y**z. We
	//    match directories with x* and recurse in each subdirectory with the
	//    pattern **y**z.
	// 2) If it is the in the second, we know that since the first ** can no
	//    longer contain any slashes, we treat it as * (this is done in
	//    matchElement). The pattern is now equivalent to x*y*/**z. We match
	//    directories with x*y* and recurse in each subdirectory with the
	//    pattern **z.
	//
	// The rules are:
	// 1) For each **, we treat it as */** and all previous ones as *. We match
	//    subdirectories with the part before /, and recurse in subdirectories
	//    with the pattern after /.
	// 2) If a literal / is encountered, we return after recursing in the
	//    subdirectories.
	for i < len(segs) {
		slash := IsSlash(segs[i])
		var first, rest []Segment
		if slash {
			// segs = x/y. Match dir with x, recurse on y.
			first, rest = segs[:i], segs[i+1:]
		} else {
			// segs = x**y. Match dir with x*, recurse on **y.
			first, rest = segs[:i+1], segs[i:]
		}

		for _, info := range infos {
			name := info.Name()
			if matchElement(first, name) && info.IsDir() {
				if !glob(rest, dir+name+"/", cb) {
					return false
				}
			}
		}

		if slash {
			// First slash cannot appear later than a slash in the pattern.
			return true
		}
		nexti()
	}

	// If we reach here, it is possible to have no slashes at all. Simply match
	// the entire pattern with all files.
	for _, info := range infos {
		name := info.Name()
		if matchElement(segs, name) {
			if !cb(dir + name) {
				return false
			}
		}
	}
	return true
}

// readDir is just like ioutil.ReadDir except that it treats an argument of ""
// as ".".
func readDir(dir string) ([]os.FileInfo, error) {
	if dir == "" {
		dir = "."
	}
	return ioutil.ReadDir(dir)
}

// matchElement matches a path element against segments, which may not contain
// any Slash segments. It treats StarStar segments as they are Star segments.
func matchElement(segs []Segment, name string) bool {
	if len(segs) == 0 {
		return name == ""
	}
	// If the name start with "." and the first segment is a Wild, only match
	// when MatchHidden is true.
	if len(name) > 0 && name[0] == '.' && IsWild(segs[0]) && !segs[0].(Wild).MatchHidden {
		return false
	}
segs:
	for len(segs) > 0 {
		// Find a chunk. A chunk is an optional Star followed by a run of
		// fixed-length segments (Literal and Question).
		var i int
		for i = 1; i < len(segs); i++ {
			if IsWild2(segs[i], Star, StarStar) {
				break
			}
		}

		chunk := segs[:i]
		startsWithStar := IsWild2(chunk[0], Star, StarStar)
		var startingStar Wild
		if startsWithStar {
			startingStar = chunk[0].(Wild)
			chunk = chunk[1:]
		}
		segs = segs[i:]

		// NOTE A quick path when len(segs) == 0 can be implemented: match
		// backwards.

		// Match at the current position. If this is the last chunk, we need to
		// make sure name is exhausted by the matching.
		ok, rest := matchFixedLength(chunk, name)
		if ok && (rest == "" || len(segs) > 0) {
			name = rest
			continue
		}

		if startsWithStar {
			// NOTE An optimization is to make the upper bound not len(names),
			// but rather len(names) - LB(# bytes segs can match)
			for i, r := range name {
				j := i + len(string(r))
				// Match name[:j] with the starting *, and the rest with chunk.
				if !startingStar.Match(r) {
					break
				}
				ok, rest := matchFixedLength(chunk, name[j:])
				if ok && (rest == "" || len(segs) > 0) {
					name = rest
					continue segs
				}
			}
		}
		return false
	}
	return name == ""
}

// matchFixedLength returns whether a run of fixed-length segments (Literal and
// Question) matches a prefix of name. It returns whether the match is
// successful and if if it is, the remaining part of name.
func matchFixedLength(segs []Segment, name string) (bool, string) {
	for _, seg := range segs {
		if name == "" {
			return false, ""
		}
		switch seg := seg.(type) {
		case Literal:
			n := len(seg.Data)
			if len(name) < n || name[:n] != seg.Data {
				return false, ""
			}
			name = name[n:]
		case Wild:
			if seg.Type == Question {
				r, n := utf8.DecodeRuneInString(name)
				if !seg.Match(r) {
					return false, ""
				}
				name = name[n:]
			} else {
				panic("matchFixedLength given non-question wild segment")
			}
		default:
			panic("matchFixedLength given non-literal non-wild segment")
		}
	}
	return true, name
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"time"
)

var (
	delay    = flag.Duration("d", 0, "delay between updates")
	duration = flag.Duration("D", 0, "duration to output continually")

	pid   = flag.Int("p", 0, "process to inspect")
	pname = flag.String("n", "", "name of process to inspec")

	socketregex = regexp.MustCompile(`\.gmx\.[0-9]+\.0`)
)

type conn struct {
	net.Conn
	*json.Decoder
	*json.Encoder
}

func listGmxProcesses(f func(file string, args interface{})) {
	dir, err := os.Open(os.TempDir())
	if err != nil {
		log.Fatalf("unable to open %s: %v", os.TempDir(), err)
	}
	pids, err := dir.Readdirnames(0)
	if err != nil {
		log.Fatalf("unable to read pids: %v", err)
	}
	for _, pid := range pids {
		if socketregex.MatchString(pid) {
			c, err := dial(filepath.Join(os.TempDir(), pid))
			if err != nil {
				continue
			}
			defer c.Close()
			c.Encode([]string{"os.args"})
			var result = make(map[string]interface{})
			if err := c.Decode(&result); err != nil {
				log.Printf("unable to decode response from %s: %v", pid, err)
				continue
			}
			if args, ok := result["os.args"]; ok {
				f(pid, args)
			}
		}
	}
}

func findGmxProcess(pname string) int {
	var found int
	listGmxProcesses(func(file string, args interface{}) {
		if argslist, ok := args.([]interface{}); ok && len(argslist) >= 1 {
			name, ok := argslist[0].(string)
			if ok {
				if name == pname {
					str_pid := file[5 : len(file)-2] // ".gmx.####.0"
					numeric_pid, err := strconv.Atoi(str_pid)
					if err == nil {
						if found == 0 {
							fmt.Printf("Using %s\t%v\n", name, args)
							found = numeric_pid
						} else if found > 0 {
							fmt.Printf("Ambiguous situation. Both %d and %d could be %s. Use -p option\n", found, numeric_pid, pname)
							found = -1
						}
					}
				}
			}
		}
	})
	if found > 0 {
		return found
	}
	return 0
}

// fetchKeys returns all the registered keys from the process.
func fetchKeys(c *conn) []string {
	// retrieve list of registered keys
	if err := c.Encode([]string{"keys"}); err != nil {
		log.Fatalf("unable to send keys request to process: %v", err)
	}
	var result = make(map[string][]string)
	if err := c.Decode(&result); err != nil {
		log.Fatalf("unable to decode keys response: %v", err)
	}
	keys, ok := result["keys"]
	if !ok {
		log.Fatalf("gmx server did not return a keys list")
	}
	sort.Strings(keys)
	return keys
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage:")
		fmt.Fprintf(os.Stderr, "  %s [OPTIONS] [REGEX PATTERN]*\n", os.Args[0])
		fmt.Fprintln(os.Stderr, "")
		fmt.Fprintln(os.Stderr, "Options:")
		flag.PrintDefaults()
	}
	flag.Parse()
	if *pid == 0 && *pname != "" {
		*pid = findGmxProcess(*pname)
	}
	if *pid == 0 {
		listGmxProcesses(func(name string, args interface{}) { fmt.Printf("%s\t%v\n", name, args) })
		return
	}
	c, err := dial(filepath.Join(os.TempDir(), fmt.Sprintf(".gmx.%d.0", *pid)))
	if err != nil {
		log.Fatalf("unable to connect to process %d: %v", *pid, err)
	}
	defer c.Close()

	// match flag.Args() as regexps
	registeredKeys := fetchKeys(c)
	var keys []string
	if len(flag.Args()) == 0 {
		// no patterns? then you get everything
		keys = registeredKeys
	} else {
		for _, a := range flag.Args() {
			r, err := regexp.Compile(a)
			if err != nil {
				log.Fatal("unable to compile regex %v: %v", a, err)
			}
			for _, k := range registeredKeys {
				if r.MatchString(k) {
					keys = append(keys, k)
				}
			}
		}
	}

	deadline := time.Now().Add(*duration)
	for {
		if err := c.Encode(keys); err != nil {
			log.Fatalf("unable to send request to process: %v", err)
		}
		var result = make(map[string]interface{})
		if err := c.Decode(&result); err != nil {
			log.Fatalf("unable to decode response: %v", err)
		}
		for _, k := range keys {
			if v, ok := result[k]; ok {
				fmt.Printf("%s: %v\n", k, v)
			}
		}
		if time.Now().After(deadline) {
			return
		}
		time.Sleep(*delay)
		fmt.Println("----") // separator line
	}
}
<commit_msg>fix typo in help text<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"time"
)

var (
	delay    = flag.Duration("d", 0, "delay between updates")
	duration = flag.Duration("D", 0, "duration to output continually")

	pid   = flag.Int("p", 0, "process to inspect")
	pname = flag.String("n", "", "name of process to inspect")

	socketregex = regexp.MustCompile(`\.gmx\.[0-9]+\.0`)
)

type conn struct {
	net.Conn
	*json.Decoder
	*json.Encoder
}

func listGmxProcesses(f func(file string, args interface{})) {
	dir, err := os.Open(os.TempDir())
	if err != nil {
		log.Fatalf("unable to open %s: %v", os.TempDir(), err)
	}
	pids, err := dir.Readdirnames(0)
	if err != nil {
		log.Fatalf("unable to read pids: %v", err)
	}
	for _, pid := range pids {
		if socketregex.MatchString(pid) {
			c, err := dial(filepath.Join(os.TempDir(), pid))
			if err != nil {
				continue
			}
			defer c.Close()
			c.Encode([]string{"os.args"})
			var result = make(map[string]interface{})
			if err := c.Decode(&result); err != nil {
				log.Printf("unable to decode response from %s: %v", pid, err)
				continue
			}
			if args, ok := result["os.args"]; ok {
				f(pid, args)
			}
		}
	}
}

func findGmxProcess(pname string) int {
	var found int
	listGmxProcesses(func(file string, args interface{}) {
		if argslist, ok := args.([]interface{}); ok && len(argslist) >= 1 {
			name, ok := argslist[0].(string)
			if ok {
				if name == pname {
					str_pid := file[5 : len(file)-2] // ".gmx.####.0"
					numeric_pid, err := strconv.Atoi(str_pid)
					if err == nil {
						if found == 0 {
							fmt.Printf("Using %s\t%v\n", name, args)
							found = numeric_pid
						} else if found > 0 {
							fmt.Printf("Ambiguous situation. Both %d and %d could be %s. Use -p option\n", found, numeric_pid, pname)
							found = -1
						}
					}
				}
			}
		}
	})
	if found > 0 {
		return found
	}
	return 0
}

// fetchKeys returns all the registered keys from the process.
func fetchKeys(c *conn) []string {
	// retrieve list of registered keys
	if err := c.Encode([]string{"keys"}); err != nil {
		log.Fatalf("unable to send keys request to process: %v", err)
	}
	var result = make(map[string][]string)
	if err := c.Decode(&result); err != nil {
		log.Fatalf("unable to decode keys response: %v", err)
	}
	keys, ok := result["keys"]
	if !ok {
		log.Fatalf("gmx server did not return a keys list")
	}
	sort.Strings(keys)
	return keys
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage:")
		fmt.Fprintf(os.Stderr, "  %s [OPTIONS] [REGEX PATTERN]*\n", os.Args[0])
		fmt.Fprintln(os.Stderr, "")
		fmt.Fprintln(os.Stderr, "Options:")
		flag.PrintDefaults()
	}
	flag.Parse()
	if *pid == 0 && *pname != "" {
		*pid = findGmxProcess(*pname)
	}
	if *pid == 0 {
		listGmxProcesses(func(name string, args interface{}) { fmt.Printf("%s\t%v\n", name, args) })
		return
	}
	c, err := dial(filepath.Join(os.TempDir(), fmt.Sprintf(".gmx.%d.0", *pid)))
	if err != nil {
		log.Fatalf("unable to connect to process %d: %v", *pid, err)
	}
	defer c.Close()

	// match flag.Args() as regexps
	registeredKeys := fetchKeys(c)
	var keys []string
	if len(flag.Args()) == 0 {
		// no patterns? then you get everything
		keys = registeredKeys
	} else {
		for _, a := range flag.Args() {
			r, err := regexp.Compile(a)
			if err != nil {
				log.Fatal("unable to compile regex %v: %v", a, err)
			}
			for _, k := range registeredKeys {
				if r.MatchString(k) {
					keys = append(keys, k)
				}
			}
		}
	}

	deadline := time.Now().Add(*duration)
	for {
		if err := c.Encode(keys); err != nil {
			log.Fatalf("unable to send request to process: %v", err)
		}
		var result = make(map[string]interface{})
		if err := c.Decode(&result); err != nil {
			log.Fatalf("unable to decode response: %v", err)
		}
		for _, k := range keys {
			if v, ok := result[k]; ok {
				fmt.Printf("%s: %v\n", k, v)
			}
		}
		if time.Now().After(deadline) {
			return
		}
		time.Sleep(*delay)
		fmt.Println("----") // separator line
	}
}
<|endoftext|>


<commit_before>// A simple database migrator for PostgreSQL.

package gomigrate

import (
	"database/sql"
	"errors"
	"io/ioutil"
	"log"
	"path/filepath"
	"sort"
)

type migrationType string

const (
	migrationTableName = "gomigrate"
	upMigration        = migrationType("up")
	downMigration      = migrationType("down")
)

var (
	InvalidMigrationFile  = errors.New("Invalid migration file")
	InvalidMigrationPair  = errors.New("Invalid pair of migration files")
	InvalidMigrationsPath = errors.New("Invalid migrations path")
	InvalidMigrationType  = errors.New("Invalid migration type")
	NoActiveMigrations    = errors.New("No active migrations to rollback")
)

type Migrator struct {
	DB             *sql.DB
	MigrationsPath string
	dbAdapter      Migratable
	migrations     map[uint64]*Migration
}

// Returns true if the migration table already exists.
func (m *Migrator) MigrationTableExists() (bool, error) {
	row := m.DB.QueryRow(m.dbAdapter.SelectMigrationTableSql(), migrationTableName)
	var tableName string
	err := row.Scan(&tableName)
	if err == sql.ErrNoRows {
		log.Print("Migrations table not found")
		return false, nil
	}
	if err != nil {
		log.Printf("Error checking for migration table: %v", err)
		return false, err
	}
	log.Print("Migrations table found")
	return true, nil
}

// Creates the migrations table if it doesn't exist.
func (m *Migrator) CreateMigrationsTable() error {
	_, err := m.DB.Query(m.dbAdapter.CreateMigrationTableSql())
	if err != nil {
		log.Fatalf("Error creating migrations table: %v", err)
	}

	log.Printf("Created migrations table: %s", migrationTableName)

	return nil
}

// Returns a new migrator.
func NewMigrator(db *sql.DB, adapter Migratable, migrationsPath string) (*Migrator, error) {
	// Normalize the migrations path.
	path := []byte(migrationsPath)
	pathLength := len(path)
	if path[pathLength-1] != '/' {
		path = append(path, '/')
	}

	log.Printf("Migrations path: %s", path)

	migrator := Migrator{
		db,
		string(path),
		adapter,
		make(map[uint64]*Migration),
	}

	// Create the migrations table if it doesn't exist.
	tableExists, err := migrator.MigrationTableExists()
	if err != nil {
		return nil, err
	}
	if !tableExists {
		if err := migrator.CreateMigrationsTable(); err != nil {
			return nil, err
		}
	}

	// Get all metadata from the database.
	if err := migrator.fetchMigrations(); err != nil {
		return nil, err
	}
	if err := migrator.getMigrationStatuses(); err != nil {
		return nil, err
	}

	return &migrator, nil
}

// Populates a migrator with a sorted list of migrations from the file system.
func (m *Migrator) fetchMigrations() error {
	pathGlob := append([]byte(m.MigrationsPath), []byte("*")...)

	matches, err := filepath.Glob(string(pathGlob))
	if err != nil {
		log.Fatalf("Error while globbing migrations: %v", err)
	}

	for _, match := range matches {
		num, migrationType, name, err := parseMigrationPath(match)
		if err != nil {
			log.Printf("Invalid migration file found: %s", match)
			continue
		}

		log.Printf("Migration file found: %s", match)

		migration, ok := m.migrations[num]
		if !ok {
			migration = &Migration{Id: num, Name: name, Status: Inactive}
			m.migrations[num] = migration
		}
		if migrationType == upMigration {
			migration.UpPath = match
		} else {
			migration.DownPath = match
		}
	}

	// Validate each migration.
	for _, migration := range m.migrations {
		if !migration.valid() {
			path := migration.UpPath
			if path == "" {
				path = migration.DownPath
			}
			log.Printf("Invalid migration pair for path: %s", path)
			return InvalidMigrationPair
		}
	}

	log.Printf("Migrations file pairs found: %v", len(m.migrations))

	return nil
}

// Queries the migration table to determine the status of each
// migration.
func (m *Migrator) getMigrationStatuses() error {
	for _, migration := range m.migrations {
		row := m.DB.QueryRow(m.dbAdapter.GetMigrationSql(), migration.Id)
		var mid uint64
		err := row.Scan(&mid)
		if err == sql.ErrNoRows {
			continue
		}
		if err != nil {
			log.Printf(
				"Error getting migration status for %s: %v",
				migration.Name,
				err,
			)
			return err
		}
		migration.Status = Active
	}
	return nil
}

// Returns a sorted list of migration ids for a given status. -1 returns
// all migrations.
func (m *Migrator) Migrations(status int) []*Migration {
	// Sort all migration ids.
	ids := make([]uint64, 0)
	for id, _ := range m.migrations {
		ids = append(ids, id)
	}
	sort.Sort(uint64slice(ids))

	// Find ids for the given status.
	migrations := make([]*Migration, 0)
	for _, id := range ids {
		migration := m.migrations[id]
		if status == -1 || migration.Status == status {
			migrations = append(migrations, migration)
		}
	}
	return migrations
}

// Applies a single migration.
func (m *Migrator) ApplyMigration(migration *Migration, mType migrationType) error {
	var path string
	if mType == upMigration {
		path = migration.UpPath
	} else if mType == downMigration {
		path = migration.DownPath
	} else {
		return InvalidMigrationType
	}

	log.Printf("Applying migration: %s", path)

	sql, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Error reading migration: %s", path)
		return err
	}
	transaction, err := m.DB.Begin()
	if err != nil {
		log.Printf("Error opening transaction: %v", err)
		return err
	}

	// Perform the migration.
	result, err := transaction.Exec(string(sql))
	if err != nil {
		log.Printf("Error executing migration: %v", err)
		if rollbackErr := transaction.Rollback(); rollbackErr != nil {
			log.Printf("Error rolling back transaction: %v", rollbackErr)
			return rollbackErr
		}
		return err
	}

	if result != nil {
		if rowsAffected, err := result.RowsAffected(); err != nil {
			log.Printf("Error getting rows affected: %v", err)
			if rollbackErr := transaction.Rollback(); rollbackErr != nil {
				log.Printf("Error rolling back transaction: %v", rollbackErr)
				return rollbackErr
			}
			return err
		} else {
			log.Printf("Rows affected: %v", rowsAffected)
		}
	}

	// Log the event.
	if mType == upMigration {
		_, err = transaction.Exec(
			m.dbAdapter.MigrationLogInsertSql(),
			migration.Id,
		)
	} else {
		_, err = transaction.Exec(
			m.dbAdapter.MigrationLogDeleteSql(),
			migration.Id,
		)
	}
	if err != nil {
		log.Printf("Error logging migration: %v", err)
		if rollbackErr := transaction.Rollback(); rollbackErr != nil {
			log.Printf("Error rolling back transaction: %v", rollbackErr)
			return rollbackErr
		}
		return err
	}

	// Commit and update the struct status.
	if err := transaction.Commit(); err != nil {
		log.Printf("Error commiting transaction: %v", err)
		return err
	}
	if mType == upMigration {
		migration.Status = Active
	} else {
		migration.Status = Inactive
	}

	return nil
}

// Applies all inactive migrations.
func (m *Migrator) Migrate() error {
	for _, migration := range m.Migrations(Inactive) {
		if err := m.ApplyMigration(migration, upMigration); err != nil {
			return err
		}
	}
	return nil
}

// Rolls back the last migration.
func (m *Migrator) Rollback() error {
	return m.RollbackN(1)
}

// Rolls back N migrations.
func (m *Migrator) RollbackN(n int) error {
	migrations := m.Migrations(Active)
	if len(migrations) == 0 {
		return NoActiveMigrations
	}

	last_migration := len(migrations) - 1 - n

	for i := len(migrations) - 1; i != last_migration; i-- {
		if err := m.ApplyMigration(migrations[i], downMigration); err != nil {
			return err
		}
	}

	return nil
}

// Rolls back all migrations.
func (m *Migrator) RollbackAll() error {
	migrations := m.Migrations(Active)
	return m.RollbackN(len(migrations))
}
<commit_msg>Don't raise an error if there aren't any active migrations<commit_after>// A simple database migrator for PostgreSQL.

package gomigrate

import (
	"database/sql"
	"errors"
	"io/ioutil"
	"log"
	"path/filepath"
	"sort"
)

type migrationType string

const (
	migrationTableName = "gomigrate"
	upMigration        = migrationType("up")
	downMigration      = migrationType("down")
)

var (
	InvalidMigrationFile  = errors.New("Invalid migration file")
	InvalidMigrationPair  = errors.New("Invalid pair of migration files")
	InvalidMigrationsPath = errors.New("Invalid migrations path")
	InvalidMigrationType  = errors.New("Invalid migration type")
	NoActiveMigrations    = errors.New("No active migrations to rollback")
)

type Migrator struct {
	DB             *sql.DB
	MigrationsPath string
	dbAdapter      Migratable
	migrations     map[uint64]*Migration
}

// Returns true if the migration table already exists.
func (m *Migrator) MigrationTableExists() (bool, error) {
	row := m.DB.QueryRow(m.dbAdapter.SelectMigrationTableSql(), migrationTableName)
	var tableName string
	err := row.Scan(&tableName)
	if err == sql.ErrNoRows {
		log.Print("Migrations table not found")
		return false, nil
	}
	if err != nil {
		log.Printf("Error checking for migration table: %v", err)
		return false, err
	}
	log.Print("Migrations table found")
	return true, nil
}

// Creates the migrations table if it doesn't exist.
func (m *Migrator) CreateMigrationsTable() error {
	_, err := m.DB.Query(m.dbAdapter.CreateMigrationTableSql())
	if err != nil {
		log.Fatalf("Error creating migrations table: %v", err)
	}

	log.Printf("Created migrations table: %s", migrationTableName)

	return nil
}

// Returns a new migrator.
func NewMigrator(db *sql.DB, adapter Migratable, migrationsPath string) (*Migrator, error) {
	// Normalize the migrations path.
	path := []byte(migrationsPath)
	pathLength := len(path)
	if path[pathLength-1] != '/' {
		path = append(path, '/')
	}

	log.Printf("Migrations path: %s", path)

	migrator := Migrator{
		db,
		string(path),
		adapter,
		make(map[uint64]*Migration),
	}

	// Create the migrations table if it doesn't exist.
	tableExists, err := migrator.MigrationTableExists()
	if err != nil {
		return nil, err
	}
	if !tableExists {
		if err := migrator.CreateMigrationsTable(); err != nil {
			return nil, err
		}
	}

	// Get all metadata from the database.
	if err := migrator.fetchMigrations(); err != nil {
		return nil, err
	}
	if err := migrator.getMigrationStatuses(); err != nil {
		return nil, err
	}

	return &migrator, nil
}

// Populates a migrator with a sorted list of migrations from the file system.
func (m *Migrator) fetchMigrations() error {
	pathGlob := append([]byte(m.MigrationsPath), []byte("*")...)

	matches, err := filepath.Glob(string(pathGlob))
	if err != nil {
		log.Fatalf("Error while globbing migrations: %v", err)
	}

	for _, match := range matches {
		num, migrationType, name, err := parseMigrationPath(match)
		if err != nil {
			log.Printf("Invalid migration file found: %s", match)
			continue
		}

		log.Printf("Migration file found: %s", match)

		migration, ok := m.migrations[num]
		if !ok {
			migration = &Migration{Id: num, Name: name, Status: Inactive}
			m.migrations[num] = migration
		}
		if migrationType == upMigration {
			migration.UpPath = match
		} else {
			migration.DownPath = match
		}
	}

	// Validate each migration.
	for _, migration := range m.migrations {
		if !migration.valid() {
			path := migration.UpPath
			if path == "" {
				path = migration.DownPath
			}
			log.Printf("Invalid migration pair for path: %s", path)
			return InvalidMigrationPair
		}
	}

	log.Printf("Migrations file pairs found: %v", len(m.migrations))

	return nil
}

// Queries the migration table to determine the status of each
// migration.
func (m *Migrator) getMigrationStatuses() error {
	for _, migration := range m.migrations {
		row := m.DB.QueryRow(m.dbAdapter.GetMigrationSql(), migration.Id)
		var mid uint64
		err := row.Scan(&mid)
		if err == sql.ErrNoRows {
			continue
		}
		if err != nil {
			log.Printf(
				"Error getting migration status for %s: %v",
				migration.Name,
				err,
			)
			return err
		}
		migration.Status = Active
	}
	return nil
}

// Returns a sorted list of migration ids for a given status. -1 returns
// all migrations.
func (m *Migrator) Migrations(status int) []*Migration {
	// Sort all migration ids.
	ids := make([]uint64, 0)
	for id, _ := range m.migrations {
		ids = append(ids, id)
	}
	sort.Sort(uint64slice(ids))

	// Find ids for the given status.
	migrations := make([]*Migration, 0)
	for _, id := range ids {
		migration := m.migrations[id]
		if status == -1 || migration.Status == status {
			migrations = append(migrations, migration)
		}
	}
	return migrations
}

// Applies a single migration.
func (m *Migrator) ApplyMigration(migration *Migration, mType migrationType) error {
	var path string
	if mType == upMigration {
		path = migration.UpPath
	} else if mType == downMigration {
		path = migration.DownPath
	} else {
		return InvalidMigrationType
	}

	log.Printf("Applying migration: %s", path)

	sql, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Error reading migration: %s", path)
		return err
	}
	transaction, err := m.DB.Begin()
	if err != nil {
		log.Printf("Error opening transaction: %v", err)
		return err
	}

	// Perform the migration.
	result, err := transaction.Exec(string(sql))
	if err != nil {
		log.Printf("Error executing migration: %v", err)
		if rollbackErr := transaction.Rollback(); rollbackErr != nil {
			log.Printf("Error rolling back transaction: %v", rollbackErr)
			return rollbackErr
		}
		return err
	}

	if result != nil {
		if rowsAffected, err := result.RowsAffected(); err != nil {
			log.Printf("Error getting rows affected: %v", err)
			if rollbackErr := transaction.Rollback(); rollbackErr != nil {
				log.Printf("Error rolling back transaction: %v", rollbackErr)
				return rollbackErr
			}
			return err
		} else {
			log.Printf("Rows affected: %v", rowsAffected)
		}
	}

	// Log the event.
	if mType == upMigration {
		_, err = transaction.Exec(
			m.dbAdapter.MigrationLogInsertSql(),
			migration.Id,
		)
	} else {
		_, err = transaction.Exec(
			m.dbAdapter.MigrationLogDeleteSql(),
			migration.Id,
		)
	}
	if err != nil {
		log.Printf("Error logging migration: %v", err)
		if rollbackErr := transaction.Rollback(); rollbackErr != nil {
			log.Printf("Error rolling back transaction: %v", rollbackErr)
			return rollbackErr
		}
		return err
	}

	// Commit and update the struct status.
	if err := transaction.Commit(); err != nil {
		log.Printf("Error commiting transaction: %v", err)
		return err
	}
	if mType == upMigration {
		migration.Status = Active
	} else {
		migration.Status = Inactive
	}

	return nil
}

// Applies all inactive migrations.
func (m *Migrator) Migrate() error {
	for _, migration := range m.Migrations(Inactive) {
		if err := m.ApplyMigration(migration, upMigration); err != nil {
			return err
		}
	}
	return nil
}

// Rolls back the last migration.
func (m *Migrator) Rollback() error {
	return m.RollbackN(1)
}

// Rolls back N migrations.
func (m *Migrator) RollbackN(n int) error {
	migrations := m.Migrations(Active)
	if len(migrations) == 0 {
		return nil
	}

	last_migration := len(migrations) - 1 - n

	for i := len(migrations) - 1; i != last_migration; i-- {
		if err := m.ApplyMigration(migrations[i], downMigration); err != nil {
			return err
		}
	}

	return nil
}

// Rolls back all migrations.
func (m *Migrator) RollbackAll() error {
	migrations := m.Migrations(Active)
	return m.RollbackN(len(migrations))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"sync"
	"text/template"
	"time"

	"github.com/BurntSushi/toml"
)

type status struct {
	Name   string `toml:"name"`
	Type   string `toml:"type"`
	Status string
	URL    string `toml:"url"`
}

type sites struct {
	Service []status
}

type Status interface {
	Check(url string) (bool, error)
}

var check map[string]Status

func readConfig(conf string) ([]status, error) {
	var config sites

	_, err := toml.DecodeFile(conf, &config)
	if err != nil {
		return nil, err
	}

	return config.Service, nil
}

func checkStatus(site_status []status) {
	count := 0
	c := make(chan int)

	for i, s := range site_status {
		site_status[i].Status = "offline"

		ck, ok := check[s.Type]
		if ok == false {
			log.Println(s.Type, s.URL, "unknown type")
			continue
		}

		count++

		go func(site status, i int) {
			healthy, err := ck.Check(site.URL)
			if err == nil && healthy {
				site_status[i].Status = "online"
			}

			if err != nil {
				log.Println(site.Type, site.URL, err)
			}

			c <- i
		}(s, i)
	}

	for i := 0; i < count; i++ {
		<-c
	}
}

func sendStatus(w http.ResponseWriter, site_status []status, file string) error {
	t, err := template.ParseFiles(file)
	if err != nil {
		return err
	}

	err = t.Execute(w, site_status)
	if err != nil {
		return err
	}

	return nil
}

var site_status []status
var next_status time.Time
var lock sync.Mutex

func statusHandler(w http.ResponseWriter, r *http.Request) {
	lock.Lock()
	defer lock.Unlock()

	host, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Println("request from", host)

	var err error

	if next_status.Before(time.Now()) {
		site_status, err = readConfig("sitecheck.conf")
		if err != nil {
			http.Error(w, "internal error", http.StatusInternalServerError)
			log.Println(err)
			return
		}

		checkStatus(site_status)

		next_status = time.Now().Add(time.Second * 30)
	}

	err = sendStatus(w, site_status, "status.html")
	if err != nil {
		http.Error(w, "internal error", http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

func init() {
	check = map[string]Status{
		"website":  new(Website),
		"etcd":     new(Etcd),
		"docker":   new(Docker),
		"registry": new(Registry),
	}
}

func main() {
	var port = flag.String("http", "", "HTTP service address (.e.g. :8080)")

	flag.Parse()

	if *port == "" {
		flag.Usage()
		return
	}

	http.HandleFunc("/", statusHandler)

	log.Fatal(http.ListenAndServe(*port, nil))
}
<commit_msg>seems more popular to omit the colon from the port parameter<commit_after>package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"sync"
	"text/template"
	"time"

	"github.com/BurntSushi/toml"
)

type status struct {
	Name   string `toml:"name"`
	Type   string `toml:"type"`
	Status string
	URL    string `toml:"url"`
}

type sites struct {
	Service []status
}

type Status interface {
	Check(url string) (bool, error)
}

var check map[string]Status

func readConfig(conf string) ([]status, error) {
	var config sites

	_, err := toml.DecodeFile(conf, &config)
	if err != nil {
		return nil, err
	}

	return config.Service, nil
}

func checkStatus(site_status []status) {
	count := 0
	c := make(chan int)

	for i, s := range site_status {
		site_status[i].Status = "offline"

		ck, ok := check[s.Type]
		if ok == false {
			log.Println(s.Type, s.URL, "unknown type")
			continue
		}

		count++

		go func(site status, i int) {
			healthy, err := ck.Check(site.URL)
			if err == nil && healthy {
				site_status[i].Status = "online"
			}

			if err != nil {
				log.Println(site.Type, site.URL, err)
			}

			c <- i
		}(s, i)
	}

	for i := 0; i < count; i++ {
		<-c
	}
}

func sendStatus(w http.ResponseWriter, site_status []status, file string) error {
	t, err := template.ParseFiles(file)
	if err != nil {
		return err
	}

	err = t.Execute(w, site_status)
	if err != nil {
		return err
	}

	return nil
}

var site_status []status
var next_status time.Time
var lock sync.Mutex

func statusHandler(w http.ResponseWriter, r *http.Request) {
	lock.Lock()
	defer lock.Unlock()

	host, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Println("request from", host)

	var err error

	if next_status.Before(time.Now()) {
		site_status, err = readConfig("sitecheck.conf")
		if err != nil {
			http.Error(w, "internal error", http.StatusInternalServerError)
			log.Println(err)
			return
		}

		checkStatus(site_status)

		next_status = time.Now().Add(time.Second * 30)
	}

	err = sendStatus(w, site_status, "status.html")
	if err != nil {
		http.Error(w, "internal error", http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

func init() {
	check = map[string]Status{
		"website":  new(Website),
		"etcd":     new(Etcd),
		"docker":   new(Docker),
		"registry": new(Registry),
	}
}

func main() {
	var port = flag.String("http", "", "HTTP service address (.e.g. 8080)")

	flag.Parse()

	if *port == "" {
		flag.Usage()
		return
	}

	http.HandleFunc("/", statusHandler)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<|endoftext|>


<commit_before>package units

import (
	"reflect"
	"runtime"
	"strings"
	"testing"
)

func TestBytesSize(t *testing.T) {
	assertEquals(t, "1 KiB", BytesSize(1024))
	assertEquals(t, "1 MiB", BytesSize(1024*1024))
	assertEquals(t, "1 MiB", BytesSize(1048576))
	assertEquals(t, "2 MiB", BytesSize(2*MiB))
	assertEquals(t, "3.42 GiB", BytesSize(3.42*GiB))
	assertEquals(t, "5.372 TiB", BytesSize(5.372*TiB))
	assertEquals(t, "2.22 PiB", BytesSize(2.22*PiB))
}

func TestHumanSize(t *testing.T) {
	assertEquals(t, "1 kB", HumanSize(1000))
	assertEquals(t, "1.024 kB", HumanSize(1024))
	assertEquals(t, "1 MB", HumanSize(1000000))
	assertEquals(t, "1.049 MB", HumanSize(1048576))
	assertEquals(t, "2 MB", HumanSize(2*MB))
	assertEquals(t, "3.42 GB", HumanSize(int64(float64(3.42*GB))))
	assertEquals(t, "5.372 TB", HumanSize(int64(float64(5.372*TB))))
	assertEquals(t, "2.22 PB", HumanSize(int64(float64(2.22*PB))))
}

func TestFromHumanSize(t *testing.T) {
	assertSuccessEquals(t, 32, FromHumanSize, "32")
	assertSuccessEquals(t, 32, FromHumanSize, "32b")
	assertSuccessEquals(t, 32, FromHumanSize, "32B")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32k")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32K")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32kb")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32Kb")
	assertSuccessEquals(t, 32*MB, FromHumanSize, "32Mb")
	assertSuccessEquals(t, 32*GB, FromHumanSize, "32Gb")
	assertSuccessEquals(t, 32*TB, FromHumanSize, "32Tb")
	assertSuccessEquals(t, 32*PB, FromHumanSize, "32Pb")

	assertError(t, FromHumanSize, "")
	assertError(t, FromHumanSize, "hello")
	assertError(t, FromHumanSize, "-32")
	assertError(t, FromHumanSize, "32.3")
	assertError(t, FromHumanSize, " 32 ")
	assertError(t, FromHumanSize, "32.3Kb")
	assertError(t, FromHumanSize, "32 mb")
	assertError(t, FromHumanSize, "32m b")
	assertError(t, FromHumanSize, "32bm")
}

func TestRAMInBytes(t *testing.T) {
	assertSuccessEquals(t, 32, RAMInBytes, "32")
	assertSuccessEquals(t, 32, RAMInBytes, "32b")
	assertSuccessEquals(t, 32, RAMInBytes, "32B")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32k")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32K")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32kb")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32Kb")
	assertSuccessEquals(t, 32*MiB, RAMInBytes, "32Mb")
	assertSuccessEquals(t, 32*GiB, RAMInBytes, "32Gb")
	assertSuccessEquals(t, 32*TiB, RAMInBytes, "32Tb")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32Pb")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32PB")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32P")

	assertError(t, RAMInBytes, "")
	assertError(t, RAMInBytes, "hello")
	assertError(t, RAMInBytes, "-32")
	assertError(t, RAMInBytes, "32.3")
	assertError(t, RAMInBytes, " 32 ")
	assertError(t, RAMInBytes, "32.3Kb")
	assertError(t, RAMInBytes, "32 mb")
	assertError(t, RAMInBytes, "32m b")
	assertError(t, RAMInBytes, "32bm")
}

func assertEquals(t *testing.T, expected, actual interface{}) {
	if expected != actual {
		t.Errorf("Expected '%v' but got '%v'", expected, actual)
	}
}

// func that maps to the parse function signatures as testing abstraction
type parseFn func(string) (int64, error)

// Define 'String()' for pretty-print
func (fn parseFn) String() string {
	fnName := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()
	return fnName[strings.LastIndex(fnName, ".")+1:]
}

func assertSuccessEquals(t *testing.T, expected int64, fn parseFn, arg string) {
	res, err := fn(arg)
	if err != nil || res != expected {
		t.Errorf("%s(\"%s\") -> expected '%d' but got '%d' with error '%v'", fn, arg, expected, res, err)
	}
}

func assertError(t *testing.T, fn parseFn, arg string) {
	res, err := fn(arg)
	if err == nil && res != -1 {
		t.Errorf("%s(\"%s\") -> expected error but got '%d'", fn, arg, res)
	}
}
<commit_msg>Revert "Fix to avoid a compile error due to float to int truncation with GCCGO"<commit_after>package units

import (
	"reflect"
	"runtime"
	"strings"
	"testing"
)

func TestBytesSize(t *testing.T) {
	assertEquals(t, "1 KiB", BytesSize(1024))
	assertEquals(t, "1 MiB", BytesSize(1024*1024))
	assertEquals(t, "1 MiB", BytesSize(1048576))
	assertEquals(t, "2 MiB", BytesSize(2*MiB))
	assertEquals(t, "3.42 GiB", BytesSize(3.42*GiB))
	assertEquals(t, "5.372 TiB", BytesSize(5.372*TiB))
	assertEquals(t, "2.22 PiB", BytesSize(2.22*PiB))
}

func TestHumanSize(t *testing.T) {
	assertEquals(t, "1 kB", HumanSize(1000))
	assertEquals(t, "1.024 kB", HumanSize(1024))
	assertEquals(t, "1 MB", HumanSize(1000000))
	assertEquals(t, "1.049 MB", HumanSize(1048576))
	assertEquals(t, "2 MB", HumanSize(2*MB))
	assertEquals(t, "3.42 GB", HumanSize(3.42*GB))
	assertEquals(t, "5.372 TB", HumanSize(5.372*TB))
	assertEquals(t, "2.22 PB", HumanSize(2.22*PB))
}

func TestFromHumanSize(t *testing.T) {
	assertSuccessEquals(t, 32, FromHumanSize, "32")
	assertSuccessEquals(t, 32, FromHumanSize, "32b")
	assertSuccessEquals(t, 32, FromHumanSize, "32B")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32k")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32K")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32kb")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32Kb")
	assertSuccessEquals(t, 32*MB, FromHumanSize, "32Mb")
	assertSuccessEquals(t, 32*GB, FromHumanSize, "32Gb")
	assertSuccessEquals(t, 32*TB, FromHumanSize, "32Tb")
	assertSuccessEquals(t, 32*PB, FromHumanSize, "32Pb")

	assertError(t, FromHumanSize, "")
	assertError(t, FromHumanSize, "hello")
	assertError(t, FromHumanSize, "-32")
	assertError(t, FromHumanSize, "32.3")
	assertError(t, FromHumanSize, " 32 ")
	assertError(t, FromHumanSize, "32.3Kb")
	assertError(t, FromHumanSize, "32 mb")
	assertError(t, FromHumanSize, "32m b")
	assertError(t, FromHumanSize, "32bm")
}

func TestRAMInBytes(t *testing.T) {
	assertSuccessEquals(t, 32, RAMInBytes, "32")
	assertSuccessEquals(t, 32, RAMInBytes, "32b")
	assertSuccessEquals(t, 32, RAMInBytes, "32B")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32k")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32K")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32kb")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32Kb")
	assertSuccessEquals(t, 32*MiB, RAMInBytes, "32Mb")
	assertSuccessEquals(t, 32*GiB, RAMInBytes, "32Gb")
	assertSuccessEquals(t, 32*TiB, RAMInBytes, "32Tb")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32Pb")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32PB")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32P")

	assertError(t, RAMInBytes, "")
	assertError(t, RAMInBytes, "hello")
	assertError(t, RAMInBytes, "-32")
	assertError(t, RAMInBytes, "32.3")
	assertError(t, RAMInBytes, " 32 ")
	assertError(t, RAMInBytes, "32.3Kb")
	assertError(t, RAMInBytes, "32 mb")
	assertError(t, RAMInBytes, "32m b")
	assertError(t, RAMInBytes, "32bm")
}

func assertEquals(t *testing.T, expected, actual interface{}) {
	if expected != actual {
		t.Errorf("Expected '%v' but got '%v'", expected, actual)
	}
}

// func that maps to the parse function signatures as testing abstraction
type parseFn func(string) (int64, error)

// Define 'String()' for pretty-print
func (fn parseFn) String() string {
	fnName := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()
	return fnName[strings.LastIndex(fnName, ".")+1:]
}

func assertSuccessEquals(t *testing.T, expected int64, fn parseFn, arg string) {
	res, err := fn(arg)
	if err != nil || res != expected {
		t.Errorf("%s(\"%s\") -> expected '%d' but got '%d' with error '%v'", fn, arg, expected, res, err)
	}
}

func assertError(t *testing.T, fn parseFn, arg string) {
	res, err := fn(arg)
	if err == nil && res != -1 {
		t.Errorf("%s(\"%s\") -> expected error but got '%d'", fn, arg, res)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
)

// Person someone having a role in the tournament
type Person struct {
	ID              string   `json:"id"`
	Name            string   `json:"name"`
	Email           string   `json:"email"`
	Nick            string   `json:"nick"`
	ColorPreference []string `json:"color_preference"`
	FacebookID      string   `json:"facebook_id"`
	FacebookToken   string   `json:"facebook_token"`
	AvatarURL       string   `json:"avatar_url"`
	Userlevel       int      `json:"userlevel"`
}

// Userlevels. Designed so that we can insert new ones in between them.
const (
	PermissionProducer    = 100
	PermissionCommentator = 50
	PermissionJudge       = 30
	PermissionPlayer      = 10
)

type score map[int]string

// ScoreSummary is a collection of scores for a Person
type ScoreSummary struct {
	Totals      score
	Tournaments map[string]score
}

func (p *Person) String() string {
	return fmt.Sprintf(
		"<Player %s (%s)>",
		p.Name,
		p.Nick,
	)
}

// JSON returns the person as a JSON representation
func (p *Person) JSON() (out []byte, err error) {
	out, err = json.Marshal(p)
	return
}

// Score gets the score of the Person
//
// Returned as a map of the total score and an array of maps - one per
// tournament participated in.
func (p *Person) Score() *ScoreSummary {
	return nil
}

// CreateFromFacebook adds a new player via Facebook login
func CreateFromFacebook(s *Server, req *FacebookAuthResponse) *Person {
	p := &Person{
		ID:            req.ID,
		FacebookID:    req.ID,
		FacebookToken: req.Token,
		Name:          req.Name,
		Email:         req.Email,
		Userlevel:     PermissionPlayer,
	}

	p.PrefillNickname()

	s.DB.SavePerson(p)

	return p
}

// PrefillNickname makes a suggestion to the nick based on the person
func (p *Person) PrefillNickname() {
	// TODO(thiderman): Move this into data files
	switch p.Name {
	case "Karl Johan Krantz":
		p.Nick = "Qrl-Astrid"
		p.Userlevel = PermissionProducer
	case "Ida Andreasson":
		p.Nick = "Queen Obscene"
		p.Userlevel = PermissionProducer
	case "Daniel Dala Tiderman":
		p.Nick = "Dala"
		p.Userlevel = PermissionProducer
	case "Lowe Thiderman":
		p.Nick = "thiderman"
		p.Userlevel = PermissionProducer
	case "Magnus Ulenius":
		p.Nick = "Goose"
		p.Userlevel = PermissionProducer
	case "Jonathan Gustafsson":
		p.Nick = "hest"
		p.Userlevel = PermissionProducer

	// Commentators
	case "Daniel McHugh":
		p.Nick = "Radcliffe"
		p.Userlevel = PermissionCommentator

	// Judges
	case "Daniele Sluijters":
		p.Nick = "Daenney"
		p.Userlevel = PermissionJudge

	// Other lovelies
	case "Agnes Skoog":
		p.Nick = "#swagnes"
	case "Mattias Aali Ahlström":
		p.Nick = "Aali"
	case "Sam Wise Ingberg":
		p.Nick = "Samselott"
	}
}

// UpdatePerson updates a person from a JoinRequest
func (p *Person) UpdatePerson(r *JoinRequest) {
	p.ID = r.ID
	p.Name = r.Name
	p.Nick = r.Nick
	p.ColorPreference = []string{r.Color}
}
<commit_msg>s/Queen Obscene/Blue/g<commit_after>package main

import (
	"encoding/json"
	"fmt"
)

// Person someone having a role in the tournament
type Person struct {
	ID              string   `json:"id"`
	Name            string   `json:"name"`
	Email           string   `json:"email"`
	Nick            string   `json:"nick"`
	ColorPreference []string `json:"color_preference"`
	FacebookID      string   `json:"facebook_id"`
	FacebookToken   string   `json:"facebook_token"`
	AvatarURL       string   `json:"avatar_url"`
	Userlevel       int      `json:"userlevel"`
}

// Userlevels. Designed so that we can insert new ones in between them.
const (
	PermissionProducer    = 100
	PermissionCommentator = 50
	PermissionJudge       = 30
	PermissionPlayer      = 10
)

type score map[int]string

// ScoreSummary is a collection of scores for a Person
type ScoreSummary struct {
	Totals      score
	Tournaments map[string]score
}

func (p *Person) String() string {
	return fmt.Sprintf(
		"<Player %s (%s)>",
		p.Name,
		p.Nick,
	)
}

// JSON returns the person as a JSON representation
func (p *Person) JSON() (out []byte, err error) {
	out, err = json.Marshal(p)
	return
}

// Score gets the score of the Person
//
// Returned as a map of the total score and an array of maps - one per
// tournament participated in.
func (p *Person) Score() *ScoreSummary {
	return nil
}

// CreateFromFacebook adds a new player via Facebook login
func CreateFromFacebook(s *Server, req *FacebookAuthResponse) *Person {
	p := &Person{
		ID:            req.ID,
		FacebookID:    req.ID,
		FacebookToken: req.Token,
		Name:          req.Name,
		Email:         req.Email,
		Userlevel:     PermissionPlayer,
	}

	p.PrefillNickname()

	s.DB.SavePerson(p)

	return p
}

// PrefillNickname makes a suggestion to the nick based on the person
func (p *Person) PrefillNickname() {
	// TODO(thiderman): Move this into data files
	switch p.Name {
	case "Karl Johan Krantz":
		p.Nick = "Qrl-Astrid"
		p.Userlevel = PermissionProducer
	case "Ida Andreasson":
		p.Nick = "Blue"
		p.Userlevel = PermissionProducer
	case "Daniel Dala Tiderman":
		p.Nick = "Dala"
		p.Userlevel = PermissionProducer
	case "Lowe Thiderman":
		p.Nick = "thiderman"
		p.Userlevel = PermissionProducer
	case "Magnus Ulenius":
		p.Nick = "Goose"
		p.Userlevel = PermissionProducer
	case "Jonathan Gustafsson":
		p.Nick = "hest"
		p.Userlevel = PermissionProducer

	// Commentators
	case "Daniel McHugh":
		p.Nick = "Radcliffe"
		p.Userlevel = PermissionCommentator

	// Judges
	case "Daniele Sluijters":
		p.Nick = "Daenney"
		p.Userlevel = PermissionJudge

	// Other lovelies
	case "Agnes Skoog":
		p.Nick = "#swagnes"
	case "Mattias Aali Ahlström":
		p.Nick = "Aali"
	case "Sam Wise Ingberg":
		p.Nick = "Samselott"
	}
}

// UpdatePerson updates a person from a JoinRequest
func (p *Person) UpdatePerson(r *JoinRequest) {
	p.ID = r.ID
	p.Name = r.Name
	p.Nick = r.Nick
	p.ColorPreference = []string{r.Color}
}
<|endoftext|>