<commit_before>package h2quic

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/utils"
	"github.com/lucas-clemente/quic-go/qerr"
	"golang.org/x/net/http2"
	"golang.org/x/net/http2/hpack"
)

type streamCreator interface {
	quic.Session
	GetOrOpenStream(protocol.StreamID) (quic.Stream, error)
}

type remoteCloser interface {
	CloseRemote(protocol.ByteCount)
}

// allows mocking of quic.Listen and quic.ListenAddr
var (
	quicListen     = quic.Listen
	quicListenAddr = quic.ListenAddr
)

// Server is a HTTP2 server listening for QUIC connections.
type Server struct {
	*http.Server

	// By providing a quic.Config, it is possible to set parameters of the QUIC connection.
	// If nil, it uses reasonable default values.
	QuicConfig *quic.Config

	// Private flag for demo, do not use
	CloseAfterFirstRequest bool

	port uint32 // used atomically

	listenerMutex sync.Mutex
	listener      quic.Listener

	supportedVersionsAsString string
}

// ListenAndServe listens on the UDP address s.Addr and calls s.Handler to handle HTTP/2 requests on incoming connections.
func (s *Server) ListenAndServe() error {
	if s.Server == nil {
		return errors.New("use of h2quic.Server without http.Server")
	}
	return s.serveImpl(s.TLSConfig, nil)
}

// ListenAndServeTLS listens on the UDP address s.Addr and calls s.Handler to handle HTTP/2 requests on incoming connections.
func (s *Server) ListenAndServeTLS(certFile, keyFile string) error {
	var err error
	certs := make([]tls.Certificate, 1)
	certs[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}
	// We currently only use the cert-related stuff from tls.Config,
	// so we don't need to make a full copy.
	config := &tls.Config{
		Certificates: certs,
	}
	return s.serveImpl(config, nil)
}

// Serve an existing UDP connection.
func (s *Server) Serve(conn net.PacketConn) error {
	return s.serveImpl(s.TLSConfig, conn)
}

func (s *Server) serveImpl(tlsConfig *tls.Config, conn net.PacketConn) error {
	if s.Server == nil {
		return errors.New("use of h2quic.Server without http.Server")
	}
	s.listenerMutex.Lock()
	if s.listener != nil {
		s.listenerMutex.Unlock()
		return errors.New("ListenAndServe may only be called once")
	}

	var ln quic.Listener
	var err error
	if conn == nil {
		ln, err = quicListenAddr(s.Addr, tlsConfig, s.QuicConfig)
	} else {
		ln, err = quicListen(conn, tlsConfig, s.QuicConfig)
	}
	if err != nil {
		s.listenerMutex.Unlock()
		return err
	}
	s.listener = ln
	s.listenerMutex.Unlock()

	for {
		sess, err := ln.Accept()
		if err != nil {
			return err
		}
		go s.handleHeaderStream(sess.(streamCreator))
	}
}

func (s *Server) handleHeaderStream(session streamCreator) {
	stream, err := session.AcceptStream()
	if err != nil {
		session.Close(qerr.Error(qerr.InvalidHeadersStreamData, err.Error()))
		return
	}

	hpackDecoder := hpack.NewDecoder(4096, nil)
	h2framer := http2.NewFramer(nil, stream)

	go func() {
		var headerStreamMutex sync.Mutex // Protects concurrent calls to Write()
		for {
			if err := s.handleRequest(session, stream, &headerStreamMutex, hpackDecoder, h2framer); err != nil {
				// QuicErrors must originate from stream.Read() returning an error.
				// In this case, the session has already logged the error, so we don't
				// need to log it again.
				if _, ok := err.(*qerr.QuicError); !ok {
					utils.Errorf("error handling h2 request: %s", err.Error())
				}
				session.Close(err)
				return
			}
		}
	}()
}

func (s *Server) handleRequest(session streamCreator, headerStream quic.Stream, headerStreamMutex *sync.Mutex, hpackDecoder *hpack.Decoder, h2framer *http2.Framer) error {
	h2frame, err := h2framer.ReadFrame()
	if err != nil {
		return qerr.Error(qerr.HeadersStreamDataDecompressFailure, "cannot read frame")
	}
	h2headersFrame, ok := h2frame.(*http2.HeadersFrame)
	if !ok {
		return qerr.Error(qerr.InvalidHeadersStreamData, "expected a header frame")
	}
	if !h2headersFrame.HeadersEnded() {
		return errors.New("http2 header continuation not implemented")
	}
	headers, err := hpackDecoder.DecodeFull(h2headersFrame.HeaderBlockFragment())
	if err != nil {
		utils.Errorf("invalid http2 headers encoding: %s", err.Error())
		return err
	}

	req, err := requestFromHeaders(headers)
	if err != nil {
		return err
	}

	req.RemoteAddr = session.RemoteAddr().String()

	if utils.Debug() {
		utils.Infof("%s %s%s, on data stream %d", req.Method, req.Host, req.RequestURI, h2headersFrame.StreamID)
	} else {
		utils.Infof("%s %s%s", req.Method, req.Host, req.RequestURI)
	}

	dataStream, err := session.GetOrOpenStream(protocol.StreamID(h2headersFrame.StreamID))
	if err != nil {
		return err
	}
	// this can happen if the client immediately closes the data stream after sending the request and the runtime processes the reset before the request
	if dataStream == nil {
		return nil
	}

	var streamEnded bool
	if h2headersFrame.StreamEnded() {
		dataStream.(remoteCloser).CloseRemote(0)
		streamEnded = true
		_, _ = dataStream.Read([]byte{0}) // read the eof
	}

	req = req.WithContext(dataStream.Context())
	reqBody := newRequestBody(dataStream)
	req.Body = reqBody

	responseWriter := newResponseWriter(headerStream, headerStreamMutex, dataStream, protocol.StreamID(h2headersFrame.StreamID))

	go func() {
		handler := s.Handler
		if handler == nil {
			handler = http.DefaultServeMux
		}
		panicked := false
		func() {
			defer func() {
				if p := recover(); p != nil {
					// Copied from net/http/server.go
					const size = 64 << 10
					buf := make([]byte, size)
					buf = buf[:runtime.Stack(buf, false)]
					utils.Errorf("http: panic serving: %v\n%s", p, buf)
					panicked = true
				}
			}()
			handler.ServeHTTP(responseWriter, req)
		}()
		if panicked {
			responseWriter.WriteHeader(500)
		} else {
			responseWriter.WriteHeader(200)
		}
		if responseWriter.dataStream != nil {
			if !streamEnded && !reqBody.requestRead {
				responseWriter.dataStream.Reset(nil)
			}
			responseWriter.dataStream.Close()
		}
		if s.CloseAfterFirstRequest {
			time.Sleep(100 * time.Millisecond)
			session.Close(nil)
		}
	}()

	return nil
}

// Close the server immediately, aborting requests and sending CONNECTION_CLOSE frames to connected clients.
// Close in combination with ListenAndServe() (instead of Serve()) may race if it is called before a UDP socket is established.
func (s *Server) Close() error {
	s.listenerMutex.Lock()
	defer s.listenerMutex.Unlock()
	if s.listener != nil {
		err := s.listener.Close()
		s.listener = nil
		return err
	}
	return nil
}

// CloseGracefully shuts down the server gracefully. The server sends a GOAWAY frame first, then waits for either timeout to trigger, or for all running requests to complete.
// CloseGracefully in combination with ListenAndServe() (instead of Serve()) may race if it is called before a UDP socket is established.
func (s *Server) CloseGracefully(timeout time.Duration) error {
	// TODO: implement
	return nil
}

// SetQuicHeaders can be used to set the proper headers that announce that this server supports QUIC.
// The values that are set depend on the port information from s.Server.Addr, and currently look like this (if Addr has port 443):
//  Alt-Svc: quic=":443"; ma=2592000; v="33,32,31,30"
func (s *Server) SetQuicHeaders(hdr http.Header) error {
	port := atomic.LoadUint32(&s.port)

	if port == 0 {
		// Extract port from s.Server.Addr
		_, portStr, err := net.SplitHostPort(s.Server.Addr)
		if err != nil {
			return err
		}
		portInt, err := net.LookupPort("tcp", portStr)
		if err != nil {
			return err
		}
		port = uint32(portInt)
		atomic.StoreUint32(&s.port, port)
	}

	if s.supportedVersionsAsString == "" {
		var versions []string
		for _, v := range protocol.SupportedVersions {
			versions = append(versions, v.ToAltSvc())
		}
		s.supportedVersionsAsString = strings.Join(versions, ",")
	}

	hdr.Add("Alt-Svc", fmt.Sprintf(`quic=":%d"; ma=2592000; v="%s"`, port, s.supportedVersionsAsString))

	return nil
}

// ListenAndServeQUIC listens on the UDP network address addr and calls the
// handler for HTTP/2 requests on incoming connections. http.DefaultServeMux is
// used when handler is nil.
func ListenAndServeQUIC(addr, certFile, keyFile string, handler http.Handler) error {
	server := &Server{
		Server: &http.Server{
			Addr:    addr,
			Handler: handler,
		},
	}
	return server.ListenAndServeTLS(certFile, keyFile)
}

// ListenAndServe listens on the given network address for both, TLS and QUIC
// connetions in parallel. It returns if one of the two returns an error.
// http.DefaultServeMux is used when handler is nil.
// The correct Alt-Svc headers for QUIC are set.
func ListenAndServe(addr, certFile, keyFile string, handler http.Handler) error {
	// Load certs
	var err error
	certs := make([]tls.Certificate, 1)
	certs[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}
	// We currently only use the cert-related stuff from tls.Config,
	// so we don't need to make a full copy.
	config := &tls.Config{
		Certificates: certs,
	}

	// Open the listeners
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return err
	}
	udpConn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		return err
	}
	defer udpConn.Close()

	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return err
	}
	tcpConn, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return err
	}
	defer tcpConn.Close()

	tlsConn := tls.NewListener(tcpConn, config)
	defer tlsConn.Close()

	// Start the servers
	httpServer := &http.Server{
		Addr:      addr,
		TLSConfig: config,
	}

	quicServer := &Server{
		Server: httpServer,
	}

	if handler == nil {
		handler = http.DefaultServeMux
	}
	httpServer.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		quicServer.SetQuicHeaders(w.Header())
		handler.ServeHTTP(w, r)
	})

	hErr := make(chan error)
	qErr := make(chan error)
	go func() {
		hErr <- httpServer.Serve(tlsConn)
	}()
	go func() {
		qErr <- quicServer.Serve(udpConn)
	}()

	select {
	case err := <-hErr:
		quicServer.Close()
		return err
	case err := <-qErr:
		// Cannot close the HTTP server or wait for requests to complete properly :/
		return err
	}
}
<commit_msg>remove superfluous goroutine<commit_after>package h2quic

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/utils"
	"github.com/lucas-clemente/quic-go/qerr"
	"golang.org/x/net/http2"
	"golang.org/x/net/http2/hpack"
)

type streamCreator interface {
	quic.Session
	GetOrOpenStream(protocol.StreamID) (quic.Stream, error)
}

type remoteCloser interface {
	CloseRemote(protocol.ByteCount)
}

// allows mocking of quic.Listen and quic.ListenAddr
var (
	quicListen     = quic.Listen
	quicListenAddr = quic.ListenAddr
)

// Server is a HTTP2 server listening for QUIC connections.
type Server struct {
	*http.Server

	// By providing a quic.Config, it is possible to set parameters of the QUIC connection.
	// If nil, it uses reasonable default values.
	QuicConfig *quic.Config

	// Private flag for demo, do not use
	CloseAfterFirstRequest bool

	port uint32 // used atomically

	listenerMutex sync.Mutex
	listener      quic.Listener

	supportedVersionsAsString string
}

// ListenAndServe listens on the UDP address s.Addr and calls s.Handler to handle HTTP/2 requests on incoming connections.
func (s *Server) ListenAndServe() error {
	if s.Server == nil {
		return errors.New("use of h2quic.Server without http.Server")
	}
	return s.serveImpl(s.TLSConfig, nil)
}

// ListenAndServeTLS listens on the UDP address s.Addr and calls s.Handler to handle HTTP/2 requests on incoming connections.
func (s *Server) ListenAndServeTLS(certFile, keyFile string) error {
	var err error
	certs := make([]tls.Certificate, 1)
	certs[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}
	// We currently only use the cert-related stuff from tls.Config,
	// so we don't need to make a full copy.
	config := &tls.Config{
		Certificates: certs,
	}
	return s.serveImpl(config, nil)
}

// Serve an existing UDP connection.
func (s *Server) Serve(conn net.PacketConn) error {
	return s.serveImpl(s.TLSConfig, conn)
}

func (s *Server) serveImpl(tlsConfig *tls.Config, conn net.PacketConn) error {
	if s.Server == nil {
		return errors.New("use of h2quic.Server without http.Server")
	}
	s.listenerMutex.Lock()
	if s.listener != nil {
		s.listenerMutex.Unlock()
		return errors.New("ListenAndServe may only be called once")
	}

	var ln quic.Listener
	var err error
	if conn == nil {
		ln, err = quicListenAddr(s.Addr, tlsConfig, s.QuicConfig)
	} else {
		ln, err = quicListen(conn, tlsConfig, s.QuicConfig)
	}
	if err != nil {
		s.listenerMutex.Unlock()
		return err
	}
	s.listener = ln
	s.listenerMutex.Unlock()

	for {
		sess, err := ln.Accept()
		if err != nil {
			return err
		}
		go s.handleHeaderStream(sess.(streamCreator))
	}
}

func (s *Server) handleHeaderStream(session streamCreator) {
	stream, err := session.AcceptStream()
	if err != nil {
		session.Close(qerr.Error(qerr.InvalidHeadersStreamData, err.Error()))
		return
	}

	hpackDecoder := hpack.NewDecoder(4096, nil)
	h2framer := http2.NewFramer(nil, stream)

	var headerStreamMutex sync.Mutex // Protects concurrent calls to Write()
	for {
		if err := s.handleRequest(session, stream, &headerStreamMutex, hpackDecoder, h2framer); err != nil {
			// QuicErrors must originate from stream.Read() returning an error.
			// In this case, the session has already logged the error, so we don't
			// need to log it again.
			if _, ok := err.(*qerr.QuicError); !ok {
				utils.Errorf("error handling h2 request: %s", err.Error())
			}
			session.Close(err)
			return
		}
	}
}

func (s *Server) handleRequest(session streamCreator, headerStream quic.Stream, headerStreamMutex *sync.Mutex, hpackDecoder *hpack.Decoder, h2framer *http2.Framer) error {
	h2frame, err := h2framer.ReadFrame()
	if err != nil {
		return qerr.Error(qerr.HeadersStreamDataDecompressFailure, "cannot read frame")
	}
	h2headersFrame, ok := h2frame.(*http2.HeadersFrame)
	if !ok {
		return qerr.Error(qerr.InvalidHeadersStreamData, "expected a header frame")
	}
	if !h2headersFrame.HeadersEnded() {
		return errors.New("http2 header continuation not implemented")
	}
	headers, err := hpackDecoder.DecodeFull(h2headersFrame.HeaderBlockFragment())
	if err != nil {
		utils.Errorf("invalid http2 headers encoding: %s", err.Error())
		return err
	}

	req, err := requestFromHeaders(headers)
	if err != nil {
		return err
	}

	req.RemoteAddr = session.RemoteAddr().String()

	if utils.Debug() {
		utils.Infof("%s %s%s, on data stream %d", req.Method, req.Host, req.RequestURI, h2headersFrame.StreamID)
	} else {
		utils.Infof("%s %s%s", req.Method, req.Host, req.RequestURI)
	}

	dataStream, err := session.GetOrOpenStream(protocol.StreamID(h2headersFrame.StreamID))
	if err != nil {
		return err
	}
	// this can happen if the client immediately closes the data stream after sending the request and the runtime processes the reset before the request
	if dataStream == nil {
		return nil
	}

	var streamEnded bool
	if h2headersFrame.StreamEnded() {
		dataStream.(remoteCloser).CloseRemote(0)
		streamEnded = true
		_, _ = dataStream.Read([]byte{0}) // read the eof
	}

	req = req.WithContext(dataStream.Context())
	reqBody := newRequestBody(dataStream)
	req.Body = reqBody

	responseWriter := newResponseWriter(headerStream, headerStreamMutex, dataStream, protocol.StreamID(h2headersFrame.StreamID))

	go func() {
		handler := s.Handler
		if handler == nil {
			handler = http.DefaultServeMux
		}
		panicked := false
		func() {
			defer func() {
				if p := recover(); p != nil {
					// Copied from net/http/server.go
					const size = 64 << 10
					buf := make([]byte, size)
					buf = buf[:runtime.Stack(buf, false)]
					utils.Errorf("http: panic serving: %v\n%s", p, buf)
					panicked = true
				}
			}()
			handler.ServeHTTP(responseWriter, req)
		}()
		if panicked {
			responseWriter.WriteHeader(500)
		} else {
			responseWriter.WriteHeader(200)
		}
		if responseWriter.dataStream != nil {
			if !streamEnded && !reqBody.requestRead {
				responseWriter.dataStream.Reset(nil)
			}
			responseWriter.dataStream.Close()
		}
		if s.CloseAfterFirstRequest {
			time.Sleep(100 * time.Millisecond)
			session.Close(nil)
		}
	}()

	return nil
}

// Close the server immediately, aborting requests and sending CONNECTION_CLOSE frames to connected clients.
// Close in combination with ListenAndServe() (instead of Serve()) may race if it is called before a UDP socket is established.
func (s *Server) Close() error {
	s.listenerMutex.Lock()
	defer s.listenerMutex.Unlock()
	if s.listener != nil {
		err := s.listener.Close()
		s.listener = nil
		return err
	}
	return nil
}

// CloseGracefully shuts down the server gracefully. The server sends a GOAWAY frame first, then waits for either timeout to trigger, or for all running requests to complete.
// CloseGracefully in combination with ListenAndServe() (instead of Serve()) may race if it is called before a UDP socket is established.
func (s *Server) CloseGracefully(timeout time.Duration) error {
	// TODO: implement
	return nil
}

// SetQuicHeaders can be used to set the proper headers that announce that this server supports QUIC.
// The values that are set depend on the port information from s.Server.Addr, and currently look like this (if Addr has port 443):
//  Alt-Svc: quic=":443"; ma=2592000; v="33,32,31,30"
func (s *Server) SetQuicHeaders(hdr http.Header) error {
	port := atomic.LoadUint32(&s.port)

	if port == 0 {
		// Extract port from s.Server.Addr
		_, portStr, err := net.SplitHostPort(s.Server.Addr)
		if err != nil {
			return err
		}
		portInt, err := net.LookupPort("tcp", portStr)
		if err != nil {
			return err
		}
		port = uint32(portInt)
		atomic.StoreUint32(&s.port, port)
	}

	if s.supportedVersionsAsString == "" {
		var versions []string
		for _, v := range protocol.SupportedVersions {
			versions = append(versions, v.ToAltSvc())
		}
		s.supportedVersionsAsString = strings.Join(versions, ",")
	}

	hdr.Add("Alt-Svc", fmt.Sprintf(`quic=":%d"; ma=2592000; v="%s"`, port, s.supportedVersionsAsString))

	return nil
}

// ListenAndServeQUIC listens on the UDP network address addr and calls the
// handler for HTTP/2 requests on incoming connections. http.DefaultServeMux is
// used when handler is nil.
func ListenAndServeQUIC(addr, certFile, keyFile string, handler http.Handler) error {
	server := &Server{
		Server: &http.Server{
			Addr:    addr,
			Handler: handler,
		},
	}
	return server.ListenAndServeTLS(certFile, keyFile)
}

// ListenAndServe listens on the given network address for both, TLS and QUIC
// connetions in parallel. It returns if one of the two returns an error.
// http.DefaultServeMux is used when handler is nil.
// The correct Alt-Svc headers for QUIC are set.
func ListenAndServe(addr, certFile, keyFile string, handler http.Handler) error {
	// Load certs
	var err error
	certs := make([]tls.Certificate, 1)
	certs[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}
	// We currently only use the cert-related stuff from tls.Config,
	// so we don't need to make a full copy.
	config := &tls.Config{
		Certificates: certs,
	}

	// Open the listeners
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return err
	}
	udpConn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		return err
	}
	defer udpConn.Close()

	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return err
	}
	tcpConn, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return err
	}
	defer tcpConn.Close()

	tlsConn := tls.NewListener(tcpConn, config)
	defer tlsConn.Close()

	// Start the servers
	httpServer := &http.Server{
		Addr:      addr,
		TLSConfig: config,
	}

	quicServer := &Server{
		Server: httpServer,
	}

	if handler == nil {
		handler = http.DefaultServeMux
	}
	httpServer.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		quicServer.SetQuicHeaders(w.Header())
		handler.ServeHTTP(w, r)
	})

	hErr := make(chan error)
	qErr := make(chan error)
	go func() {
		hErr <- httpServer.Serve(tlsConn)
	}()
	go func() {
		qErr <- quicServer.Serve(udpConn)
	}()

	select {
	case err := <-hErr:
		quicServer.Close()
		return err
	case err := <-qErr:
		// Cannot close the HTTP server or wait for requests to complete properly :/
		return err
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"reflect"
	"strings"
	"testing"
)

func TestGetAllRecordsOfType(t *testing.T) {
	types := map[string]string{
		"posts": `[
					{
					  "id": 1,
					  "title": "Testing",
					  "author": "Foo"
					},
					{
					  "id": 2,
					  "title": "Testing Post ID 2",
					  "author": "Bar"
					}
				  ]`,
		"comments": `[
						{
						  "id": 1,
						  "body": "Testing",
						  "postId": 1
						},
						{
						  "id": 2,
						  "body": "Testing Comment ID 2",
						  "postId": 2
						}
					  ]`,
	}

	for recordType, expectedJson := range types {
		testGetRequest(t, "/"+recordType, expectedJson, http.StatusOK, true, false)
	}

	testGetRequest(t, "/invalid", "", http.StatusNotFound, true, false)
}

func TestGetRecord(t *testing.T) {
	resp, err := http.Get("http://" + TestServerAddr + "/posts")

	if err != nil {
		t.Error(err)
		return
	}

	defer resp.Body.Close()
}

func TestPostRecord(t *testing.T) {

}

func TestPutRecord(t *testing.T) {

}

func TestDeleteRecord(t *testing.T) {

}

func TestGetAll(t *testing.T) {

}

func testGetRequest(t *testing.T, path string, expectedJson string, expectedStatus int, useArray bool, compareBody bool) {
	resp, err := http.Get("http://" + TestServerAddr + path)

	if err != nil {
		t.Error(err)
		return
	}

	defer resp.Body.Close()

	if resp.StatusCode != expectedStatus {
		t.Errorf("Unexpected status code. Expected %d, got %d\n", expectedStatus, resp.StatusCode)
	}

	// Seems a little weird to have this as a parameter, but it prevents duplication of the above code
	if !compareBody {
		return
	}

	match, err := jsonResponseMatchesActual(resp, expectedJson, useArray)

	if !match {
		// TODO: make this print out the data mismatch? should probably just print the raw maps
		t.Error("Data mismatch for path", path)
	}

	if err != nil {
		t.Error(err)
	}
}

func jsonResponseMatchesActual(resp *http.Response, expected string, useArray bool) (bool, error) {
	var (
		expectedData interface{}
		actualData   interface{}
	)

	if useArray {
		expectedData = make(map[string]interface{})
	} else {
		expectedData = []interface{}{}
	}

	if contentType := resp.Header.Get("Content-Type"); contentType != "application/json" {
		return false, fmt.Errorf("Unexpected Content-Type %s\n", contentType)
	}

	if useArray {
		actualData = make(map[string]interface{})
	} else {
		actualData = []interface{}{}
	}

	err := decodeJson(strings.NewReader(expected), &expectedData)

	if err != nil {
		return false, err
	}

	err = decodeJson(resp.Body, &actualData)

	if err != nil {
		return false, err
	}

	return reflect.DeepEqual(expectedData, actualData), nil
}
<commit_msg>Complete TestGetRecord<commit_after>package main

import (
	"fmt"
	"net/http"
	"reflect"
	"strings"
	"testing"
)

func TestGetAllRecordsOfType(t *testing.T) {
	types := map[string]string{
		"posts": `[
					{
					  "id": 1,
					  "title": "Testing",
					  "author": "Foo"
					},
					{
					  "id": 2,
					  "title": "Testing Post ID 2",
					  "author": "Bar"
					}
				  ]`,
		"comments": `[
						{
						  "id": 1,
						  "body": "Testing",
						  "postId": 1
						},
						{
						  "id": 2,
						  "body": "Testing Comment ID 2",
						  "postId": 2
						}
					  ]`,
	}

	for recordType, expectedJson := range types {
		testGetRequest(t, "/"+recordType, expectedJson, http.StatusOK, true, false)
	}

	testGetRequest(t, "/invalid", "", http.StatusNotFound, true, false)
}

func TestGetRecord(t *testing.T) {
	paths := map[string]string{
		"/posts/2": `{
					  "id": 2,
					  "title": "Testing Post ID 2",
					  "author": "Bar"
					}`,
		"/comments/1": `{
						  "id": 1,
						  "body": "Testing",
						  "postId": 1
						}`,
	}

	for path, expectedJson := range paths {
		testGetRequest(t, path, expectedJson, http.StatusOK, true, false)
	}

	invalidPaths := []string{
		"/posts/-1",
		"/posts/9000",
		"/comments/-1",
		"/comments/9000",
	}

	for _, path := range invalidPaths {
		testGetRequest(t, path, "", http.StatusNotFound, true, false)
	}
}

func TestPostRecord(t *testing.T) {

}

func TestPutRecord(t *testing.T) {

}

func TestDeleteRecord(t *testing.T) {

}

func TestGetAll(t *testing.T) {

}

func testGetRequest(t *testing.T, path string, expectedJson string, expectedStatus int, useArray bool, compareBody bool) {
	resp, err := http.Get("http://" + TestServerAddr + path)

	if err != nil {
		t.Error(err)
		return
	}

	defer resp.Body.Close()

	if resp.StatusCode != expectedStatus {
		t.Errorf("Unexpected status code. Expected %d, got %d\n", expectedStatus, resp.StatusCode)
	}

	// Seems a little weird to have this as a parameter, but it prevents duplication of the above code
	if !compareBody {
		return
	}

	match, err := jsonResponseMatchesActual(resp, expectedJson, useArray)

	if !match {
		// TODO: make this print out the data mismatch? should probably just print the raw maps
		t.Error("Data mismatch for path", path)
	}

	if err != nil {
		t.Error(err)
	}
}

func jsonResponseMatchesActual(resp *http.Response, expected string, useArray bool) (bool, error) {
	var (
		expectedData interface{}
		actualData   interface{}
	)

	if useArray {
		expectedData = make(map[string]interface{})
	} else {
		expectedData = []interface{}{}
	}

	if contentType := resp.Header.Get("Content-Type"); contentType != "application/json" {
		return false, fmt.Errorf("Unexpected Content-Type %s\n", contentType)
	}

	if useArray {
		actualData = make(map[string]interface{})
	} else {
		actualData = []interface{}{}
	}

	err := decodeJson(strings.NewReader(expected), &expectedData)

	if err != nil {
		return false, err
	}

	err = decodeJson(resp.Body, &actualData)

	if err != nil {
		return false, err
	}

	return reflect.DeepEqual(expectedData, actualData), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease_test

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"strings"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/lease"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestMultiReadProxy(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// A ReadProxy that wraps another, calling CheckInvariants before and after
// each action.
type checkingReadProxy struct {
	Wrapped lease.ReadProxy
}

func (crp *checkingReadProxy) Size() (size int64) {
	crp.Wrapped.CheckInvariants()
	defer crp.Wrapped.CheckInvariants()

	size = crp.Wrapped.Size()
	return
}

func (crp *checkingReadProxy) ReadAt(
	ctx context.Context,
	p []byte,
	off int64) (n int, err error) {
	crp.Wrapped.CheckInvariants()
	defer crp.Wrapped.CheckInvariants()

	n, err = crp.Wrapped.ReadAt(ctx, p, off)
	return
}

func (crp *checkingReadProxy) Upgrade(
	ctx context.Context) (rwl lease.ReadWriteLease, err error) {
	crp.Wrapped.CheckInvariants()
	rwl, err = crp.Wrapped.Upgrade(ctx)
	return
}

func (crp *checkingReadProxy) Destroy() {
	crp.Wrapped.CheckInvariants()
	crp.Wrapped.Destroy()
}

func (crp *checkingReadProxy) CheckInvariants() {
	crp.Wrapped.CheckInvariants()
}

// A range to read, the contents we expect to get back, and a matcher for the
// returned error. Special case: a nil matcher means Equals(nil).
type readAtTestCase struct {
	start            int64
	limit            int64
	errMatcher       Matcher
	expectedContents string
}

func runReadAtTestCases(
	rp lease.ReadProxy,
	cases []readAtTestCase) {
	for i, tc := range cases {
		desc := fmt.Sprintf("Test case %d: [%d, %d)", i, tc.start, tc.limit)

		AssertLe(tc.start, tc.limit)
		buf := make([]byte, tc.limit-tc.start)

		n, err := rp.ReadAt(context.Background(), buf, tc.start)
		AssertEq(tc.expectedContents, string(buf[:n]), "%s", desc)

		if tc.errMatcher == nil {
			AssertEq(nil, err, "%s", desc)
		} else {
			ExpectThat(err, tc.errMatcher, desc)
		}
	}
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type MultiReadProxyTest struct {
	ctx context.Context

	// Canned content and errors returned by the refreshers.
	refresherContents []string
	refresherErrors   []error

	leaser lease.FileLeaser
	proxy  *checkingReadProxy
}

var _ SetUpInterface = &MultiReadProxyTest{}
var _ TearDownInterface = &MultiReadProxyTest{}

func init() { RegisterTestSuite(&MultiReadProxyTest{}) }

func (t *MultiReadProxyTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.leaser = lease.NewFileLeaser("", math.MaxInt64)

	// Set up default refresher contents and nil errors.
	t.refresherContents = []string{
		"taco",
		"burrito",
		"enchilada",
	}
	t.refresherErrors = make([]error, len(t.refresherContents))

	// Create the proxy.
	t.resetProxy()
}

func (t *MultiReadProxyTest) TearDown() {
	// Make sure nothing goes crazy.
	if t.proxy != nil {
		t.proxy.Destroy()
	}
}

// Recreate refreshers using makeRefreshers and reset the proxy.
func (t *MultiReadProxyTest) resetProxy() {
	t.proxy = &checkingReadProxy{
		Wrapped: lease.NewMultiReadProxy(
			t.leaser,
			t.makeRefreshers(),
			nil),
	}
}

// Create refreshers based on the current contents of t.refresherContents.
// t.refresherErrors will be inspected only when Refresh is called.
func (t *MultiReadProxyTest) makeRefreshers() (refreshers []lease.Refresher) {
	for i := range t.refresherContents {
		iCopy := i
		contents := t.refresherContents[i]

		r := &funcRefresher{
			N: int64(len(contents)),
			F: func(ctx context.Context) (rc io.ReadCloser, err error) {
				rc = ioutil.NopCloser(strings.NewReader(contents))
				err = t.refresherErrors[iCopy]
				return
			},
		}

		refreshers = append(refreshers, r)
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *MultiReadProxyTest) SizeZero_NoRefreshers() {
	t.refresherContents = []string{}
	t.refresherErrors = []error{}
	t.resetProxy()

	// Size
	ExpectEq(0, t.proxy.Size())

	// ReadAt
	eofMatcher := Equals(io.EOF)
	testCases := []readAtTestCase{
		readAtTestCase{0, 0, eofMatcher, ""},
		readAtTestCase{0, 10, eofMatcher, ""},
		readAtTestCase{5, 10, eofMatcher, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) SizeZero_WithRefreshers() {
	t.refresherContents = []string{"", "", "", ""}
	t.refresherErrors = make([]error, len(t.refresherContents))
	t.resetProxy()

	// Size
	ExpectEq(0, t.proxy.Size())

	// ReadAt
	eofMatcher := Equals(io.EOF)
	testCases := []readAtTestCase{
		readAtTestCase{0, 0, eofMatcher, ""},
		readAtTestCase{0, 10, eofMatcher, ""},
		readAtTestCase{5, 10, eofMatcher, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) Size() {
	var expected int64
	for _, contents := range t.refresherContents {
		expected += int64(len(contents))
	}

	ExpectEq(expected, t.proxy.Size())
}

func (t *MultiReadProxyTest) ReadAt_NegativeOffset() {
	// Test cases.
	m := Error(HasSubstr("Invalid offset"))
	testCases := []readAtTestCase{
		readAtTestCase{-1, 0, m, ""},
		readAtTestCase{-1, 1, m, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) ReadAt_OneRefresherReturnsError() {
	AssertThat(
		t.refresherContents,
		ElementsAre(
			"taco",
			"burrito",
			"enchilada",
		))

	AssertEq(4, len(t.refresherContents[0]))
	AssertEq(7, len(t.refresherContents[1]))
	AssertEq(9, len(t.refresherContents[2]))

	// Configure an error for the middle read lease.
	someErr := errors.New("foobar")
	t.refresherErrors[1] = someErr

	// Test cases.
	someErrMatcher := Error(HasSubstr(someErr.Error()))
	eofMatcher := Equals(io.EOF)

	testCases := []readAtTestCase{
		// First read lease only.
		readAtTestCase{0, 0, nil, ""},
		readAtTestCase{0, 1, nil, "t"},
		readAtTestCase{0, 4, nil, "taco"},
		readAtTestCase{1, 4, nil, "aco"},
		readAtTestCase{4, 4, nil, ""},

		// First and second read leases.
		readAtTestCase{0, 5, someErrMatcher, "taco"},
		readAtTestCase{1, 11, someErrMatcher, "aco"},

		// All read leases.
		readAtTestCase{0, 20, someErrMatcher, "taco"},
		readAtTestCase{1, 20, someErrMatcher, "aco"},
		readAtTestCase{1, 100, someErrMatcher, "aco"},

		// Second read lease only.
		readAtTestCase{4, 4, nil, ""},
		readAtTestCase{4, 5, someErrMatcher, ""},
		readAtTestCase{4, 11, someErrMatcher, ""},

		// Second and third read leases.
		readAtTestCase{4, 12, someErrMatcher, ""},
		readAtTestCase{4, 20, someErrMatcher, ""},
		readAtTestCase{5, 100, someErrMatcher, ""},

		// Third read lease only.
		readAtTestCase{11, 20, nil, "enchilada"},
		readAtTestCase{11, 100, eofMatcher, "enchilada"},
		readAtTestCase{12, 20, nil, "nchilada"},
		readAtTestCase{19, 20, nil, "a"},
		readAtTestCase{20, 20, eofMatcher, ""},

		// Past end.
		readAtTestCase{21, 21, eofMatcher, ""},
		readAtTestCase{21, 22, eofMatcher, ""},
		readAtTestCase{21, 100, eofMatcher, ""},
		readAtTestCase{100, 1000, eofMatcher, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) ReadAt_AllSuccessful() {
	AssertThat(
		t.refresherContents,
		ElementsAre(
			"taco",
			"burrito",
			"enchilada",
		))

	AssertEq(4, len(t.refresherContents[0]))
	AssertEq(7, len(t.refresherContents[1]))
	AssertEq(9, len(t.refresherContents[2]))

	// Test cases.
	eofMatcher := Equals(io.EOF)
	testCases := []readAtTestCase{
		// First read lease only.
		readAtTestCase{0, 0, nil, ""},
		readAtTestCase{0, 1, nil, "t"},
		readAtTestCase{0, 4, nil, "taco"},
		readAtTestCase{1, 4, nil, "aco"},
		readAtTestCase{4, 4, nil, ""},

		// First and second read leases.
		readAtTestCase{0, 5, nil, "tacob"},
		readAtTestCase{1, 11, nil, "acoburrito"},

		// All read leases.
		readAtTestCase{0, 20, nil, "tacoburritoenchilada"},
		readAtTestCase{1, 19, nil, "acoburritoenchilad"},
		readAtTestCase{3, 17, nil, "oburritoenchil"},

		// Second read lease only.
		readAtTestCase{4, 4, nil, ""},
		readAtTestCase{4, 5, nil, "b"},
		readAtTestCase{4, 11, nil, "burrito"},

		// Second and third read leases.
		readAtTestCase{4, 12, nil, "burritoe"},
		readAtTestCase{4, 20, nil, "burritoenchilada"},
		readAtTestCase{5, 100, eofMatcher, "urritoenchilada"},

		// Third read lease only.
		readAtTestCase{11, 20, nil, "enchilada"},
		readAtTestCase{11, 100, eofMatcher, "enchilada"},
		readAtTestCase{12, 20, nil, "nchilada"},
		readAtTestCase{19, 20, nil, "a"},
		readAtTestCase{20, 20, eofMatcher, ""},

		// Past end.
		readAtTestCase{21, 21, eofMatcher, ""},
		readAtTestCase{21, 22, eofMatcher, ""},
		readAtTestCase{21, 100, eofMatcher, ""},
		readAtTestCase{100, 1000, eofMatcher, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) ReadAt_ContentAlreadyCached() {
	AssertThat(
		t.refresherContents,
		ElementsAre(
			"taco",
			"burrito",
			"enchilada",
		))

	// Read the entire contents, causing read leases to be issued for each
	// sub-proxy.
	buf := make([]byte, 1024)
	n, err := t.proxy.ReadAt(context.Background(), buf, 0)

	AssertThat(err, AnyOf(nil, io.EOF))
	AssertEq("tacoburritoenchilada", string(buf[:n]))

	// Set up all refreshers to return errors when invoked.
	for i, _ := range t.refresherErrors {
		t.refresherErrors[i] = errors.New("foo")
	}

	// Despite this, the content should still be available.
	n, err = t.proxy.ReadAt(context.Background(), buf, 0)

	AssertThat(err, AnyOf(nil, io.EOF))
	AssertEq("tacoburritoenchilada", string(buf[:n]))
}

func (t *MultiReadProxyTest) Upgrade_OneRefresherReturnsError() {
	AssertTrue(false, "TODO")
}

func (t *MultiReadProxyTest) Upgrade_AllSuccessful() {
	AssertThat(
		t.refresherContents,
		ElementsAre(
			"taco",
			"burrito",
			"enchilada",
		))

	// Upgrade
	rwl, err := t.proxy.Upgrade(context.Background())
	t.proxy = nil
	AssertEq(nil, err)

	defer func() { rwl.Downgrade().Revoke() }()

	// Check the contents of the read/write lease.
	_, err = rwl.Seek(0, 0)
	AssertEq(nil, err)

	contents, err := ioutil.ReadAll(rwl)
	AssertEq(nil, err)
	ExpectEq("tacoburritoenchilada", string(contents))
}

func (t *MultiReadProxyTest) Upgrade_ContentAlreadyCached() {
	AssertTrue(false, "TODO")
}

func (t *MultiReadProxyTest) InitialReadLeaseValid() {
	// TODO(jacobsa): Test both ReadAt and Upgrade.
	AssertTrue(false, "TODO")
}

func (t *MultiReadProxyTest) InitialReadLeaseRevoked() {
	// TODO(jacobsa): Test both ReadAt and Upgrade.
	AssertTrue(false, "TODO")
}
<commit_msg>MultiReadProxyTest.Upgrade_OneRefresherReturnsError<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease_test

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"strings"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/lease"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestMultiReadProxy(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// A ReadProxy that wraps another, calling CheckInvariants before and after
// each action.
type checkingReadProxy struct {
	Wrapped lease.ReadProxy
}

func (crp *checkingReadProxy) Size() (size int64) {
	crp.Wrapped.CheckInvariants()
	defer crp.Wrapped.CheckInvariants()

	size = crp.Wrapped.Size()
	return
}

func (crp *checkingReadProxy) ReadAt(
	ctx context.Context,
	p []byte,
	off int64) (n int, err error) {
	crp.Wrapped.CheckInvariants()
	defer crp.Wrapped.CheckInvariants()

	n, err = crp.Wrapped.ReadAt(ctx, p, off)
	return
}

func (crp *checkingReadProxy) Upgrade(
	ctx context.Context) (rwl lease.ReadWriteLease, err error) {
	crp.Wrapped.CheckInvariants()
	rwl, err = crp.Wrapped.Upgrade(ctx)
	return
}

func (crp *checkingReadProxy) Destroy() {
	crp.Wrapped.CheckInvariants()
	crp.Wrapped.Destroy()
}

func (crp *checkingReadProxy) CheckInvariants() {
	crp.Wrapped.CheckInvariants()
}

// A range to read, the contents we expect to get back, and a matcher for the
// returned error. Special case: a nil matcher means Equals(nil).
type readAtTestCase struct {
	start            int64
	limit            int64
	errMatcher       Matcher
	expectedContents string
}

func runReadAtTestCases(
	rp lease.ReadProxy,
	cases []readAtTestCase) {
	for i, tc := range cases {
		desc := fmt.Sprintf("Test case %d: [%d, %d)", i, tc.start, tc.limit)

		AssertLe(tc.start, tc.limit)
		buf := make([]byte, tc.limit-tc.start)

		n, err := rp.ReadAt(context.Background(), buf, tc.start)
		AssertEq(tc.expectedContents, string(buf[:n]), "%s", desc)

		if tc.errMatcher == nil {
			AssertEq(nil, err, "%s", desc)
		} else {
			ExpectThat(err, tc.errMatcher, desc)
		}
	}
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type MultiReadProxyTest struct {
	ctx context.Context

	// Canned content and errors returned by the refreshers.
	refresherContents []string
	refresherErrors   []error

	leaser lease.FileLeaser
	proxy  *checkingReadProxy
}

var _ SetUpInterface = &MultiReadProxyTest{}
var _ TearDownInterface = &MultiReadProxyTest{}

func init() { RegisterTestSuite(&MultiReadProxyTest{}) }

func (t *MultiReadProxyTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.leaser = lease.NewFileLeaser("", math.MaxInt64)

	// Set up default refresher contents and nil errors.
	t.refresherContents = []string{
		"taco",
		"burrito",
		"enchilada",
	}
	t.refresherErrors = make([]error, len(t.refresherContents))

	// Create the proxy.
	t.resetProxy()
}

func (t *MultiReadProxyTest) TearDown() {
	// Make sure nothing goes crazy.
	if t.proxy != nil {
		t.proxy.Destroy()
	}
}

// Recreate refreshers using makeRefreshers and reset the proxy.
func (t *MultiReadProxyTest) resetProxy() {
	t.proxy = &checkingReadProxy{
		Wrapped: lease.NewMultiReadProxy(
			t.leaser,
			t.makeRefreshers(),
			nil),
	}
}

// Create refreshers based on the current contents of t.refresherContents.
// t.refresherErrors will be inspected only when Refresh is called.
func (t *MultiReadProxyTest) makeRefreshers() (refreshers []lease.Refresher) {
	for i := range t.refresherContents {
		iCopy := i
		contents := t.refresherContents[i]

		r := &funcRefresher{
			N: int64(len(contents)),
			F: func(ctx context.Context) (rc io.ReadCloser, err error) {
				rc = ioutil.NopCloser(strings.NewReader(contents))
				err = t.refresherErrors[iCopy]
				return
			},
		}

		refreshers = append(refreshers, r)
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *MultiReadProxyTest) SizeZero_NoRefreshers() {
	t.refresherContents = []string{}
	t.refresherErrors = []error{}
	t.resetProxy()

	// Size
	ExpectEq(0, t.proxy.Size())

	// ReadAt
	eofMatcher := Equals(io.EOF)
	testCases := []readAtTestCase{
		readAtTestCase{0, 0, eofMatcher, ""},
		readAtTestCase{0, 10, eofMatcher, ""},
		readAtTestCase{5, 10, eofMatcher, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) SizeZero_WithRefreshers() {
	t.refresherContents = []string{"", "", "", ""}
	t.refresherErrors = make([]error, len(t.refresherContents))
	t.resetProxy()

	// Size
	ExpectEq(0, t.proxy.Size())

	// ReadAt
	eofMatcher := Equals(io.EOF)
	testCases := []readAtTestCase{
		readAtTestCase{0, 0, eofMatcher, ""},
		readAtTestCase{0, 10, eofMatcher, ""},
		readAtTestCase{5, 10, eofMatcher, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) Size() {
	var expected int64
	for _, contents := range t.refresherContents {
		expected += int64(len(contents))
	}

	ExpectEq(expected, t.proxy.Size())
}

func (t *MultiReadProxyTest) ReadAt_NegativeOffset() {
	// Test cases.
	m := Error(HasSubstr("Invalid offset"))
	testCases := []readAtTestCase{
		readAtTestCase{-1, 0, m, ""},
		readAtTestCase{-1, 1, m, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) ReadAt_OneRefresherReturnsError() {
	AssertThat(
		t.refresherContents,
		ElementsAre(
			"taco",
			"burrito",
			"enchilada",
		))

	AssertEq(4, len(t.refresherContents[0]))
	AssertEq(7, len(t.refresherContents[1]))
	AssertEq(9, len(t.refresherContents[2]))

	// Configure an error for the middle read lease.
	someErr := errors.New("foobar")
	t.refresherErrors[1] = someErr

	// Test cases.
	someErrMatcher := Error(HasSubstr(someErr.Error()))
	eofMatcher := Equals(io.EOF)

	testCases := []readAtTestCase{
		// First read lease only.
		readAtTestCase{0, 0, nil, ""},
		readAtTestCase{0, 1, nil, "t"},
		readAtTestCase{0, 4, nil, "taco"},
		readAtTestCase{1, 4, nil, "aco"},
		readAtTestCase{4, 4, nil, ""},

		// First and second read leases.
		readAtTestCase{0, 5, someErrMatcher, "taco"},
		readAtTestCase{1, 11, someErrMatcher, "aco"},

		// All read leases.
		readAtTestCase{0, 20, someErrMatcher, "taco"},
		readAtTestCase{1, 20, someErrMatcher, "aco"},
		readAtTestCase{1, 100, someErrMatcher, "aco"},

		// Second read lease only.
		readAtTestCase{4, 4, nil, ""},
		readAtTestCase{4, 5, someErrMatcher, ""},
		readAtTestCase{4, 11, someErrMatcher, ""},

		// Second and third read leases.
		readAtTestCase{4, 12, someErrMatcher, ""},
		readAtTestCase{4, 20, someErrMatcher, ""},
		readAtTestCase{5, 100, someErrMatcher, ""},

		// Third read lease only.
		readAtTestCase{11, 20, nil, "enchilada"},
		readAtTestCase{11, 100, eofMatcher, "enchilada"},
		readAtTestCase{12, 20, nil, "nchilada"},
		readAtTestCase{19, 20, nil, "a"},
		readAtTestCase{20, 20, eofMatcher, ""},

		// Past end.
		readAtTestCase{21, 21, eofMatcher, ""},
		readAtTestCase{21, 22, eofMatcher, ""},
		readAtTestCase{21, 100, eofMatcher, ""},
		readAtTestCase{100, 1000, eofMatcher, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) ReadAt_AllSuccessful() {
	AssertThat(
		t.refresherContents,
		ElementsAre(
			"taco",
			"burrito",
			"enchilada",
		))

	AssertEq(4, len(t.refresherContents[0]))
	AssertEq(7, len(t.refresherContents[1]))
	AssertEq(9, len(t.refresherContents[2]))

	// Test cases.
	eofMatcher := Equals(io.EOF)
	testCases := []readAtTestCase{
		// First read lease only.
		readAtTestCase{0, 0, nil, ""},
		readAtTestCase{0, 1, nil, "t"},
		readAtTestCase{0, 4, nil, "taco"},
		readAtTestCase{1, 4, nil, "aco"},
		readAtTestCase{4, 4, nil, ""},

		// First and second read leases.
		readAtTestCase{0, 5, nil, "tacob"},
		readAtTestCase{1, 11, nil, "acoburrito"},

		// All read leases.
		readAtTestCase{0, 20, nil, "tacoburritoenchilada"},
		readAtTestCase{1, 19, nil, "acoburritoenchilad"},
		readAtTestCase{3, 17, nil, "oburritoenchil"},

		// Second read lease only.
		readAtTestCase{4, 4, nil, ""},
		readAtTestCase{4, 5, nil, "b"},
		readAtTestCase{4, 11, nil, "burrito"},

		// Second and third read leases.
		readAtTestCase{4, 12, nil, "burritoe"},
		readAtTestCase{4, 20, nil, "burritoenchilada"},
		readAtTestCase{5, 100, eofMatcher, "urritoenchilada"},

		// Third read lease only.
		readAtTestCase{11, 20, nil, "enchilada"},
		readAtTestCase{11, 100, eofMatcher, "enchilada"},
		readAtTestCase{12, 20, nil, "nchilada"},
		readAtTestCase{19, 20, nil, "a"},
		readAtTestCase{20, 20, eofMatcher, ""},

		// Past end.
		readAtTestCase{21, 21, eofMatcher, ""},
		readAtTestCase{21, 22, eofMatcher, ""},
		readAtTestCase{21, 100, eofMatcher, ""},
		readAtTestCase{100, 1000, eofMatcher, ""},
	}

	runReadAtTestCases(t.proxy, testCases)
}

func (t *MultiReadProxyTest) ReadAt_ContentAlreadyCached() {
	AssertThat(
		t.refresherContents,
		ElementsAre(
			"taco",
			"burrito",
			"enchilada",
		))

	// Read the entire contents, causing read leases to be issued for each
	// sub-proxy.
	buf := make([]byte, 1024)
	n, err := t.proxy.ReadAt(context.Background(), buf, 0)

	AssertThat(err, AnyOf(nil, io.EOF))
	AssertEq("tacoburritoenchilada", string(buf[:n]))

	// Set up all refreshers to return errors when invoked.
	for i, _ := range t.refresherErrors {
		t.refresherErrors[i] = errors.New("foo")
	}

	// Despite this, the content should still be available.
	n, err = t.proxy.ReadAt(context.Background(), buf, 0)

	AssertThat(err, AnyOf(nil, io.EOF))
	AssertEq("tacoburritoenchilada", string(buf[:n]))
}

func (t *MultiReadProxyTest) Upgrade_OneRefresherReturnsError() {
	AssertEq(3, len(t.refresherErrors))

	// Configure an error for the middle read lease.
	t.refresherErrors[1] = errors.New("foobar")

	// Upgrade
	_, err := t.proxy.Upgrade(context.Background())
	t.proxy = nil

	ExpectThat(err, Error(HasSubstr("foobar")))
}

func (t *MultiReadProxyTest) Upgrade_AllSuccessful() {
	AssertThat(
		t.refresherContents,
		ElementsAre(
			"taco",
			"burrito",
			"enchilada",
		))

	// Upgrade
	rwl, err := t.proxy.Upgrade(context.Background())
	t.proxy = nil
	AssertEq(nil, err)

	defer func() { rwl.Downgrade().Revoke() }()

	// Check the contents of the read/write lease.
	_, err = rwl.Seek(0, 0)
	AssertEq(nil, err)

	contents, err := ioutil.ReadAll(rwl)
	AssertEq(nil, err)
	ExpectEq("tacoburritoenchilada", string(contents))
}

func (t *MultiReadProxyTest) Upgrade_ContentAlreadyCached() {
	AssertTrue(false, "TODO")
}

func (t *MultiReadProxyTest) InitialReadLeaseValid() {
	// TODO(jacobsa): Test both ReadAt and Upgrade.
	AssertTrue(false, "TODO")
}

func (t *MultiReadProxyTest) InitialReadLeaseRevoked() {
	// TODO(jacobsa): Test both ReadAt and Upgrade.
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>package sudoku

import (
	"fmt"
	"log"
	"strings"
)

//DifficultySignals is a collection of names to float64 values, representing
//the various signals extracted from a SolveDirections, and used for the
//Difficulty calculation. Generally not useful to package users.
type DifficultySignals map[string]float64

//A difficulty signal generator can return more than one difficutly signal, so
//it doesn't just return float64 Each signal generator should always return a
//map with the SAME keys--so if you've called it once you know what the next
//calls will have as keys.
type difficultySignalGenerator func(directions SolveDirections) DifficultySignals

const _DIFFICULTY_WEIGHT_FILENAME = "difficulties.csv"

var difficultySignalGenerators []difficultySignalGenerator

//These are the weights that will be used to turn a list of signals into a
//difficulty. starting weights are set in hs_difficulty_weights.go, which is
//auto-generated. Generate those now: go:generate python util/difficulty-
//convert.py
var difficultySignalWeights map[string]float64

func init() {
	difficultySignalGenerators = []difficultySignalGenerator{
		signalTechnique,
		signalNumberOfSteps,
		signalTechniquePercentage,
		signalPercentageFilledSteps,
		signalNumberUnfilled,
		signalStepsUntilNonFill,
	}
}

//Stats returns a printout of interesting statistics about the
//SolveDirections, including number of steps, difficulty (based on this solve
//description alone), how unrelated the cells in subsequent steps are, and the
//values of all of the signals used to generate the difficulty.
func (self SolveDirections) Stats() []string {
	//TODO: test this.
	techniqueCount := make(map[string]int)
	var lastStep *SolveStep
	dissimilarityAccum := 0.0
	for _, step := range self.Steps {
		if lastStep != nil {
			dissimilarityAccum += step.TargetCells.chainDissimilarity(lastStep.TargetCells)
		}
		techniqueCount[step.Technique.Name()] += 1
		lastStep = step
	}
	dissimilarityAccum /= float64(len(self.Steps))

	var result []string

	//TODO: use a standard divider across the codebase
	divider := "-------------------------"

	result = append(result, divider)
	//TODO: we shouldn't even include this... it's not meaningful to report the difficulty of a single solve.
	result = append(result, fmt.Sprintf("Difficulty : %f", self.Signals().difficulty()))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Step count: %d", len(self.Steps)))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Avg Dissimilarity: %f", dissimilarityAccum))
	result = append(result, divider)

	//We want a stable ordering for technique counts.
	for _, technique := range AllTechniques {
		//TODO: pad the technique name with enough spaces so the colon lines up.
		result = append(result, fmt.Sprintf("%s : %d", technique.Name(), techniqueCount[technique.Name()]))
	}

	result = append(result, divider)

	return result
}

//Description returns a comprehensive prose description of the
//SolveDirections, including reasoning for each step, that if followed would
//lead to the grid being solved. Unlike Walkthrough, Description() does not
//include diagrams for each step.
func (self SolveDirections) Description() []string {

	if len(self.Steps) == 0 {
		return []string{""}
	}

	descriptions := make([]string, len(self.Steps))

	for i, step := range self.Steps {
		intro := ""
		switch i {
		case 0:
			intro = "First, "
		case len(self.Steps) - 1:
			intro = "Finally, "
		default:
			//TODO: switch between "then" and "next" randomly.
			intro = "Next, "
		}
		descriptions[i] = intro + strings.ToLower(step.Description())

	}
	return descriptions
}

//Walkthrough prints an exhaustive set of human-readable directions that
//includes diagrams at each step to make it easier to follow.
func (self SolveDirections) Walkthrough() string {

	//TODO: test this.

	if len(self.Steps) == 0 {
		return "The puzzle could not be solved with any of the techniques we're aware of."
	}

	clone := self.Grid()
	defer clone.Done()

	DIVIDER := "\n\n--------------------------------------------\n\n"

	intro := fmt.Sprintf("This will take %d steps to solve.", len(self.Steps))

	intro += "\nWhen you start, your grid looks like this:\n"

	intro += clone.Diagram()

	intro += "\n"

	intro += DIVIDER

	descriptions := self.Description()

	results := make([]string, len(self.Steps))

	for i, description := range descriptions {

		result := description + "\n"
		result += "After doing that, your grid will look like: \n\n"

		self.Steps[i].Apply(clone)

		result += clone.Diagram()

		results[i] = result
	}

	return intro + strings.Join(results, DIVIDER) + DIVIDER + "Now the puzzle is solved."
}

//Signals returns the DifficultySignals for this set of SolveDirections.
func (self SolveDirections) Signals() DifficultySignals {
	//Because of the contract of a DifficultySignalGenerator (that it always
	//returns the same keys), as long as DifficultySignalGenerators stays
	//constant it's reasonable for callers to assume that one call to
	//Signals() will return all of the string keys you'll see any time you
	//call Signals()
	result := DifficultySignals{}
	for _, generator := range difficultySignalGenerators {
		result.add(generator(self))
	}
	return result
}

//This will overwrite colliding values
//TODO: this is confusingly named
func (self DifficultySignals) add(other DifficultySignals) {
	for key, val := range other {
		self[key] = val
	}
}

//For keys in both, will sum them together.
//TODO: this is confusingly named (compared to Add)  Do we really need both Sum and Add?
func (self DifficultySignals) sum(other DifficultySignals) {
	for key, val := range other {
		self[key] += val
	}
}

func (self DifficultySignals) difficulty() float64 {
	accum := 0.0

	if constant, ok := difficultySignalWeights["Constant"]; ok {
		accum = constant
	} else {
		log.Println("Didn't have the constant term loaded.")
	}

	for signal, val := range self {
		//We can discard the OK because 0 is a reasonable thing to do with weights we aren't aware of.
		weight, _ := difficultySignalWeights[signal]

		accum += val * weight
	}

	if accum < 0.0 {
		log.Println("Accumuldated difficulty snapped to 0.0:", accum)
		accum = 0.0
	}

	if accum > 1.0 {
		log.Println("Accumulated difficulty snapped to 1.0:", accum)
		accum = 1.0
	}

	return accum
}

//Rest of file is different Signals

//TODO: now that SolveDirections includes gridSnapshot, think if there are any
//additional Signals we can generate.

//This technique returns a count of how many each type of technique is seen.
//Different techniques are different "difficulties" so seeing more of a hard
//technique will Lead to a higher overall difficulty.
func signalTechnique(directions SolveDirections) DifficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := DifficultySignals{}
	for _, techniqueName := range AllTechniqueVariants {
		result[techniqueName+" Count"] = 0.0
	}
	for _, step := range directions.Steps {
		result[step.TechniqueVariant()+" Count"]++
	}
	return result
}

//This signal is just number of steps. More steps is PROBABLY a harder puzzle.
func signalNumberOfSteps(directions SolveDirections) DifficultySignals {
	return DifficultySignals{
		"Number of Steps": float64(len(directions.Steps)),
	}
}

//This signal is like signalTechnique, except it returns the count divided by
//the TOTAL number of steps.
func signalTechniquePercentage(directions SolveDirections) DifficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := DifficultySignals{}
	for _, techniqueName := range AllTechniqueVariants {
		result[techniqueName+" Percentage"] = 0.0
	}

	count := len(directions.Steps)

	if count == 0 {
		return result
	}

	for _, step := range directions.Steps {
		result[step.TechniqueVariant()+" Percentage"]++
	}

	//Now normalize all of them
	for name := range result {
		result[name] /= float64(count)
	}

	return result
}

//This signal is how many steps are filled out of all steps. Presumably harder
//puzzles will have more non-fill steps.
func signalPercentageFilledSteps(directions SolveDirections) DifficultySignals {
	numerator := 0.0
	denominator := float64(len(directions.Steps))

	for _, step := range directions.Steps {
		if step.Technique.IsFill() {
			numerator += 1.0
		}
	}

	return DifficultySignals{
		"Percentage Fill Steps": numerator / denominator,
	}
}

//This signal is how many cells are unfilled at the beginning. Presumably
//harder puzzles will have fewer cells filled (although obviously this isn't
//necessarily true)
func signalNumberUnfilled(directions SolveDirections) DifficultySignals {

	//We don't have access to the underlying grid, so we'll just count how
	//many fill steps (since each can only add one number, and no numbers are
	//ever unfilled)

	count := 0.0
	for _, step := range directions.Steps {
		if step.Technique.IsFill() {
			count++
		}
	}

	return DifficultySignals{
		"Number Unfilled Cells": count,
	}
}

//This signal is how many steps into the solve directions before you encounter
//your first non-fill step. Non-fill steps are harder, so this signal captures
//how easy the start of the puzzle is.
func signalStepsUntilNonFill(directions SolveDirections) DifficultySignals {
	count := 0.0
	for _, step := range directions.Steps {
		if !step.Technique.IsFill() {
			break
		}
		count++
	}

	return DifficultySignals{
		"Steps Until Nonfill": count,
	}

}
<commit_msg>Changed the default message for zero-len SolveDirections in SolveDirections.Description.<commit_after>package sudoku

import (
	"fmt"
	"log"
	"strings"
)

//DifficultySignals is a collection of names to float64 values, representing
//the various signals extracted from a SolveDirections, and used for the
//Difficulty calculation. Generally not useful to package users.
type DifficultySignals map[string]float64

//A difficulty signal generator can return more than one difficutly signal, so
//it doesn't just return float64 Each signal generator should always return a
//map with the SAME keys--so if you've called it once you know what the next
//calls will have as keys.
type difficultySignalGenerator func(directions SolveDirections) DifficultySignals

const _DIFFICULTY_WEIGHT_FILENAME = "difficulties.csv"

var difficultySignalGenerators []difficultySignalGenerator

//These are the weights that will be used to turn a list of signals into a
//difficulty. starting weights are set in hs_difficulty_weights.go, which is
//auto-generated. Generate those now: go:generate python util/difficulty-
//convert.py
var difficultySignalWeights map[string]float64

func init() {
	difficultySignalGenerators = []difficultySignalGenerator{
		signalTechnique,
		signalNumberOfSteps,
		signalTechniquePercentage,
		signalPercentageFilledSteps,
		signalNumberUnfilled,
		signalStepsUntilNonFill,
	}
}

//Stats returns a printout of interesting statistics about the
//SolveDirections, including number of steps, difficulty (based on this solve
//description alone), how unrelated the cells in subsequent steps are, and the
//values of all of the signals used to generate the difficulty.
func (self SolveDirections) Stats() []string {
	//TODO: test this.
	techniqueCount := make(map[string]int)
	var lastStep *SolveStep
	dissimilarityAccum := 0.0
	for _, step := range self.Steps {
		if lastStep != nil {
			dissimilarityAccum += step.TargetCells.chainDissimilarity(lastStep.TargetCells)
		}
		techniqueCount[step.Technique.Name()] += 1
		lastStep = step
	}
	dissimilarityAccum /= float64(len(self.Steps))

	var result []string

	//TODO: use a standard divider across the codebase
	divider := "-------------------------"

	result = append(result, divider)
	//TODO: we shouldn't even include this... it's not meaningful to report the difficulty of a single solve.
	result = append(result, fmt.Sprintf("Difficulty : %f", self.Signals().difficulty()))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Step count: %d", len(self.Steps)))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Avg Dissimilarity: %f", dissimilarityAccum))
	result = append(result, divider)

	//We want a stable ordering for technique counts.
	for _, technique := range AllTechniques {
		//TODO: pad the technique name with enough spaces so the colon lines up.
		result = append(result, fmt.Sprintf("%s : %d", technique.Name(), techniqueCount[technique.Name()]))
	}

	result = append(result, divider)

	return result
}

//Description returns a comprehensive prose description of the
//SolveDirections, including reasoning for each step, that if followed would
//lead to the grid being solved. Unlike Walkthrough, Description() does not
//include diagrams for each step.
func (self SolveDirections) Description() []string {

	if len(self.Steps) == 0 {
		return []string{"The puzzle is already solved."}
	}

	descriptions := make([]string, len(self.Steps))

	for i, step := range self.Steps {
		intro := ""
		switch i {
		case 0:
			intro = "First, "
		case len(self.Steps) - 1:
			intro = "Finally, "
		default:
			//TODO: switch between "then" and "next" randomly.
			intro = "Next, "
		}
		descriptions[i] = intro + strings.ToLower(step.Description())

	}
	return descriptions
}

//Walkthrough prints an exhaustive set of human-readable directions that
//includes diagrams at each step to make it easier to follow.
func (self SolveDirections) Walkthrough() string {

	//TODO: test this.

	if len(self.Steps) == 0 {
		return "The puzzle could not be solved with any of the techniques we're aware of."
	}

	clone := self.Grid()
	defer clone.Done()

	DIVIDER := "\n\n--------------------------------------------\n\n"

	intro := fmt.Sprintf("This will take %d steps to solve.", len(self.Steps))

	intro += "\nWhen you start, your grid looks like this:\n"

	intro += clone.Diagram()

	intro += "\n"

	intro += DIVIDER

	descriptions := self.Description()

	results := make([]string, len(self.Steps))

	for i, description := range descriptions {

		result := description + "\n"
		result += "After doing that, your grid will look like: \n\n"

		self.Steps[i].Apply(clone)

		result += clone.Diagram()

		results[i] = result
	}

	return intro + strings.Join(results, DIVIDER) + DIVIDER + "Now the puzzle is solved."
}

//Signals returns the DifficultySignals for this set of SolveDirections.
func (self SolveDirections) Signals() DifficultySignals {
	//Because of the contract of a DifficultySignalGenerator (that it always
	//returns the same keys), as long as DifficultySignalGenerators stays
	//constant it's reasonable for callers to assume that one call to
	//Signals() will return all of the string keys you'll see any time you
	//call Signals()
	result := DifficultySignals{}
	for _, generator := range difficultySignalGenerators {
		result.add(generator(self))
	}
	return result
}

//This will overwrite colliding values
//TODO: this is confusingly named
func (self DifficultySignals) add(other DifficultySignals) {
	for key, val := range other {
		self[key] = val
	}
}

//For keys in both, will sum them together.
//TODO: this is confusingly named (compared to Add)  Do we really need both Sum and Add?
func (self DifficultySignals) sum(other DifficultySignals) {
	for key, val := range other {
		self[key] += val
	}
}

func (self DifficultySignals) difficulty() float64 {
	accum := 0.0

	if constant, ok := difficultySignalWeights["Constant"]; ok {
		accum = constant
	} else {
		log.Println("Didn't have the constant term loaded.")
	}

	for signal, val := range self {
		//We can discard the OK because 0 is a reasonable thing to do with weights we aren't aware of.
		weight, _ := difficultySignalWeights[signal]

		accum += val * weight
	}

	if accum < 0.0 {
		log.Println("Accumuldated difficulty snapped to 0.0:", accum)
		accum = 0.0
	}

	if accum > 1.0 {
		log.Println("Accumulated difficulty snapped to 1.0:", accum)
		accum = 1.0
	}

	return accum
}

//Rest of file is different Signals

//TODO: now that SolveDirections includes gridSnapshot, think if there are any
//additional Signals we can generate.

//This technique returns a count of how many each type of technique is seen.
//Different techniques are different "difficulties" so seeing more of a hard
//technique will Lead to a higher overall difficulty.
func signalTechnique(directions SolveDirections) DifficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := DifficultySignals{}
	for _, techniqueName := range AllTechniqueVariants {
		result[techniqueName+" Count"] = 0.0
	}
	for _, step := range directions.Steps {
		result[step.TechniqueVariant()+" Count"]++
	}
	return result
}

//This signal is just number of steps. More steps is PROBABLY a harder puzzle.
func signalNumberOfSteps(directions SolveDirections) DifficultySignals {
	return DifficultySignals{
		"Number of Steps": float64(len(directions.Steps)),
	}
}

//This signal is like signalTechnique, except it returns the count divided by
//the TOTAL number of steps.
func signalTechniquePercentage(directions SolveDirections) DifficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := DifficultySignals{}
	for _, techniqueName := range AllTechniqueVariants {
		result[techniqueName+" Percentage"] = 0.0
	}

	count := len(directions.Steps)

	if count == 0 {
		return result
	}

	for _, step := range directions.Steps {
		result[step.TechniqueVariant()+" Percentage"]++
	}

	//Now normalize all of them
	for name := range result {
		result[name] /= float64(count)
	}

	return result
}

//This signal is how many steps are filled out of all steps. Presumably harder
//puzzles will have more non-fill steps.
func signalPercentageFilledSteps(directions SolveDirections) DifficultySignals {
	numerator := 0.0
	denominator := float64(len(directions.Steps))

	for _, step := range directions.Steps {
		if step.Technique.IsFill() {
			numerator += 1.0
		}
	}

	return DifficultySignals{
		"Percentage Fill Steps": numerator / denominator,
	}
}

//This signal is how many cells are unfilled at the beginning. Presumably
//harder puzzles will have fewer cells filled (although obviously this isn't
//necessarily true)
func signalNumberUnfilled(directions SolveDirections) DifficultySignals {

	//We don't have access to the underlying grid, so we'll just count how
	//many fill steps (since each can only add one number, and no numbers are
	//ever unfilled)

	count := 0.0
	for _, step := range directions.Steps {
		if step.Technique.IsFill() {
			count++
		}
	}

	return DifficultySignals{
		"Number Unfilled Cells": count,
	}
}

//This signal is how many steps into the solve directions before you encounter
//your first non-fill step. Non-fill steps are harder, so this signal captures
//how easy the start of the puzzle is.
func signalStepsUntilNonFill(directions SolveDirections) DifficultySignals {
	count := 0.0
	for _, step := range directions.Steps {
		if !step.Technique.IsFill() {
			break
		}
		count++
	}

	return DifficultySignals{
		"Steps Until Nonfill": count,
	}

}
<|endoftext|>


<commit_before>// santiago - webhook dispatching service
// https://github.com/topfreegames/santiago
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package metadata

//VERSION identifies current version of the application
var VERSION = "1.1.0"
<commit_msg>release 1.2.0<commit_after>// santiago - webhook dispatching service
// https://github.com/topfreegames/santiago
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package metadata

//VERSION identifies current version of the application
var VERSION = "1.2.0"
<|endoftext|>


<commit_before>package match

import (
	"reflect"
	"regexp"
	"testing"
)

// Matcher is a helper object for chaining matcher conditions.
type Matcher struct {
	t     *testing.T
	value interface{}
}

// Fatal causes the unit test to immediately fail is one of the previous
// conditions has fail.
// Note: This will only affect previous conditions, not subsequent ones. You
// can build complex conditions this way.
// Ex: match.IsNotNil(t, value).Fatal().Equals(other)
func (m *Matcher) Fatal() *Matcher {
	if m.t.Failed() {
		m.t.FailNow()
	}
	return m
}

// IsNil asserts that the matched value is nil.
func (m *Matcher) IsNil() *Matcher {
	if m.value != nil {
		m.t.Errorf("expected == nil, got %v", m.value)
	}
	return m
}

// IsNotNil asserts that the matched value is not nil.
func (m *Matcher) IsNotNil() *Matcher {
	if m.value == nil {
		m.t.Error("expected != nil, got nil")
	}
	return m
}

// Equals asserts that the matched value is equal to value.
func (m *Matcher) Equals(value interface{}) *Matcher {
	if !reflect.DeepEqual(m.value, value) {
		m.t.Errorf("expected == %v, got %v", value, m.value)
	}
	return m
}

// NotEquals asserts that the match value is not equal to value.
func (m *Matcher) NotEquals(value interface{}) *Matcher {
	if reflect.DeepEqual(m.value, value) {
		m.t.Errorf("expected != %v, got %v", value, m.value)
	}
	return m
}

// LessThan asserts that the matched value is less than value.
// Note: This function will only work on numeric values.
// If a non-numeric value is passed, a fatal error will be thrown.
func (m *Matcher) LessThan(value interface{}) *Matcher {
	ok := true
	rv := reflect.ValueOf(m.value)
	switch rv.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		ok = rv.Int() < reflect.ValueOf(value).Int()
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		ok = rv.Uint() < uint64(reflect.ValueOf(value).Int())
	case reflect.Float32, reflect.Float64:
		ok = rv.Float() < reflect.ValueOf(value).Float()
	default:
		m.t.Fatalf("expected numeric value, got %v", m.value)
	}
	if !ok {
		m.t.Errorf("expected < %v, got %v", value, m.value)
	}
	return m
}

// GreaterThan asserts that the matched value is greater then value.
// Note: This function will only work on numeric values.
// If a non-numeric value is passed, a fatal error will be thrown.
func (m *Matcher) GreaterThan(value interface{}) *Matcher {
	ok := true
	rv := reflect.ValueOf(m.value)
	switch rv.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		ok = rv.Int() > reflect.ValueOf(value).Int()
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		ok = rv.Uint() > uint64(reflect.ValueOf(value).Int())
	case reflect.Float32, reflect.Float64:
		ok = rv.Float() > reflect.ValueOf(value).Float()
	default:
		m.t.Fatalf("expected numeric value, got %v", m.value)
	}
	if !ok {
		m.t.Errorf("expected < %v, got %v", value, m.value)
	}
	return m
}

// Contains asserts that the matched value contains value.
// If used on an array, asserts that the array contains value.
// If used on a map, asserts that the map contains a key with value.
// If used on a string, asserts that the string contains value.
func (m *Matcher) Contains(value interface{}) *Matcher {
	// TODO: Implement Matcher.Contains
	return m
}

// Matches asserts that the matched value matches the regex in pattern.
// Note: If the regex cannot be compiled, a fatal error will be thrown.
// If used on a non-string value, a fatal error will be thrown.
func (m *Matcher) Matches(pattern string) *Matcher {
	rv := reflect.ValueOf(m.value)
	if rv.Kind() != reflect.String {
		m.t.Fatalf("expected string, got %s", rv.Kind().String())
	}
	re, err := regexp.Compile(pattern)
	if err != nil {
		m.t.Fatal(err)
	}
	if !re.MatchString(rv.String()) {
		m.t.Errorf("expected %#q to match pattern %#q", rv.String(), pattern)
	}
	return m
}

// KindOf asserts that the matched value of the kind.
func (m *Matcher) KindOf(kind reflect.Kind) *Matcher {
	rv := reflect.ValueOf(m.value)
	if rv.Kind() != kind {
		m.t.Errorf("expected kind %s, got %s", kind.String(), rv.Kind().String())
	}
	return m
}
<commit_msg>Implement Contains matcher<commit_after>package match

import (
	"reflect"
	"regexp"
	"strings"
	"testing"
)

// Matcher is a helper object for chaining matcher conditions.
type Matcher struct {
	t     *testing.T
	value interface{}
}

// Fatal causes the unit test to immediately fail is one of the previous
// conditions has fail.
// Note: This will only affect previous conditions, not subsequent ones. You
// can build complex conditions this way.
// Ex: match.IsNotNil(t, value).Fatal().Equals(other)
func (m *Matcher) Fatal() *Matcher {
	if m.t.Failed() {
		m.t.FailNow()
	}
	return m
}

// IsNil asserts that the matched value is nil.
func (m *Matcher) IsNil() *Matcher {
	if m.value != nil {
		m.t.Errorf("expected == nil, got %v", m.value)
	}
	return m
}

// IsNotNil asserts that the matched value is not nil.
func (m *Matcher) IsNotNil() *Matcher {
	if m.value == nil {
		m.t.Error("expected != nil, got nil")
	}
	return m
}

// Equals asserts that the matched value is equal to value.
func (m *Matcher) Equals(value interface{}) *Matcher {
	if !reflect.DeepEqual(m.value, value) {
		m.t.Errorf("expected == %v, got %v", value, m.value)
	}
	return m
}

// NotEquals asserts that the match value is not equal to value.
func (m *Matcher) NotEquals(value interface{}) *Matcher {
	if reflect.DeepEqual(m.value, value) {
		m.t.Errorf("expected != %v, got %v", value, m.value)
	}
	return m
}

// LessThan asserts that the matched value is less than value.
// Note: This function will only work on numeric values.
// If a non-numeric value is passed, a fatal error will be thrown.
func (m *Matcher) LessThan(value interface{}) *Matcher {
	ok := true
	rv := reflect.ValueOf(m.value)
	switch rv.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		ok = rv.Int() < reflect.ValueOf(value).Int()
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		ok = rv.Uint() < uint64(reflect.ValueOf(value).Int())
	case reflect.Float32, reflect.Float64:
		ok = rv.Float() < reflect.ValueOf(value).Float()
	default:
		m.t.Fatalf("expected numeric value, got %v", m.value)
	}
	if !ok {
		m.t.Errorf("expected < %v, got %v", value, m.value)
	}
	return m
}

// GreaterThan asserts that the matched value is greater then value.
// Note: This function will only work on numeric values.
// If a non-numeric value is passed, a fatal error will be thrown.
func (m *Matcher) GreaterThan(value interface{}) *Matcher {
	ok := true
	rv := reflect.ValueOf(m.value)
	switch rv.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		ok = rv.Int() > reflect.ValueOf(value).Int()
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		ok = rv.Uint() > uint64(reflect.ValueOf(value).Int())
	case reflect.Float32, reflect.Float64:
		ok = rv.Float() > reflect.ValueOf(value).Float()
	default:
		m.t.Fatalf("expected numeric value, got %v", m.value)
	}
	if !ok {
		m.t.Errorf("expected < %v, got %v", value, m.value)
	}
	return m
}

// Contains asserts that the matched value contains value.
// If used on an array, asserts that the array contains value.
// If used on a map, asserts that the map contains a key with value.
// If used on a string, asserts that the string contains value.
func (m *Matcher) Contains(value interface{}) *Matcher {
	rv := reflect.ValueOf(m.value)
	switch rv.Kind() {
	case reflect.Array, reflect.Slice:
		ok := false
		for i := 0; i < rv.Len(); i++ {
			if rv.Index(i).Interface() == value {
				ok = true
				break
			}
		}
		if !ok {
			m.t.Errorf("expected array to contain %v", value)
		}
	case reflect.Map:
		ok := false
		for _, key := range rv.MapKeys() {
			if key.Interface() == value {
				ok = true
				break
			}
		}
		if !ok {
			m.t.Errorf("expected map to have key %v", value)
		}
	case reflect.String:
		if !strings.Contains(rv.String(), value.(string)) {
			m.t.Errorf("expected %#q to contain %#q", rv.String(), value.(string))
		}
	default:
		m.t.Errorf("expected array, map or string, got %s", rv.Kind().String())
	}
	return m
}

// Matches asserts that the matched value matches the regex in pattern.
// Note: If the regex cannot be compiled, a fatal error will be thrown.
// If used on a non-string value, a fatal error will be thrown.
func (m *Matcher) Matches(pattern string) *Matcher {
	rv := reflect.ValueOf(m.value)
	if rv.Kind() != reflect.String {
		m.t.Fatalf("expected string, got %s", rv.Kind().String())
	}
	re, err := regexp.Compile(pattern)
	if err != nil {
		m.t.Fatal(err)
	}
	if !re.MatchString(rv.String()) {
		m.t.Errorf("expected %#q to match pattern %#q", rv.String(), pattern)
	}
	return m
}

// KindOf asserts that the matched value of the kind.
func (m *Matcher) KindOf(kind reflect.Kind) *Matcher {
	rv := reflect.ValueOf(m.value)
	if rv.Kind() != kind {
		m.t.Errorf("expected kind %s, got %s", kind.String(), rv.Kind().String())
	}
	return m
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/afex/hystrix-go/hystrix"
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: true, expire: false},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "tagtypes", fieldcount: 0, hash: false, expire: false},
		{base: "imageboards", fieldcount: 0, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// redis circuitbreaker
		hystrix.Do("cache", func() (err error) {

			// Trim leading / from path and split
			params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

			// get the keyname
			key, ok := RedisKeyIndex[params[0]]
			if !ok {
				c.Next()
				return
			}

			// set the key minus the base
			key.SetKey(params[1:]...)

			result, err = key.Get()
			if err == redis.ErrCacheMiss {
				// go to the controller
				c.Next()

				// Check if there was an error from the controller
				_, controllerError := c.Get("controllerError")
				if controllerError {
					c.Abort()
					return
				}

				err = key.Set(c.MustGet("data").([]byte))
				if err != nil {
					c.Error(err)
					c.Abort()
					return
				}

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			return

		}, func(err error) error {
			c.Next()
			return nil
		})

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	if r.fieldcount == 0 {
		r.key = r.base
		return
	}

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		err = redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		err = redis.RedisCache.Set(r.key, data)
	}

	if r.expire {
		return redis.RedisCache.Expire(r.key, 600)
	}

	return
}
<commit_msg>try hystrix on cache middleware<commit_after>package middleware

import (
	"github.com/afex/hystrix-go/hystrix"
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: true, expire: false},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "tagtypes", fieldcount: 0, hash: false, expire: false},
		{base: "imageboards", fieldcount: 0, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// redis circuitbreaker
		err := hystrix.Do("cache", func() (err error) {

			// Trim leading / from path and split
			params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

			// get the keyname
			key, ok := RedisKeyIndex[params[0]]
			if !ok {
				c.Next()
				return
			}

			// set the key minus the base
			key.SetKey(params[1:]...)

			result, err = key.Get()
			if err == redis.ErrCacheMiss {
				// go to the controller
				c.Next()

				// Check if there was an error from the controller
				_, controllerError := c.Get("controllerError")
				if controllerError {
					c.Abort()
					return
				}

				err = key.Set(c.MustGet("data").([]byte))
				if err != nil {
					c.Error(err)
					c.Abort()
					return
				}

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			return

		}, func(err error) error {
			c.Next()
			return nil
		})
		if err != nil {
			c.Error(err)
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	if r.fieldcount == 0 {
		r.key = r.base
		return
	}

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		err = redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		err = redis.RedisCache.Set(r.key, data)
	}

	if r.expire {
		return redis.RedisCache.Expire(r.key, 600)
	}

	return
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// Trim leading / from path and split
		request := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

		// get the keyname
		key, ok := redis.RedisKeyIndex[request[0]]
		if !ok {
			c.Next()
			return
		}

		// set the key minus the base
		key.SetKey(request[1:]...)

		// check the cache
		result, err := key.Get()
		if result == nil {
			// go to the controller if it wasnt found
			c.Next()

			// Check if there was an error from the controller
			_, controllerError := c.Get("controllerError")
			if controllerError {
				c.Abort()
				return
			}

			// set the data returned from the controller
			err = key.Set(c.MustGet("data").([]byte))
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			return

		}
		if err != nil {
			c.Error(err)
			c.Abort()
			return
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}
<commit_msg>setkey returns rediskey<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// Trim leading / from path and split
		request := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

		// get the keyname
		key, ok := redis.RedisKeyIndex[request[0]]
		if !ok {
			c.Next()
			return
		}

		// check the cache
		result, err := key.SetKey(request[1:]...).Get()
		if result == nil {
			// go to the controller if it wasnt found
			c.Next()

			// Check if there was an error from the controller
			_, controllerError := c.Get("controllerError")
			if controllerError {
				c.Abort()
				return
			}

			// set the data returned from the controller
			err = key.Set(c.MustGet("data").([]byte))
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			return

		}
		if err != nil {
			c.Error(err)
			c.Abort()
			return
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}
<|endoftext|>


<commit_before>package mtrparser

import (
	"math"
	"net"
	"strconv"
	"strings"
	"time"
)

func stdDev(timings []time.Duration, avg time.Duration) time.Duration {
	//taken from https://github.com/ae6rt/golang-examples/blob/master/goeg/src/statistics_ans/statistics.go
	if len(timings) < 2 {
		return time.Duration(0)
	}
	mean := float64(avg)
	total := 0.0
	for _, t := range timings {
		number := float64(t)
		total += math.Pow(number-mean, 2)
	}
	variance := total / float64(len(timings)-1)
	std := math.Sqrt(variance)
	return time.Duration(std)
}

type MtrHop struct {
	IP       []string
	Host     []string
	ASN      []string
	Timings  []time.Duration //In Json they become nanosecond
	Avg      time.Duration
	Loss     int
	SD       time.Duration
	Sent     int
	Received int
	Last     time.Duration
	Best     time.Duration
	Worst    time.Duration
}

func (hop *MtrHop) Summarize(count int) {
	//After the Timings block has been populated.
	hop.Sent = count
	hop.Received = len(hop.Timings)
	if len(hop.Timings) > 0 {
		hop.Last = hop.Timings[len(hop.Timings)-1]
		hop.Best = hop.Timings[0]
		hop.Worst = hop.Timings[0]
	}
	for _, t := range hop.Timings {
		hop.Avg += t / time.Duration(hop.Received)
		if hop.Best > t {
			hop.Best = t
		}
		if hop.Worst < t {
			hop.Worst = t
		}
	}
	hop.SD = stdDev(hop.Timings, hop.Avg)
	hop.Loss = (100 * (hop.Sent - hop.Received)) / hop.Sent
	//Populate ips
	hop.Host = make([]string, len(hop.IP))
	for idx, ip := range hop.IP {
		addr, err := net.LookupAddr(ip)
		if err == nil {
			//log.Println(addr[0], err)
			if len(addr) > 0 {
				hop.Host[idx] = addr[0]
			}
		}
	}
}

type MTROutPut struct {
	Hops     []*MtrHop
	HopCount int
}

type rawhop struct {
	datatype string
	idx      int
	value    string
}

//raw is the output from mtr command, count is the -c argument, default 10 in mtr
func NewMTROutPut(raw string, count int) (*MTROutPut, error) {
	//last hop comes in multiple times... https://github.com/traviscross/mtr/blob/master/FORMATS
	out := &MTROutPut{}
	rawhops := make([]rawhop, 0)
	//Store each line of output in rawhop structure
	for _, line := range strings.Split(raw, "\n") {
		things := strings.Split(line, " ")
		if len(things) == 3 {
			//log.Println(things)
			idx, err := strconv.Atoi(things[1])
			if err != nil {
				return nil, err
			}
			data := rawhop{
				datatype: things[0],
				idx:      idx,
				value:    things[2],
			}
			rawhops = append(rawhops, data)
			//Number of hops = highest index+1
			if out.HopCount < (idx + 1) {
				out.HopCount = idx + 1
			}
		}
	}
	out.Hops = make([]*MtrHop, out.HopCount)
	for idx, _ := range out.Hops {
		out.Hops[idx] = &MtrHop{
			Timings: make([]time.Duration, 0),
			IP:      make([]string, 0),
			Host:    make([]string, 0),
		}
		//hop.Timings = make([]time.Duration, 0)
	}
	for _, data := range rawhops {
		switch data.datatype {
		case "h":
			out.Hops[data.idx].IP = append(out.Hops[data.idx].IP, data.value)
		case "d":
			//Not entirely sure if multiple IPs. Better use -n in mtr and resolve later in summarize.
			out.Hops[data.idx].Host = append(out.Hops[data.idx].Host, data.value)
		case "p":
			t, err := strconv.Atoi(data.value)
			if err != nil {
				return nil, err
			}
			out.Hops[data.idx].Timings = append(out.Hops[data.idx].Timings, time.Duration(t)*time.Microsecond)
		}
	}
	//Filter dupe last hops
	finalidx := 0
	previousip := ""
	for idx, hop := range out.Hops {
		if len(hop.IP) > 0 {
			if hop.IP[0] != previousip {
				previousip = hop.IP[0]
				finalidx = idx + 1
			}
		}
	}
	out.Hops = out.Hops[0:finalidx]
	for _, hop := range out.Hops {
		hop.Summarize(count)
	}
	return out, nil
}
<commit_msg>dont use hostnames from mtr, use -n to make mtr faster<commit_after>package mtrparser

import (
	"math"
	"net"
	"strconv"
	"strings"
	"time"
)

func stdDev(timings []time.Duration, avg time.Duration) time.Duration {
	//taken from https://github.com/ae6rt/golang-examples/blob/master/goeg/src/statistics_ans/statistics.go
	if len(timings) < 2 {
		return time.Duration(0)
	}
	mean := float64(avg)
	total := 0.0
	for _, t := range timings {
		number := float64(t)
		total += math.Pow(number-mean, 2)
	}
	variance := total / float64(len(timings)-1)
	std := math.Sqrt(variance)
	return time.Duration(std)
}

type MtrHop struct {
	IP       []string
	Host     []string
	ASN      []string
	Timings  []time.Duration //In Json they become nanosecond
	Avg      time.Duration
	Loss     int
	SD       time.Duration
	Sent     int
	Received int
	Last     time.Duration
	Best     time.Duration
	Worst    time.Duration
}

func (hop *MtrHop) Summarize(count int) {
	//After the Timings block has been populated.
	hop.Sent = count
	hop.Received = len(hop.Timings)
	if len(hop.Timings) > 0 {
		hop.Last = hop.Timings[len(hop.Timings)-1]
		hop.Best = hop.Timings[0]
		hop.Worst = hop.Timings[0]
	}
	for _, t := range hop.Timings {
		hop.Avg += t / time.Duration(hop.Received)
		if hop.Best > t {
			hop.Best = t
		}
		if hop.Worst < t {
			hop.Worst = t
		}
	}
	hop.SD = stdDev(hop.Timings, hop.Avg)
	hop.Loss = (100 * (hop.Sent - hop.Received)) / hop.Sent
	//Populate ips
	hop.Host = make([]string, len(hop.IP))
	for idx, ip := range hop.IP {
		addr, err := net.LookupAddr(ip)
		if err == nil {
			//log.Println(addr[0], err)
			if len(addr) > 0 {
				hop.Host[idx] = addr[0]
			}
		}
	}
}

type MTROutPut struct {
	Hops     []*MtrHop
	HopCount int
}

type rawhop struct {
	datatype string
	idx      int
	value    string
}

//raw is the output from mtr command, count is the -c argument, default 10 in mtr
func NewMTROutPut(raw string, count int) (*MTROutPut, error) {
	//last hop comes in multiple times... https://github.com/traviscross/mtr/blob/master/FORMATS
	out := &MTROutPut{}
	rawhops := make([]rawhop, 0)
	//Store each line of output in rawhop structure
	for _, line := range strings.Split(raw, "\n") {
		things := strings.Split(line, " ")
		if len(things) == 3 {
			//log.Println(things)
			idx, err := strconv.Atoi(things[1])
			if err != nil {
				return nil, err
			}
			data := rawhop{
				datatype: things[0],
				idx:      idx,
				value:    things[2],
			}
			rawhops = append(rawhops, data)
			//Number of hops = highest index+1
			if out.HopCount < (idx + 1) {
				out.HopCount = idx + 1
			}
		}
	}
	out.Hops = make([]*MtrHop, out.HopCount)
	for idx, _ := range out.Hops {
		out.Hops[idx] = &MtrHop{
			Timings: make([]time.Duration, 0),
			IP:      make([]string, 0),
			Host:    make([]string, 0),
		}
		//hop.Timings = make([]time.Duration, 0)
	}
	for _, data := range rawhops {
		switch data.datatype {
		case "h":
			out.Hops[data.idx].IP = append(out.Hops[data.idx].IP, data.value)
		//case "d":
			//Not entirely sure if multiple IPs. Better use -n in mtr and resolve later in summarize.
			//out.Hops[data.idx].Host = append(out.Hops[data.idx].Host, data.value)
		case "p":
			t, err := strconv.Atoi(data.value)
			if err != nil {
				return nil, err
			}
			out.Hops[data.idx].Timings = append(out.Hops[data.idx].Timings, time.Duration(t)*time.Microsecond)
		}
	}
	//Filter dupe last hops
	finalidx := 0
	previousip := ""
	for idx, hop := range out.Hops {
		if len(hop.IP) > 0 {
			if hop.IP[0] != previousip {
				previousip = hop.IP[0]
				finalidx = idx + 1
			}
		}
	}
	out.Hops = out.Hops[0:finalidx]
	for _, hop := range out.Hops {
		hop.Summarize(count)
	}
	return out, nil
}
<|endoftext|>