<commit_before>// Package mucp transparently forwards the incoming request using a go-micro client.
package mucp

import (
	"context"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"

	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/codec"
	"github.com/micro/go-micro/codec/bytes"
	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/errors"
	"github.com/micro/go-micro/proxy"
	"github.com/micro/go-micro/router"
	"github.com/micro/go-micro/server"
)

// Proxy will transparently proxy requests to an endpoint.
// If no endpoint is specified it will call a service using the client.
type Proxy struct {
	// embed options
	options.Options

	// Endpoint specifies the fixed service endpoint to call.
	Endpoint string

	// The client to use for outbound requests in the local network
	Client client.Client

	// Links are used for outbound requests not in the local network
	Links map[string]client.Client

	// The router for routes
	Router router.Router

	// A fib of routes service:address
	sync.RWMutex
	Routes map[string]map[uint64]router.Route

	// The channel to monitor watcher errors
	errChan chan error
}

// read client request and write to server
func readLoop(r server.Request, s client.Stream) error {
	// request to backend server
	req := s.Request()

	for {
		// get data from client
		//  no need to decode it
		body, err := r.Read()
		if err == io.EOF {
			return nil
		}

		if err != nil {
			return err
		}

		// get the header from client
		hdr := r.Header()
		msg := &codec.Message{
			Type:   codec.Request,
			Header: hdr,
			Body:   body,
		}

		// write the raw request
		err = req.Codec().Write(msg, nil)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// toNodes returns a list of node addresses from given routes
func toNodes(routes []router.Route) []string {
	var nodes []string
	for _, node := range routes {
		address := node.Address
		if len(node.Gateway) > 0 {
			address = node.Gateway
		}
		nodes = append(nodes, address)
	}
	return nodes
}

func (p *Proxy) getLink(r router.Route) (client.Client, error) {
	if r.Link == "local" || len(p.Links) == 0 {
		return p.Client, nil
	}
	l, ok := p.Links[r.Link]
	if !ok {
		return nil, errors.InternalServerError("go.micro.proxy", "link not found")
	}
	return l, nil
}

func (p *Proxy) getRoute(service string) ([]router.Route, error) {
	toSlice := func(r map[uint64]router.Route) []router.Route {
		var routes []router.Route
		for _, v := range r {
			routes = append(routes, v)
		}
		return routes
	}

	// lookup the route cache first
	p.Lock()
	routes, ok := p.Routes[service]
	if ok {
		p.Unlock()
		return toSlice(routes), nil
	}
	p.Unlock()

	// lookup the routes in the router
	results, err := p.Router.Lookup(router.NewQuery(router.QueryService(service)))
	if err != nil {
		// check the status of the router
		if status := p.Router.Status(); status.Code == router.Error {
			return nil, status.Error
		}
		// otherwise return the error
		return nil, err
	}

	// update the proxy cache
	p.Lock()
	for _, route := range results {
		// create if does not exist
		if _, ok := p.Routes[service]; !ok {
			p.Routes[service] = make(map[uint64]router.Route)
		}
		p.Routes[service][route.Hash()] = route
	}
	routes = p.Routes[service]
	p.Unlock()

	return toSlice(routes), nil
}

// manageRouteCache applies action on a given route to Proxy route cache
func (p *Proxy) manageRouteCache(route router.Route, action string) error {
	switch action {
	case "create", "update":
		if _, ok := p.Routes[route.Service]; !ok {
			p.Routes[route.Service] = make(map[uint64]router.Route)
		}
		p.Routes[route.Service][route.Hash()] = route
	case "delete":
		if _, ok := p.Routes[route.Service]; !ok {
			return fmt.Errorf("route not found")
		}
		delete(p.Routes[route.Service], route.Hash())
	default:
		return fmt.Errorf("unknown action: %s", action)
	}

	return nil
}

// watchRoutes watches service routes and updates proxy cache
func (p *Proxy) watchRoutes() {
	// this is safe to do as the only way watchRoutes returns is
	// when some error is written into error channel - we want to bail then
	defer close(p.errChan)

	// route watcher
	w, err := p.Router.Watch()
	if err != nil {
		p.errChan <- err
		return
	}

	for {
		event, err := w.Next()
		if err != nil {
			p.errChan <- err
			return
		}

		p.Lock()
		if err := p.manageRouteCache(event.Route, fmt.Sprintf("%s", event.Type)); err != nil {
			// TODO: should we bail here?
			p.Unlock()
			continue
		}
		p.Unlock()
	}
}

func (p *Proxy) SendRequest(ctx context.Context, req client.Request, rsp client.Response) error {
	return errors.InternalServerError("go.micro.proxy", "SendRequest is unsupported")
}

// ServeRequest honours the server.Router interface
func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server.Response) error {
	// determine if its local routing
	var local bool
	// address to call
	var addresses []string
	// routes
	var routes []router.Route
	// service name to call
	service := req.Service()
	// endpoint to call
	endpoint := req.Endpoint()

	// are we network routing or local routing
	if len(p.Links) == 0 {
		local = true
	}

	// call a specific backend endpoint either by name or address
	if len(p.Endpoint) > 0 {
		// address:port
		if parts := strings.Split(p.Endpoint, ":"); len(parts) > 1 {
			addresses = []string{p.Endpoint}
		} else {
			// get route for endpoint from router
			addr, err := p.getRoute(p.Endpoint)
			if err != nil {
				return err
			}
			// set the address
			routes = addr
			// set the name
			service = p.Endpoint
		}
	} else {
		// no endpoint was specified just lookup the route
		// get route for endpoint from router
		addr, err := p.getRoute(service)
		if err != nil {
			return err
		}
		routes = addr
	}

	// if the address is already set just serve it
	// TODO: figure it out if we should know to pick a link
	if len(addresses) > 0 {
		// serve the normal way
		return p.serveRequest(ctx, p.Client, service, endpoint, req, rsp, client.WithAddress(addresses...))
	}

	// sort the routes in order of metric
	sort.Slice(routes, func(i, j int) bool { return routes[i].Metric < routes[j].Metric })

	// there's no links e.g we're local routing then just serve it with addresses
	if local {
		var opts []client.CallOption

		// set address if available via routes or specific endpoint
		if len(routes) > 0 {
			addresses := toNodes(routes)
			opts = append(opts, client.WithAddress(addresses...))
		}

		// serve the normal way
		return p.serveRequest(ctx, p.Client, service, endpoint, req, rsp, opts...)
	}

	var gerr error

	// we're routing globally with multiple links
	// so we need to pick a link per route
	for _, route := range routes {
		// pick the link or error out
		link, err := p.getLink(route)
		if err != nil {
			// ok let's try again
			gerr = err
			continue
		}

		// set the address to call
		addresses := toNodes([]router.Route{route})

		// do the request with the link
		gerr = p.serveRequest(ctx, link, service, endpoint, req, rsp, client.WithAddress(addresses...))
		// return on no error since we succeeded
		if gerr == nil {
			return nil
		}

		// return where the context deadline was exceeded
		if gerr == context.Canceled || gerr == context.DeadlineExceeded {
			return err
		}

		// otherwise attempt to do it all over again
	}

	// if we got here something went really badly wrong
	return gerr
}

func (p *Proxy) serveRequest(ctx context.Context, link client.Client, service, endpoint string, req server.Request, rsp server.Response, opts ...client.CallOption) error {
	// read initial request
	body, err := req.Read()
	if err != nil {
		return err
	}

	// create new request with raw bytes body
	creq := link.NewRequest(service, endpoint, &bytes.Frame{body}, client.WithContentType(req.ContentType()))

	// not a stream so make a client.Call request
	if !req.Stream() {
		crsp := new(bytes.Frame)

		// make a call to the backend
		if err := link.Call(ctx, creq, crsp, opts...); err != nil {
			return err
		}

		// write the response
		if err := rsp.Write(crsp.Data); err != nil {
			return err
		}

		return nil
	}

	// create new stream
	stream, err := link.Stream(ctx, creq, opts...)
	if err != nil {
		return err
	}
	defer stream.Close()

	// create client request read loop if streaming
	go readLoop(req, stream)

	// get raw response
	resp := stream.Response()

	// route watcher error
	var watchErr error

	// create server response write loop
	for {
		select {
		case err := <-p.errChan:
			if err != nil {
				watchErr = err
			}
			return watchErr
		default:
			// read backend response body
			body, err := resp.Read()
			if err == io.EOF {
				return nil
			} else if err != nil {
				return err
			}

			// read backend response header
			hdr := resp.Header()

			// write raw response header to client
			rsp.WriteHeader(hdr)

			// write raw response body to client
			err = rsp.Write(body)
			if err == io.EOF {
				return nil
			} else if err != nil {
				return err
			}
		}
	}

	return nil
}

// NewSingleHostProxy returns a proxy which sends requests to a single backend
func NewSingleHostProxy(endpoint string) *Proxy {
	return &Proxy{
		Options:  options.NewOptions(),
		Endpoint: endpoint,
	}
}

// NewProxy returns a new proxy which will route based on mucp headers
func NewProxy(opts ...options.Option) proxy.Proxy {
	p := new(Proxy)
	p.Links = map[string]client.Client{}
	p.Options = options.NewOptions(opts...)
	p.Options.Init(options.WithString("mucp"))

	// get endpoint
	ep, ok := p.Options.Values().Get("proxy.endpoint")
	if ok {
		p.Endpoint = ep.(string)
	}

	// get client
	c, ok := p.Options.Values().Get("proxy.client")
	if ok {
		p.Client = c.(client.Client)
	}

	// set the default client
	if p.Client == nil {
		p.Client = client.DefaultClient
	}

	// get client
	links, ok := p.Options.Values().Get("proxy.links")
	if ok {
		p.Links = links.(map[string]client.Client)
	}

	// get router
	r, ok := p.Options.Values().Get("proxy.router")
	if ok {
		p.Router = r.(router.Router)
	}

	// create default router and start it
	if p.Router == nil {
		p.Router = router.DefaultRouter
	}

	// routes cache
	p.Routes = make(map[string]map[uint64]router.Route)

	// watch router service routes
	p.errChan = make(chan error, 1)
	go p.watchRoutes()

	return p
}
<commit_msg>change where we order the routes<commit_after>// Package mucp transparently forwards the incoming request using a go-micro client.
package mucp

import (
	"context"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"

	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/codec"
	"github.com/micro/go-micro/codec/bytes"
	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/errors"
	"github.com/micro/go-micro/proxy"
	"github.com/micro/go-micro/router"
	"github.com/micro/go-micro/server"
)

// Proxy will transparently proxy requests to an endpoint.
// If no endpoint is specified it will call a service using the client.
type Proxy struct {
	// embed options
	options.Options

	// Endpoint specifies the fixed service endpoint to call.
	Endpoint string

	// The client to use for outbound requests in the local network
	Client client.Client

	// Links are used for outbound requests not in the local network
	Links map[string]client.Client

	// The router for routes
	Router router.Router

	// A fib of routes service:address
	sync.RWMutex
	Routes map[string]map[uint64]router.Route

	// The channel to monitor watcher errors
	errChan chan error
}

// read client request and write to server
func readLoop(r server.Request, s client.Stream) error {
	// request to backend server
	req := s.Request()

	for {
		// get data from client
		//  no need to decode it
		body, err := r.Read()
		if err == io.EOF {
			return nil
		}

		if err != nil {
			return err
		}

		// get the header from client
		hdr := r.Header()
		msg := &codec.Message{
			Type:   codec.Request,
			Header: hdr,
			Body:   body,
		}

		// write the raw request
		err = req.Codec().Write(msg, nil)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// toNodes returns a list of node addresses from given routes
func toNodes(routes []router.Route) []string {
	var nodes []string
	for _, node := range routes {
		address := node.Address
		if len(node.Gateway) > 0 {
			address = node.Gateway
		}
		nodes = append(nodes, address)
	}
	return nodes
}

func (p *Proxy) getLink(r router.Route) (client.Client, error) {
	if r.Link == "local" || len(p.Links) == 0 {
		return p.Client, nil
	}
	l, ok := p.Links[r.Link]
	if !ok {
		return nil, errors.InternalServerError("go.micro.proxy", "link not found")
	}
	return l, nil
}

func (p *Proxy) getRoute(service string) ([]router.Route, error) {
	toSlice := func(r map[uint64]router.Route) []router.Route {
		var routes []router.Route
		for _, v := range r {
			routes = append(routes, v)
		}

		// sort the routes in order of metric
		sort.Slice(routes, func(i, j int) bool { return routes[i].Metric < routes[j].Metric })

		return routes
	}

	// lookup the route cache first
	p.Lock()
	routes, ok := p.Routes[service]
	if ok {
		p.Unlock()
		return toSlice(routes), nil
	}
	p.Unlock()

	// lookup the routes in the router
	results, err := p.Router.Lookup(router.NewQuery(router.QueryService(service)))
	if err != nil {
		// check the status of the router
		if status := p.Router.Status(); status.Code == router.Error {
			return nil, status.Error
		}
		// otherwise return the error
		return nil, err
	}

	// update the proxy cache
	p.Lock()
	for _, route := range results {
		// create if does not exist
		if _, ok := p.Routes[service]; !ok {
			p.Routes[service] = make(map[uint64]router.Route)
		}
		p.Routes[service][route.Hash()] = route
	}
	routes = p.Routes[service]
	p.Unlock()

	return toSlice(routes), nil
}

// manageRouteCache applies action on a given route to Proxy route cache
func (p *Proxy) manageRouteCache(route router.Route, action string) error {
	switch action {
	case "create", "update":
		if _, ok := p.Routes[route.Service]; !ok {
			p.Routes[route.Service] = make(map[uint64]router.Route)
		}
		p.Routes[route.Service][route.Hash()] = route
	case "delete":
		if _, ok := p.Routes[route.Service]; !ok {
			return fmt.Errorf("route not found")
		}
		delete(p.Routes[route.Service], route.Hash())
	default:
		return fmt.Errorf("unknown action: %s", action)
	}

	return nil
}

// watchRoutes watches service routes and updates proxy cache
func (p *Proxy) watchRoutes() {
	// this is safe to do as the only way watchRoutes returns is
	// when some error is written into error channel - we want to bail then
	defer close(p.errChan)

	// route watcher
	w, err := p.Router.Watch()
	if err != nil {
		p.errChan <- err
		return
	}

	for {
		event, err := w.Next()
		if err != nil {
			p.errChan <- err
			return
		}

		p.Lock()
		if err := p.manageRouteCache(event.Route, fmt.Sprintf("%s", event.Type)); err != nil {
			// TODO: should we bail here?
			p.Unlock()
			continue
		}
		p.Unlock()
	}
}

func (p *Proxy) SendRequest(ctx context.Context, req client.Request, rsp client.Response) error {
	return errors.InternalServerError("go.micro.proxy", "SendRequest is unsupported")
}

// ServeRequest honours the server.Router interface
func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server.Response) error {
	// determine if its local routing
	var local bool
	// address to call
	var addresses []string
	// routes
	var routes []router.Route
	// service name to call
	service := req.Service()
	// endpoint to call
	endpoint := req.Endpoint()

	// are we network routing or local routing
	if len(p.Links) == 0 {
		local = true
	}

	// call a specific backend endpoint either by name or address
	if len(p.Endpoint) > 0 {
		// address:port
		if parts := strings.Split(p.Endpoint, ":"); len(parts) > 1 {
			addresses = []string{p.Endpoint}
		} else {
			// get route for endpoint from router
			addr, err := p.getRoute(p.Endpoint)
			if err != nil {
				return err
			}
			// set the address
			routes = addr
			// set the name
			service = p.Endpoint
		}
	} else {
		// no endpoint was specified just lookup the route
		// get route for endpoint from router
		addr, err := p.getRoute(service)
		if err != nil {
			return err
		}
		routes = addr
	}

	// if the address is already set just serve it
	// TODO: figure it out if we should know to pick a link
	if len(addresses) > 0 {
		// serve the normal way
		return p.serveRequest(ctx, p.Client, service, endpoint, req, rsp, client.WithAddress(addresses...))
	}

	// there's no links e.g we're local routing then just serve it with addresses
	if local {
		var opts []client.CallOption

		// set address if available via routes or specific endpoint
		if len(routes) > 0 {
			addresses := toNodes(routes)
			opts = append(opts, client.WithAddress(addresses...))
		}

		// serve the normal way
		return p.serveRequest(ctx, p.Client, service, endpoint, req, rsp, opts...)
	}

	var gerr error

	// we're routing globally with multiple links
	// so we need to pick a link per route
	for _, route := range routes {
		// pick the link or error out
		link, err := p.getLink(route)
		if err != nil {
			// ok let's try again
			gerr = err
			continue
		}

		// set the address to call
		addresses := toNodes([]router.Route{route})

		// do the request with the link
		gerr = p.serveRequest(ctx, link, service, endpoint, req, rsp, client.WithAddress(addresses...))
		// return on no error since we succeeded
		if gerr == nil {
			return nil
		}

		// return where the context deadline was exceeded
		if gerr == context.Canceled || gerr == context.DeadlineExceeded {
			return err
		}

		// otherwise attempt to do it all over again
	}

	// if we got here something went really badly wrong
	return gerr
}

func (p *Proxy) serveRequest(ctx context.Context, link client.Client, service, endpoint string, req server.Request, rsp server.Response, opts ...client.CallOption) error {
	// read initial request
	body, err := req.Read()
	if err != nil {
		return err
	}

	// create new request with raw bytes body
	creq := link.NewRequest(service, endpoint, &bytes.Frame{body}, client.WithContentType(req.ContentType()))

	// not a stream so make a client.Call request
	if !req.Stream() {
		crsp := new(bytes.Frame)

		// make a call to the backend
		if err := link.Call(ctx, creq, crsp, opts...); err != nil {
			return err
		}

		// write the response
		if err := rsp.Write(crsp.Data); err != nil {
			return err
		}

		return nil
	}

	// create new stream
	stream, err := link.Stream(ctx, creq, opts...)
	if err != nil {
		return err
	}
	defer stream.Close()

	// create client request read loop if streaming
	go readLoop(req, stream)

	// get raw response
	resp := stream.Response()

	// route watcher error
	var watchErr error

	// create server response write loop
	for {
		select {
		case err := <-p.errChan:
			if err != nil {
				watchErr = err
			}
			return watchErr
		default:
			// read backend response body
			body, err := resp.Read()
			if err == io.EOF {
				return nil
			} else if err != nil {
				return err
			}

			// read backend response header
			hdr := resp.Header()

			// write raw response header to client
			rsp.WriteHeader(hdr)

			// write raw response body to client
			err = rsp.Write(body)
			if err == io.EOF {
				return nil
			} else if err != nil {
				return err
			}
		}
	}

	return nil
}

// NewSingleHostProxy returns a proxy which sends requests to a single backend
func NewSingleHostProxy(endpoint string) *Proxy {
	return &Proxy{
		Options:  options.NewOptions(),
		Endpoint: endpoint,
	}
}

// NewProxy returns a new proxy which will route based on mucp headers
func NewProxy(opts ...options.Option) proxy.Proxy {
	p := new(Proxy)
	p.Links = map[string]client.Client{}
	p.Options = options.NewOptions(opts...)
	p.Options.Init(options.WithString("mucp"))

	// get endpoint
	ep, ok := p.Options.Values().Get("proxy.endpoint")
	if ok {
		p.Endpoint = ep.(string)
	}

	// get client
	c, ok := p.Options.Values().Get("proxy.client")
	if ok {
		p.Client = c.(client.Client)
	}

	// set the default client
	if p.Client == nil {
		p.Client = client.DefaultClient
	}

	// get client
	links, ok := p.Options.Values().Get("proxy.links")
	if ok {
		p.Links = links.(map[string]client.Client)
	}

	// get router
	r, ok := p.Options.Values().Get("proxy.router")
	if ok {
		p.Router = r.(router.Router)
	}

	// create default router and start it
	if p.Router == nil {
		p.Router = router.DefaultRouter
	}

	// routes cache
	p.Routes = make(map[string]map[uint64]router.Route)

	// watch router service routes
	p.errChan = make(chan error, 1)
	go p.watchRoutes()

	return p
}
<|endoftext|>


<commit_before>package message

import (
	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	blocks "github.com/jbenet/go-ipfs/blocks"
	pb "github.com/jbenet/go-ipfs/exchange/bitswap/message/internal/pb"
	netmsg "github.com/jbenet/go-ipfs/net/message"
	nm "github.com/jbenet/go-ipfs/net/message"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
)

// TODO move message.go into the bitswap package
// TODO move bs/msg/internal/pb to bs/internal/pb and rename pb package to bitswap_pb

type BitSwapMessage interface {
	Wantlist() []u.Key
	Blocks() []blocks.Block
	AddWanted(k u.Key)
	AppendBlock(b blocks.Block)
	Exportable
}

type Exportable interface {
	ToProto() *pb.Message
	ToNet(p peer.Peer) (nm.NetMessage, error)
}

// message wraps a proto message for convenience
type message struct {
	wantlist map[u.Key]struct{}
	blocks   []blocks.Block
}

func New() BitSwapMessage {
	return &message{
		wantlist: make(map[u.Key]struct{}),
	}
}

func newMessageFromProto(pbm pb.Message) BitSwapMessage {
	m := New()
	for _, s := range pbm.GetWantlist() {
		m.AddWanted(u.Key(s))
	}
	for _, d := range pbm.GetBlocks() {
		b := blocks.NewBlock(d)
		m.AppendBlock(*b)
	}
	return m
}

// TODO(brian): convert these into keys
func (m *message) Wantlist() []u.Key {
	wl := make([]u.Key, 0)
	for k, _ := range m.wantlist {
		wl = append(wl, k)
	}
	return wl
}

// TODO(brian): convert these into blocks
func (m *message) Blocks() []blocks.Block {
	return m.blocks
}

func (m *message) AddWanted(k u.Key) {
	m.wantlist[k] = struct{}{}
}

func (m *message) AppendBlock(b blocks.Block) {
	m.blocks = append(m.blocks, b)
}

func FromNet(nmsg netmsg.NetMessage) (BitSwapMessage, error) {
	pb := new(pb.Message)
	if err := proto.Unmarshal(nmsg.Data(), pb); err != nil {
		return nil, err
	}
	m := newMessageFromProto(*pb)
	return m, nil
}

func (m *message) ToProto() *pb.Message {
	pb := new(pb.Message)
	for _, k := range m.Wantlist() {
		pb.Wantlist = append(pb.Wantlist, string(k))
	}
	for _, b := range m.Blocks() {
		pb.Blocks = append(pb.Blocks, b.Data)
	}
	return pb
}

func (m *message) ToNet(p peer.Peer) (nm.NetMessage, error) {
	return nm.FromObject(p, m.ToProto())
}
<commit_msg>style(bitswap/message) rename struct<commit_after>package message

import (
	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	blocks "github.com/jbenet/go-ipfs/blocks"
	pb "github.com/jbenet/go-ipfs/exchange/bitswap/message/internal/pb"
	netmsg "github.com/jbenet/go-ipfs/net/message"
	nm "github.com/jbenet/go-ipfs/net/message"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
)

// TODO move message.go into the bitswap package
// TODO move bs/msg/internal/pb to bs/internal/pb and rename pb package to bitswap_pb

type BitSwapMessage interface {
	Wantlist() []u.Key
	Blocks() []blocks.Block
	AddWanted(k u.Key)
	AppendBlock(b blocks.Block)
	Exportable
}

type Exportable interface {
	ToProto() *pb.Message
	ToNet(p peer.Peer) (nm.NetMessage, error)
}

// message wraps a proto message for convenience
type impl struct {
	wantlist map[u.Key]struct{}
	blocks   []blocks.Block
}

func New() BitSwapMessage {
	return &impl{
		wantlist: make(map[u.Key]struct{}),
	}
}

func newMessageFromProto(pbm pb.Message) BitSwapMessage {
	m := New()
	for _, s := range pbm.GetWantlist() {
		m.AddWanted(u.Key(s))
	}
	for _, d := range pbm.GetBlocks() {
		b := blocks.NewBlock(d)
		m.AppendBlock(*b)
	}
	return m
}

// TODO(brian): convert these into keys
func (m *impl) Wantlist() []u.Key {
	wl := make([]u.Key, 0)
	for k, _ := range m.wantlist {
		wl = append(wl, k)
	}
	return wl
}

// TODO(brian): convert these into blocks
func (m *impl) Blocks() []blocks.Block {
	return m.blocks
}

func (m *impl) AddWanted(k u.Key) {
	m.wantlist[k] = struct{}{}
}

func (m *impl) AppendBlock(b blocks.Block) {
	m.blocks = append(m.blocks, b)
}

func FromNet(nmsg netmsg.NetMessage) (BitSwapMessage, error) {
	pb := new(pb.Message)
	if err := proto.Unmarshal(nmsg.Data(), pb); err != nil {
		return nil, err
	}
	m := newMessageFromProto(*pb)
	return m, nil
}

func (m *impl) ToProto() *pb.Message {
	pb := new(pb.Message)
	for _, k := range m.Wantlist() {
		pb.Wantlist = append(pb.Wantlist, string(k))
	}
	for _, b := range m.Blocks() {
		pb.Blocks = append(pb.Blocks, b.Data)
	}
	return pb
}

func (m *impl) ToNet(p peer.Peer) (nm.NetMessage, error) {
	return nm.FromObject(p, m.ToProto())
}
<|endoftext|>


<commit_before>package errorutil

import "regexp"

// IsExitStatusError ...
func IsExitStatusError(err error) bool {
	return IsExitStatusErrorStr(err.Error())
}

// IsExitStatusErrorStr ...
func IsExitStatusErrorStr(errString string) bool {
	// example exit status error string: exit status 1
	var rex = regexp.MustCompile(`^exit status [0-9]{1,3}$`)
	return rex.MatchString(errString)
}
<commit_msg>CmdExitCodeFromError (#28)<commit_after>package errorutil

import (
	"errors"
	"os/exec"
	"regexp"
	"syscall"
)

// IsExitStatusError ...
func IsExitStatusError(err error) bool {
	return IsExitStatusErrorStr(err.Error())
}

// IsExitStatusErrorStr ...
func IsExitStatusErrorStr(errString string) bool {
	// example exit status error string: exit status 1
	var rex = regexp.MustCompile(`^exit status [0-9]{1,3}$`)
	return rex.MatchString(errString)
}

// CmdExitCodeFromError ...
func CmdExitCodeFromError(err error) (int, error) {
	cmdExitCode := 0
	if err != nil {
		if exitError, ok := err.(*exec.ExitError); ok {
			waitStatus, ok := exitError.Sys().(syscall.WaitStatus)
			if !ok {
				return 1, errors.New("Failed to cast exit status")
			}
			cmdExitCode = waitStatus.ExitStatus()
		}
		return cmdExitCode, nil
	}
	return 0, nil
}
<|endoftext|>


<commit_before>package human_test

import (
	"fmt"
	"github.com/anexia-it/go-human"
	"net"
	"os"
)

type SimpleChild struct {
	Name      string  // no tag
	Property1 uint64  `human:"-"`          // Ignored
	Property2 float64 `human:",omitempty"` // Omitted if empty
}

type SimpleTest struct {
	Var1  string //no tag
	Var2  int    `human:"variable_2"`
	Child SimpleChild
}

type MapTest struct {
	Val1      uint64
	Map       map[string]SimpleChild
	StructMap map[SimpleChild]uint8
}

type SliceTest struct {
	IntSlice    []int
	StructSlice []SimpleChild
}

type MapSliceTest struct {
	StructMapSlice []map[string]int
}

type address struct {
	Ip net.IP
}

type TagFailTest struct {
	Test int `human:"&§/$"`
}

type AnonymousFieldTest struct {
	int
	Text string
}

func ExampleEncoder_Encode_SimpleOmitEmpty() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}
	testStruct := SimpleTest{
		Var1: "v1",
		Var2: 2,
		Child: SimpleChild{
			Name:      "theChild",
			Property1: 3, // should be ignored
			Property2: 0, // empty, should be omitted
		},
	}

	// Output: Var1: v1
	// variable_2: 2
	// Child:
	//   Name: theChild

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}

func ExampleEncoder_Encode_Simple() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}
	testStruct := SimpleTest{
		Var1: "v1",
		Var2: 2,
		Child: SimpleChild{
			Name:      "theChild",
			Property1: 3, // should be ignored
			Property2: 4.5,
		},
	}

	// Output: Var1: v1
	// variable_2: 2
	// Child:
	//   Name: theChild
	//   Property2: 4.5

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}

func ExampleEncoder_Encode_SimpleMap() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	child1 := SimpleChild{
		Name:      "Person1",
		Property2: 4.5,
		Property1: 0, // should be ignored
	}

	child2 := SimpleChild{
		Name: "Person2",
	}
	stringMap := map[string]SimpleChild{
		"One": child1,
		"Two": child2,
	}
	structMap := map[SimpleChild]uint8{
		child1: 1,
		child2: 2,
	}
	testStruct := MapTest{
		Map:       stringMap,
		StructMap: structMap,
	}

	// Output: Val1: 0
	// Map:
	//   * One: Name: Person1
	//     Property2: 4.5
	//   * Two: Name: Person2
	// StructMap:
	//   * {Person1 0 4.5}: 1
	//   * {Person2 0 0}: 2

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}

func ExampleEncoder_Encode_SimpleSlice() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	child1 := SimpleChild{
		Name:      "Person1",
		Property2: 4.5,
		Property1: 0, // should be ignored
	}

	child2 := SimpleChild{
		Name: "Person2",
	}
	structSlice := []SimpleChild{child1, child2}
	testStruct := SliceTest{
		IntSlice:    []int{1, 2, 3, 4, 5},
		StructSlice: structSlice,
	}

	// Output: IntSlice:
	//   * 1
	//   * 2
	//   * 3
	//   * 4
	//   * 5
	// StructSlice:
	//   * Name: Person1
	//     Property2: 4.5
	//   * Name: Person2

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}

func ExampleEncoder_Encode_StructMapSlice() {
	enc, err := human.NewEncoder(os.Stdout, human.OptionListSymbols("+", "-"))
	if err != nil {
		return
	}

	mapSliceElement1 := map[string]int{
		"one": 1,
		"two": 2,
		"tenthousandonehundredfourtytwo": 10142,
	}
	slice := []map[string]int{mapSliceElement1, mapSliceElement1}
	testStruct := MapSliceTest{
		StructMapSlice: slice,
	}

	//Output: StructMapSlice:
	//   +
	//     - one: 1
	//     - tenthousandonehundredfourtytwo: 10142
	//     - two: 2
	//   +
	//     - one: 1
	//     - tenthousandonehundredfourtytwo: 10142
	//     - two: 2
	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}

}

func ExampleEncoder_Encode_TextMarshaler() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	// Output: Ip:[49 50 55 46 48 46 48 46 49]

	addr := address{
		Ip: net.ParseIP("127.0.0.1"),
	}
	if err := enc.Encode(addr); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}

}

func ExampleEncoder_Encode_MapFieldError() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	testStruct := TagFailTest{
		Test: 1,
	}

	// Output: ERROR: 1 error occurred:
	//
	// * Invalid tag: '&§/$'
	//
	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}

}

func ExampleEncoder_Encode_AnonymousFiled() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	//anonymous int field is ignored
	testStruct := AnonymousFieldTest{
		Text: "test",
	}
	// Output: Text: test

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}
<commit_msg>fixed go vet issues<commit_after>package human_test

import (
	"fmt"
	"github.com/anexia-it/go-human"
	"net"
	"os"
)

type SimpleChild struct {
	Name      string  // no tag
	Property1 uint64  `human:"-"`          // Ignored
	Property2 float64 `human:",omitempty"` // Omitted if empty
}

type SimpleTest struct {
	Var1  string //no tag
	Var2  int    `human:"variable_2"`
	Child SimpleChild
}

type MapTest struct {
	Val1      uint64
	Map       map[string]SimpleChild
	StructMap map[SimpleChild]uint8
}

type SliceTest struct {
	IntSlice    []int
	StructSlice []SimpleChild
}

type MapSliceTest struct {
	StructMapSlice []map[string]int
}

type address struct {
	Ip net.IP
}

type TagFailTest struct {
	Test int `human:"&§/$"`
}

type AnonymousFieldTest struct {
	int
	Text string
}

func ExampleEncoder_Encode_simpleOmitEmpty() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}
	testStruct := SimpleTest{
		Var1: "v1",
		Var2: 2,
		Child: SimpleChild{
			Name:      "theChild",
			Property1: 3, // should be ignored
			Property2: 0, // empty, should be omitted
		},
	}

	// Output: Var1: v1
	// variable_2: 2
	// Child:
	//   Name: theChild

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}

func ExampleEncoder_Encode_simple() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}
	testStruct := SimpleTest{
		Var1: "v1",
		Var2: 2,
		Child: SimpleChild{
			Name:      "theChild",
			Property1: 3, // should be ignored
			Property2: 4.5,
		},
	}

	// Output: Var1: v1
	// variable_2: 2
	// Child:
	//   Name: theChild
	//   Property2: 4.5

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}

func ExampleEncoder_Encode_simpleMap() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	child1 := SimpleChild{
		Name:      "Person1",
		Property2: 4.5,
		Property1: 0, // should be ignored
	}

	child2 := SimpleChild{
		Name: "Person2",
	}
	stringMap := map[string]SimpleChild{
		"One": child1,
		"Two": child2,
	}
	structMap := map[SimpleChild]uint8{
		child1: 1,
		child2: 2,
	}
	testStruct := MapTest{
		Map:       stringMap,
		StructMap: structMap,
	}

	// Output: Val1: 0
	// Map:
	//   * One: Name: Person1
	//     Property2: 4.5
	//   * Two: Name: Person2
	// StructMap:
	//   * {Person1 0 4.5}: 1
	//   * {Person2 0 0}: 2

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}

func ExampleEncoder_Encode_simpleSlice() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	child1 := SimpleChild{
		Name:      "Person1",
		Property2: 4.5,
		Property1: 0, // should be ignored
	}

	child2 := SimpleChild{
		Name: "Person2",
	}
	structSlice := []SimpleChild{child1, child2}
	testStruct := SliceTest{
		IntSlice:    []int{1, 2, 3, 4, 5},
		StructSlice: structSlice,
	}

	// Output: IntSlice:
	//   * 1
	//   * 2
	//   * 3
	//   * 4
	//   * 5
	// StructSlice:
	//   * Name: Person1
	//     Property2: 4.5
	//   * Name: Person2

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}

func ExampleEncoder_Encode_structMapSlice() {
	enc, err := human.NewEncoder(os.Stdout, human.OptionListSymbols("+", "-"))
	if err != nil {
		return
	}

	mapSliceElement1 := map[string]int{
		"one": 1,
		"two": 2,
		"tenthousandonehundredfourtytwo": 10142,
	}
	slice := []map[string]int{mapSliceElement1, mapSliceElement1}
	testStruct := MapSliceTest{
		StructMapSlice: slice,
	}

	//Output: StructMapSlice:
	//   +
	//     - one: 1
	//     - tenthousandonehundredfourtytwo: 10142
	//     - two: 2
	//   +
	//     - one: 1
	//     - tenthousandonehundredfourtytwo: 10142
	//     - two: 2
	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}

}

func ExampleEncoder_Encode_textMarshaler() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	// Output: Ip:[49 50 55 46 48 46 48 46 49]

	addr := address{
		Ip: net.ParseIP("127.0.0.1"),
	}
	if err := enc.Encode(addr); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}

}

func ExampleEncoder_Encode_mapFieldError() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	testStruct := TagFailTest{
		Test: 1,
	}

	// Output: ERROR: 1 error occurred:
	//
	// * Invalid tag: '&§/$'
	//
	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}

}

func ExampleEncoder_Encode_anonymousFiled() {
	enc, err := human.NewEncoder(os.Stdout)
	if err != nil {
		return
	}

	//anonymous int field is ignored
	testStruct := AnonymousFieldTest{
		Text: "test",
	}
	// Output: Text: test

	if err := enc.Encode(testStruct); err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
		return
	}
}
<|endoftext|>


<commit_before>package corehttp

import (
	"bufio"
	"fmt"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"

	core "github.com/ipfs/go-ipfs/core"

	protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
	peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer"
	inet "gx/ipfs/QmfDPh144WGBqRxZb1TGDHerbMnZATrHZggAPw7putNnBq/go-libp2p-net"
)

// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer
func ProxyOption() ServeOption {
	return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
		mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) {
			// parse request
			parsedRequest, err := parseRequest(request)
			if err != nil {
				handleError(w, "Failed to parse request", err, 400)
				return
			}

			// open connect to peer
			stream, err := ipfsNode.P2P.PeerHost.NewStream(ipfsNode.Context(), parsedRequest.target, protocol.ID("/x/"+parsedRequest.name))
			if err != nil {
				msg := fmt.Sprintf("Failed to open stream '%v' to target peer '%v'", parsedRequest.name, parsedRequest.target)
				handleError(w, msg, err, 500)
				return
			}
			//send proxy request and response to client
			newReverseHTTPProxy(parsedRequest, &stream).ServeHTTP(w, request)
		})
		return mux, nil
	}
}

type proxyRequest struct {
	target   peer.ID
	name     string
	httpPath string // path to send to the proxy-host
}

// from the url path parse the peer-ID, name and http path
// /proxy/http/$peer_id/$name/$http_path
func parseRequest(request *http.Request) (*proxyRequest, error) {
	path := request.URL.Path

	split := strings.SplitN(path, "/", 6)
	if len(split) < 6 {
		return nil, fmt.Errorf("Invalid request path '%s'", path)
	}

	peerID, err := peer.IDB58Decode(split[3])

	if err != nil {
		return nil, err
	}

	return &proxyRequest{peerID, split[4], "/" + split[5]}, nil
}

func handleError(w http.ResponseWriter, msg string, err error, code int) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%s: %s\n", msg, err)
	log.Warningf("server error: %s: %s", err)
}

func newReverseHTTPProxy(req *proxyRequest, streamToPeer *inet.Stream) *httputil.ReverseProxy {
	director := func(r *http.Request) {
		r.URL.Path = req.httpPath //the scheme etc. doesn't matter
	}

	return &httputil.ReverseProxy{
		Director:  director,
		Transport: &roundTripper{streamToPeer}}
}

type roundTripper struct {
	stream *inet.Stream
}

func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {

	sendRequest := func() {
		err := req.Write(*rt.stream)
		if err != nil {
			(*(rt.stream)).Close()
		}
		if req.Body != nil {
			req.Body.Close()
		}
	}
	//send request while reading response
	go sendRequest()
	s := bufio.NewReader(*rt.stream)
	return http.ReadResponse(s, req)
}
<commit_msg>Remove unnecessary pointer usage.<commit_after>package corehttp

import (
	"bufio"
	"fmt"
	"net"
	"io"
	"net/http"
	"net/http/httputil"
	"strings"

	core "github.com/ipfs/go-ipfs/core"

	protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
	peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer"
	inet "gx/ipfs/QmfDPh144WGBqRxZb1TGDHerbMnZATrHZggAPw7putNnBq/go-libp2p-net"
)

// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer
func ProxyOption() ServeOption {
	return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
		mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) {
			// parse request
			parsedRequest, err := parseRequest(request)
			if err != nil {
				handleError(w, "Failed to parse request", err, 400)
				return
			}

			// open connect to peer
			stream, err := ipfsNode.P2P.PeerHost.NewStream(ipfsNode.Context(), parsedRequest.target, protocol.ID("/x/"+parsedRequest.name))
			if err != nil {
				msg := fmt.Sprintf("Failed to open stream '%v' to target peer '%v'", parsedRequest.name, parsedRequest.target)
				handleError(w, msg, err, 500)
				return
			}
			//send proxy request and response to client
			newReverseHTTPProxy(parsedRequest, stream).ServeHTTP(w, request)
		})
		return mux, nil
	}
}

type proxyRequest struct {
	target   peer.ID
	name     string
	httpPath string // path to send to the proxy-host
}

// from the url path parse the peer-ID, name and http path
// /proxy/http/$peer_id/$name/$http_path
func parseRequest(request *http.Request) (*proxyRequest, error) {
	path := request.URL.Path

	split := strings.SplitN(path, "/", 6)
	if len(split) < 6 {
		return nil, fmt.Errorf("Invalid request path '%s'", path)
	}

	peerID, err := peer.IDB58Decode(split[3])

	if err != nil {
		return nil, err
	}

	return &proxyRequest{peerID, split[4], "/" + split[5]}, nil
}

func handleError(w http.ResponseWriter, msg string, err error, code int) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%s: %s\n", msg, err)
	log.Warningf("server error: %s: %s", err)
}

func newReverseHTTPProxy(req *proxyRequest, streamToPeer inet.Stream) *httputil.ReverseProxy {
	director := func(r *http.Request) {
		r.URL.Path = req.httpPath //the scheme etc. doesn't matter
	}

	return &httputil.ReverseProxy{
		Director:  director,
		Transport: &roundTripper{streamToPeer}}
}

type roundTripper struct {
	stream inet.Stream
}

// we wrap the response body and close the stream
// only when it's closed.
type respBody struct {
	io.ReadCloser
	stream inet.Stream
}

// Closes the response's body and the connection.
func (rb *respBody) Close() error {
	rb.stream.Close()
	return rb.ReadCloser.Close()
}

func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {

	sendRequest := func() {
		err := req.Write(rt.stream)
		if err != nil {
			rt.stream.Close()
		}
		if req.Body != nil {
			req.Body.Close()
		}
	}
	//send request while reading response
	go sendRequest()
	s := bufio.NewReader(rt.stream)

	resp, err := http.ReadResponse(s, req)
	if err != nil {
		return resp, err
	}

	resp.Body = &respBody{
		ReadCloser: resp.Body,
		stream:       rt.stream,
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package models

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/SpectoLabs/hoverfly/core/handlers/v1"
	. "github.com/SpectoLabs/hoverfly/core/util"
	"github.com/tdewolff/minify"
	"github.com/tdewolff/minify/json"
	"github.com/tdewolff/minify/xml"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"github.com/SpectoLabs/hoverfly/core/handlers/v2"
)

const (
	contentTypeJSON = "application/json"
	contentTypeXML  = "application/xml"
	otherType       = "otherType"
)

var (
	rxJSON = regexp.MustCompile("[/+]json$")
	rxXML  = regexp.MustCompile("[/+]xml$")
	// mime types which will not be base 64 encoded when exporting as JSON
	supportedMimeTypes = [...]string{"text", "plain", "css", "html", "json", "xml", "js", "javascript"}
	minifiers          *minify.M
)

func init() {
	// GetNewMinifiers - sets minify.M with prepared xml/json minifiers
	minifiers = minify.New()
	minifiers.AddFuncRegexp(regexp.MustCompile("[/+]xml$"), xml.Minify)
	minifiers.AddFuncRegexp(regexp.MustCompile("[/+]json$"), json.Minify)
}

// Payload structure holds request and response structure
type RequestResponsePair struct {
	Response ResponseDetails `json:"response"`
	Request  RequestDetails  `json:"request"`
}

func (this RequestResponsePair) Id() string {
	return this.Request.Hash()
}

func (this RequestResponsePair) IdWithoutHost() string {
	return this.Request.HashWithoutHost()
}

// Encode method encodes all exported Payload fields to bytes
func (this *RequestResponsePair) Encode() ([]byte, error) {
	buf := new(bytes.Buffer)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(this)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (this *RequestResponsePair) ConvertToRequestResponsePairView() *v1.RequestResponsePairView {
	return &v1.RequestResponsePairView{Response: this.Response.ConvertToResponseDetailsView(), Request: this.Request.ConvertToRequestDetailsView()}
}

func (this *RequestResponsePair) ConvertToV2RequestResponsePairView() v2.RequestResponsePairView {
	return v2.RequestResponsePairView{Response: this.Response.ConvertToV2ResponseDetailsView(), Request: this.Request.ConvertToV2RequestDetailsView()}
}

// NewPayloadFromBytes decodes supplied bytes into Payload structure
func NewRequestResponsePairFromBytes(data []byte) (*RequestResponsePair, error) {
	var pair *RequestResponsePair
	buf := bytes.NewBuffer(data)
	dec := gob.NewDecoder(buf)
	err := dec.Decode(&pair)
	if err != nil {
		return nil, err
	}
	return pair, nil
}

func NewRequestResponsePairFromRequestResponsePairView(pairView v1.RequestResponsePairView) RequestResponsePair {
	return RequestResponsePair{
		Response: NewResponseDetailsFromResponseDetailsView(pairView.Response),
		Request:  NewRequestDetailsFromRequestDetailsView(pairView.Request),
	}
}

// RequestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type RequestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Scheme      string              `json:"scheme"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	Headers     map[string][]string `json:"headers"`
}

func NewRequestDetailsFromHttpRequest(req *http.Request) (RequestDetails, error) {
	if req.Body == nil {
		req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	reqBody, err := extractRequestBody(req)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  "capture",
		}).Error("Got error while reading request body")
		return RequestDetails{}, err
	}

	requestDetails := RequestDetails{
		Path:        req.URL.Path,
		Method:      req.Method,
		Destination: req.Host,
		Scheme:      req.URL.Scheme,
		Query:       req.URL.RawQuery,
		Body:        string(reqBody),
		Headers:     req.Header,
	}
	return requestDetails, nil
}

func extractRequestBody(req *http.Request) (extract []byte, err error) {
	save := req.Body
	savecl := req.ContentLength

	save, req.Body, err = CopyBody(req.Body)
	if err != nil {
		return
	}

	defer req.Body.Close()
	extract, err = ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}

	req.Body = save
	req.ContentLength = savecl
	return extract, nil
}

func CopyBody(body io.ReadCloser) (resp1, resp2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(body); err != nil {
		return nil, nil, err
	}
	if err = body.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func NewRequestDetailsFromRequestDetailsView(data v1.RequestDetailsView) RequestDetails {
	return RequestDetails{
		Path:        PointerToString(data.Path),
		Method:      PointerToString(data.Method),
		Destination: PointerToString(data.Destination),
		Scheme:      PointerToString(data.Scheme),
		Query:       PointerToString(data.Query),
		Body:        PointerToString(data.Body),
		Headers:     data.Headers,
	}
}

func (this *RequestDetails) ConvertToRequestDetailsView() v1.RequestDetailsView {
	s := "recording"
	return v1.RequestDetailsView{
		RequestType: &s,
		Path:        &this.Path,
		Method:      &this.Method,
		Destination: &this.Destination,
		Scheme:      &this.Scheme,
		Query:       &this.Query,
		Body:        &this.Body,
		Headers:     this.Headers,
	}
}

func (this *RequestDetails) ConvertToV2RequestDetailsView() v2.RequestDetailsView {
	s := "recording"
	return v2.RequestDetailsView{
		RequestType: &s,
		Path:        &this.Path,
		Method:      &this.Method,
		Destination: &this.Destination,
		Scheme:      &this.Scheme,
		Query:       &this.Query,
		Body:        &this.Body,
		Headers:     this.Headers,
	}
}

func (r *RequestDetails) concatenate(withHost bool) string {
	var buffer bytes.Buffer

	if withHost {
		buffer.WriteString(r.Destination)
	}

	buffer.WriteString(r.Path)
	buffer.WriteString(r.Method)
	buffer.WriteString(r.Query)
	if len(r.Body) > 0 {
		ct := r.getContentType()

		if ct == contentTypeJSON || ct == contentTypeXML {
			buffer.WriteString(r.minifyBody(ct))
		} else {
			log.WithFields(log.Fields{
				"content-type": r.Headers["Content-Type"],
			}).Debug("unknown content type")

			buffer.WriteString(r.Body)
		}
	}

	return buffer.String()
}

func (r *RequestDetails) minifyBody(mediaType string) (minified string) {
	var err error
	minified, err = minifiers.String(mediaType, r.Body)
	if err != nil {
		log.WithFields(log.Fields{
			"error":       err.Error(),
			"destination": r.Destination,
			"path":        r.Path,
			"method":      r.Method,
		}).Errorf("failed to minify request body, media type given: %s. Request matching might fail", mediaType)
		return r.Body
	}
	log.Debugf("body minified, mediatype: %s", mediaType)
	return minified
}

func (r *RequestDetails) Hash() string {
	h := md5.New()
	io.WriteString(h, r.concatenate(true))
	return fmt.Sprintf("%x", h.Sum(nil))
}
func (r *RequestDetails) HashWithoutHost() string {
	h := md5.New()
	io.WriteString(h, r.concatenate(false))
	return fmt.Sprintf("%x", h.Sum(nil))
}

func (r *RequestDetails) getContentType() string {
	for _, v := range r.Headers["Content-Type"] {
		if rxJSON.MatchString(v) {
			return contentTypeJSON
		}
		if rxXML.MatchString(v) {
			return contentTypeXML
		}
	}
	return otherType
}

// ResponseDetails structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type ResponseDetails struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

func NewResponseDetailsFromResponseDetailsView(data v1.ResponseDetailsView) ResponseDetails {
	body := data.Body

	if data.EncodedBody == true {
		decoded, _ := base64.StdEncoding.DecodeString(data.Body)
		body = string(decoded)
	}

	return ResponseDetails{Status: data.Status, Body: body, Headers: data.Headers}
}

func (r *ResponseDetails) ConvertToResponseDetailsView() v1.ResponseDetailsView {
	needsEncoding := false

	// Check headers for gzip
	contentEncodingValues := r.Headers["Content-Encoding"]
	if len(contentEncodingValues) > 0 {
		needsEncoding = true
	} else {
		mimeType := http.DetectContentType([]byte(r.Body))
		needsEncoding = true
		for _, v := range supportedMimeTypes {
			if strings.Contains(mimeType, v) {
				needsEncoding = false
				break
			}
		}
	}

	// If contains gzip, base64 encode
	body := r.Body
	if needsEncoding {
		body = base64.StdEncoding.EncodeToString([]byte(r.Body))
	}

	return v1.ResponseDetailsView{Status: r.Status, Body: body, Headers: r.Headers, EncodedBody: needsEncoding}
}

func (r *ResponseDetails) ConvertToV2ResponseDetailsView() v2.ResponseDetailsView {
	needsEncoding := false

	// Check headers for gzip
	contentEncodingValues := r.Headers["Content-Encoding"]
	if len(contentEncodingValues) > 0 {
		needsEncoding = true
	} else {
		mimeType := http.DetectContentType([]byte(r.Body))
		needsEncoding = true
		for _, v := range supportedMimeTypes {
			if strings.Contains(mimeType, v) {
				needsEncoding = false
				break
			}
		}
	}

	// If contains gzip, base64 encode
	body := r.Body
	if needsEncoding {
		body = base64.StdEncoding.EncodeToString([]byte(r.Body))
	}

	return v2.ResponseDetailsView{Status: r.Status, Body: body, Headers: r.Headers, EncodedBody: needsEncoding}
}

<commit_msg>Added some comments to explain a bit of a backwards bit of code<commit_after>package models

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/SpectoLabs/hoverfly/core/handlers/v1"
	"github.com/SpectoLabs/hoverfly/core/handlers/v2"
	. "github.com/SpectoLabs/hoverfly/core/util"
	"github.com/tdewolff/minify"
	"github.com/tdewolff/minify/json"
	"github.com/tdewolff/minify/xml"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
)

const (
	contentTypeJSON = "application/json"
	contentTypeXML  = "application/xml"
	otherType       = "otherType"
)

var (
	rxJSON = regexp.MustCompile("[/+]json$")
	rxXML  = regexp.MustCompile("[/+]xml$")
	// mime types which will not be base 64 encoded when exporting as JSON
	supportedMimeTypes = [...]string{"text", "plain", "css", "html", "json", "xml", "js", "javascript"}
	minifiers          *minify.M
)

func init() {
	// GetNewMinifiers - sets minify.M with prepared xml/json minifiers
	minifiers = minify.New()
	minifiers.AddFuncRegexp(regexp.MustCompile("[/+]xml$"), xml.Minify)
	minifiers.AddFuncRegexp(regexp.MustCompile("[/+]json$"), json.Minify)
}

// Payload structure holds request and response structure
type RequestResponsePair struct {
	Response ResponseDetails `json:"response"`
	Request  RequestDetails  `json:"request"`
}

func (this RequestResponsePair) Id() string {
	return this.Request.Hash()
}

func (this RequestResponsePair) IdWithoutHost() string {
	return this.Request.HashWithoutHost()
}

// Encode method encodes all exported Payload fields to bytes
func (this *RequestResponsePair) Encode() ([]byte, error) {
	buf := new(bytes.Buffer)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(this)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (this *RequestResponsePair) ConvertToRequestResponsePairView() *v1.RequestResponsePairView {
	return &v1.RequestResponsePairView{Response: this.Response.ConvertToResponseDetailsView(), Request: this.Request.ConvertToRequestDetailsView()}
}

func (this *RequestResponsePair) ConvertToV2RequestResponsePairView() v2.RequestResponsePairView {
	return v2.RequestResponsePairView{Response: this.Response.ConvertToV2ResponseDetailsView(), Request: this.Request.ConvertToV2RequestDetailsView()}
}

// NewPayloadFromBytes decodes supplied bytes into Payload structure
func NewRequestResponsePairFromBytes(data []byte) (*RequestResponsePair, error) {
	var pair *RequestResponsePair
	buf := bytes.NewBuffer(data)
	dec := gob.NewDecoder(buf)
	err := dec.Decode(&pair)
	if err != nil {
		return nil, err
	}
	return pair, nil
}

func NewRequestResponsePairFromRequestResponsePairView(pairView v1.RequestResponsePairView) RequestResponsePair {
	return RequestResponsePair{
		Response: NewResponseDetailsFromResponseDetailsView(pairView.Response),
		Request:  NewRequestDetailsFromRequestDetailsView(pairView.Request),
	}
}

// RequestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type RequestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Scheme      string              `json:"scheme"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	Headers     map[string][]string `json:"headers"`
}

func NewRequestDetailsFromHttpRequest(req *http.Request) (RequestDetails, error) {
	if req.Body == nil {
		req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	reqBody, err := extractRequestBody(req)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  "capture",
		}).Error("Got error while reading request body")
		return RequestDetails{}, err
	}

	requestDetails := RequestDetails{
		Path:        req.URL.Path,
		Method:      req.Method,
		Destination: req.Host,
		Scheme:      req.URL.Scheme,
		Query:       req.URL.RawQuery,
		Body:        string(reqBody),
		Headers:     req.Header,
	}
	return requestDetails, nil
}

func extractRequestBody(req *http.Request) (extract []byte, err error) {
	save := req.Body
	savecl := req.ContentLength

	save, req.Body, err = CopyBody(req.Body)
	if err != nil {
		return
	}

	defer req.Body.Close()
	extract, err = ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}

	req.Body = save
	req.ContentLength = savecl
	return extract, nil
}

func CopyBody(body io.ReadCloser) (resp1, resp2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(body); err != nil {
		return nil, nil, err
	}
	if err = body.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func NewRequestDetailsFromRequestDetailsView(data v1.RequestDetailsView) RequestDetails {
	return RequestDetails{
		Path:        PointerToString(data.Path),
		Method:      PointerToString(data.Method),
		Destination: PointerToString(data.Destination),
		Scheme:      PointerToString(data.Scheme),
		Query:       PointerToString(data.Query),
		Body:        PointerToString(data.Body),
		Headers:     data.Headers,
	}
}

func (this *RequestDetails) ConvertToRequestDetailsView() v1.RequestDetailsView {
	s := "recording"
	return v1.RequestDetailsView{
		RequestType: &s,
		Path:        &this.Path,
		Method:      &this.Method,
		Destination: &this.Destination,
		Scheme:      &this.Scheme,
		Query:       &this.Query,
		Body:        &this.Body,
		Headers:     this.Headers,
	}
}

func (this *RequestDetails) ConvertToV2RequestDetailsView() v2.RequestDetailsView {
	s := "recording"
	return v2.RequestDetailsView{
		RequestType: &s,
		Path:        &this.Path,
		Method:      &this.Method,
		Destination: &this.Destination,
		Scheme:      &this.Scheme,
		Query:       &this.Query,
		Body:        &this.Body,
		Headers:     this.Headers,
	}
}

func (r *RequestDetails) concatenate(withHost bool) string {
	var buffer bytes.Buffer

	if withHost {
		buffer.WriteString(r.Destination)
	}

	buffer.WriteString(r.Path)
	buffer.WriteString(r.Method)
	buffer.WriteString(r.Query)
	if len(r.Body) > 0 {
		ct := r.getContentType()

		if ct == contentTypeJSON || ct == contentTypeXML {
			buffer.WriteString(r.minifyBody(ct))
		} else {
			log.WithFields(log.Fields{
				"content-type": r.Headers["Content-Type"],
			}).Debug("unknown content type")

			buffer.WriteString(r.Body)
		}
	}

	return buffer.String()
}

func (r *RequestDetails) minifyBody(mediaType string) (minified string) {
	var err error
	minified, err = minifiers.String(mediaType, r.Body)
	if err != nil {
		log.WithFields(log.Fields{
			"error":       err.Error(),
			"destination": r.Destination,
			"path":        r.Path,
			"method":      r.Method,
		}).Errorf("failed to minify request body, media type given: %s. Request matching might fail", mediaType)
		return r.Body
	}
	log.Debugf("body minified, mediatype: %s", mediaType)
	return minified
}

func (r *RequestDetails) Hash() string {
	h := md5.New()
	io.WriteString(h, r.concatenate(true))
	return fmt.Sprintf("%x", h.Sum(nil))
}
func (r *RequestDetails) HashWithoutHost() string {
	h := md5.New()
	io.WriteString(h, r.concatenate(false))
	return fmt.Sprintf("%x", h.Sum(nil))
}

func (r *RequestDetails) getContentType() string {
	for _, v := range r.Headers["Content-Type"] {
		if rxJSON.MatchString(v) {
			return contentTypeJSON
		}
		if rxXML.MatchString(v) {
			return contentTypeXML
		}
	}
	return otherType
}

// ResponseDetails structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type ResponseDetails struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

func NewResponseDetailsFromResponseDetailsView(data v1.ResponseDetailsView) ResponseDetails {
	body := data.Body

	if data.EncodedBody == true {
		decoded, _ := base64.StdEncoding.DecodeString(data.Body)
		body = string(decoded)
	}

	return ResponseDetails{Status: data.Status, Body: body, Headers: data.Headers}
}

// This function will create a JSON appriopriate version of ResponseDetails for the v1 API
// If the response headers indicate that the content is encoded, or it has a non-matching
// supported mimetype, we base64 encode it.
func (r *ResponseDetails) ConvertToResponseDetailsView() v1.ResponseDetailsView {
	needsEncoding := false

	// Check headers for gzip
	contentEncodingValues := r.Headers["Content-Encoding"]
	if len(contentEncodingValues) > 0 {
		needsEncoding = true
	} else {
		mimeType := http.DetectContentType([]byte(r.Body))
		needsEncoding = true
		for _, v := range supportedMimeTypes {
			if strings.Contains(mimeType, v) {
				needsEncoding = false
				break
			}
		}
	}

	// If contains gzip, base64 encode
	body := r.Body
	if needsEncoding {
		body = base64.StdEncoding.EncodeToString([]byte(r.Body))
	}

	return v1.ResponseDetailsView{Status: r.Status, Body: body, Headers: r.Headers, EncodedBody: needsEncoding}
}
// This function will create a JSON appriopriate version of ResponseDetails for the v2 API
// If the response headers indicate that the content is encoded, or it has a non-matching
// supported mimetype, we base64 encode it.
func (r *ResponseDetails) ConvertToV2ResponseDetailsView() v2.ResponseDetailsView {
	needsEncoding := false

	// Check headers for gzip
	contentEncodingValues := r.Headers["Content-Encoding"]
	if len(contentEncodingValues) > 0 {
		needsEncoding = true
	} else {
		mimeType := http.DetectContentType([]byte(r.Body))
		needsEncoding = true
		for _, v := range supportedMimeTypes {
			if strings.Contains(mimeType, v) {
				needsEncoding = false
				break
			}
		}
	}

	// If contains gzip, base64 encode
	body := r.Body
	if needsEncoding {
		body = base64.StdEncoding.EncodeToString([]byte(r.Body))
	}

	return v2.ResponseDetailsView{Status: r.Status, Body: body, Headers: r.Headers, EncodedBody: needsEncoding}
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package monitor implements the monitor service. A monitor repeatedly polls a
// key-transparency server's Mutations API and signs Map Roots if it could
// reconstruct clients can query.
package monitor

import (
	"bytes"
	"errors"
	"math/big"

	"github.com/golang/glog"
	"github.com/google/keytransparency/core/mutator/entry"
	"github.com/google/trillian"
	"github.com/google/trillian/merkle"
	"github.com/google/trillian/storage"
	"github.com/google/trillian/types"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
	statuspb "google.golang.org/genproto/googleapis/rpc/status"
)

var (
	// ErrInconsistentProofs occurs when the server returned different hashes
	// for the same inclusion proof node in the tree.
	ErrInconsistentProofs = errors.New("inconsistent inclusion proofs")
	// ErrInvalidLogConsistencyProof occurs when the log consistency proof does
	// not verify.
	ErrInvalidLogConsistencyProof = errors.New("invalid log consistency proof")
	// ErrInvalidLogInclusion occurs if the inclusion proof for the signed map
	// root into the log does not verify.
	ErrInvalidLogInclusion = errors.New("invalid log inclusion proof")
	// ErrInvalidLogSignature occurs if the log roots signature does not verify.
	ErrInvalidLogSignature = errors.New("invalid signature on log root")
	// ErrInvalidMapSignature occurs if the map roots signature does not verify.
	ErrInvalidMapSignature = errors.New("invalid signature on map root")
	// ErrInvalidMutation occurs when verification failed because of an invalid
	// mutation.
	ErrInvalidMutation = errors.New("invalid mutation")
	// ErrNotMatchingMapRoot occurs when the reconstructed root differs from the
	// one we received from the server.
	ErrNotMatchingMapRoot = errors.New("recreated root does not match")
)

// ErrList is a list of errors.
type ErrList []error

// AppendStatus adds a status errord, or the error about adding
// the status if the latter is not nil.
func (e *ErrList) AppendStatus(s *status.Status, err error) {
	if err != nil {
		*e = append(*e, err)
	} else {
		*e = append(*e, s.Err())
	}
}

// AppendErr adds a generic error to the list.
func (e *ErrList) appendErr(err ...error) {
	*e = append(*e, err...)
}

// Proto converts all the errors to statuspb.Status.
// If the original error was not a status.Status, we use codes.Unknown.
func (e *ErrList) Proto() []*statuspb.Status {
	errs := make([]*statuspb.Status, 0, len(*e))
	for _, err := range *e {
		if s, ok := status.FromError(err); ok {
			errs = append(errs, s.Proto())
			continue
		}
		errs = append(errs, status.Newf(codes.Unknown, "%v", err).Proto())
	}
	return errs
}

func (m *Monitor) verifyMutations(muts []*pb.MutationProof, oldRoot *trillian.SignedMapRoot, expectedNewRoot *types.MapRootV1) []error {
	errs := ErrList{}
	oldProofNodes := make(map[string][]byte)
	newLeaves := make([]merkle.HStar2LeafHash, 0, len(muts))
	glog.Infof("verifyMutations() called with %v mutations.", len(muts))

	for _, mut := range muts {
		oldLeaf, err := entry.FromLeafValue(mut.GetLeafProof().GetLeaf().GetLeafValue())
		if err != nil {
			errs.AppendStatus(status.Newf(codes.DataLoss, "could not decode leaf: %v", err).WithDetails(mut.GetLeafProof().GetLeaf()))
		}

		// verify that the provided leaf’s inclusion proof goes to revision e-1:
		index := mut.GetLeafProof().GetLeaf().GetIndex()
		if err := m.mapVerifier.VerifyMapLeafInclusion(oldRoot, mut.GetLeafProof()); err != nil {
			glog.Infof("VerifyMapInclusionProof(%x): %v", index, err)
			errs.AppendStatus(status.Newf(codes.DataLoss, "invalid  map inclusion proof: %v", err).WithDetails(mut.GetLeafProof()))
		}

		// compute the new leaf
		newValue, err := entry.MutateFn(oldLeaf, mut.GetMutation())
		if err != nil {
			glog.Infof("Mutation did not verify: %v", err)
			errs.AppendStatus(status.Newf(codes.DataLoss, "invalid mutation: %v", err).WithDetails(mut.GetMutation()))
		}
		leafNodeID := storage.NewNodeIDFromPrefixSuffix(index, storage.Suffix{}, m.mapVerifier.Hasher.BitLen())
		leaf, err := entry.ToLeafValue(newValue)
		if err != nil {
			glog.Infof("Failed to serialize: %v", err)
			errs.AppendStatus(status.Newf(codes.DataLoss, "failed to serialize: %v", err).WithDetails(newValue))
		}

		// BUG(gdbelvin): Proto serializations are not idempotent.
		// - Upgrade the hasher to use ObjectHash.
		// - Use deep compare between the tree and the computed value.
		leafHash, err := m.mapVerifier.Hasher.HashLeaf(m.mapVerifier.MapID, index, leaf)
		if err != nil {
			errs.appendErr(err)
		}
		newLeaves = append(newLeaves, merkle.HStar2LeafHash{
			Index:    leafNodeID.BigInt(),
			LeafHash: leafHash,
		})

		// store the proof hashes locally to recompute the tree below:
		sibIDs := leafNodeID.Siblings()
		proofs := mut.GetLeafProof().GetInclusion()
		for level, sibID := range sibIDs {
			proof := proofs[level]
			if p, ok := oldProofNodes[sibID.String()]; ok {
				// sanity check: for each mut overlapping proof nodes should be
				// equal:
				if !bytes.Equal(p, proof) {
					// this is really odd and should never happen
					errs.appendErr(ErrInconsistentProofs)
				}
			} else {
				if len(proof) > 0 {
					oldProofNodes[sibID.String()] = proof
				}
			}
		}
	}

	if err := m.validateMapRoot(expectedNewRoot, newLeaves, oldProofNodes); err != nil {
		errs.appendErr(err)
	}

	return errs
}

func (m *Monitor) validateMapRoot(newRoot *types.MapRootV1, mutatedLeaves []merkle.HStar2LeafHash, oldProofNodes map[string][]byte) error {
	// compute the new root using local intermediate hashes from revision e
	// (above proof hashes):
	hs2 := merkle.NewHStar2(m.mapVerifier.MapID, m.mapVerifier.Hasher)
	rootHash, err := hs2.HStar2Nodes([]byte{}, m.mapVerifier.Hasher.BitLen(), mutatedLeaves,
		func(depth int, index *big.Int) ([]byte, error) { //nolint:unparam
			nID := storage.NewNodeIDFromBigInt(depth, index, m.mapVerifier.Hasher.BitLen())
			if p, ok := oldProofNodes[nID.String()]; ok {
				return p, nil
			}
			return nil, nil
		}, nil)

	if err != nil {
		glog.Errorf("hs2.HStar2Nodes(_): %v", err)
		return ErrNotMatchingMapRoot
	}

	// verify rootHash
	if !bytes.Equal(rootHash, newRoot.RootHash) {
		return ErrNotMatchingMapRoot
	}

	return nil
}
<commit_msg>Follow google/trillian#1507 (#1260)<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package monitor implements the monitor service. A monitor repeatedly polls a
// key-transparency server's Mutations API and signs Map Roots if it could
// reconstruct clients can query.
package monitor

import (
	"bytes"
	"errors"
	"math/big"

	"github.com/golang/glog"
	"github.com/google/keytransparency/core/mutator/entry"
	"github.com/google/trillian"
	"github.com/google/trillian/merkle"
	"github.com/google/trillian/storage"
	"github.com/google/trillian/types"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
	statuspb "google.golang.org/genproto/googleapis/rpc/status"
)

var (
	// ErrInconsistentProofs occurs when the server returned different hashes
	// for the same inclusion proof node in the tree.
	ErrInconsistentProofs = errors.New("inconsistent inclusion proofs")
	// ErrInvalidLogConsistencyProof occurs when the log consistency proof does
	// not verify.
	ErrInvalidLogConsistencyProof = errors.New("invalid log consistency proof")
	// ErrInvalidLogInclusion occurs if the inclusion proof for the signed map
	// root into the log does not verify.
	ErrInvalidLogInclusion = errors.New("invalid log inclusion proof")
	// ErrInvalidLogSignature occurs if the log roots signature does not verify.
	ErrInvalidLogSignature = errors.New("invalid signature on log root")
	// ErrInvalidMapSignature occurs if the map roots signature does not verify.
	ErrInvalidMapSignature = errors.New("invalid signature on map root")
	// ErrInvalidMutation occurs when verification failed because of an invalid
	// mutation.
	ErrInvalidMutation = errors.New("invalid mutation")
	// ErrNotMatchingMapRoot occurs when the reconstructed root differs from the
	// one we received from the server.
	ErrNotMatchingMapRoot = errors.New("recreated root does not match")
)

// ErrList is a list of errors.
type ErrList []error

// AppendStatus adds a status errord, or the error about adding
// the status if the latter is not nil.
func (e *ErrList) AppendStatus(s *status.Status, err error) {
	if err != nil {
		*e = append(*e, err)
	} else {
		*e = append(*e, s.Err())
	}
}

// AppendErr adds a generic error to the list.
func (e *ErrList) appendErr(err ...error) {
	*e = append(*e, err...)
}

// Proto converts all the errors to statuspb.Status.
// If the original error was not a status.Status, we use codes.Unknown.
func (e *ErrList) Proto() []*statuspb.Status {
	errs := make([]*statuspb.Status, 0, len(*e))
	for _, err := range *e {
		if s, ok := status.FromError(err); ok {
			errs = append(errs, s.Proto())
			continue
		}
		errs = append(errs, status.Newf(codes.Unknown, "%v", err).Proto())
	}
	return errs
}

func (m *Monitor) verifyMutations(muts []*pb.MutationProof, oldRoot *trillian.SignedMapRoot, expectedNewRoot *types.MapRootV1) []error {
	errs := ErrList{}
	oldProofNodes := make(map[string][]byte)
	newLeaves := make([]merkle.HStar2LeafHash, 0, len(muts))
	glog.Infof("verifyMutations() called with %v mutations.", len(muts))

	for _, mut := range muts {
		oldLeaf, err := entry.FromLeafValue(mut.GetLeafProof().GetLeaf().GetLeafValue())
		if err != nil {
			errs.AppendStatus(status.Newf(codes.DataLoss, "could not decode leaf: %v", err).WithDetails(mut.GetLeafProof().GetLeaf()))
		}

		// verify that the provided leaf’s inclusion proof goes to revision e-1:
		index := mut.GetLeafProof().GetLeaf().GetIndex()
		if err := m.mapVerifier.VerifyMapLeafInclusion(oldRoot, mut.GetLeafProof()); err != nil {
			glog.Infof("VerifyMapInclusionProof(%x): %v", index, err)
			errs.AppendStatus(status.Newf(codes.DataLoss, "invalid  map inclusion proof: %v", err).WithDetails(mut.GetLeafProof()))
		}

		// compute the new leaf
		newValue, err := entry.MutateFn(oldLeaf, mut.GetMutation())
		if err != nil {
			glog.Infof("Mutation did not verify: %v", err)
			errs.AppendStatus(status.Newf(codes.DataLoss, "invalid mutation: %v", err).WithDetails(mut.GetMutation()))
		}
		leafNodeID := storage.NewNodeIDFromPrefixSuffix(index, storage.EmptySuffix, m.mapVerifier.Hasher.BitLen())
		leaf, err := entry.ToLeafValue(newValue)
		if err != nil {
			glog.Infof("Failed to serialize: %v", err)
			errs.AppendStatus(status.Newf(codes.DataLoss, "failed to serialize: %v", err).WithDetails(newValue))
		}

		// BUG(gdbelvin): Proto serializations are not idempotent.
		// - Upgrade the hasher to use ObjectHash.
		// - Use deep compare between the tree and the computed value.
		leafHash, err := m.mapVerifier.Hasher.HashLeaf(m.mapVerifier.MapID, index, leaf)
		if err != nil {
			errs.appendErr(err)
		}
		newLeaves = append(newLeaves, merkle.HStar2LeafHash{
			Index:    leafNodeID.BigInt(),
			LeafHash: leafHash,
		})

		// store the proof hashes locally to recompute the tree below:
		sibIDs := leafNodeID.Siblings()
		proofs := mut.GetLeafProof().GetInclusion()
		for level, sibID := range sibIDs {
			proof := proofs[level]
			if p, ok := oldProofNodes[sibID.String()]; ok {
				// sanity check: for each mut overlapping proof nodes should be
				// equal:
				if !bytes.Equal(p, proof) {
					// this is really odd and should never happen
					errs.appendErr(ErrInconsistentProofs)
				}
			} else {
				if len(proof) > 0 {
					oldProofNodes[sibID.String()] = proof
				}
			}
		}
	}

	if err := m.validateMapRoot(expectedNewRoot, newLeaves, oldProofNodes); err != nil {
		errs.appendErr(err)
	}

	return errs
}

func (m *Monitor) validateMapRoot(newRoot *types.MapRootV1, mutatedLeaves []merkle.HStar2LeafHash, oldProofNodes map[string][]byte) error {
	// compute the new root using local intermediate hashes from revision e
	// (above proof hashes):
	hs2 := merkle.NewHStar2(m.mapVerifier.MapID, m.mapVerifier.Hasher)
	rootHash, err := hs2.HStar2Nodes([]byte{}, m.mapVerifier.Hasher.BitLen(), mutatedLeaves,
		func(depth int, index *big.Int) ([]byte, error) { //nolint:unparam
			nID := storage.NewNodeIDFromBigInt(depth, index, m.mapVerifier.Hasher.BitLen())
			if p, ok := oldProofNodes[nID.String()]; ok {
				return p, nil
			}
			return nil, nil
		}, nil)

	if err != nil {
		glog.Errorf("hs2.HStar2Nodes(_): %v", err)
		return ErrNotMatchingMapRoot
	}

	// verify rootHash
	if !bytes.Equal(rootHash, newRoot.RootHash) {
		return ErrNotMatchingMapRoot
	}

	return nil
}
<|endoftext|>


<commit_before>package session

import (
	"crypto/tls"
	"fmt"
	"net/url"
	"time"

	"github.com/catatsuy/isucon6-final/bench/http"
	"github.com/catatsuy/isucon6-final/bench/http/cookiejar"
)

const (
	DefaultTimeout      = time.Duration(5) * time.Second
	MaxIdleConnsPerHost = 6
)

type Session struct {
	Scheme    string
	Host      string
	UserAgent string
	Client    *http.Client
	Transport *http.Transport
}

func New(baseURL string) *Session {
	s := &Session{}

	s.Transport = &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
		MaxIdleConnsPerHost: MaxIdleConnsPerHost,
	}

	jar, _ := cookiejar.New(nil)

	s.Client = &http.Client{
		Transport: s.Transport,
		Jar:       jar,
		Timeout:   DefaultTimeout,
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return fmt.Errorf("redirect attempted")
		},
	}

	s.UserAgent = "benchmarker"

	u, err := url.Parse(baseURL)
	if err != nil {
		panic(err) // should be cared at initialization
	}
	s.Scheme = u.Scheme
	s.Host = u.Host

	return s
}

func (s *Session) Bye() {
	s.Transport.CloseIdleConnections()
}
<commit_msg>Refactor RoomWatcher<commit_after>package session

import (
	"crypto/tls"
	"fmt"
	"net/url"
	"time"

	"github.com/catatsuy/isucon6-final/bench/http"
	"github.com/catatsuy/isucon6-final/bench/http/cookiejar"
)

const (
	DefaultTimeout      = time.Duration(10) * time.Second
	MaxIdleConnsPerHost = 6
)

type Session struct {
	Scheme    string
	Host      string
	UserAgent string
	Client    *http.Client
	Transport *http.Transport
}

func New(baseURL string) *Session {
	s := &Session{}

	s.Transport = &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
		MaxIdleConnsPerHost: MaxIdleConnsPerHost,
	}

	jar, _ := cookiejar.New(nil)

	s.Client = &http.Client{
		Transport: s.Transport,
		Jar:       jar,
		Timeout:   DefaultTimeout,
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return fmt.Errorf("redirect attempted")
		},
	}

	s.UserAgent = "benchmarker"

	u, err := url.Parse(baseURL)
	if err != nil {
		panic(err) // should be cared at initialization
	}
	s.Scheme = u.Scheme
	s.Host = u.Host

	return s
}

func (s *Session) Bye() {
	s.Transport.CloseIdleConnections()
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013 Zhen, LLC. http://zhen.io. All rights reserved.
 * Use of this source code is governed by the Apache 2.0 license.
 *
 */

package benchtools

import (
	"os"
	"io"
	"bytes"
	"log"
	"fmt"
	"time"
	"compress/gzip"
	"compress/lzw"
	"runtime/pprof"
	"code.google.com/p/snappy-go/snappy"
	"github.com/reducedb/encoding"
	"github.com/reducedb/encoding/cursor"
)

func TestCodec(codec encoding.Integer, in []int32, sizes []int) {
	for _, k := range sizes {
		if k > len(in) {
			continue
		}

		dur, out, err := Compress(codec, in[:k], k)
		if err != nil {
			log.Fatal(err)
		}

		dur2, out2, err2 := Uncompress(codec, out, k)
		if err2 != nil {
			log.Fatal(err2)
		}

		//log.Printf("benchtools/TestCodec: %f %.2f %.2f\n", float64(len(out)*32)/float64(k), (float64(k)/(float64(dur)/1000000000.0)/1000000.0), (float64(k)/(float64(dur2)/1000000000.0)/1000000.0))
		fmt.Printf("%f %.2f %.2f\n", float64(len(out)*32)/float64(k), (float64(k)/(float64(dur)/1000000000.0)/1000000.0), (float64(k)/(float64(dur2)/1000000000.0)/1000000.0))

		for i := 0; i < k; i++ {
			if in[i] != out2[i] {
				log.Fatalf("benchtools/TestCodec: Problem recovering. index = %d, in = %d, recovered = %d, original length = %d, recovered length = %d\n", i, in[i], out2[i], k, len(out2))
			}
		}
	}
}

func PprofCodec(codec encoding.Integer, in []int32, sizes []int) {
	for _, k := range sizes {
		if k > len(in) {
			continue
		}

		dur, out, err := PprofCompress(codec, in[:k], k)
		if err != nil {
			log.Fatal(err)
		}

		dur2, out2, err2 := PprofUncompress(codec, out, k)
		if err2 != nil {
			log.Fatal(err2)
		}

		log.Printf("benchtools/PprofCodec: %f %.2f %.2f\n", float64(len(out)*32)/float64(k), (float64(k)/(float64(dur)/1000000000.0)/1000000.0), (float64(k)/(float64(dur2)/1000000000.0)/1000000.0))

		for i := 0; i < k; i++ {
			if in[i] != out2[i] {
				log.Fatalf("benchtools/PprofCodec: Problem recovering. index = %d, in = %d, recovered = %d, original length = %d, recovered length = %d\n", i, in[i], out2[i], k, len(out2))
			}
		}
	}
}

func Compress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
	now := time.Now()
	out, err = runCompress(codec, in, length)
	if err != nil {
		return 0, nil, err
	}

	return time.Since(now).Nanoseconds(), out, nil
}

func Uncompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
	now := time.Now()
	out, err = runUncompress(codec, in, length)
	if err != nil {
		return 0, nil, err
	}

	return time.Since(now).Nanoseconds(), out, nil
}

func PprofCompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
	f, e := os.Create("cpu.compress.pprof")
	if e != nil {
		log.Fatal(e)
	}
	defer f.Close()

	pprof.StartCPUProfile(f)
	duration, out, err = Compress(codec, in, length)
	pprof.StopCPUProfile()

	return
}

func PprofUncompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
	f, e := os.Create("cpu.uncompress.pprof")
	if e != nil {
		log.Fatal(e)
	}
	defer f.Close()

	pprof.StartCPUProfile(f)
	duration, out, err = Uncompress(codec, in, length)
	pprof.StopCPUProfile()

	return
}

func RunTestGzip(data []byte) {
	log.Printf("encoding/RunTestGzip: Testing comprssion Gzip\n")

	var compressed bytes.Buffer
	w := gzip.NewWriter(&compressed)
	defer w.Close()
	now := time.Now()
	w.Write(data)

	cl := compressed.Len()
	log.Printf("encoding/RunTestGzip: Compressed from %d bytes to %d bytes in %d ns\n", len(data), cl, time.Since(now).Nanoseconds())

	recovered := make([]byte, len(data))
	r, _ := gzip.NewReader(&compressed)
	defer r.Close()

	total := 0
	n := 100
	var err error = nil
	for err != io.EOF && n != 0 {
		n, err = r.Read(recovered[total:])
		total += n
	}
	log.Printf("encoding/RunTestGzip: Uncompressed from %d bytes to %d bytes in %d ns\n", cl, len(recovered), time.Since(now).Nanoseconds())
}

func RunTestLZW(data []byte) {
	log.Printf("encoding/RunTestLZW: Testing comprssion LZW\n")

	var compressed bytes.Buffer
	w := lzw.NewWriter(&compressed, lzw.MSB, 8)
	defer w.Close()
	now := time.Now()
	w.Write(data)

	cl := compressed.Len()
	log.Printf("encoding/RunTestLZW: Compressed from %d bytes to %d bytes in %d ns\n", len(data), cl, time.Since(now).Nanoseconds())

	recovered := make([]byte, len(data))
	r := lzw.NewReader(&compressed, lzw.MSB, 8)
	defer r.Close()

	total := 0
	n := 100
	var err error = nil
	for err != io.EOF && n != 0 {
		n, err = r.Read(recovered[total:])
		total += n
	}
	log.Printf("encoding/RunTestLZW: Uncompressed from %d bytes to %d bytes in %d ns\n", cl, len(recovered), time.Since(now).Nanoseconds())
}

func RunTestSnappy(data []byte) {
	log.Printf("encoding/RunTestSnappy: Testing comprssion Snappy\n")

	now := time.Now()
	e, err := snappy.Encode(nil, data)
	if err != nil {
		log.Fatalf("encoding/RunTestSnappy: encoding error: %v\n", err)
	}
	log.Printf("encoding/RunTestSnappy: Compressed from %d bytes to %d bytes in %d ns\n", len(data), len(e), time.Since(now).Nanoseconds())

	d, err := snappy.Decode(nil, e)
	if err != nil {
		log.Fatalf("encoding/RunTestSnappy: decoding error: %v\n", err)
	}
	log.Printf("encoding/RunTestSnappy: Uncompressed from %d bytes to %d bytes in %d ns\n", len(e), len(d), time.Since(now).Nanoseconds())

	if !bytes.Equal(data, d) {
		log.Fatalf("encoding/RunTestSnappy: roundtrip mismatch\n")
	}
}

func runCompress(codec encoding.Integer, in []int32, length int) (out []int32, err error) {
	out = make([]int32, length*2)
	inpos := cursor.New()
	outpos := cursor.New()

	if err = codec.Compress(in, inpos, len(in), out, outpos); err != nil {
		return nil, err
	}

	out = out[:outpos.Get()]
	return out, nil
}

func runUncompress(codec encoding.Integer, in []int32, length int) (out []int32, err error) {
	out = make([]int32, length)
	inpos := cursor.New()
	outpos := cursor.New()

	if err = codec.Uncompress(in, inpos, len(in), out, outpos); err != nil {
		return nil, err
	}

	out = out[:outpos.Get()]
	return out, nil
}
<commit_msg>took out extra/unneeded compress/decompress functions<commit_after>/*
 * Copyright (c) 2013 Zhen, LLC. http://zhen.io. All rights reserved.
 * Use of this source code is governed by the Apache 2.0 license.
 *
 */

package benchtools

import (
	"os"
	"io"
	"bytes"
	"log"
	"fmt"
	"time"
	"compress/gzip"
	"compress/lzw"
	"runtime/pprof"
	"code.google.com/p/snappy-go/snappy"
	"github.com/reducedb/encoding"
	"github.com/reducedb/encoding/cursor"
)

func TestCodec(codec encoding.Integer, in []int32, sizes []int) {
	for _, k := range sizes {
		if k > len(in) {
			continue
		}

		dur, out, err := Compress(codec, in[:k], k)
		if err != nil {
			log.Fatal(err)
		}

		dur2, out2, err2 := Uncompress(codec, out, k)
		if err2 != nil {
			log.Fatal(err2)
		}

		//log.Printf("benchtools/TestCodec: %f %.2f %.2f\n", float64(len(out)*32)/float64(k), (float64(k)/(float64(dur)/1000000000.0)/1000000.0), (float64(k)/(float64(dur2)/1000000000.0)/1000000.0))
		fmt.Printf("%f %.2f %.2f\n", float64(len(out)*32)/float64(k), (float64(k)/(float64(dur)/1000000000.0)/1000000.0), (float64(k)/(float64(dur2)/1000000000.0)/1000000.0))

		for i := 0; i < k; i++ {
			if in[i] != out2[i] {
				log.Fatalf("benchtools/TestCodec: Problem recovering. index = %d, in = %d, recovered = %d, original length = %d, recovered length = %d\n", i, in[i], out2[i], k, len(out2))
			}
		}
	}
}

func PprofCodec(codec encoding.Integer, in []int32, sizes []int) {
	for _, k := range sizes {
		if k > len(in) {
			continue
		}

		dur, out, err := PprofCompress(codec, in[:k], k)
		if err != nil {
			log.Fatal(err)
		}

		dur2, out2, err2 := PprofUncompress(codec, out, k)
		if err2 != nil {
			log.Fatal(err2)
		}

		log.Printf("benchtools/PprofCodec: %f %.2f %.2f\n", float64(len(out)*32)/float64(k), (float64(k)/(float64(dur)/1000000000.0)/1000000.0), (float64(k)/(float64(dur2)/1000000000.0)/1000000.0))

		for i := 0; i < k; i++ {
			if in[i] != out2[i] {
				log.Fatalf("benchtools/PprofCodec: Problem recovering. index = %d, in = %d, recovered = %d, original length = %d, recovered length = %d\n", i, in[i], out2[i], k, len(out2))
			}
		}
	}
}

func PprofCompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
	f, e := os.Create("cpu.compress.pprof")
	if e != nil {
		log.Fatal(e)
	}
	defer f.Close()

	pprof.StartCPUProfile(f)
	duration, out, err = Compress(codec, in, length)
	pprof.StopCPUProfile()

	return
}

func PprofUncompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
	f, e := os.Create("cpu.uncompress.pprof")
	if e != nil {
		log.Fatal(e)
	}
	defer f.Close()

	pprof.StartCPUProfile(f)
	duration, out, err = Uncompress(codec, in, length)
	pprof.StopCPUProfile()

	return
}

func RunTestGzip(data []byte) {
	log.Printf("encoding/RunTestGzip: Testing comprssion Gzip\n")

	var compressed bytes.Buffer
	w := gzip.NewWriter(&compressed)
	defer w.Close()
	now := time.Now()
	w.Write(data)

	cl := compressed.Len()
	log.Printf("encoding/RunTestGzip: Compressed from %d bytes to %d bytes in %d ns\n", len(data), cl, time.Since(now).Nanoseconds())

	recovered := make([]byte, len(data))
	r, _ := gzip.NewReader(&compressed)
	defer r.Close()

	total := 0
	n := 100
	var err error = nil
	for err != io.EOF && n != 0 {
		n, err = r.Read(recovered[total:])
		total += n
	}
	log.Printf("encoding/RunTestGzip: Uncompressed from %d bytes to %d bytes in %d ns\n", cl, len(recovered), time.Since(now).Nanoseconds())
}

func RunTestLZW(data []byte) {
	log.Printf("encoding/RunTestLZW: Testing comprssion LZW\n")

	var compressed bytes.Buffer
	w := lzw.NewWriter(&compressed, lzw.MSB, 8)
	defer w.Close()
	now := time.Now()
	w.Write(data)

	cl := compressed.Len()
	log.Printf("encoding/RunTestLZW: Compressed from %d bytes to %d bytes in %d ns\n", len(data), cl, time.Since(now).Nanoseconds())

	recovered := make([]byte, len(data))
	r := lzw.NewReader(&compressed, lzw.MSB, 8)
	defer r.Close()

	total := 0
	n := 100
	var err error = nil
	for err != io.EOF && n != 0 {
		n, err = r.Read(recovered[total:])
		total += n
	}
	log.Printf("encoding/RunTestLZW: Uncompressed from %d bytes to %d bytes in %d ns\n", cl, len(recovered), time.Since(now).Nanoseconds())
}

func RunTestSnappy(data []byte) {
	log.Printf("encoding/RunTestSnappy: Testing comprssion Snappy\n")

	now := time.Now()
	e, err := snappy.Encode(nil, data)
	if err != nil {
		log.Fatalf("encoding/RunTestSnappy: encoding error: %v\n", err)
	}
	log.Printf("encoding/RunTestSnappy: Compressed from %d bytes to %d bytes in %d ns\n", len(data), len(e), time.Since(now).Nanoseconds())

	d, err := snappy.Decode(nil, e)
	if err != nil {
		log.Fatalf("encoding/RunTestSnappy: decoding error: %v\n", err)
	}
	log.Printf("encoding/RunTestSnappy: Uncompressed from %d bytes to %d bytes in %d ns\n", len(e), len(d), time.Since(now).Nanoseconds())

	if !bytes.Equal(data, d) {
		log.Fatalf("encoding/RunTestSnappy: roundtrip mismatch\n")
	}
}

func Compress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
	out = make([]int32, length*2)
	inpos := cursor.New()
	outpos := cursor.New()

	now := time.Now()
	if err = codec.Compress(in, inpos, len(in), out, outpos); err != nil {
		return 0, nil, err
	}

    return time.Since(now).Nanoseconds(), out[:outpos.Get()], nil
}

func Uncompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
	out = make([]int32, length)
	inpos := cursor.New()
	outpos := cursor.New()

	now := time.Now()
	if err = codec.Uncompress(in, inpos, len(in), out, outpos); err != nil {
		return 0, nil, err
	}

    return time.Since(now).Nanoseconds(), out[:outpos.Get()], nil
}
<|endoftext|>